[feature]change url crawler to Generators,add province selector

Author: imquanquan

.gitignore

@@ -105,3 +105,5 @@ venv.bak/
 
 # idea
 .idea/
+
+test1.py

test.py

@@ -28,8 +28,15 @@
 # 登录
 login_driver = tyc_login.TianyanchaLogin(PHONE, PASSWORD)
 
+"""
+爬取省份列表
+省份缩写可以查看天眼查 url 格式
+"""
+
+PROVINCE = ['gd']
+
 # 初始化 URL 爬虫
-url_crawler = tyc_urls_crawler.TianyanchaUrlsCrawler(login_driver.driver, KEYWORDS, STATUS)
+url_crawler = tyc_urls_crawler.TianyanchaUrlsCrawler(login_driver.driver, KEYWORDS, STATUS, PROVINCE)
 
 # 爬取返回 URL 列表
 urls = []

tyc_spider/tyc_urls_crawler.py

@@ -11,7 +11,7 @@ class TianyanchaUrlsCrawler:
     """
     爬详细页链接，传入登录后的 driver 以及关键字
     """
-    def __init__(self, driver, keywords, status=None):
+    def __init__(self, driver, keywords, status=None, province=None):
         """
         :param driver: selenium webdriver
         :param keywords: str
@@ -32,31 +32,39 @@ def __init__(self, driver, keywords, status=None):
         if not status:
             self.status = ['1', '2', '3', '4', '5']
         print(self.status)
-        self.url_fmt = 'https://www.tianyancha.com/search/os{status}/p{page_num}?key={keyword}&searchType=company'
+        if not province:
+            self.province = ['']
+        else:
+            self.province = province
+        self.url_fmt = 'https://{province}.tianyancha.com/search/' \
+                       'os{status}/p{page_num}?key={keyword}&searchType=company'
         self.url_list = []
 
     def crawl_urls(self):
-        for keyword in self.keywords:
-            for status in self.status:
-                pages = self.get_page_num(keyword, status)
-                for page_num in range(1, int(pages)+1):
-                    url = self.url_fmt.format(page_num=page_num, keyword=keyword, status=status)
-                    self.driver.get(url)
-                    sleep(randint(300, 600)/100)
+        for province in self.province:
+            for keyword in self.keywords:
+                for status in self.status:
+                    pages = self.get_page_num(keyword, status, province)
+                    for page_num in range(1, int(pages)+1):
+                        url = self.url_fmt.format(page_num=page_num, keyword=keyword, status=status, province=province)
+                        self.driver.get(url)
+                        sleep(randint(300, 600)/100)
 
-                    soup = BeautifulSoup(self.driver.page_source, 'lxml')
-                    links = soup.findAll('a', class_="name ")
-                    print(keyword, page_num)
-                    for link in links:
-                        yield link['href']
+                        soup = BeautifulSoup(self.driver.page_source, 'lxml')
+                        links = soup.findAll('a', class_="name ")
+                        print(keyword, page_num)
+                        for link in links:
+                            yield link['href']
 
-    def get_page_num(self, keyword, status):
+    def get_page_num(self, keyword, status, province):
         """
         爬取页数
         :param keyword: str
+        :param status: str
+        :param province: str
         :return: int
         """
-        url = self.url_fmt.format(page_num=1, keyword=keyword, status=status)
+        url = self.url_fmt.format(page_num=1, keyword=keyword, status=status, province=province)
 
         try:
             self.driver.get(url)