update

Python3WebSpider · May 4, 2020 · 3a2cf9e · 3a2cf9e
1 parent d3fd457
commit 3a2cf9e
Show file tree

Hide file tree

Showing 5 changed files with 11 additions and 7 deletions.
diff --git a/deployment-comment.yml b/deployment-comment.yml
@@ -6,7 +6,7 @@ metadata:
 ---
 apiVersion: v1
 items:
-  - apiVersion: extensions/v1beta1
+  - apiVersion: apps/v1
     kind: Deployment
     metadata:
       annotations:
@@ -19,6 +19,9 @@ items:
       namespace: crawler
     spec:
       replicas: 10
+      selector:
+        matchLabels:
+          io.kompose.service: crawler-weibo-comment
       revisionHistoryLimit: 1
       strategy: {}
       template:
@@ -53,7 +56,7 @@ items:
                       key: universal
                 - name: PROXYPOOL_ENABLED
                   value: 'true'
-              image: germey/crawler-weibo-comment:${TAG}
+              image: germey/crawler-weibo-comment
               name: crawler-weibo-comment
               resources:
                 limits:

diff --git a/deployment-single.yml b/deployment-single.yml
@@ -4,7 +4,7 @@ metadata:
   name: crawler-weibo-single
   namespace: crawler
 spec:
-  schedule: "* */1 * * *"
+  schedule: "*/20 * * * *"
   jobTemplate:
     spec:
       template:
@@ -16,7 +16,7 @@ spec:
                 - single
               env:
                 - name: WEIBO_WEIBO_ID
-                  value: '4496797961825543'
+                  value: '4467107636950632'
                 - name: WEIBO_COOKIES
                   valueFrom:
                     secretKeyRef:

diff --git a/weibo/middlewares.py b/weibo/middlewares.py
@@ -78,7 +78,7 @@ def get_random_proxy(self):
         :return:
         """
         try:
-            if self.auth:
+            if getattr(self, 'auth') and self.auth:
                 response = requests.get(self.proxypool_url, timeout=5, auth=self.auth)
             else:
                 response = requests.get(self.proxypool_url, timeout=5)

diff --git a/weibo/settings.py b/weibo/settings.py
@@ -136,7 +136,7 @@
 
 # definition of proxy
 PROXYPOOL_URL = env.str('PROXYPOOL_URL')
-PROXYTUNNEL_URL = env.str('PROXYTUNNEL_URL')
+# PROXYTUNNEL_URL = env.str('PROXYTUNNEL_URL')
 
 # definition of elasticsearch
 ELASTICSEARCH_CONNECTION_STRING = env.str('ELASTICSEARCH_CONNECTION_STRING')
diff --git a/weibo/spiders/single.py b/weibo/spiders/single.py
@@ -12,6 +12,7 @@
 START_COMMENT_ID = env.str('START_COMMENT_ID', None)
 print('START_COMMENT_ID', START_COMMENT_ID)
 
+
 class SingleSpider(Spider):
     """
     comment spider of single weibo
@@ -45,7 +46,7 @@ class SingleSpider(Spider):
         "X-Requested-With": "XMLHttpRequest",
     }
 
-    cookies = env.str('WEIBO_COOKIES')
+    cookies = env.str('WEIBO_COOKIES', '')
     page = 1
 
     def start_requests(self):