updated unit tests

_ext/python/cli/actions/upload.py

@@ -24,13 +24,18 @@ def upload(args: Namespace):
 
     # variables
     name = args.name if args.name is not None else cfg.name
-    col_name = args.col_name if args.col_name is not None else cfg.col_name
+    description = args.description if args.description is not None else cfg.description
+    mode = args.mode if args.mode is not None else cfg.mode
+    priority = args.priority if args.priority is not None else cfg.priority
     cmd = args.cmd if args.cmd is not None else cfg.cmd
+    param = args.param if args.param is not None else cfg.param
+    col_name = args.col_name if args.col_name is not None else cfg.col_name
 
     # create spider
     if args.create:
         try:
-            _id = create_spider(name=name, col_name=col_name, cmd=cmd)
+            _id = create_spider(name=name, description=description, mode=mode, priority=priority, cmd=cmd, param=param,
+                                col_name=col_name)
             print_color(f'created spider {name} (id: {_id})', tag='success', tag_color='green', color='white')
         except HttpException:
             print_color(f'create spider {name} failed', tag='error', tag_color='red', color='white')
@@ -71,7 +76,7 @@ def upload(args: Namespace):
     print_color(f'failed: {stats["error"]}', tag='info', tag_color='cyan', color='white')
 
 
-def create_spider(name: str, col_name: str = None, cmd: str = None, description: str = None, mode: str = None,
+def create_spider(name: str, description: str = None, col_name: str = None, mode: str = None, cmd: str = None,
                   param: str = None, priority: int = None) -> str:
     # results collection name
     if col_name is None:

_ext/python/crawlab/config/spider.py

@@ -15,10 +15,26 @@ def __init__(self, value: Dict = None):
     def name(self):
         return self.get('name')
 
+    @property
+    def description(self):
+        return self.get('description')
+
+    @property
+    def mode(self):
+        return self.get('mode')
+
+    @property
+    def priority(self):
+        return self.get('priority')
+
     @property
     def cmd(self):
         return self.get('cmd')
 
+    @property
+    def param(self):
+        return self.get('param')
+
     @property
     def col_name(self):
         return self.get('col_name')

_ext/python/test/cli_action/upload_test.py

@@ -12,44 +12,90 @@
 
 
 class CliActionUploadTestCase(unittest.TestCase):
+    endpoint = os.environ.get('CRAWLAB_API_ADDRESS') or 'http://localhost:8000'
+
     @staticmethod
-    def test_upload():
-        dir_path = os.path.join(tempfile.gettempdir(), 'test_spider')
+    def _setup():
+        name = 'test_spider' + f'_{int(datetime.now().timestamp())}'
+        dir_path = os.path.join(tempfile.gettempdir(), name)
         if not os.path.exists(dir_path):
             os.makedirs(dir_path)
+        os.chdir(dir_path)
+        return name, dir_path
+
+    def test_upload(self):
+        name, dir_path = self._setup()
         with open(os.path.join(dir_path, 'main.py'), 'w') as f:
             f.write('print(\'hello world\')')
-        os.chdir(dir_path)
-        name = 'test_spider' + f'_{int(datetime.now().timestamp())}'
         description = 'test_description_' + f'_{int(datetime.now().timestamp())}'
+        mode = 'random'
+        priority = 1
         cmd = 'echo hello'
         param = 'test'
-        args = Namespace(
+        login(Namespace(
+            username='admin',
+            password='admin',
+            api_address=self.endpoint,
+        ))
+        upload(Namespace(
             id=None,
             dir=None,
             name=name,
             description=description,
+            mode=mode,
+            priority=priority,
             cmd=cmd,
-            col_name=None,
             param=param,
+            col_name=None,
             create=True,
-        )
-        endpoint = os.environ.get('CRAWLAB_API_ADDRESS') or 'http://localhost:8000'
-        login(Namespace(
-            username='admin',
-            password='admin',
-            api_address=endpoint,
         ))
-        upload(args)
 
-        res = requests.get(f'{endpoint}/spiders', headers={'Authorization': config.data.get("token")},
+        res = requests.get(f'{self.endpoint}/spiders', headers={'Authorization': config.data.get("token")},
                            params={'size': 1, 'page': 1, 'sort': '[]'})
         assert res.status_code == 200
         data = res.json().get('data')
         assert len(data) == 1
         spider = data[0]
         assert spider.get('name') == name
-        requests.delete(f'{endpoint}/spiders/{spider.get("_id")}')
+        assert spider.get('description') == description
+        assert spider.get('mode') == mode
+        assert spider.get('cmd') == cmd
+        assert spider.get('param') == param
+        requests.delete(f'{self.endpoint}/spiders/{spider.get("_id")}',
+                        headers={'Authorization': config.data.get("token")})
+
+    def test_upload_with_crawlab_json(self):
+        name, dir_path = self._setup()
+        with open(os.path.join(dir_path, 'main.py'), 'w') as f:
+            f.write('print(\'hello world\')')
+        with open(os.path.join(dir_path, 'crawlab.json'), 'w') as f:
+            f.write('{"name":"%s"}' % name)
+        login(Namespace(
+            username='admin',
+            password='admin',
+            api_address=self.endpoint,
+        ))
+        upload(Namespace(
+            id=None,
+            dir=None,
+            name=name,
+            description=None,
+            mode=None,
+            priority=None,
+            cmd=None,
+            param=None,
+            col_name=None,
+            create=True,
+        ))
+        res = requests.get(f'{self.endpoint}/spiders', headers={'Authorization': config.data.get("token")},
+                           params={'size': 10, 'page': 1, 'sort': '[]'})
+        assert res.status_code == 200
+        data = res.json().get('data')
+        assert len(data) == 10
+        spider = data[0]
+        assert spider.get('name') == name
+        requests.delete(f'{self.endpoint}/spiders/{spider.get("_id")}',
+                        headers={'Authorization': config.data.get("token")})
 
 
 if __name__ == '__main__':