diff --git a/_ext/python/cli/actions/upload.py b/_ext/python/cli/actions/upload.py index 7ba0807..544423e 100644 --- a/_ext/python/cli/actions/upload.py +++ b/_ext/python/cli/actions/upload.py @@ -24,13 +24,18 @@ def upload(args: Namespace): # variables name = args.name if args.name is not None else cfg.name - col_name = args.col_name if args.col_name is not None else cfg.col_name + description = args.description if args.description is not None else cfg.description + mode = args.mode if args.mode is not None else cfg.mode + priority = args.priority if args.priority is not None else cfg.priority cmd = args.cmd if args.cmd is not None else cfg.cmd + param = args.param if args.param is not None else cfg.param + col_name = args.col_name if args.col_name is not None else cfg.col_name # create spider if args.create: try: - _id = create_spider(name=name, col_name=col_name, cmd=cmd) + _id = create_spider(name=name, description=description, mode=mode, priority=priority, cmd=cmd, param=param, + col_name=col_name) print_color(f'created spider {name} (id: {_id})', tag='success', tag_color='green', color='white') except HttpException: print_color(f'create spider {name} failed', tag='error', tag_color='red', color='white') @@ -71,7 +76,7 @@ def upload(args: Namespace): print_color(f'failed: {stats["error"]}', tag='info', tag_color='cyan', color='white') -def create_spider(name: str, col_name: str = None, cmd: str = None, description: str = None, mode: str = None, +def create_spider(name: str, description: str = None, col_name: str = None, mode: str = None, cmd: str = None, param: str = None, priority: int = None) -> str: # results collection name if col_name is None: diff --git a/_ext/python/crawlab/config/spider.py b/_ext/python/crawlab/config/spider.py index dd10bbf..2f7982d 100644 --- a/_ext/python/crawlab/config/spider.py +++ b/_ext/python/crawlab/config/spider.py @@ -15,10 +15,26 @@ def __init__(self, value: Dict = None): def name(self): return self.get('name') + @property + def description(self): + return self.get('description') + + @property + def mode(self): + return self.get('mode') + + @property + def priority(self): + return self.get('priority') + @property def cmd(self): return self.get('cmd') + @property + def param(self): + return self.get('param') + @property def col_name(self): return self.get('col_name') diff --git a/_ext/python/test/cli_action/upload_test.py b/_ext/python/test/cli_action/upload_test.py index 15aa90c..1fd2345 100644 --- a/_ext/python/test/cli_action/upload_test.py +++ b/_ext/python/test/cli_action/upload_test.py @@ -12,44 +12,90 @@ class CliActionUploadTestCase(unittest.TestCase): + endpoint = os.environ.get('CRAWLAB_API_ADDRESS') or 'http://localhost:8000' + @staticmethod - def test_upload(): - dir_path = os.path.join(tempfile.gettempdir(), 'test_spider') + def _setup(): + name = 'test_spider' + f'_{int(datetime.now().timestamp())}' + dir_path = os.path.join(tempfile.gettempdir(), name) if not os.path.exists(dir_path): os.makedirs(dir_path) + os.chdir(dir_path) + return name, dir_path + + def test_upload(self): + name, dir_path = self._setup() with open(os.path.join(dir_path, 'main.py'), 'w') as f: f.write('print(\'hello world\')') - os.chdir(dir_path) - name = 'test_spider' + f'_{int(datetime.now().timestamp())}' description = 'test_description_' + f'_{int(datetime.now().timestamp())}' + mode = 'random' + priority = 1 cmd = 'echo hello' param = 'test' - args = Namespace( + login(Namespace( + username='admin', + password='admin', + api_address=self.endpoint, + )) + upload(Namespace( id=None, dir=None, name=name, description=description, + mode=mode, + priority=priority, cmd=cmd, - col_name=None, param=param, + col_name=None, create=True, - ) - endpoint = os.environ.get('CRAWLAB_API_ADDRESS') or 'http://localhost:8000' - login(Namespace( - username='admin', - password='admin', - api_address=endpoint, )) - upload(args) - res = requests.get(f'{endpoint}/spiders', headers={'Authorization': config.data.get("token")}, + res = requests.get(f'{self.endpoint}/spiders', headers={'Authorization': config.data.get("token")}, params={'size': 1, 'page': 1, 'sort': '[]'}) assert res.status_code == 200 data = res.json().get('data') assert len(data) == 1 spider = data[0] assert spider.get('name') == name - requests.delete(f'{endpoint}/spiders/{spider.get("_id")}') + assert spider.get('description') == description + assert spider.get('mode') == mode + assert spider.get('cmd') == cmd + assert spider.get('param') == param + requests.delete(f'{self.endpoint}/spiders/{spider.get("_id")}', + headers={'Authorization': config.data.get("token")}) + + def test_upload_with_crawlab_json(self): + name, dir_path = self._setup() + with open(os.path.join(dir_path, 'main.py'), 'w') as f: + f.write('print(\'hello world\')') + with open(os.path.join(dir_path, 'crawlab.json'), 'w') as f: + f.write('{"name":"%s"}' % name) + login(Namespace( + username='admin', + password='admin', + api_address=self.endpoint, + )) + upload(Namespace( + id=None, + dir=None, + name=name, + description=None, + mode=None, + priority=None, + cmd=None, + param=None, + col_name=None, + create=True, + )) + res = requests.get(f'{self.endpoint}/spiders', headers={'Authorization': config.data.get("token")}, + params={'size': 10, 'page': 1, 'sort': '[]'}) + assert res.status_code == 200 + data = res.json().get('data') + assert len(data) == 10 + spider = data[0] + assert spider.get('name') == name + requests.delete(f'{self.endpoint}/spiders/{spider.get("_id")}', + headers={'Authorization': config.data.get("token")}) if __name__ == '__main__':