From ccb95fccc5f4ea7a348126ff0664c9bc935cd339 Mon Sep 17 00:00:00 2001 From: Danilo Bardusco Date: Tue, 4 Jul 2023 14:05:39 -0300 Subject: [PATCH 1/3] Fixed request.json verification that were preventing requests to complete --- scrapydweb/views/baseview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapydweb/views/baseview.py b/scrapydweb/views/baseview.py index 96af6d9..e9b3f74 100644 --- a/scrapydweb/views/baseview.py +++ b/scrapydweb/views/baseview.py @@ -69,7 +69,7 @@ def __init__(self, *args, **kwargs): self.logger.debug('request.args of %s\n%s', request.url, self.json_dumps(request.args)) if request.form: self.logger.debug('request.form from %s\n%s', request.url, self.json_dumps(request.form)) - if request.json: + if request.is_json: self.logger.debug('request.json from %s\n%s', request.url, self.json_dumps(request.json)) if request.files: self.logger.debug('request.files from %s\n\n %s\n', request.url, request.files) From ea0770c34926ee0bfa1db2528b6247f4d952ec0b Mon Sep 17 00:00:00 2001 From: Danilo Bardusco Date: Tue, 4 Jul 2023 14:56:24 -0300 Subject: [PATCH 2/3] Fixed application context issue. Recent versions of SQLAlchemy no longer support the bind parameter. --- scrapydweb/__init__.py | 7 ++++--- scrapydweb/run.py | 2 +- scrapydweb/utils/check_app_config.py | 7 ++++--- tests/test_a_factory.py | 4 ++-- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/scrapydweb/__init__.py b/scrapydweb/__init__.py index 9ae757d..e159834 100644 --- a/scrapydweb/__init__.py +++ b/scrapydweb/__init__.py @@ -118,9 +118,10 @@ def handle_db(app): # self.app = app # if app is not None: # self.init_app(app) - db.app = app # https://github.com/viniciuschiele/flask-apscheduler/blob/master/examples/flask_context.py - db.init_app(app) # http://flask-sqlalchemy.pocoo.org/2.3/contexts/ - db.create_all() + with app.app_context(): + db.app = app # https://github.com/viniciuschiele/flask-apscheduler/blob/master/examples/flask_context.py + db.init_app(app) # http://flask-sqlalchemy.pocoo.org/2.3/contexts/ + db.create_all() # https://blog.miguelgrinberg.com/post/the-flask-mega-tutorial-part-vii-error-handling @app.teardown_request diff --git a/scrapydweb/run.py b/scrapydweb/run.py index cdb9c2b..74e1483 100644 --- a/scrapydweb/run.py +++ b/scrapydweb/run.py @@ -40,7 +40,7 @@ def main(): # "scrapydweb -h" ends up here update_app_config(app.config, args) try: - check_app_config(app.config) + check_app_config(app.config, app) except AssertionError as err: logger.error("Check app config fail: ") sys.exit(u"\n{err}\n\nCheck and update your settings in {path}\n".format( diff --git a/scrapydweb/utils/check_app_config.py b/scrapydweb/utils/check_app_config.py index 6da43a5..e32972e 100644 --- a/scrapydweb/utils/check_app_config.py +++ b/scrapydweb/utils/check_app_config.py @@ -35,7 +35,7 @@ """, re.X) -def check_app_config(config): +def check_app_config(config, app): def check_assert(key, default, is_instance, allow_zero=True, non_empty=False, containing_type=None): if is_instance is int: if allow_zero: @@ -110,8 +110,9 @@ def check_assert(key, default, is_instance, allow_zero=True, non_empty=False, co # Note that check_app_config() is executed multiple times in test if node not in jobs_table_map: jobs_table_map[node] = create_jobs_table(re.sub(STRICT_NAME_PATTERN, '_', scrapyd_server)) - db.create_all(bind='jobs') - logger.debug("Created %s tables for JobsView", len(jobs_table_map)) + with app.app_context(): + db.create_all() + logger.debug("Created %s tables for JobsView", len(jobs_table_map)) check_assert('LOCAL_SCRAPYD_LOGS_DIR', '', str) check_assert('LOCAL_SCRAPYD_SERVER', '', str) diff --git a/tests/test_a_factory.py b/tests/test_a_factory.py index 868b0ad..5b53693 100644 --- a/tests/test_a_factory.py +++ b/tests/test_a_factory.py @@ -43,7 +43,7 @@ def test_check_app_config(app, client): # ['username:password@127.0.0.1:6800', ] app.config['SCRAPYD_SERVERS'] = app.config['_SCRAPYD_SERVERS'] - check_app_config(app.config) + check_app_config(app.config, app) strings = [] @@ -66,7 +66,7 @@ def test_check_app_config(app, client): # ['username:password@127.0.0.1:6800', ] app.config['SCRAPYD_SERVERS'] = app.config['_SCRAPYD_SERVERS'] - check_app_config(app.config) + check_app_config(app.config, app) assert app.config['LOGPARSER_PID'] is None assert app.config['POLL_PID'] is None From a97c0851bfea69c5b128cf8bb32947a50b6fc6a7 Mon Sep 17 00:00:00 2001 From: Danilo Bardusco Date: Wed, 5 Jul 2023 16:19:27 -0300 Subject: [PATCH 3/3] Fix telegram message --- scrapydweb/views/utilities/send_text.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapydweb/views/utilities/send_text.py b/scrapydweb/views/utilities/send_text.py index 6a8f112..584ab26 100644 --- a/scrapydweb/views/utilities/send_text.py +++ b/scrapydweb/views/utilities/send_text.py @@ -59,7 +59,7 @@ def __init__(self): self.text = self.view_args['text'] or request.args.get('text', None) if not self.text: - self.text = self.json_dumps(self.form) if self.form else 'test' + self.text = self.form.get('text') if self.form else 'test' self.logger.debug('text: %s', self.text) self.js = {}