Merge pull request #55 from qld-gov-au/QOLDEV-490-fast-load-enhancements

[QOLDEV-490] fix timeout extension
qld-gov-au · Aug 7, 2023 · 340778b · 340778b
2 parents 03989cc + 388bc7b
commit 340778b
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 5 deletions.
diff --git a/ckanext/xloader/action.py b/ckanext/xloader/action.py
@@ -152,10 +152,11 @@ def xloader_submit(context, data_dict):
             'original_url': resource_dict.get('url'),
         }
     }
-    timeout = config.get('ckanext.xloader.job_timeout', '3600')
-    if not utils.datastore_resource_exists(res_id):
-        # Expand timeout for resources that have to be type-guessed
-        timeout = timeout * 3
+    # Expand timeout for resources that have to be type-guessed
+    timeout = config.get(
+        'ckanext.xloader.job_timeout',
+        '3600' if utils.datastore_resource_exists(res_id) else '10800')
+    log.debug("Timeout for XLoading resource %s is %s", res_id, timeout)
 
     try:
         job = enqueue_job(

diff --git a/ckanext/xloader/jobs.py b/ckanext/xloader/jobs.py
@@ -211,7 +211,7 @@ def tabulator_load():
     use_type_guessing = asbool(
         config.get('ckanext.xloader.use_type_guessing', config.get(
             'ckanext.xloader.just_load_with_messytables', False))) \
-        and datastore_resource_exists(resource['id']) \
+        and not datastore_resource_exists(resource['id']) \
         and os.path.getsize(tmp_file.name) <= MAX_TYPE_GUESSING_LENGTH
     logger.info("'use_type_guessing' mode is: %s", use_type_guessing)
     try: