Merge pull request #364 from EBI-Metagenomics/develop

Develop

emgapi/management/commands/populate_metagenomics_exchange.py

@@ -131,6 +131,9 @@ def process_to_index_and_update_records(self, analyses_to_index_and_update):
                         mgya=annotation_job.accession,
                         sequence_accession=sequence_accession,
                     )
+                    if not response:
+                        logging.warning(f"Error occurred {annotation_job}")
+                        continue
                     if response.ok:
                         logging.info(f"Successfully added {annotation_job}")
                         registry_id, metadata_match = self.mgx_api.check_analysis(
@@ -171,9 +174,19 @@ def process_to_index_and_update_records(self, analyses_to_index_and_update):
                         else:
                             logging.error(f"Analysis {annotation_job} update failed")
                     else:
-                        logging.debug(
-                            f"No edit for {annotation_job}, metadata is correct"
-                        )
+                        if annotation_job.mgx_accession and annotation_job.last_mgx_indexed:
+                            logging.info(
+                                f"No edit for {annotation_job}, metadata is correct"
+                            )
+                        else:
+                            logging.info(
+                                f"Metadata is correct but {annotation_job} is missing in DB. Adding."
+                            )
+                            annotation_job.mgx_accession = registry_id
+                            annotation_job.last_mgx_indexed = (
+                                    timezone.now() + timedelta(minutes=1)
+                            )
+                            jobs_to_update.append(annotation_job)
 
             AnalysisJob.objects.bulk_update(
                 jobs_to_update,

emgapi/metagenomics_exchange.py

@@ -17,8 +17,9 @@
 import logging
 
 import requests
+import time
 from django.conf import settings
-from requests.exceptions import HTTPError
+from requests.exceptions import HTTPError, JSONDecodeError
 
 
 class MetagenomicsExchangeAPI:
@@ -106,16 +107,38 @@ def add_analysis(self, mgya: str, sequence_accession: str):
             The response object from the API request.
         """
         data = self.generate_metadata(mgya, sequence_accession)
-        try:
-            response = self.post_request(endpoint="datasets", data=data)
-        except HTTPError as http_error:
+        max_retries = 3
+        wait_time = 2  # seconds
+
+        for attempt in range(max_retries):
             try:
-                response_json = http_error.response.json()
-                logging.error(f"API response content: {response_json}")
-            except:
-                pass
-            raise http_error
-        return response
+                response = self.post_request(endpoint="datasets", data=data)
+                response.raise_for_status()  # Ensure we raise for HTTP errors
+                return response
+            except HTTPError as http_error:
+                if http_error.response.status_code == 500:
+                    logging.error(f"HTTP 500 error on attempt {attempt + 1} of {max_retries}: {http_error}")
+                    if attempt < max_retries - 1:
+                        time.sleep(wait_time)  # Wait before retrying
+                        continue
+                    else:
+                        logging.error("Max retries reached. Failing the request.")
+                else:
+                    # For other HTTP errors, do not retry and log the error
+                    try:
+                        response_json = http_error.response.json()
+                        logging.error(f"API response content: {response_json}")
+                    except JSONDecodeError:  # Catch JSON decoding errors
+                        logging.error(f"Failed to decode JSON from response: {http_error.response.text}")
+                    except Exception as e:
+                        logging.error(f"Unexpected error: {e}")
+                    # Log the HTTP status code and the error message
+                    logging.error(f"HTTPError occurred: {http_error}")
+                    return None
+            except Exception as e:
+                logging.error(f"An unexpected error occurred: {e}")
+                return None
+        return None
 
     def check_analysis(self, mgya: str, sequence_accession: str, metadata=None):
         """Check if a sequence exists in the M. Exchange

emgcli/__init__.py

@@ -1 +1 @@
-__version__: str = "2.4.47"
+__version__: str = "2.4.48"

pyproject.toml

@@ -117,7 +117,7 @@ max-line-length = 119
 """
 
 [tool.bumpversion]
-current_version = "2.4.47"
+current_version = "2.4.48"
 
 [[tool.bumpversion.files]]
 filename = "emgcli/__init__.py"