Merge pull request #100 from genomic-medicine-sweden/dev

Iris v.5.2.2

.gitignore

@@ -148,7 +148,7 @@ queries/*.json
 !queries/query_template.json
 proxies/*.json
 !proxies/proxy_template.json
-tests/*.ini
+tests/*/*.ini
 !tests/test_conf_template.ini
 
 .DS_Store

NGPIris/hcp/hcp.py

@@ -67,15 +67,37 @@ def __init__(self, credentials_path : str, use_ssl : bool = False, proxy_path :
         self.hcp = credentials_handler.hcp
         self.endpoint = "https://" + self.hcp["endpoint"]
 
+        # A lookup table for GMC names to HCP tenant names
+        gmc_tenant_map = {
+            "gmc-joint" : "vgtn0008",
+            "gmc-west" : "vgtn0012",
+            "gmc-southeast" : "vgtn0014",
+            "gmc-south" : "vgtn0015",
+            "gmc-orebro" : "vgtn0016",
+            "gmc-karolinska" : "vgtn0017",
+            "gmc-north" : "vgtn0018",
+            "gmc-uppsala" : "vgtn0019"
+        }
+
         self.tenant = None
-        for endpoint_format_string in ["https://{}.ngp-fs1000.vgregion.se", "https://{}.ngp-fs2000.vgregion.se", "https://{}.ngp-fs3000.vgregion.se", "https://{}.vgregion.sjunet.org"]:
+        for endpoint_format_string in ["https://{}.ngp-fs1000.vgregion.se", "https://{}.ngp-fs2000.vgregion.se", "https://{}.ngp-fs3000.vgregion.se", "https://{}.hcp1.vgregion.se", "https://{}.vgregion.sjunet.org"]:
             tenant_parse = parse(endpoint_format_string, self.endpoint) 
             if type(tenant_parse) is Result:
-                self.tenant = str(tenant_parse[0])
+                tenant = str(tenant_parse[0])
+                if endpoint_format_string == "https://{}.vgregion.sjunet.org": # Check if endpoint is Sjunet
+                    mapped_tenant = gmc_tenant_map.get(tenant)
+                    if mapped_tenant:
+                        self.tenant = mapped_tenant
+                    else:
+                        raise RuntimeError("The provided tenant name, \"" + tenant + "\", could is not a valid tenant name. Hint: did you spell it correctly?")
+                else:
+                    self.tenant = tenant
+
                 break
 
+
         if not self.tenant:
-            raise RuntimeError("Unable to parse endpoint. Make sure that you have entered the correct endpoint in your credentials JSON file. Hint: The endpoint should *not* contain \"https://\" or port numbers")
+            raise RuntimeError("Unable to parse endpoint, \"" + self.endpoint + "\". Make sure that you have entered the correct endpoint in your credentials JSON file. Hints:\n - The endpoint should *not* contain \"https://\" or port numbers\n - Is the endpoint spelled correctly?")
         self.base_request_url = self.endpoint + ":9090/mapi/tenants/" + self.tenant
         self.aws_access_key_id = self.hcp["aws_access_key_id"]
         self.aws_secret_access_key = self.hcp["aws_secret_access_key"]
@@ -126,7 +148,7 @@ def __init__(self, credentials_path : str, use_ssl : bool = False, proxy_path :
         else:
             self.transfer_config = TransferConfig(
                 multipart_threshold = 10 * _MB,
-                max_concurrency = 60,
+                max_concurrency = 30,
                 multipart_chunksize = 40 * _MB,
                 use_threads = True
             )
@@ -262,6 +284,11 @@ def list_objects(self, path_key : str = "", name_only : bool = False, files_only
 
         pages_filtered = pages.search(filter_string)
         for object in pages_filtered:
+            # If there are no objects in the bucket, then `object` will be None,
+            # which means that we should break out of the for loop
+            if not object:
+                break
+
             # Split the object key by "/"
             split_object = object["Key"].split("/")
             # Check if the object is within the specified path_key depth

pyproject.toml

@@ -1,11 +1,11 @@
 [project]
 name = "NGPIris"
-version = "5.2.0"
+version = "5.2.2"
 readme = "README.md"
 dependencies = [
     "requests >= 2.31.0",
     "urllib3 == 1.26.19",
-    "boto3 >= 1.26.76",
+    "boto3 == 1.35.81",
     "parse >= 1.19.1",
     "RapidFuzz >= 3.10.1",
     "tqdm >= 4.66.2",

tests/test_hcp.py

@@ -132,14 +132,23 @@ def test_download_folder(custom_config : CustomConfig) -> None:
     test_mount_bucket(custom_config)
     custom_config.hcp_h.download_folder("a folder of data/", custom_config.result_path) 
 
-def test_search_objects_in_bucket(custom_config : CustomConfig) -> None:
+def test_search_in_bucket(custom_config : CustomConfig) -> None:
     test_mount_bucket(custom_config)
     test_file = Path(custom_config.test_file_path).name 
-    custom_config.hcp_h.search_objects_in_bucket(test_file) 
+    custom_config.hcp_h.search_in_bucket(test_file) 
 
-def test_search_objects_in_bucket_without_mounting(custom_config : CustomConfig) -> None:
+def test_search_in_bucket_without_mounting(custom_config : CustomConfig) -> None:
     _hcp_h = custom_config.hcp_h 
-    _without_mounting(_hcp_h, HCPHandler.search_objects_in_bucket)
+    _without_mounting(_hcp_h, HCPHandler.search_in_bucket)
+
+def test_fuzzy_search_in_bucket(custom_config : CustomConfig) -> None:
+    test_mount_bucket(custom_config)
+    test_file = Path(custom_config.test_file_path).name 
+    custom_config.hcp_h.fuzzy_search_in_bucket(test_file) 
+
+def test_fuzzy_search_in_bucket_without_mounting(custom_config : CustomConfig) -> None:
+    _hcp_h = custom_config.hcp_h 
+    _without_mounting(_hcp_h, HCPHandler.fuzzy_search_in_bucket)
 
 def test_get_object_acl(custom_config : CustomConfig) -> None:
     test_mount_bucket(custom_config)