From 7e07d62fa130880fb6e22dedec5cbba85fd5d0db Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Thu, 27 Jun 2024 09:53:09 +0200 Subject: [PATCH 01/53] Create pytype_and_pytest.yml --- .github/workflows/pytype_and_pytest.yml | 33 +++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/pytype_and_pytest.yml diff --git a/.github/workflows/pytype_and_pytest.yml b/.github/workflows/pytype_and_pytest.yml new file mode 100644 index 0000000..a7a79a6 --- /dev/null +++ b/.github/workflows/pytype_and_pytest.yml @@ -0,0 +1,33 @@ + +name: PyType and PyTest + +on: + push: + branches: [ "dev" ] + pull_request: + branches: [ "dev" ] + +permissions: + contents: read + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.11" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytype pytest + - name: Run pytype + run: | + pytype + - name: Test with pytest + run: | + pytest From 203df1f678e1f73fa0b6a9cd442179bc235b5dca Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Thu, 27 Jun 2024 10:04:42 +0200 Subject: [PATCH 02/53] Update pytype_and_pytest.yml --- .github/workflows/pytype_and_pytest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytype_and_pytest.yml b/.github/workflows/pytype_and_pytest.yml index a7a79a6..fafc19e 100644 --- a/.github/workflows/pytype_and_pytest.yml +++ b/.github/workflows/pytype_and_pytest.yml @@ -3,9 +3,9 @@ name: PyType and PyTest on: push: - branches: [ "dev" ] + branches: [ "dev", "master" ] pull_request: - branches: [ "dev" ] + branches: [ "dev", "master" ] permissions: contents: read From 716836071fb04ebede2b3224bf2a8c16a9c93e42 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Thu, 27 Jun 2024 10:35:32 +0200 Subject: [PATCH 03/53] Update pytype_and_pytest.yml --- .github/workflows/pytype_and_pytest.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pytype_and_pytest.yml b/.github/workflows/pytype_and_pytest.yml index fafc19e..5cb1fb5 100644 --- a/.github/workflows/pytype_and_pytest.yml +++ b/.github/workflows/pytype_and_pytest.yml @@ -25,6 +25,7 @@ jobs: run: | python -m pip install --upgrade pip pip install pytype pytest + pip install . - name: Run pytype run: | pytype From 4070e7f7b6755cbf192abc9c55a838d657489057 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Thu, 27 Jun 2024 10:40:55 +0200 Subject: [PATCH 04/53] Update pytype_and_pytest.yml --- .github/workflows/pytype_and_pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytype_and_pytest.yml b/.github/workflows/pytype_and_pytest.yml index 5cb1fb5..56bc1e8 100644 --- a/.github/workflows/pytype_and_pytest.yml +++ b/.github/workflows/pytype_and_pytest.yml @@ -11,7 +11,7 @@ permissions: contents: read jobs: - build: + pytype_and_pytest: runs-on: ubuntu-latest From 0fb3eb67538437ae203b383730289590f4e6729e Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Thu, 28 Nov 2024 16:33:59 +0100 Subject: [PATCH 05/53] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e281fc7..6ac38b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "NGPIris" -version = "5.1.0" +version = "5.1.1" readme = "README.md" dependencies = [ "requests >= 2.31.0", From 5401526b71855899c0ae3cc18604cee107b50d45 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Thu, 28 Nov 2024 16:34:27 +0100 Subject: [PATCH 06/53] Fix endpoint parsing problem --- NGPIris/hcp/hcp.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/NGPIris/hcp/hcp.py b/NGPIris/hcp/hcp.py index da80ba7..e235afa 100644 --- a/NGPIris/hcp/hcp.py +++ b/NGPIris/hcp/hcp.py @@ -62,10 +62,15 @@ def __init__(self, credentials_path : str, use_ssl : bool = False, proxy_path : credentials_handler = CredentialsHandler(credentials_path) self.hcp = credentials_handler.hcp self.endpoint = "https://" + self.hcp["endpoint"] - tenant_parse = parse("https://{}.hcp1.vgregion.se", self.endpoint) - if type(tenant_parse) is Result: - self.tenant = str(tenant_parse[0]) - else: # pragma: no cover + + self.tenant = None + for endpoint_format_string in ["https://{}.ngp-fs2000.vgregion.se", "https://{}.vgregion.sjunet.org"]: + tenant_parse = parse(endpoint_format_string, self.endpoint) + if type(tenant_parse) is Result: + self.tenant = str(tenant_parse[0]) + break + + if not self.tenant: raise RuntimeError("Unable to parse endpoint. Make sure that you have entered the correct endpoint in your credentials JSON file. Hint: The endpoint should *not* contain \"https://\" or port numbers") self.base_request_url = self.endpoint + ":9090/mapi/tenants/" + self.tenant self.aws_access_key_id = self.hcp["aws_access_key_id"] From 1dbd90f0e6fd24edbedb9ab8024b7ac8f5c17def Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Wed, 4 Dec 2024 09:37:48 +0100 Subject: [PATCH 07/53] Update .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 8d80134..2d07e78 100644 --- a/.gitignore +++ b/.gitignore @@ -142,7 +142,7 @@ build*/ *dist*/ dl/* -credentials/*.json +credentials/* !credentials/credentials_template.json queries/*.json !queries/query_template.json From a4121860bb4c57c18932d5f249a520d100800afe Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Wed, 11 Dec 2024 09:57:13 +0100 Subject: [PATCH 08/53] Update hcp.py --- NGPIris/hcp/hcp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NGPIris/hcp/hcp.py b/NGPIris/hcp/hcp.py index e235afa..3b1e8c8 100644 --- a/NGPIris/hcp/hcp.py +++ b/NGPIris/hcp/hcp.py @@ -64,7 +64,7 @@ def __init__(self, credentials_path : str, use_ssl : bool = False, proxy_path : self.endpoint = "https://" + self.hcp["endpoint"] self.tenant = None - for endpoint_format_string in ["https://{}.ngp-fs2000.vgregion.se", "https://{}.vgregion.sjunet.org"]: + for endpoint_format_string in ["https://{}.ngp-fs1000.vgregion.se", "https://{}.ngp-fs2000.vgregion.se", "https://{}.ngp-fs3000.vgregion.se", "https://{}.vgregion.sjunet.org"]: tenant_parse = parse(endpoint_format_string, self.endpoint) if type(tenant_parse) is Result: self.tenant = str(tenant_parse[0]) From 97a3384c539471cbbd83ee9d9cdf9e2d9c5c85ef Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Thu, 12 Dec 2024 13:27:40 +0100 Subject: [PATCH 09/53] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6ac38b7..e7548be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "NGPIris" -version = "5.1.1" +version = "5.1.1.dev1" readme = "README.md" dependencies = [ "requests >= 2.31.0", From bf1ed2e5af7c8baff7c8d7c99e93b04eb9d61f1f Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Thu, 12 Dec 2024 14:09:26 +0100 Subject: [PATCH 10/53] Update pyproject.toml --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e7548be..b96f0f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,11 +1,10 @@ [project] name = "NGPIris" -version = "5.1.1.dev1" +version = "5.1.1.dev2" readme = "README.md" dependencies = [ "requests >= 2.31.0", "urllib3 == 1.26.19", - "requests >= 2.31.0", "boto3 >= 1.26.76", "parse >= 1.19.1", "tqdm >= 4.66.2", From 8f3ed95b18f1422aae5a2340ff55a7331e3fd232 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Fri, 13 Dec 2024 11:25:50 +0100 Subject: [PATCH 11/53] Update __init__.py --- NGPIris/cli/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NGPIris/cli/__init__.py b/NGPIris/cli/__init__.py index 700a921..825dfe4 100644 --- a/NGPIris/cli/__init__.py +++ b/NGPIris/cli/__init__.py @@ -229,6 +229,9 @@ def simple_search(context : Context, bucket : str, search_string : str, case_sen """ Make simple search using substrings in a bucket/namespace on the HCP. + NOTE: This command does not use the HCI. Instead, it uses a linear search of + all the objects in the HCP. As such, this search might be slow. + BUCKET is the name of the bucket in which to make the search. SEARCH_STRING is any string that is to be used for the search. From e080ba683245c841ad776e1ebcdbfa054a459c5e Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Fri, 13 Dec 2024 11:25:54 +0100 Subject: [PATCH 12/53] Update README.md --- README.md | 110 +----------------------------------------------------- 1 file changed, 1 insertion(+), 109 deletions(-) diff --git a/README.md b/README.md index 03ef204..a8b19ae 100644 --- a/README.md +++ b/README.md @@ -149,116 +149,8 @@ Commands: test-connection Test the connection to a bucket/namespace. upload Upload files to an HCP bucket/namespace. ``` +Each sub-command has its own help message and is displayed by `iris path/to/your/credentials.json sub-command --help` -#### The `delete-folder` command -``` -Usage: iris CREDENTIALS delete-folder [OPTIONS] FOLDER BUCKET - - Delete a folder from an HCP bucket/namespace. - - FOLDER is the name of the folder to be deleted. - - BUCKET is the name of the bucket where the folder to be deleted exist. - -Options: - --help Show this message and exit. -``` - -#### The `delete-object` command -``` -Usage: iris CREDENTIALS delete-object [OPTIONS] OBJECT BUCKET - - Delete an object from an HCP bucket/namespace. - - OBJECT is the name of the object to be deleted. - - BUCKET is the name of the bucket where the object to be deleted exist. - -Options: - --help Show this message and exit. -``` - -#### The `download` command -``` -Usage: iris CREDENTIALS download [OPTIONS] OBJECT BUCKET LOCAL_PATH - - Download files from an HCP bucket/namespace. - - OBJECT is the name of the object to be downloaded. - - BUCKET is the name of the upload destination bucket. - - LOCAL_PATH is the path to where the downloaded objects are to be stored - locally. - -Options: - --help Show this message and exit. -``` - -#### The `list-buckets` command -``` -Usage: iris CREDENTIALS list-buckets [OPTIONS] - - List the available buckets/namespaces on the HCP. - -Options: - --help Show this message and exit. -``` - -#### The `list-objects` command -``` -Usage: iris CREDENTIALS list-objects [OPTIONS] BUCKET - - List the objects in a certain bucket/namespace on the HCP. - - BUCKET is the name of the bucket in which to list its objects. - -Options: - -no, --name-only BOOLEAN Output only the name of the objects instead of all - the associated metadata - --help Show this message and exit. -``` - -#### The `simple-search` command -``` -Usage: iris CREDENTIALS simple-search [OPTIONS] BUCKET SEARCH_STRING - - Make simple search using substrings in a bucket/namespace on the HCP. - - BUCKET is the name of the bucket in which to make the search. - - SEARCH_STRING is any string that is to be used for the search. - -Options: - -cs, --case_sensitive BOOLEAN Use case sensitivity? - --help Show this message and exit. -``` - -#### The `test-connection` command -``` -Usage: iris CREDENTIALS test-connection [OPTIONS] BUCKET - - Test the connection to a bucket/namespace. - - BUCKET is the name of the bucket for which a connection test should be made. - -Options: - --help Show this message and exit. -``` - -#### The `upload` command -``` -Usage: iris CREDENTIALS upload [OPTIONS] FILE_OR_FOLDER BUCKET - - Upload files to an HCP bucket/namespace. - - FILE-OR-FOLDER is the path to the file or folder of files to be uploaded. - - BUCKET is the name of the upload destination bucket. - -Options: - --help Show this message and exit. -``` ## Testing Assuming that the repository has been cloned, run the following tests: ```shell From 74696cb99e250145c90a47aa4e1d8c741362cebb Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Fri, 13 Dec 2024 11:25:57 +0100 Subject: [PATCH 13/53] Update Tutorial.md --- docs/Tutorial.md | 73 +++++++++++++++++++++--------------------------- 1 file changed, 32 insertions(+), 41 deletions(-) diff --git a/docs/Tutorial.md b/docs/Tutorial.md index 944daa0..5d84be9 100644 --- a/docs/Tutorial.md +++ b/docs/Tutorial.md @@ -15,6 +15,7 @@ - [The `HCPHandler` class](#the-hcphandler-class) - [Example use cases](#example-use-cases-1) - [Listing buckets/namespaces](#listing-bucketsnamespaces-1) + - [Listing objects in a bucket/namespace](#listing-objects-in-a-bucketnamespace) - [Downloading a file](#downloading-a-file-1) - [Uploading a file](#uploading-a-file-1) - [Uploading a folder](#uploading-a-folder) @@ -27,13 +28,16 @@ - [Look up information of an index](#look-up-information-of-an-index) - [Make queries](#make-queries) --- + +This tutorial was updated for `NGPIris 5.1`. + ## Introduction -IRIS 5 is a complete overhaul of the previous versions of IRIS, mainly in terms of its codebase. The general functionality like download from and upload to the HCP are still here, but might differ from previous versions from what you are used to. This document will hopefully shed some light on what you (the user) can expect and how your workflow with IRIS might change in comparison to previous versions of IRIS. +IRIS 5 is a complete overhaul of the previous versions of IRIS, mainly in terms of its codebase. The general functionality like download from and upload to the HCP are still here, but might differ from previous versions from what you are used to. This document will hopefully shed some light on what users can expect and how your workflow with IRIS might change in comparison to previous versions of IRIS. IRIS 5, like previous versions of IRIS, consists of two main parts: a Python package and an associated Command Line Interface (CLI), which are described below. ## CLI -IRIS 5 features a CLI like recent versions of IRIS. However, the new CLI is a bit different compared to before; the general structure of subcommands for the `iris` command are totally different, but it still has the subcommands you would come to expect. A new command, `iris_generate_credentials_file`, has also been added. It will generate an empty credentials file that can be filled in with your own NGPr credentials. +IRIS 5 features a CLI like recent versions of IRIS. However, the new CLI is a bit different compared to before; the general structure of subcommands for the `iris` command are vastly different, but it still has the subcommands you would come to expect. A new, separate, command called `iris_generate_credentials_file` has also been added. It will generate an empty credentials file that can be filled in with your own NGPr credentials. ### The `iris` command @@ -52,7 +56,7 @@ Options: Commands: delete-folder Delete a folder from an HCP bucket/namespace. delete-object Delete an object from an HCP bucket/namespace. - download Download files from an HCP bucket/namespace. + download Download a file or folder from an HCP bucket/namespace. list-buckets List the available buckets/namespaces on the HCP. list-objects List the objects in a certain bucket/namespace on the... simple-search Make simple search using substrings in a... @@ -61,42 +65,13 @@ Commands: ``` * `delete-folder`: Deletes a folder on the HCP * `delete-object`: Deletes an object on the HCP -* `download`: - * Downloads a file from a bucket/namespace on the HCP - * `iris path/to/credentials.json download --help`: - * ```cmd - Usage: iris CREDENTIALS download [OPTIONS] OBJECT BUCKET LOCAL_PATH - - Download files from an HCP bucket/namespace. - - OBJECT is the name of the object to be downloaded. - - BUCKET is the name of the upload destination bucket. - - LOCAL_PATH is the path to where the downloaded objects are to be stored - locally. - - Options: - --help Show this message and exit. - ``` +* `download`: Downloads a file or folder from a bucket/namespace on the HCP * `list-buckets`: Lists all buckets that the user is allowed to see * `list-objects`: Lists all objects that the user is allowed to see * `simple-search`: Performs a simple search using a substring in order to find matching objects in a bucket/namespace -* `upload`: - * Uploads either a file or a folder to a bucket/namespace on the HCP - * `iris path/to/credentials.json upload --help`: - * ```cmd - Usage: iris CREDENTIALS upload [OPTIONS] FILE_OR_FOLDER BUCKET - - Upload files to an HCP bucket/namespace. - - FILE-OR-FOLDER is the path to the file or folder of files to be uploaded. +* `test-connection`: Used for testing your connection to the HCP +* `upload`: Uploads a file or a folder to a bucket/namespace on the HCP - BUCKET is the name of the upload destination bucket. - - Options: - --help Show this message and exit. - ``` #### Example use cases The following subsections contain examples of simple use cases for IRIS 5. Of course, correct paths and bucket names should be replaced for your circumstances. ##### Listing buckets/namespaces @@ -105,11 +80,11 @@ iris path/to/your/credentials.json list-buckets ``` ##### Downloading a file ```shell -iris path/to/your/credentials.json download path/to/your/file/on/the/bucket the_name_of_the_bucket path/on/your/local/machine +iris path/to/your/credentials.json download the_name_of_the_bucket path/to/your/file/in/the/bucket path/on/your/local/machine ``` ##### Uploading a file ```shell -iris path/to/your/credentials.json upload destination/path/on/the/bucket the_name_of_the_bucket path/to/your/file/on/your/local/machine +iris path/to/your/credentials.json upload the_name_of_the_bucket destination/path/in/the/bucket path/to/your/file/on/your/local/machine ``` ##### Searching for a file By default, the `simple-search` command is case insensitive: @@ -122,11 +97,11 @@ iris path/to/your/credentials.json simple-search --case_sensitive True the_name_ ``` ##### Delete a file ```shell -iris path/to/your/credentials.json delete-object path/to/your/file/on/the/bucket the_name_of_the_bucket +iris path/to/your/credentials.json delete-object the_name_of_the_bucket path/to/your/file/in/the/bucket ``` ##### Delete a folder ```shell -iris path/to/your/credentials.json delete-folder path/to/your/folder/on/the/bucket/ the_name_of_the_bucket +iris path/to/your/credentials.json delete-folder the_name_of_the_bucket path/to/your/folder/on/the/bucket/ ``` ### The `iris_generate_credentials_file` command @@ -140,8 +115,9 @@ Usage: iris_generate_credentials_file [OPTIONS] plaintext. Options: - --path TEXT Path for where to put the new credentials file - --name TEXT Custom name for the credentials file + --path TEXT Path for where to put the new credentials file. + --name TEXT Custom name for the credentials file. Will filter out + everything after a "." character, if any exist. --help Show this message and exit. ``` Simply running `iris_generate_credentials_file` will generate a blank credentials file (which is just a JSON file) like the following: @@ -184,6 +160,7 @@ hcp_h = HCPHandler("credentials.json") hcp_h.mount_bucket("myBucket") ``` + #### Example use cases ##### Listing buckets/namespaces There is no need for mounting a bucket when listing all available buckets. However, credentials are still needed. As such, we can list all buckets with the following: @@ -194,6 +171,20 @@ hcp_h = HCPHandler("credentials.json") print(hcp_h.list_buckets()) ``` +##### Listing objects in a bucket/namespace +Since there might be many objects in a given bucket, a regular Python list would be memory inefficient. As such, a `Generator` is returned instead. Since `Generator`s are lazy objects, if we want to explicitly want all the objects we must first cast it to a `list` +```python +from NGPIris.hcp import HCPHandler + +hcp_h = HCPHandler("credentials.json") + +hcp_h.mount_bucket("myBucket") + +objects_generator = hcp_h.list_objects() + +print(list(objects_generator)) +``` + ##### Downloading a file ```python from NGPIris.hcp import HCPHandler From c2031c4ef9074dce4382d6d7c931c5d436bd040b Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Fri, 13 Dec 2024 15:38:27 +0100 Subject: [PATCH 14/53] Add fuzzy search --- NGPIris/hcp/hcp.py | 30 ++++++++++++++++++++++++------ pyproject.toml | 1 + 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/NGPIris/hcp/hcp.py b/NGPIris/hcp/hcp.py index 3b1e8c8..88ae114 100644 --- a/NGPIris/hcp/hcp.py +++ b/NGPIris/hcp/hcp.py @@ -34,6 +34,11 @@ search, Result ) +from rapidfuzz import ( + fuzz, + process, + utils +) from requests import get from urllib3 import disable_warnings from tqdm import tqdm @@ -499,13 +504,26 @@ def search_objects_in_bucket(self, search_string : str, case_sensitive : bool = search_result : list[str] = [] for key in self.list_objects(name_only = True): parse_object = search( + @check_mounted + def fuzzy_search_in_bucket(self, search_string : str, name_only : bool = True, case_sensitive : bool = False, threshold : int = 80): #-> Generator[str, None, None]: + if case_sensitive: + processor = None + else: + processor=utils.default_process + + if not name_only: + full_list = list(self.list_objects()) + for item, score, index in process.extract_iter( search_string, - key, - case_sensitive = case_sensitive - ) - if type(parse_object) is Result: - search_result.append(key) - return search_result + self.list_objects(name_only = True), + scorer = fuzz.partial_ratio, + processor = processor + ): + if score >= threshold: + if name_only: + yield item + else: + yield full_list[index] @check_mounted def get_object_acl(self, key : str) -> dict: diff --git a/pyproject.toml b/pyproject.toml index b96f0f5..329dcf9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,7 @@ dependencies = [ "urllib3 == 1.26.19", "boto3 >= 1.26.76", "parse >= 1.19.1", + "RapidFuzz >= 3.10.1", "tqdm >= 4.66.2", "click >= 8.1.7", "bitmath == 1.3.3.1", From ed4d26fd658351aca65913b71cfafc1a7c9e1f1c Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Fri, 13 Dec 2024 15:39:19 +0100 Subject: [PATCH 15/53] Update hcp.py --- NGPIris/hcp/hcp.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/NGPIris/hcp/hcp.py b/NGPIris/hcp/hcp.py index 88ae114..535b2a4 100644 --- a/NGPIris/hcp/hcp.py +++ b/NGPIris/hcp/hcp.py @@ -505,14 +505,22 @@ def search_objects_in_bucket(self, search_string : str, case_sensitive : bool = for key in self.list_objects(name_only = True): parse_object = search( @check_mounted - def fuzzy_search_in_bucket(self, search_string : str, name_only : bool = True, case_sensitive : bool = False, threshold : int = 80): #-> Generator[str, None, None]: + def fuzzy_search_in_bucket( + self, + search_string : str, + name_only : bool = True, + case_sensitive : bool = False, + threshold : int = 80 + ) -> Generator: + if case_sensitive: processor = None else: - processor=utils.default_process + processor = utils.default_process if not name_only: full_list = list(self.list_objects()) + for item, score, index in process.extract_iter( search_string, self.list_objects(name_only = True), From a7785cb414f041a3ba039cfa2459e59c3e8c2953 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Fri, 13 Dec 2024 15:39:47 +0100 Subject: [PATCH 16/53] Make `search_objects_in_bucket` use generators --- NGPIris/hcp/hcp.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/NGPIris/hcp/hcp.py b/NGPIris/hcp/hcp.py index 535b2a4..a253ba3 100644 --- a/NGPIris/hcp/hcp.py +++ b/NGPIris/hcp/hcp.py @@ -31,7 +31,6 @@ from json import dumps from parse import ( parse, - search, Result ) from rapidfuzz import ( @@ -488,7 +487,7 @@ def delete_folder(self, key : str, verbose : bool = True) -> None: self.delete_objects(object_path_in_folder + [key], verbose = verbose) @check_mounted - def search_objects_in_bucket(self, search_string : str, case_sensitive : bool = False) -> list[str]: + def search_objects_in_bucket(self, search_string : str, case_sensitive : bool = False) -> Generator[dict, None, None]: """ Simple search method using substrings in order to find certain objects. Case insensitive by default. Does not utilise the HCI @@ -501,9 +500,11 @@ def search_objects_in_bucket(self, search_string : str, case_sensitive : bool = :return: List of object names that match the in some way to the object names :rtype: list[str] """ - search_result : list[str] = [] - for key in self.list_objects(name_only = True): - parse_object = search( + paginator : Paginator = self.s3_client.get_paginator("list_objects_v2") + pages : PageIterator = paginator.paginate(Bucket = self.bucket_name) + for object in pages.search("Contents[?contains(Key, '" + search_string + "')][]"): + yield object + @check_mounted def fuzzy_search_in_bucket( self, From c2566ac852ce115d0f3e1e31d9b9239fa0a8dc79 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Fri, 13 Dec 2024 16:07:19 +0100 Subject: [PATCH 17/53] Update hcp.py --- NGPIris/hcp/hcp.py | 63 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 19 deletions(-) diff --git a/NGPIris/hcp/hcp.py b/NGPIris/hcp/hcp.py index a253ba3..cbf5355 100644 --- a/NGPIris/hcp/hcp.py +++ b/NGPIris/hcp/hcp.py @@ -177,7 +177,7 @@ def test_connection(self, bucket_name : str = "") -> dict: elif bucket_name: pass else: - raise RuntimeError("No bucket selected. Either use `mount_bucket` first or supply the optional `bucket_name` paramter for `test_connection`") + raise RuntimeError("No bucket selected. Either use `mount_bucket` first or supply the optional `bucket_name` parameter for `test_connection`") try: response = dict(self.s3_client.head_bucket(Bucket = bucket_name)) except EndpointConnectionError as e: # pragma: no cover @@ -465,18 +465,18 @@ def delete_object(self, key : str, verbose : bool = True) -> None: @check_mounted def delete_folder(self, key : str, verbose : bool = True) -> None: """ - Delete a folder of objects in the mounted bucket. If there are subfolders, a RuntimeError is raisesd + Delete a folder of objects in the mounted bucket. If there are subfolders, a RuntimeError is raised :param key: The folder of objects to be deleted :type key: str :param verbose: Print the result of the deletion. defaults to True :type verbose: bool, optional - :raises RuntimeError: If there are subfolders, a RuntimeError is raisesd + :raises RuntimeError: If there are subfolders, a RuntimeError is raised """ if key[-1] != "/": key += "/" object_path_in_folder = [] - for s in self.search_objects_in_bucket(key): + for s in self.search_in_bucket(key): parse_object = parse(key + "{}", s) if type(parse_object) is Result: object_path_in_folder.append(s) @@ -487,32 +487,57 @@ def delete_folder(self, key : str, verbose : bool = True) -> None: self.delete_objects(object_path_in_folder + [key], verbose = verbose) @check_mounted - def search_objects_in_bucket(self, search_string : str, case_sensitive : bool = False) -> Generator[dict, None, None]: + def search_in_bucket( + self, + search_string : str, + name_only : bool = True, + case_sensitive : bool = False + ) -> Generator: """ - Simple search method using substrings in order to find certain objects. Case insensitive by default. Does not utilise the HCI + Simple search method using exact substrings in order to find certain + objects. Case insensitive by default. Does not utilise the HCI :param search_string: Substring to be used in the search :type search_string: str + :param name_only: If True, yield only a the object names. If False, yield the full metadata about each object. Defaults to False. + :type name_only: bool, optional + :param case_sensitive: Case sensitivity. Defaults to False :type case_sensitive: bool, optional - :return: List of object names that match the in some way to the object names - :rtype: list[str] + :return: A generator of objects based on the search string + :rtype: Generator """ - paginator : Paginator = self.s3_client.get_paginator("list_objects_v2") - pages : PageIterator = paginator.paginate(Bucket = self.bucket_name) - for object in pages.search("Contents[?contains(Key, '" + search_string + "')][]"): - yield object + return self.fuzzy_search_in_bucket(search_string, name_only, case_sensitive, 100) + @check_mounted def fuzzy_search_in_bucket( - self, - search_string : str, - name_only : bool = True, - case_sensitive : bool = False, - threshold : int = 80 - ) -> Generator: + self, + search_string : str, + name_only : bool = True, + case_sensitive : bool = False, + threshold : int = 80 + ) -> Generator: + """ + Fuzzy search implementation based on the `RapidFuzz` library. + + :param search_string: Substring to be used in the search + :type search_string: str + + :param name_only: If True, yield only a the object names. If False, yield the full metadata about each object. Defaults to False. + :type name_only: bool, optional + + :param case_sensitive: Case sensitivity. Defaults to False + :type case_sensitive: bool, optional + + :param threshold: The fuzzy search similarity score. Defaults to 80 + :type threshold: int, optional + + :return: A generator of objects based on the search string + :rtype: Generator + """ if case_sensitive: processor = None @@ -521,7 +546,7 @@ def fuzzy_search_in_bucket( if not name_only: full_list = list(self.list_objects()) - + for item, score, index in process.extract_iter( search_string, self.list_objects(name_only = True), From afaba28a3aab9e23ac43eecc6657430a88a4e29e Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Mon, 16 Dec 2024 13:11:33 +0100 Subject: [PATCH 18/53] Update __init__.py --- NGPIris/cli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NGPIris/cli/__init__.py b/NGPIris/cli/__init__.py index 825dfe4..3fc1bd4 100644 --- a/NGPIris/cli/__init__.py +++ b/NGPIris/cli/__init__.py @@ -238,7 +238,7 @@ def simple_search(context : Context, bucket : str, search_string : str, case_sen """ hcph : HCPHandler = get_HCPHandler(context) hcph.mount_bucket(bucket) - list_of_results = hcph.search_objects_in_bucket(search_string, case_sensitive) + list_of_results = hcph.search_in_bucket(search_string, case_sensitive) click.echo("Search results:") for result in list_of_results: click.echo("- " + result) From b3df429008723197d85cd47d4c18f33c506ca923 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Mon, 16 Dec 2024 13:33:37 +0100 Subject: [PATCH 19/53] Add fuzzy search command --- NGPIris/cli/__init__.py | 68 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 4 deletions(-) diff --git a/NGPIris/cli/__init__.py b/NGPIris/cli/__init__.py index 3fc1bd4..06361f3 100644 --- a/NGPIris/cli/__init__.py +++ b/NGPIris/cli/__init__.py @@ -222,10 +222,18 @@ def list_objects(context : Context, bucket : str, name_only : bool): "-cs", "--case_sensitive", help = "Use case sensitivity? Default value is False", - default = False + default = False, + is_flag = True +) +@click.option( + "-v", + "--verbose", + help = "Get a verbose output of files. Default value is False, since it might be slower", + default = False, + is_flag = True ) @click.pass_context -def simple_search(context : Context, bucket : str, search_string : str, case_sensitive : bool): +def simple_search(context : Context, bucket : str, search_string : str, case_sensitive : bool, verbose : bool): """ Make simple search using substrings in a bucket/namespace on the HCP. @@ -238,10 +246,62 @@ def simple_search(context : Context, bucket : str, search_string : str, case_sen """ hcph : HCPHandler = get_HCPHandler(context) hcph.mount_bucket(bucket) - list_of_results = hcph.search_in_bucket(search_string, case_sensitive) + list_of_results = hcph.search_in_bucket( + search_string, + name_only = (not verbose), + case_sensitive = case_sensitive + ) + click.echo("Search results:") + for result in list_of_results: + click.echo(result) + +@cli.command() +@click.argument("bucket") +@click.argument("search_string") +@click.option( + "-cs", + "--case_sensitive", + help = "Use case sensitivity? Default value is False", + default = False, + is_flag = True +) +@click.option( + "-v", + "--verbose", + help = "Get a verbose output of files. Default value is False, since it might be slower", + default = False, + is_flag = True +) +@click.option( + "-t", + "--threshold", + help = "Set the threshold for the fuzzy search score. Default value is 80", + default = 80 +) +@click.pass_context +def fuzzy_search(context : Context, bucket : str, search_string : str, case_sensitive : bool, verbose : bool, threshold : int): + """ + Make a fuzzy search using a search string in a bucket/namespace on the HCP. + + NOTE: This command does not use the HCI. Instead, it uses the RapidFuzz + library in order to find objects in the HCP. As such, this search might + be slow. + + BUCKET is the name of the bucket in which to make the search. + + SEARCH_STRING is any string that is to be used for the search. + """ + hcph : HCPHandler = get_HCPHandler(context) + hcph.mount_bucket(bucket) + list_of_results = hcph.fuzzy_search_in_bucket( + search_string, + name_only = (not verbose), + case_sensitive = case_sensitive, + threshold = threshold + ) click.echo("Search results:") for result in list_of_results: - click.echo("- " + result) + click.echo(result) @cli.command() @click.argument("bucket") From 8556ecc2502e9ed8480d32aa92c896a5da5fbbba Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Mon, 16 Dec 2024 14:59:45 +0100 Subject: [PATCH 20/53] Fix `delete_objects` and `delete_folder` --- NGPIris/hcp/hcp.py | 51 +++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/NGPIris/hcp/hcp.py b/NGPIris/hcp/hcp.py index cbf5355..b8404aa 100644 --- a/NGPIris/hcp/hcp.py +++ b/NGPIris/hcp/hcp.py @@ -430,25 +430,24 @@ def delete_objects(self, keys : list[str], verbose : bool = True) -> None: :type verbose: bool, optional """ object_list = [] + does_not_exist = [] for key in keys: - object_list.append({"Key" : key}) - - deletion_dict = {"Objects": object_list} + if self.object_exists(key): + object_list.append({"Key" : key}) + else: + does_not_exist.append(key) - response : dict = self.s3_client.delete_objects( - Bucket = self.bucket_name, - Delete = deletion_dict - ) - if verbose: - print(dumps(response, indent=4)) - - deleted_dict_list : list[dict] = response["Deleted"] - does_not_exist = [] - for deleted_dict in deleted_dict_list: - if not "VersionId" in deleted_dict: - does_not_exist.append("- " + key + "\n") - if does_not_exist: - print("The following could not be deleted because they didn't exist: \n" + "".join(does_not_exist)) + if object_list: + deletion_dict = {"Objects": object_list} + response : dict = self.s3_client.delete_objects( + Bucket = self.bucket_name, + Delete = deletion_dict + ) + if verbose: + print(dumps(response, indent=4)) + + if verbose and does_not_exist: + print("The following could not be deleted because they didn't exist: \n" + "\n".join(does_not_exist)) @check_mounted def delete_object(self, key : str, verbose : bool = True) -> None: @@ -475,16 +474,16 @@ def delete_folder(self, key : str, verbose : bool = True) -> None: """ if key[-1] != "/": key += "/" - object_path_in_folder = [] - for s in self.search_in_bucket(key): - parse_object = parse(key + "{}", s) - if type(parse_object) is Result: - object_path_in_folder.append(s) - - for object_path in object_path_in_folder: - if object_path[-1] == "/": + + objects : list[str] = list(self.list_objects(key, name_only = True)) + + if not objects: + raise RuntimeError("\"" + key + "\"" + " is not a valid path") #TODO: change this error + + for object_path in objects: + if (object_path[-1] == "/") and (not object_path == key): # `objects` might contain key, in which case everything is fine raise RuntimeError("There are subfolders in this folder. Please remove these first, before deleting this one") - self.delete_objects(object_path_in_folder + [key], verbose = verbose) + self.delete_objects(objects, verbose = verbose) @check_mounted def search_in_bucket( From 6fe3673b3e23804556728d93e02c743392507372 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Tue, 17 Dec 2024 08:54:56 +0100 Subject: [PATCH 21/53] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 329dcf9..8788c68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "NGPIris" -version = "5.1.1.dev2" +version = "5.1.1.dev3" readme = "README.md" dependencies = [ "requests >= 2.31.0", From 5bb4905e7ed875157383d7040905a55bb7896383 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Tue, 17 Dec 2024 09:02:34 +0100 Subject: [PATCH 22/53] Update hcp.py --- NGPIris/hcp/hcp.py | 1 - 1 file changed, 1 deletion(-) diff --git a/NGPIris/hcp/hcp.py b/NGPIris/hcp/hcp.py index b8404aa..aaa9356 100644 --- a/NGPIris/hcp/hcp.py +++ b/NGPIris/hcp/hcp.py @@ -510,7 +510,6 @@ def search_in_bucket( """ return self.fuzzy_search_in_bucket(search_string, name_only, case_sensitive, 100) - @check_mounted def fuzzy_search_in_bucket( self, From 63831af0c68b884cb8a4ca0bc7e0424f20e0955f Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Tue, 17 Dec 2024 09:41:22 +0100 Subject: [PATCH 23/53] Update __init__.py --- NGPIris/cli/__init__.py | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/NGPIris/cli/__init__.py b/NGPIris/cli/__init__.py index 06361f3..23ad557 100644 --- a/NGPIris/cli/__init__.py +++ b/NGPIris/cli/__init__.py @@ -19,14 +19,14 @@ def format_list(list_of_things : list) -> str: list_of_buckets = list(map(lambda s : s + "\n", list_of_things)) return "".join(list_of_buckets).strip("\n") -def _list_objects_generator(hcph : HCPHandler, name_only : bool) -> Generator[str, Any, None]: +def _list_objects_generator(hcph : HCPHandler, path : str, name_only : bool) -> Generator[str, Any, None]: """ Handle object list as a paginator that `click` can handle. It works slightly different from `list_objects` in `hcp.py` in order to make the output printable in a terminal """ paginator : Paginator = hcph.s3_client.get_paginator("list_objects_v2") - pages : PageIterator = paginator.paginate(Bucket = hcph.bucket_name) + pages : PageIterator = paginator.paginate(Bucket = hcph.bucket_name, Prefix = path) (nb_of_cols, _) = get_terminal_size() max_width = floor(nb_of_cols / 5) if (not name_only): @@ -55,6 +55,11 @@ def _list_objects_generator(hcph : HCPHandler, name_only : bool) -> Generator[st def object_is_folder(object_path : str, hcph : HCPHandler) -> bool: return (object_path[-1] == "/") and (hcph.get_object(object_path)["ContentLength"] == 0) +def add_trailing_slash(path : str) -> str: + if not path[-1] == "/": + path += "/" + return path + @click.group() @click.argument("credentials") @click.version_option(package_name = "NGPIris") @@ -198,22 +203,43 @@ def list_buckets(context : Context): @cli.command() @click.argument("bucket") +@click.argument("path", required = False) @click.option( "-no", "--name-only", help = "Output only the name of the objects instead of all the associated metadata", - default = False + default = False, + is_flag = True +) +@click.option( + "-p", + "--pagination", + help = "Output as a paginator", + default = False, + is_flag = True +) ) @click.pass_context -def list_objects(context : Context, bucket : str, name_only : bool): +def list_objects(context : Context, bucket : str, path : str, name_only : bool, pagination : bool): """ List the objects in a certain bucket/namespace on the HCP. BUCKET is the name of the bucket in which to list its objects. + + PATH is an optional argument for where to list the objects """ hcph : HCPHandler = get_HCPHandler(context) hcph.mount_bucket(bucket) - click.echo_via_pager(_list_objects_generator(hcph, name_only)) + path_with_slash = add_trailing_slash(path) + + if not hcph.object_exists(path_with_slash): + raise RuntimeError("Path does not exist") + + if pagination: + click.echo_via_pager(_list_objects_generator(hcph, path_with_slash, name_only)) + else: + for obj in hcph.list_objects(path_with_slash, name_only): + click.echo(obj) @cli.command() @click.argument("bucket") From 579656be86c96d3ddcb3917c19661629773289ed Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Tue, 17 Dec 2024 14:30:07 +0100 Subject: [PATCH 24/53] Update hcp.py --- NGPIris/hcp/hcp.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/NGPIris/hcp/hcp.py b/NGPIris/hcp/hcp.py index aaa9356..127e166 100644 --- a/NGPIris/hcp/hcp.py +++ b/NGPIris/hcp/hcp.py @@ -46,6 +46,8 @@ from typing import Generator +from icecream import ic + _KB = 1024 _MB = _KB * _KB @@ -234,7 +236,7 @@ def list_buckets(self) -> list[str]: return list_of_buckets @check_mounted - def list_objects(self, path_key : str = "", name_only : bool = False) -> Generator: + def list_objects(self, path_key : str = "", name_only : bool = False, files_only : bool = False) -> Generator: """ List all objects in the mounted bucket as a generator. If one wishes to get the result as a list, use :py:function:`list` to type cast the generator @@ -247,12 +249,23 @@ def list_objects(self, path_key : str = "", name_only : bool = False) -> Generat :rtype: Generator """ paginator : Paginator = self.s3_client.get_paginator("list_objects_v2") - pages : PageIterator = paginator.paginate(Bucket = self.bucket_name) - for object in pages.search("Contents[?starts_with(Key, '" + path_key + "')][]"): - if name_only: - yield str(object["Key"]) - else: - yield object + pages : PageIterator = paginator.paginate(Bucket = self.bucket_name, Prefix = path_key) + + if files_only: + filter_string = "Contents[?!ends_with(Key, '/')][]" + else: + filter_string = "Contents[*][]" + + split_path_key = len(path_key.split("/")) + 1 + + pages_filtered = pages.search(filter_string) + for object in pages_filtered: + split_object = object["Key"].split("/") + if len(split_object) <= split_path_key: + if name_only: + yield str(object["Key"]) + else: + yield object @check_mounted def get_object(self, key : str) -> dict: From 6fadbf7d56345f7a08227652023d7a4dec807e8d Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Tue, 17 Dec 2024 14:30:09 +0100 Subject: [PATCH 25/53] Update __init__.py --- NGPIris/cli/__init__.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/NGPIris/cli/__init__.py b/NGPIris/cli/__init__.py index 23ad557..798a5bc 100644 --- a/NGPIris/cli/__init__.py +++ b/NGPIris/cli/__init__.py @@ -19,7 +19,7 @@ def format_list(list_of_things : list) -> str: list_of_buckets = list(map(lambda s : s + "\n", list_of_things)) return "".join(list_of_buckets).strip("\n") -def _list_objects_generator(hcph : HCPHandler, path : str, name_only : bool) -> Generator[str, Any, None]: +def _list_objects_generator(hcph : HCPHandler, path : str, name_only : bool, files_only : bool) -> Generator[str, Any, None]: """ Handle object list as a paginator that `click` can handle. It works slightly different from `list_objects` in `hcp.py` in order to make the output @@ -36,7 +36,11 @@ def _list_objects_generator(hcph : HCPHandler, path : str, name_only : bool) -> tablefmt = "plain", stralign = "center" ) + "\n" + "-"*nb_of_cols + "\n" - for object in pages.search("Contents[?!ends_with(Key, '/')][]"): # filter objects that does not end with "/" + if files_only: + filter_string = "Contents[?!ends_with(Key, '/')][]" # filter objects that does not end with "/" + else: + filter_string = "Contents[*][]" + for object in pages.search(filter_string): if name_only: yield str(object["Key"]) + "\n" else: @@ -218,9 +222,15 @@ def list_buckets(context : Context): default = False, is_flag = True ) +@click.option( + "-fo", + "--files-only", + help = "Output only file objects", + default = False, + is_flag = True ) @click.pass_context -def list_objects(context : Context, bucket : str, path : str, name_only : bool, pagination : bool): +def list_objects(context : Context, bucket : str, path : str, name_only : bool, pagination : bool, files_only : bool): """ List the objects in a certain bucket/namespace on the HCP. @@ -230,15 +240,18 @@ def list_objects(context : Context, bucket : str, path : str, name_only : bool, """ hcph : HCPHandler = get_HCPHandler(context) hcph.mount_bucket(bucket) - path_with_slash = add_trailing_slash(path) + if path: + path_with_slash = add_trailing_slash(path) - if not hcph.object_exists(path_with_slash): - raise RuntimeError("Path does not exist") + if not hcph.object_exists(path_with_slash): + raise RuntimeError("Path does not exist") + else: + path_with_slash = "" if pagination: - click.echo_via_pager(_list_objects_generator(hcph, path_with_slash, name_only)) + click.echo_via_pager(_list_objects_generator(hcph, path_with_slash, name_only, files_only)) else: - for obj in hcph.list_objects(path_with_slash, name_only): + for obj in hcph.list_objects(path_with_slash, name_only, files_only): click.echo(obj) @cli.command() From 9b7cbd6adc1aab7fc85dc083d2d9f900632727c7 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Tue, 17 Dec 2024 14:37:32 +0100 Subject: [PATCH 26/53] Update hcp.py --- NGPIris/hcp/hcp.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/NGPIris/hcp/hcp.py b/NGPIris/hcp/hcp.py index 127e166..e73f9d7 100644 --- a/NGPIris/hcp/hcp.py +++ b/NGPIris/hcp/hcp.py @@ -245,6 +245,8 @@ def list_objects(self, path_key : str = "", name_only : bool = False, files_only :type path_key: str, optional :param name_only: If True, yield only a the object names. If False, yield the full metadata about each object. Defaults to False. :type name_only: bool, optional + :param files_only: If true, only yield file objects. Defaults to False + :type files_only: bool, optional :yield: A generator of all objects in a bucket :rtype: Generator """ @@ -260,8 +262,14 @@ def list_objects(self, path_key : str = "", name_only : bool = False, files_only pages_filtered = pages.search(filter_string) for object in pages_filtered: + # Split the object key by "/" split_object = object["Key"].split("/") + # Check if the object is within the specified path_key depth if len(split_object) <= split_path_key: + # Skip objects that are not at the desired depth + if (len(split_object) == split_path_key) and split_object[-1]: + continue + if name_only: yield str(object["Key"]) else: From d201317cf066562240b4e700b9d6f55ba986f87c Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Tue, 17 Dec 2024 14:38:21 +0100 Subject: [PATCH 27/53] Update hcp.py --- NGPIris/hcp/hcp.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/NGPIris/hcp/hcp.py b/NGPIris/hcp/hcp.py index e73f9d7..9250402 100644 --- a/NGPIris/hcp/hcp.py +++ b/NGPIris/hcp/hcp.py @@ -46,8 +46,6 @@ from typing import Generator -from icecream import ic - _KB = 1024 _MB = _KB * _KB From 90b04ace6cc9eb3db2834d637a420bda3919b0de Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Tue, 17 Dec 2024 15:43:30 +0100 Subject: [PATCH 28/53] Update hcp.py --- NGPIris/hcp/hcp.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/NGPIris/hcp/hcp.py b/NGPIris/hcp/hcp.py index 9250402..3e39974 100644 --- a/NGPIris/hcp/hcp.py +++ b/NGPIris/hcp/hcp.py @@ -178,6 +178,8 @@ def test_connection(self, bucket_name : str = "") -> dict: pass else: raise RuntimeError("No bucket selected. Either use `mount_bucket` first or supply the optional `bucket_name` parameter for `test_connection`") + + response = {} try: response = dict(self.s3_client.head_bucket(Bucket = bucket_name)) except EndpointConnectionError as e: # pragma: no cover @@ -574,7 +576,7 @@ def fuzzy_search_in_bucket( if name_only: yield item else: - yield full_list[index] + yield full_list[index] # type: ignore @check_mounted def get_object_acl(self, key : str) -> dict: From 5ef6380e70b0a4ba5d9a27828838c56c27b47dc8 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Tue, 17 Dec 2024 15:53:56 +0100 Subject: [PATCH 29/53] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8788c68..35e6d8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "NGPIris" -version = "5.1.1.dev3" +version = "5.1.1.dev4" readme = "README.md" dependencies = [ "requests >= 2.31.0", From 5ae82436e832ebad9e1f67bb5eb42534832d5f56 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Wed, 18 Dec 2024 10:39:24 +0100 Subject: [PATCH 30/53] Update __init__.py --- NGPIris/cli/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NGPIris/cli/__init__.py b/NGPIris/cli/__init__.py index 798a5bc..5ff176f 100644 --- a/NGPIris/cli/__init__.py +++ b/NGPIris/cli/__init__.py @@ -94,6 +94,8 @@ def upload(context : Context, bucket : str, source : str, destination : str): """ hcph : HCPHandler = get_HCPHandler(context) hcph.mount_bucket(bucket) + source = add_trailing_slash(source) + destination = add_trailing_slash(destination) if Path(source).is_dir(): hcph.upload_folder(source, destination) else: From 51aeb459a2392e76ac52390e2edfe7e664900683 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Wed, 18 Dec 2024 10:46:01 +0100 Subject: [PATCH 31/53] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 35e6d8f..1f97b34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "NGPIris" -version = "5.1.1.dev4" +version = "5.2.0.dev1" readme = "README.md" dependencies = [ "requests >= 2.31.0", From 421fae9bb37d32f070a92a3fe625bea53db61dc0 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Wed, 18 Dec 2024 12:18:56 +0100 Subject: [PATCH 32/53] Fix single file upload --- NGPIris/cli/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/NGPIris/cli/__init__.py b/NGPIris/cli/__init__.py index 5ff176f..432d19e 100644 --- a/NGPIris/cli/__init__.py +++ b/NGPIris/cli/__init__.py @@ -94,11 +94,13 @@ def upload(context : Context, bucket : str, source : str, destination : str): """ hcph : HCPHandler = get_HCPHandler(context) hcph.mount_bucket(bucket) - source = add_trailing_slash(source) destination = add_trailing_slash(destination) if Path(source).is_dir(): + source = add_trailing_slash(source) hcph.upload_folder(source, destination) else: + file_name = Path(source).name + destination += file_name hcph.upload_file(source, destination) @cli.command() From fa521ab50bbabfc414c95f020eb5269a0428f9cc Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Wed, 18 Dec 2024 13:30:32 +0100 Subject: [PATCH 33/53] Update hcp.py --- NGPIris/hcp/hcp.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NGPIris/hcp/hcp.py b/NGPIris/hcp/hcp.py index 3e39974..4143d71 100644 --- a/NGPIris/hcp/hcp.py +++ b/NGPIris/hcp/hcp.py @@ -403,6 +403,9 @@ def upload_file(self, local_file_path : str, key : str = "") -> None: file_name = Path(local_file_path).name key = file_name + if "\\" in local_file_path: + raise RuntimeError("The \\ character is not allowed in the file path") + if self.object_exists(key): raise ObjectAlreadyExist("The object \"" + key + "\" already exist in the mounted bucket") else: From 4cee7fd50cf04df392bd5d59f8e1c4b3b6c07424 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Wed, 18 Dec 2024 13:36:26 +0100 Subject: [PATCH 34/53] Update hcp.py --- NGPIris/hcp/hcp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NGPIris/hcp/hcp.py b/NGPIris/hcp/hcp.py index 4143d71..159480e 100644 --- a/NGPIris/hcp/hcp.py +++ b/NGPIris/hcp/hcp.py @@ -404,7 +404,7 @@ def upload_file(self, local_file_path : str, key : str = "") -> None: key = file_name if "\\" in local_file_path: - raise RuntimeError("The \\ character is not allowed in the file path") + raise RuntimeError("The \"\\\" character is not allowed in the file path") if self.object_exists(key): raise ObjectAlreadyExist("The object \"" + key + "\" already exist in the mounted bucket") From 50dcf99f757e5863a1a16b72842115763b9755c2 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Fri, 3 Jan 2025 13:56:43 +0100 Subject: [PATCH 35/53] Replace `pytype_and_pytest.yml` with just `pytype.yml` --- .github/workflows/{pytype_and_pytest.yml => pytype.yml} | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) rename .github/workflows/{pytype_and_pytest.yml => pytype.yml} (87%) diff --git a/.github/workflows/pytype_and_pytest.yml b/.github/workflows/pytype.yml similarity index 87% rename from .github/workflows/pytype_and_pytest.yml rename to .github/workflows/pytype.yml index 56bc1e8..b53b455 100644 --- a/.github/workflows/pytype_and_pytest.yml +++ b/.github/workflows/pytype.yml @@ -1,9 +1,9 @@ -name: PyType and PyTest +name: PyType on: push: - branches: [ "dev", "master" ] + branches: [ "dev" ] pull_request: branches: [ "dev", "master" ] @@ -11,12 +11,13 @@ permissions: contents: read jobs: - pytype_and_pytest: - + pytype: + name: Run PyType runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Set up Python 3.10 uses: actions/setup-python@v3 with: From 0f39a8fae9638009a9086eddb3f240286be20518 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Fri, 3 Jan 2025 14:01:44 +0100 Subject: [PATCH 36/53] Update pytype.yml --- .github/workflows/pytype.yml | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pytype.yml b/.github/workflows/pytype.yml index b53b455..937c7ec 100644 --- a/.github/workflows/pytype.yml +++ b/.github/workflows/pytype.yml @@ -3,9 +3,10 @@ name: PyType on: push: - branches: [ "dev" ] + branches: ["dev"] pull_request: - branches: [ "dev", "master" ] + branches: ["dev", "master"] + workflow_dispatch: permissions: contents: read @@ -18,18 +19,21 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up Python 3.10 - uses: actions/setup-python@v3 + - name: Set up Python + uses: actions/setup-python@v5 with: python-version: "3.11" + + - name: Create and start virtual environment + run: | + python -m venv venv + source venv/bin/activate + - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install pytype pytest + pip install pytype pip install . + - name: Run pytype run: | - pytype - - name: Test with pytest - run: | - pytest + pytype \ No newline at end of file From d9dce1fa32d4edbb199acc77fc975b65d004a158 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Fri, 3 Jan 2025 14:09:12 +0100 Subject: [PATCH 37/53] Create pytest.yml --- .github/workflows/pytest.yml | 37 ++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/pytest.yml diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml new file mode 100644 index 0000000..a6941d1 --- /dev/null +++ b/.github/workflows/pytest.yml @@ -0,0 +1,37 @@ + +name: PyTest + +on: + pull_request: + branches: [ "dev", "master" ] + workflow_dispatch: + +permissions: + contents: read + +jobs: + pytest: + name: Run PyTest + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Create and start virtual environment + run: | + python -m venv venv + source venv/bin/activate + + - name: Install dependencies + run: | + pip install pytest + pip install . + + - name: Run pytest + run: | + pytest \ No newline at end of file From e42c74c84f5511f62ce0002a84f8ad3a9a8e86a7 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Fri, 3 Jan 2025 15:04:24 +0100 Subject: [PATCH 38/53] Revert "Create pytest.yml" This reverts commit d9dce1fa32d4edbb199acc77fc975b65d004a158. --- .github/workflows/pytest.yml | 37 ------------------------------------ 1 file changed, 37 deletions(-) delete mode 100644 .github/workflows/pytest.yml diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml deleted file mode 100644 index a6941d1..0000000 --- a/.github/workflows/pytest.yml +++ /dev/null @@ -1,37 +0,0 @@ - -name: PyTest - -on: - pull_request: - branches: [ "dev", "master" ] - workflow_dispatch: - -permissions: - contents: read - -jobs: - pytest: - name: Run PyTest - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Create and start virtual environment - run: | - python -m venv venv - source venv/bin/activate - - - name: Install dependencies - run: | - pip install pytest - pip install . - - - name: Run pytest - run: | - pytest \ No newline at end of file From 7b482786b17338126e9fb77f7b9d45cd76414b59 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Tue, 7 Jan 2025 14:51:27 +0100 Subject: [PATCH 39/53] Update test_conf_template.ini --- tests/test_conf_template.ini | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_conf_template.ini b/tests/test_conf_template.ini index 9bd1287..97b4b90 100644 --- a/tests/test_conf_template.ini +++ b/tests/test_conf_template.ini @@ -1,3 +1,6 @@ -[hcp_tests] +[General] +credentials_path = + +[HCP_tests] bucket = data_test_file = 80MB_test_file \ No newline at end of file From 2bc75749f0c3c767ea578307daf326c5608c4302 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Tue, 7 Jan 2025 14:51:41 +0100 Subject: [PATCH 40/53] Update test suites --- pyproject.toml | 4 ++++ tests/conftest.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ tests/test_hci.py | 6 +++++- tests/test_hcp.py | 39 ++++++++++++++++++++++++++++++--------- 4 files changed, 85 insertions(+), 10 deletions(-) create mode 100644 tests/conftest.py diff --git a/pyproject.toml b/pyproject.toml index b96f0f5..46658d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,10 @@ inputs = ["NGPIris"] pythonpath = [ "." ] +testpaths = [ + "tests" +] +addopts = "--strict-markers" # Adds command-line options filterwarnings = "ignore::urllib3.connectionpool.InsecureRequestWarning" [project.scripts] diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..f7b9a59 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,46 @@ + +from pytest import FixtureRequest, fixture, UsageError, Config +from configparser import ConfigParser +from NGPIris.hcp import HCPHandler +from icecream import ic + +def set_section(config : Config, parser : ConfigParser, section : str): + parse_dict = dict(parser.items(section)) + for k, v in parse_dict.items(): + setattr(config, k, v) + + +def pytest_addoption(parser): + parser.addoption( + "--config", + action="store", + default=None, + help="Path to the configuration file (e.g., path/to/config.ini)" + ) + +def pytest_configure(config : Config): + config_path = config.getoption("--config") + if not config_path: + raise UsageError("--config argument is required.") + else: + parser = ConfigParser() + parser.read(str(config_path)) + + setattr(config, "hcp_h", HCPHandler(parser.get("General", "credentials_path"))) + set_section(config, parser, "HCP_tests") + + #config.hcp_h = HCPHandler(parser.get("General", "credentials_path")) # type: ignore + #config.hcp_h = parser.get("General", "credentials_path") # type: ignore + + + +@fixture(scope = "session") +def get_ini_config(request : FixtureRequest): + config_path = request.config.getoption("--config") + if not config_path: + raise UsageError("--config argument is required.") + else: + parser = ConfigParser() + parser.read(str(config_path)) + return parser + diff --git a/tests/test_hci.py b/tests/test_hci.py index dd64a8a..a9f7982 100644 --- a/tests/test_hci.py +++ b/tests/test_hci.py @@ -1,10 +1,14 @@ +from configparser import ConfigParser from NGPIris.hci import HCIHandler from random import randint from json import dump from os import remove -hci_h = HCIHandler("credentials/testCredentials.json") +ini_config = ConfigParser() +ini_config.read("tests/test_conf.ini") + +hci_h = HCIHandler(ini_config.get("General", "credentials_path")) hci_h.request_token() def test_list_index_names_type() -> None: diff --git a/tests/test_hcp.py b/tests/test_hcp.py index 8fac4ba..95c5b88 100644 --- a/tests/test_hcp.py +++ b/tests/test_hcp.py @@ -6,17 +6,19 @@ from shutil import rmtree from filecmp import cmp -hcp_h = HCPHandler("credentials/testCredentials.json") +# --------------------------- Helper fucntions --------------------------------- -ini_config = ConfigParser() -ini_config.read("tests/test_conf.ini") +#def _get_hcp_handler(config_parser : ConfigParser) -> HCPHandler: +# return HCPHandler(config_parser.get("General", "credentials_path")) -test_bucket = ini_config.get("hcp_tests", "bucket") +def _get_all_config(config_parser : ConfigParser) -> dict: + return dict(config_parser.items("HCP_tests")) -test_file = ini_config.get("hcp_tests","data_test_file") -test_file_path = "tests/data/" + test_file - -result_path = "tests/data/results/" +#def _get_test_bucket(config_parser : ConfigParser) -> str: +# return config_parser.get("hcp_tests", "bucket") +# +#def _get_test_file_path(config_parser : ConfigParser) -> str: +# return config_parser.get("hcp_tests","data_test_file") def _without_mounting(test : Callable) -> None: try: @@ -26,7 +28,26 @@ def _without_mounting(test : Callable) -> None: else: # pragma: no cover assert False -def test_list_buckets() -> None: +# --------------------------- Global variables --------------------------------- + +#hcp_h = None +# +#test_bucket = None +# +#test_file = None +# +#test_file_path = None +# +#result_path = None + +# --------------------------- Test suite --------------------------------------- + +def test_blank(pytestconfig) -> None: + hcp_h = pytestconfig.hcp_h + print(hcp_h) + +def test_list_buckets(pytestconfig) -> None: + hcp_h = pytestconfig.hcp_h assert hcp_h.list_buckets() def test_mount_bucket() -> None: From ce061028ad0a479e78a04160ed4cec7d2cb3a542 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Tue, 7 Jan 2025 15:48:00 +0100 Subject: [PATCH 41/53] Update conftest.py --- tests/conftest.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index f7b9a59..5172a49 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,6 @@ from pytest import FixtureRequest, fixture, UsageError, Config from configparser import ConfigParser from NGPIris.hcp import HCPHandler -from icecream import ic def set_section(config : Config, parser : ConfigParser, section : str): parse_dict = dict(parser.items(section)) @@ -28,12 +27,7 @@ def pytest_configure(config : Config): setattr(config, "hcp_h", HCPHandler(parser.get("General", "credentials_path"))) set_section(config, parser, "HCP_tests") - - #config.hcp_h = HCPHandler(parser.get("General", "credentials_path")) # type: ignore - #config.hcp_h = parser.get("General", "credentials_path") # type: ignore - - @fixture(scope = "session") def get_ini_config(request : FixtureRequest): config_path = request.config.getoption("--config") From 3cc69e354bb56ac08990fb380f23870225973951 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Tue, 7 Jan 2025 15:48:27 +0100 Subject: [PATCH 42/53] Update test_hcp.py --- tests/test_hcp.py | 249 ++++++++++++++++++++++------------------------ 1 file changed, 119 insertions(+), 130 deletions(-) diff --git a/tests/test_hcp.py b/tests/test_hcp.py index 95c5b88..5170817 100644 --- a/tests/test_hcp.py +++ b/tests/test_hcp.py @@ -1,6 +1,7 @@ from typing import Callable -from NGPIris.hcp import HCPHandler + +from pytest import Config from configparser import ConfigParser from pathlib import Path from shutil import rmtree @@ -28,48 +29,31 @@ def _without_mounting(test : Callable) -> None: else: # pragma: no cover assert False -# --------------------------- Global variables --------------------------------- - -#hcp_h = None -# -#test_bucket = None -# -#test_file = None -# -#test_file_path = None -# -#result_path = None - # --------------------------- Test suite --------------------------------------- -def test_blank(pytestconfig) -> None: - hcp_h = pytestconfig.hcp_h - print(hcp_h) - -def test_list_buckets(pytestconfig) -> None: - hcp_h = pytestconfig.hcp_h - assert hcp_h.list_buckets() +def test_list_buckets(pytestconfig : Config) -> None: + assert pytestconfig.hcp_h.list_buckets() # type: ignore -def test_mount_bucket() -> None: - hcp_h.mount_bucket(test_bucket) +def test_mount_bucket(pytestconfig : Config) -> None: + pytestconfig.hcp_h.mount_bucket(pytestconfig.test_bucket) # type: ignore -def test_mount_nonexisting_bucket() -> None: +def test_mount_nonexisting_bucket(pytestconfig : Config) -> None: try: - hcp_h.mount_bucket("aBucketThatDoesNotExist") + pytestconfig.hcp_h.mount_bucket("aBucketThatDoesNotExist") # type: ignore except: assert True else: # pragma: no cover assert False -def test_test_connection() -> None: - test_mount_bucket() - hcp_h.test_connection() +def test_test_connection(pytestconfig : Config) -> None: + test_mount_bucket(pytestconfig) + pytestconfig.hcp_h.test_connection() # type: ignore -def test_test_connection_with_bucket_name() -> None: - hcp_h.test_connection(bucket_name = test_bucket) +def test_test_connection_with_bucket_name(pytestconfig : Config) -> None: + pytestconfig.hcp_h.test_connection(bucket_name = pytestconfig.test_bucket) # type: ignore -def test_test_connection_without_mounting_bucket() -> None: - _hcp_h = HCPHandler("credentials/testCredentials.json") +def test_test_connection_without_mounting_bucket(pytestconfig : Config) -> None: + _hcp_h = pytestconfig.hcp_h # type: ignore try: _hcp_h.test_connection() except: @@ -77,161 +61,166 @@ def test_test_connection_without_mounting_bucket() -> None: else: # pragma: no cover assert False -def test_list_objects() -> None: - test_mount_bucket() - assert type(list(hcp_h.list_objects())) == list +def test_list_objects(pytestconfig : Config) -> None: + test_mount_bucket(pytestconfig) + assert type(list(pytestconfig.hcp_h.list_objects())) == list # type: ignore -def test_list_objects_without_mounting() -> None: - _hcp_h = HCPHandler("credentials/testCredentials.json") +def test_list_objects_without_mounting(pytestconfig : Config) -> None: + _hcp_h = pytestconfig.hcp_h # type: ignore _without_mounting(_hcp_h.list_objects) -def test_upload_file() -> None: - test_mount_bucket() - hcp_h.upload_file(test_file_path) +def test_upload_file(pytestconfig : Config) -> None: + test_mount_bucket(pytestconfig) + pytestconfig.hcp_h.upload_file(pytestconfig.test_file_path) # type: ignore -def test_upload_file_without_mounting() -> None: - _hcp_h = HCPHandler("credentials/testCredentials.json") +def test_upload_file_without_mounting(pytestconfig : Config) -> None: + _hcp_h = pytestconfig.hcp_h # type: ignore _without_mounting(_hcp_h.upload_file) -def test_upload_file_in_sub_directory() -> None: - test_mount_bucket() - hcp_h.upload_file(test_file_path, "a_sub_directory/a_file") +def test_upload_file_in_sub_directory(pytestconfig : Config) -> None: + test_mount_bucket(pytestconfig) + pytestconfig.hcp_h.upload_file(pytestconfig.test_file_path, "a_sub_directory/a_file") # type: ignore -def test_upload_nonexistent_file() -> None: - test_mount_bucket() +def test_upload_nonexistent_file(pytestconfig : Config) -> None: + test_mount_bucket(pytestconfig) try: - hcp_h.upload_file("tests/data/aTestFileThatDoesNotExist") + pytestconfig.hcp_h.upload_file("tests/data/aTestFileThatDoesNotExist") # type: ignore except: assert True else: # pragma: no cover assert False -def test_upload_folder() -> None: - test_mount_bucket() - hcp_h.upload_folder("tests/data/a folder of data/", "a folder of data/") +def test_upload_folder(pytestconfig : Config) -> None: + test_mount_bucket(pytestconfig) + pytestconfig.hcp_h.upload_folder("tests/data/a folder of data/", "a folder of data/") # type: ignore -def test_upload_folder_without_mounting() -> None: - _hcp_h = HCPHandler("credentials/testCredentials.json") +def test_upload_folder_without_mounting(pytestconfig : Config) -> None: + _hcp_h = pytestconfig.hcp_h # type: ignore _without_mounting(_hcp_h.upload_folder) -def test_upload_nonexisting_folder() -> None: - test_mount_bucket() +def test_upload_nonexisting_folder(pytestconfig : Config) -> None: + test_mount_bucket(pytestconfig) try: - hcp_h.upload_folder("tests/data/aFolderOfFilesThatDoesNotExist") + pytestconfig.hcp_h.upload_folder("tests/data/aFolderOfFilesThatDoesNotExist") # type: ignore except: assert True else: # pragma: no cover assert False -def test_get_file() -> None: - test_mount_bucket() - assert hcp_h.object_exists("a_sub_directory/a_file") - assert hcp_h.get_object("a_sub_directory/a_file") +def test_get_file(pytestconfig : Config) -> None: + test_mount_bucket(pytestconfig) + assert pytestconfig.hcp_h.object_exists("a_sub_directory/a_file") # type: ignore + assert pytestconfig.hcp_h.get_object("a_sub_directory/a_file") # type: ignore -def test_get_folder_without_mounting() -> None: - _hcp_h = HCPHandler("credentials/testCredentials.json") +def test_get_folder_without_mounting(pytestconfig : Config) -> None: + _hcp_h = pytestconfig.hcp_h # type: ignore _without_mounting(_hcp_h.object_exists) _without_mounting(_hcp_h.get_object) -def test_get_file_in_sub_directory() -> None: - test_mount_bucket() - assert hcp_h.object_exists(test_file) - assert hcp_h.get_object(test_file) - -def test_download_file() -> None: - test_mount_bucket() - Path(result_path).mkdir() - hcp_h.download_file(test_file, result_path + test_file) - assert cmp(result_path + test_file, test_file_path) - -def test_download_file_without_mounting() -> None: - _hcp_h = HCPHandler("credentials/testCredentials.json") +def test_get_file_in_sub_directory(pytestconfig : Config) -> None: + test_mount_bucket(pytestconfig) + test_file = Path(pytestconfig.test_file_path).name # type: ignore + assert pytestconfig.hcp_h.object_exists(test_file) # type: ignore + assert pytestconfig.hcp_h.get_object(test_file) # type: ignore + +def test_download_file(pytestconfig : Config) -> None: + test_mount_bucket(pytestconfig) + Path(pytestconfig.result_path).mkdir() # type: ignore + test_file = Path(pytestconfig.test_file_path).name # type: ignore + pytestconfig.hcp_h.download_file(test_file, pytestconfig.result_path + test_file) # type: ignore + assert cmp(pytestconfig.result_path + test_file, pytestconfig.test_file_path) # type: ignore + +def test_download_file_without_mounting(pytestconfig : Config) -> None: + _hcp_h = pytestconfig.hcp_h # type: ignore _without_mounting(_hcp_h.download_file) -def test_download_nonexistent_file() -> None: - test_mount_bucket() +def test_download_nonexistent_file(pytestconfig : Config) -> None: + test_mount_bucket(pytestconfig) try: - hcp_h.download_file("aFileThatDoesNotExist", result_path + "aFileThatDoesNotExist") + pytestconfig.hcp_h.download_file("aFileThatDoesNotExist", pytestconfig.result_path + "aFileThatDoesNotExist") # type: ignore except: assert True else: # pragma: no cover assert False -def test_download_folder() -> None: - test_mount_bucket() - hcp_h.download_folder("a folder of data/", result_path) +def test_download_folder(pytestconfig : Config) -> None: + test_mount_bucket(pytestconfig) + pytestconfig.hcp_h.download_folder("a folder of data/", pytestconfig.result_path) # type: ignore -def test_search_objects_in_bucket() -> None: - test_mount_bucket() - hcp_h.search_objects_in_bucket(test_file) +def test_search_objects_in_bucket(pytestconfig : Config) -> None: + test_mount_bucket(pytestconfig) + test_file = Path(pytestconfig.test_file_path).name # type: ignore + pytestconfig.hcp_h.search_objects_in_bucket(test_file) # type: ignore -def test_search_objects_in_bucket_without_mounting() -> None: - _hcp_h = HCPHandler("credentials/testCredentials.json") +def test_search_objects_in_bucket_without_mounting(pytestconfig : Config) -> None: + _hcp_h = pytestconfig.hcp_h # type: ignore _without_mounting(_hcp_h.search_objects_in_bucket) -def test_get_object_acl() -> None: - test_mount_bucket() - hcp_h.get_object_acl(test_file) +def test_get_object_acl(pytestconfig : Config) -> None: + test_mount_bucket(pytestconfig) + test_file = Path(pytestconfig.test_file_path).name # type: ignore + pytestconfig.hcp_h.get_object_acl(test_file) # type: ignore -def test_get_object_acl_without_mounting() -> None: - _hcp_h = HCPHandler("credentials/testCredentials.json") +def test_get_object_acl_without_mounting(pytestconfig : Config) -> None: + _hcp_h = pytestconfig.hcp_h # type: ignore _without_mounting(_hcp_h.get_object_acl) -def test_get_bucket_acl() -> None: - test_mount_bucket() - hcp_h.get_bucket_acl() +def test_get_bucket_acl(pytestconfig : Config) -> None: + test_mount_bucket(pytestconfig) + pytestconfig.hcp_h.get_bucket_acl() # type: ignore -def test_get_bucket_acl_without_mounting() -> None: - _hcp_h = HCPHandler("credentials/testCredentials.json") +def test_get_bucket_acl_without_mounting(pytestconfig : Config) -> None: + _hcp_h = pytestconfig.hcp_h # type: ignore _without_mounting(_hcp_h.get_bucket_acl) -#def test_modify_single_object_acl() -> None: -# test_mount_bucket() -# hcp_h.modify_single_object_acl() +#def test_modify_single_object_acl(pytestconfig : Config) -> None: +# test_mount_bucket(pytestconfig) +# pytestconfig.hcp_h.modify_single_object_acl() # -#def test_modify_single_bucket_acl() -> None: -# test_mount_bucket() -# hcp_h.modify_single_bucket_acl() +#def test_modify_single_bucket_acl(pytestconfig : Config) -> None: +# test_mount_bucket(pytestconfig) +# pytestconfig.hcp_h.modify_single_bucket_acl() # -#def test_modify_object_acl() -> None: -# test_mount_bucket() -# hcp_h.modify_object_acl() +#def test_modify_object_acl(pytestconfig : Config) -> None: +# test_mount_bucket(pytestconfig) +# pytestconfig.hcp_h.modify_object_acl() # -#def test_modify_bucket_acl() -> None: -# test_mount_bucket() -# hcp_h.modify_bucket_acl() - -def test_delete_file() -> None: - test_mount_bucket() - hcp_h.delete_object(test_file) - hcp_h.delete_object("a_sub_directory/a_file") - hcp_h.delete_object("a_sub_directory") - -def test_delete_file_without_mounting() -> None: - _hcp_h = HCPHandler("credentials/testCredentials.json") +#def test_modify_bucket_acl(pytestconfig : Config) -> None: +# test_mount_bucket(pytestconfig) +# pytestconfig.hcp_h.modify_bucket_acl() + +def test_delete_file(pytestconfig : Config) -> None: + test_mount_bucket(pytestconfig) + test_file = Path(pytestconfig.test_file_path).name # type: ignore + pytestconfig.hcp_h.delete_object(test_file) # type: ignore + pytestconfig.hcp_h.delete_object("a_sub_directory/a_file") # type: ignore + pytestconfig.hcp_h.delete_object("a_sub_directory") # type: ignore + +def test_delete_file_without_mounting(pytestconfig : Config) -> None: + _hcp_h = pytestconfig.hcp_h # type: ignore _without_mounting(_hcp_h.delete_object) -def test_delete_folder_with_sub_directory() -> None: - test_mount_bucket() - hcp_h.upload_file(test_file_path, "a folder of data/a sub dir/a file") +def test_delete_folder_with_sub_directory(pytestconfig : Config) -> None: + test_mount_bucket(pytestconfig) + pytestconfig.hcp_h.upload_file(pytestconfig.test_file_path, "a folder of data/a sub dir/a file") # type: ignore try: - hcp_h.delete_folder("a folder of data/") + pytestconfig.hcp_h.delete_folder("a folder of data/") # type: ignore except: assert True else: # pragma: no cover assert False - hcp_h.delete_folder("a folder of data/a sub dir/") + pytestconfig.hcp_h.delete_folder("a folder of data/a sub dir/") # type: ignore -def test_delete_folder() -> None: - test_mount_bucket() - hcp_h.delete_folder("a folder of data/") +def test_delete_folder(pytestconfig : Config) -> None: + test_mount_bucket(pytestconfig) + pytestconfig.hcp_h.delete_folder("a folder of data/") # type: ignore -def test_delete_folder_without_mounting() -> None: - _hcp_h = HCPHandler("credentials/testCredentials.json") +def test_delete_folder_without_mounting(pytestconfig : Config) -> None: + _hcp_h = pytestconfig.hcp_h # type: ignore _without_mounting(_hcp_h.delete_folder) -def test_delete_nonexistent_files() -> None: - hcp_h.delete_objects(["some", "files", "that", "does", "not", "exist"]) +def test_delete_nonexistent_files(pytestconfig : Config) -> None: + pytestconfig.hcp_h.delete_objects(["some", "files", "that", "does", "not", "exist"]) # type: ignore -def test_clean_up() -> None: - rmtree(result_path) \ No newline at end of file +def test_clean_up(pytestconfig : Config) -> None: + rmtree(pytestconfig.result_path) # type: ignore \ No newline at end of file From 9cc5dbc5a98a8c705f2142d4a67469d0ef45f01b Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Tue, 7 Jan 2025 16:22:20 +0100 Subject: [PATCH 43/53] Add `DynamicConfig` for making the test suite a little more type safe --- tests/conftest.py | 32 ++++--- tests/test_hcp.py | 236 +++++++++++++++++++++++----------------------- 2 files changed, 138 insertions(+), 130 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 5172a49..8e31855 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,13 +1,24 @@ -from pytest import FixtureRequest, fixture, UsageError, Config +from pytest import fixture, UsageError, Config from configparser import ConfigParser from NGPIris.hcp import HCPHandler +from typing import Any def set_section(config : Config, parser : ConfigParser, section : str): parse_dict = dict(parser.items(section)) for k, v in parse_dict.items(): setattr(config, k, v) +class DynamicConfig(Config): + def __init__(self, config: Config): + self._config = config # Store the original pytest Config object + + def __getattr__(self, name: str) -> Any: + # Provide default behavior for dynamic attributes + try: + return super().__getattribute__(name) + except AttributeError: + raise AttributeError(type(self).__name__ + " object has no attribute " + name) def pytest_addoption(parser): parser.addoption( @@ -25,16 +36,11 @@ def pytest_configure(config : Config): parser = ConfigParser() parser.read(str(config_path)) - setattr(config, "hcp_h", HCPHandler(parser.get("General", "credentials_path"))) - set_section(config, parser, "HCP_tests") - -@fixture(scope = "session") -def get_ini_config(request : FixtureRequest): - config_path = request.config.getoption("--config") - if not config_path: - raise UsageError("--config argument is required.") - else: - parser = ConfigParser() - parser.read(str(config_path)) - return parser + dynamic_config = DynamicConfig(config) + setattr(dynamic_config, "hcp_h", HCPHandler(parser.get("General", "credentials_path"))) + set_section(dynamic_config, parser, "HCP_tests") + +@fixture +def dynamic_config(pytestconfig : Config) -> DynamicConfig: + return DynamicConfig(pytestconfig) \ No newline at end of file diff --git a/tests/test_hcp.py b/tests/test_hcp.py index 5170817..38153d6 100644 --- a/tests/test_hcp.py +++ b/tests/test_hcp.py @@ -7,6 +7,8 @@ from shutil import rmtree from filecmp import cmp +from conftest import DynamicConfig + # --------------------------- Helper fucntions --------------------------------- #def _get_hcp_handler(config_parser : ConfigParser) -> HCPHandler: @@ -31,29 +33,29 @@ def _without_mounting(test : Callable) -> None: # --------------------------- Test suite --------------------------------------- -def test_list_buckets(pytestconfig : Config) -> None: - assert pytestconfig.hcp_h.list_buckets() # type: ignore +def test_list_buckets(dynamic_config : DynamicConfig) -> None: + assert dynamic_config.hcp_h.list_buckets() -def test_mount_bucket(pytestconfig : Config) -> None: - pytestconfig.hcp_h.mount_bucket(pytestconfig.test_bucket) # type: ignore +def test_mount_bucket(dynamic_config : DynamicConfig) -> None: + dynamic_config.hcp_h.mount_bucket(dynamic_config.test_bucket) -def test_mount_nonexisting_bucket(pytestconfig : Config) -> None: +def test_mount_nonexisting_bucket(dynamic_config : DynamicConfig) -> None: try: - pytestconfig.hcp_h.mount_bucket("aBucketThatDoesNotExist") # type: ignore + dynamic_config.hcp_h.mount_bucket("aBucketThatDoesNotExist") except: assert True else: # pragma: no cover assert False -def test_test_connection(pytestconfig : Config) -> None: - test_mount_bucket(pytestconfig) - pytestconfig.hcp_h.test_connection() # type: ignore +def test_test_connection(dynamic_config : DynamicConfig) -> None: + test_mount_bucket(dynamic_config) + dynamic_config.hcp_h.test_connection() -def test_test_connection_with_bucket_name(pytestconfig : Config) -> None: - pytestconfig.hcp_h.test_connection(bucket_name = pytestconfig.test_bucket) # type: ignore +def test_test_connection_with_bucket_name(dynamic_config : DynamicConfig) -> None: + dynamic_config.hcp_h.test_connection(bucket_name = dynamic_config.test_bucket) -def test_test_connection_without_mounting_bucket(pytestconfig : Config) -> None: - _hcp_h = pytestconfig.hcp_h # type: ignore +def test_test_connection_without_mounting_bucket(dynamic_config : DynamicConfig) -> None: + _hcp_h = dynamic_config.hcp_h try: _hcp_h.test_connection() except: @@ -61,166 +63,166 @@ def test_test_connection_without_mounting_bucket(pytestconfig : Config) -> None: else: # pragma: no cover assert False -def test_list_objects(pytestconfig : Config) -> None: - test_mount_bucket(pytestconfig) - assert type(list(pytestconfig.hcp_h.list_objects())) == list # type: ignore +def test_list_objects(dynamic_config : DynamicConfig) -> None: + test_mount_bucket(dynamic_config) + assert type(list(dynamic_config.hcp_h.list_objects())) == list -def test_list_objects_without_mounting(pytestconfig : Config) -> None: - _hcp_h = pytestconfig.hcp_h # type: ignore +def test_list_objects_without_mounting(dynamic_config : DynamicConfig) -> None: + _hcp_h = dynamic_config.hcp_h _without_mounting(_hcp_h.list_objects) -def test_upload_file(pytestconfig : Config) -> None: - test_mount_bucket(pytestconfig) - pytestconfig.hcp_h.upload_file(pytestconfig.test_file_path) # type: ignore +def test_upload_file(dynamic_config : DynamicConfig) -> None: + test_mount_bucket(dynamic_config) + dynamic_config.hcp_h.upload_file(dynamic_config.test_file_path) -def test_upload_file_without_mounting(pytestconfig : Config) -> None: - _hcp_h = pytestconfig.hcp_h # type: ignore +def test_upload_file_without_mounting(dynamic_config : DynamicConfig) -> None: + _hcp_h = dynamic_config.hcp_h _without_mounting(_hcp_h.upload_file) -def test_upload_file_in_sub_directory(pytestconfig : Config) -> None: - test_mount_bucket(pytestconfig) - pytestconfig.hcp_h.upload_file(pytestconfig.test_file_path, "a_sub_directory/a_file") # type: ignore +def test_upload_file_in_sub_directory(dynamic_config : DynamicConfig) -> None: + test_mount_bucket(dynamic_config) + dynamic_config.hcp_h.upload_file(dynamic_config.test_file_path, "a_sub_directory/a_file") -def test_upload_nonexistent_file(pytestconfig : Config) -> None: - test_mount_bucket(pytestconfig) +def test_upload_nonexistent_file(dynamic_config : DynamicConfig) -> None: + test_mount_bucket(dynamic_config) try: - pytestconfig.hcp_h.upload_file("tests/data/aTestFileThatDoesNotExist") # type: ignore + dynamic_config.hcp_h.upload_file("tests/data/aTestFileThatDoesNotExist") except: assert True else: # pragma: no cover assert False -def test_upload_folder(pytestconfig : Config) -> None: - test_mount_bucket(pytestconfig) - pytestconfig.hcp_h.upload_folder("tests/data/a folder of data/", "a folder of data/") # type: ignore +def test_upload_folder(dynamic_config : DynamicConfig) -> None: + test_mount_bucket(dynamic_config) + dynamic_config.hcp_h.upload_folder("tests/data/a folder of data/", "a folder of data/") -def test_upload_folder_without_mounting(pytestconfig : Config) -> None: - _hcp_h = pytestconfig.hcp_h # type: ignore +def test_upload_folder_without_mounting(dynamic_config : DynamicConfig) -> None: + _hcp_h = dynamic_config.hcp_h _without_mounting(_hcp_h.upload_folder) -def test_upload_nonexisting_folder(pytestconfig : Config) -> None: - test_mount_bucket(pytestconfig) +def test_upload_nonexisting_folder(dynamic_config : DynamicConfig) -> None: + test_mount_bucket(dynamic_config) try: - pytestconfig.hcp_h.upload_folder("tests/data/aFolderOfFilesThatDoesNotExist") # type: ignore + dynamic_config.hcp_h.upload_folder("tests/data/aFolderOfFilesThatDoesNotExist") except: assert True else: # pragma: no cover assert False -def test_get_file(pytestconfig : Config) -> None: - test_mount_bucket(pytestconfig) - assert pytestconfig.hcp_h.object_exists("a_sub_directory/a_file") # type: ignore - assert pytestconfig.hcp_h.get_object("a_sub_directory/a_file") # type: ignore +def test_get_file(dynamic_config : DynamicConfig) -> None: + test_mount_bucket(dynamic_config) + assert dynamic_config.hcp_h.object_exists("a_sub_directory/a_file") + assert dynamic_config.hcp_h.get_object("a_sub_directory/a_file") -def test_get_folder_without_mounting(pytestconfig : Config) -> None: - _hcp_h = pytestconfig.hcp_h # type: ignore +def test_get_folder_without_mounting(dynamic_config : DynamicConfig) -> None: + _hcp_h = dynamic_config.hcp_h _without_mounting(_hcp_h.object_exists) _without_mounting(_hcp_h.get_object) -def test_get_file_in_sub_directory(pytestconfig : Config) -> None: - test_mount_bucket(pytestconfig) - test_file = Path(pytestconfig.test_file_path).name # type: ignore - assert pytestconfig.hcp_h.object_exists(test_file) # type: ignore - assert pytestconfig.hcp_h.get_object(test_file) # type: ignore - -def test_download_file(pytestconfig : Config) -> None: - test_mount_bucket(pytestconfig) - Path(pytestconfig.result_path).mkdir() # type: ignore - test_file = Path(pytestconfig.test_file_path).name # type: ignore - pytestconfig.hcp_h.download_file(test_file, pytestconfig.result_path + test_file) # type: ignore - assert cmp(pytestconfig.result_path + test_file, pytestconfig.test_file_path) # type: ignore - -def test_download_file_without_mounting(pytestconfig : Config) -> None: - _hcp_h = pytestconfig.hcp_h # type: ignore +def test_get_file_in_sub_directory(dynamic_config : DynamicConfig) -> None: + test_mount_bucket(dynamic_config) + test_file = Path(dynamic_config.test_file_path).name + assert dynamic_config.hcp_h.object_exists(test_file) + assert dynamic_config.hcp_h.get_object(test_file) + +def test_download_file(dynamic_config : DynamicConfig) -> None: + test_mount_bucket(dynamic_config) + Path(dynamic_config.result_path).mkdir() + test_file = Path(dynamic_config.test_file_path).name + dynamic_config.hcp_h.download_file(test_file, dynamic_config.result_path + test_file) + assert cmp(dynamic_config.result_path + test_file, dynamic_config.test_file_path) + +def test_download_file_without_mounting(dynamic_config : DynamicConfig) -> None: + _hcp_h = dynamic_config.hcp_h _without_mounting(_hcp_h.download_file) -def test_download_nonexistent_file(pytestconfig : Config) -> None: - test_mount_bucket(pytestconfig) +def test_download_nonexistent_file(dynamic_config : DynamicConfig) -> None: + test_mount_bucket(dynamic_config) try: - pytestconfig.hcp_h.download_file("aFileThatDoesNotExist", pytestconfig.result_path + "aFileThatDoesNotExist") # type: ignore + dynamic_config.hcp_h.download_file("aFileThatDoesNotExist", dynamic_config.result_path + "aFileThatDoesNotExist") except: assert True else: # pragma: no cover assert False -def test_download_folder(pytestconfig : Config) -> None: - test_mount_bucket(pytestconfig) - pytestconfig.hcp_h.download_folder("a folder of data/", pytestconfig.result_path) # type: ignore +def test_download_folder(dynamic_config : DynamicConfig) -> None: + test_mount_bucket(dynamic_config) + dynamic_config.hcp_h.download_folder("a folder of data/", dynamic_config.result_path) -def test_search_objects_in_bucket(pytestconfig : Config) -> None: - test_mount_bucket(pytestconfig) - test_file = Path(pytestconfig.test_file_path).name # type: ignore - pytestconfig.hcp_h.search_objects_in_bucket(test_file) # type: ignore +def test_search_objects_in_bucket(dynamic_config : DynamicConfig) -> None: + test_mount_bucket(dynamic_config) + test_file = Path(dynamic_config.test_file_path).name + dynamic_config.hcp_h.search_objects_in_bucket(test_file) -def test_search_objects_in_bucket_without_mounting(pytestconfig : Config) -> None: - _hcp_h = pytestconfig.hcp_h # type: ignore +def test_search_objects_in_bucket_without_mounting(dynamic_config : DynamicConfig) -> None: + _hcp_h = dynamic_config.hcp_h _without_mounting(_hcp_h.search_objects_in_bucket) -def test_get_object_acl(pytestconfig : Config) -> None: - test_mount_bucket(pytestconfig) - test_file = Path(pytestconfig.test_file_path).name # type: ignore - pytestconfig.hcp_h.get_object_acl(test_file) # type: ignore +def test_get_object_acl(dynamic_config : DynamicConfig) -> None: + test_mount_bucket(dynamic_config) + test_file = Path(dynamic_config.test_file_path).name + dynamic_config.hcp_h.get_object_acl(test_file) -def test_get_object_acl_without_mounting(pytestconfig : Config) -> None: - _hcp_h = pytestconfig.hcp_h # type: ignore +def test_get_object_acl_without_mounting(dynamic_config : DynamicConfig) -> None: + _hcp_h = dynamic_config.hcp_h _without_mounting(_hcp_h.get_object_acl) -def test_get_bucket_acl(pytestconfig : Config) -> None: - test_mount_bucket(pytestconfig) - pytestconfig.hcp_h.get_bucket_acl() # type: ignore +def test_get_bucket_acl(dynamic_config : DynamicConfig) -> None: + test_mount_bucket(dynamic_config) + dynamic_config.hcp_h.get_bucket_acl() -def test_get_bucket_acl_without_mounting(pytestconfig : Config) -> None: - _hcp_h = pytestconfig.hcp_h # type: ignore +def test_get_bucket_acl_without_mounting(dynamic_config : DynamicConfig) -> None: + _hcp_h = dynamic_config.hcp_h _without_mounting(_hcp_h.get_bucket_acl) -#def test_modify_single_object_acl(pytestconfig : Config) -> None: -# test_mount_bucket(pytestconfig) -# pytestconfig.hcp_h.modify_single_object_acl() +#def test_modify_single_object_acl(dynamic_config : DynamicConfig) -> None: +# test_mount_bucket(dynamic_config) +# dynamic_config.hcp_h.modify_single_object_acl() # -#def test_modify_single_bucket_acl(pytestconfig : Config) -> None: -# test_mount_bucket(pytestconfig) -# pytestconfig.hcp_h.modify_single_bucket_acl() +#def test_modify_single_bucket_acl(dynamic_config : DynamicConfig) -> None: +# test_mount_bucket(dynamic_config) +# dynamic_config.hcp_h.modify_single_bucket_acl() # -#def test_modify_object_acl(pytestconfig : Config) -> None: -# test_mount_bucket(pytestconfig) -# pytestconfig.hcp_h.modify_object_acl() +#def test_modify_object_acl(dynamic_config : DynamicConfig) -> None: +# test_mount_bucket(dynamic_config) +# dynamic_config.hcp_h.modify_object_acl() # -#def test_modify_bucket_acl(pytestconfig : Config) -> None: -# test_mount_bucket(pytestconfig) -# pytestconfig.hcp_h.modify_bucket_acl() - -def test_delete_file(pytestconfig : Config) -> None: - test_mount_bucket(pytestconfig) - test_file = Path(pytestconfig.test_file_path).name # type: ignore - pytestconfig.hcp_h.delete_object(test_file) # type: ignore - pytestconfig.hcp_h.delete_object("a_sub_directory/a_file") # type: ignore - pytestconfig.hcp_h.delete_object("a_sub_directory") # type: ignore - -def test_delete_file_without_mounting(pytestconfig : Config) -> None: - _hcp_h = pytestconfig.hcp_h # type: ignore +#def test_modify_bucket_acl(dynamic_config : DynamicConfig) -> None: +# test_mount_bucket(dynamic_config) +# dynamic_config.hcp_h.modify_bucket_acl() + +def test_delete_file(dynamic_config : DynamicConfig) -> None: + test_mount_bucket(dynamic_config) + test_file = Path(dynamic_config.test_file_path).name + dynamic_config.hcp_h.delete_object(test_file) + dynamic_config.hcp_h.delete_object("a_sub_directory/a_file") + dynamic_config.hcp_h.delete_object("a_sub_directory") + +def test_delete_file_without_mounting(dynamic_config : DynamicConfig) -> None: + _hcp_h = dynamic_config.hcp_h _without_mounting(_hcp_h.delete_object) -def test_delete_folder_with_sub_directory(pytestconfig : Config) -> None: - test_mount_bucket(pytestconfig) - pytestconfig.hcp_h.upload_file(pytestconfig.test_file_path, "a folder of data/a sub dir/a file") # type: ignore +def test_delete_folder_with_sub_directory(dynamic_config : DynamicConfig) -> None: + test_mount_bucket(dynamic_config) + dynamic_config.hcp_h.upload_file(dynamic_config.test_file_path, "a folder of data/a sub dir/a file") try: - pytestconfig.hcp_h.delete_folder("a folder of data/") # type: ignore + dynamic_config.hcp_h.delete_folder("a folder of data/") except: assert True else: # pragma: no cover assert False - pytestconfig.hcp_h.delete_folder("a folder of data/a sub dir/") # type: ignore + dynamic_config.hcp_h.delete_folder("a folder of data/a sub dir/") -def test_delete_folder(pytestconfig : Config) -> None: - test_mount_bucket(pytestconfig) - pytestconfig.hcp_h.delete_folder("a folder of data/") # type: ignore +def test_delete_folder(dynamic_config : DynamicConfig) -> None: + test_mount_bucket(dynamic_config) + dynamic_config.hcp_h.delete_folder("a folder of data/") -def test_delete_folder_without_mounting(pytestconfig : Config) -> None: - _hcp_h = pytestconfig.hcp_h # type: ignore +def test_delete_folder_without_mounting(dynamic_config : DynamicConfig) -> None: + _hcp_h = dynamic_config.hcp_h _without_mounting(_hcp_h.delete_folder) -def test_delete_nonexistent_files(pytestconfig : Config) -> None: - pytestconfig.hcp_h.delete_objects(["some", "files", "that", "does", "not", "exist"]) # type: ignore +def test_delete_nonexistent_files(dynamic_config : DynamicConfig) -> None: + dynamic_config.hcp_h.delete_objects(["some", "files", "that", "does", "not", "exist"]) -def test_clean_up(pytestconfig : Config) -> None: - rmtree(pytestconfig.result_path) # type: ignore \ No newline at end of file +def test_clean_up(dynamic_config : DynamicConfig) -> None: + rmtree(dynamic_config.result_path) \ No newline at end of file From 49dd22318701ea1736ede6954e47fbb318a059d8 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Wed, 8 Jan 2025 09:33:05 +0100 Subject: [PATCH 44/53] Update conftest.py --- tests/conftest.py | 30 +++++++----------------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 8e31855..754610c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,23 +4,12 @@ from NGPIris.hcp import HCPHandler from typing import Any -def set_section(config : Config, parser : ConfigParser, section : str): +def set_section(config : Config, parser : ConfigParser, section : str) -> None: parse_dict = dict(parser.items(section)) for k, v in parse_dict.items(): - setattr(config, k, v) + setattr(config, k, v) # config.k = v -class DynamicConfig(Config): - def __init__(self, config: Config): - self._config = config # Store the original pytest Config object - - def __getattr__(self, name: str) -> Any: - # Provide default behavior for dynamic attributes - try: - return super().__getattribute__(name) - except AttributeError: - raise AttributeError(type(self).__name__ + " object has no attribute " + name) - -def pytest_addoption(parser): +def pytest_addoption(parser) -> None: parser.addoption( "--config", action="store", @@ -28,7 +17,7 @@ def pytest_addoption(parser): help="Path to the configuration file (e.g., path/to/config.ini)" ) -def pytest_configure(config : Config): +def pytest_configure(config : Config) -> None: config_path = config.getoption("--config") if not config_path: raise UsageError("--config argument is required.") @@ -36,11 +25,6 @@ def pytest_configure(config : Config): parser = ConfigParser() parser.read(str(config_path)) - dynamic_config = DynamicConfig(config) - - setattr(dynamic_config, "hcp_h", HCPHandler(parser.get("General", "credentials_path"))) - set_section(dynamic_config, parser, "HCP_tests") - -@fixture -def dynamic_config(pytestconfig : Config) -> DynamicConfig: - return DynamicConfig(pytestconfig) \ No newline at end of file + setattr(config, "hcp_h", HCPHandler(parser.get("General", "credentials_path"))) + set_section(config, parser, "HCP_tests") + \ No newline at end of file From 8d7108f5e4ea5d110c27c7507bd252c8eb35f36c Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Wed, 8 Jan 2025 09:53:56 +0100 Subject: [PATCH 45/53] Fix type safety --- tests/conftest.py | 36 +++++-- tests/test_hcp.py | 236 +++++++++++++++++++++++----------------------- 2 files changed, 148 insertions(+), 124 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 754610c..9ddf6a1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,22 +1,38 @@ - -from pytest import fixture, UsageError, Config +from pytest import Config, fixture, UsageError from configparser import ConfigParser -from NGPIris.hcp import HCPHandler from typing import Any +from NGPIris.hcp import HCPHandler + +class CustomConfig: + """A typed wrapper around pytest.Config for dynamic attributes.""" + def __init__(self, pytest_config : Config): + self._config = pytest_config + + @property + def hcp_h(self) -> HCPHandler: + """Access the HCPHandler instance.""" + return getattr(self._config, "hcp_h") + + def __getattr__(self, name : str) -> Any: + """Dynamically get attributes set during pytest configuration.""" + return getattr(self._config, name) + def set_section(config : Config, parser : ConfigParser, section : str) -> None: parse_dict = dict(parser.items(section)) for k, v in parse_dict.items(): - setattr(config, k, v) # config.k = v + setattr(config, k, v) # Adds attributes dynamically to pytest.Config + def pytest_addoption(parser) -> None: parser.addoption( "--config", action="store", default=None, - help="Path to the configuration file (e.g., path/to/config.ini)" + help="Path to the configuration file (e.g., path/to/config.ini)", ) + def pytest_configure(config : Config) -> None: config_path = config.getoption("--config") if not config_path: @@ -25,6 +41,14 @@ def pytest_configure(config : Config) -> None: parser = ConfigParser() parser.read(str(config_path)) + # Dynamically add an HCPHandler instance to config setattr(config, "hcp_h", HCPHandler(parser.get("General", "credentials_path"))) + + # Dynamically add all key-value pairs from "HCP_tests" section set_section(config, parser, "HCP_tests") - \ No newline at end of file + + +@fixture +def custom_config(pytestconfig : Config) -> CustomConfig: + """Provide the typed wrapper for pytest.Config.""" + return CustomConfig(pytestconfig) diff --git a/tests/test_hcp.py b/tests/test_hcp.py index 38153d6..d5577ed 100644 --- a/tests/test_hcp.py +++ b/tests/test_hcp.py @@ -7,7 +7,7 @@ from shutil import rmtree from filecmp import cmp -from conftest import DynamicConfig +from conftest import CustomConfig # --------------------------- Helper fucntions --------------------------------- @@ -33,29 +33,29 @@ def _without_mounting(test : Callable) -> None: # --------------------------- Test suite --------------------------------------- -def test_list_buckets(dynamic_config : DynamicConfig) -> None: - assert dynamic_config.hcp_h.list_buckets() +def test_list_buckets(custom_config : CustomConfig) -> None: + assert custom_config.hcp_h.list_buckets() -def test_mount_bucket(dynamic_config : DynamicConfig) -> None: - dynamic_config.hcp_h.mount_bucket(dynamic_config.test_bucket) +def test_mount_bucket(custom_config : CustomConfig) -> None: + custom_config.hcp_h.mount_bucket(custom_config.test_bucket) -def test_mount_nonexisting_bucket(dynamic_config : DynamicConfig) -> None: +def test_mount_nonexisting_bucket(custom_config : CustomConfig) -> None: try: - dynamic_config.hcp_h.mount_bucket("aBucketThatDoesNotExist") + custom_config.hcp_h.mount_bucket("aBucketThatDoesNotExist") except: assert True else: # pragma: no cover assert False -def test_test_connection(dynamic_config : DynamicConfig) -> None: - test_mount_bucket(dynamic_config) - dynamic_config.hcp_h.test_connection() +def test_test_connection(custom_config : CustomConfig) -> None: + test_mount_bucket(custom_config) + custom_config.hcp_h.test_connection() -def test_test_connection_with_bucket_name(dynamic_config : DynamicConfig) -> None: - dynamic_config.hcp_h.test_connection(bucket_name = dynamic_config.test_bucket) +def test_test_connection_with_bucket_name(custom_config : CustomConfig) -> None: + custom_config.hcp_h.test_connection(bucket_name = custom_config.test_bucket) -def test_test_connection_without_mounting_bucket(dynamic_config : DynamicConfig) -> None: - _hcp_h = dynamic_config.hcp_h +def test_test_connection_without_mounting_bucket(custom_config : CustomConfig) -> None: + _hcp_h = custom_config.hcp_h try: _hcp_h.test_connection() except: @@ -63,166 +63,166 @@ def test_test_connection_without_mounting_bucket(dynamic_config : DynamicConfig) else: # pragma: no cover assert False -def test_list_objects(dynamic_config : DynamicConfig) -> None: - test_mount_bucket(dynamic_config) - assert type(list(dynamic_config.hcp_h.list_objects())) == list +def test_list_objects(custom_config : CustomConfig) -> None: + test_mount_bucket(custom_config) + assert type(list(custom_config.hcp_h.list_objects())) == list -def test_list_objects_without_mounting(dynamic_config : DynamicConfig) -> None: - _hcp_h = dynamic_config.hcp_h +def test_list_objects_without_mounting(custom_config : CustomConfig) -> None: + _hcp_h = custom_config.hcp_h _without_mounting(_hcp_h.list_objects) -def test_upload_file(dynamic_config : DynamicConfig) -> None: - test_mount_bucket(dynamic_config) - dynamic_config.hcp_h.upload_file(dynamic_config.test_file_path) +def test_upload_file(custom_config : CustomConfig) -> None: + test_mount_bucket(custom_config) + custom_config.hcp_h.upload_file(custom_config.test_file_path) -def test_upload_file_without_mounting(dynamic_config : DynamicConfig) -> None: - _hcp_h = dynamic_config.hcp_h +def test_upload_file_without_mounting(custom_config : CustomConfig) -> None: + _hcp_h = custom_config.hcp_h _without_mounting(_hcp_h.upload_file) -def test_upload_file_in_sub_directory(dynamic_config : DynamicConfig) -> None: - test_mount_bucket(dynamic_config) - dynamic_config.hcp_h.upload_file(dynamic_config.test_file_path, "a_sub_directory/a_file") +def test_upload_file_in_sub_directory(custom_config : CustomConfig) -> None: + test_mount_bucket(custom_config) + custom_config.hcp_h.upload_file(custom_config.test_file_path, "a_sub_directory/a_file") -def test_upload_nonexistent_file(dynamic_config : DynamicConfig) -> None: - test_mount_bucket(dynamic_config) +def test_upload_nonexistent_file(custom_config : CustomConfig) -> None: + test_mount_bucket(custom_config) try: - dynamic_config.hcp_h.upload_file("tests/data/aTestFileThatDoesNotExist") + custom_config.hcp_h.upload_file("tests/data/aTestFileThatDoesNotExist") except: assert True else: # pragma: no cover assert False -def test_upload_folder(dynamic_config : DynamicConfig) -> None: - test_mount_bucket(dynamic_config) - dynamic_config.hcp_h.upload_folder("tests/data/a folder of data/", "a folder of data/") +def test_upload_folder(custom_config : CustomConfig) -> None: + test_mount_bucket(custom_config) + custom_config.hcp_h.upload_folder("tests/data/a folder of data/", "a folder of data/") -def test_upload_folder_without_mounting(dynamic_config : DynamicConfig) -> None: - _hcp_h = dynamic_config.hcp_h +def test_upload_folder_without_mounting(custom_config : CustomConfig) -> None: + _hcp_h = custom_config.hcp_h _without_mounting(_hcp_h.upload_folder) -def test_upload_nonexisting_folder(dynamic_config : DynamicConfig) -> None: - test_mount_bucket(dynamic_config) +def test_upload_nonexisting_folder(custom_config : CustomConfig) -> None: + test_mount_bucket(custom_config) try: - dynamic_config.hcp_h.upload_folder("tests/data/aFolderOfFilesThatDoesNotExist") + custom_config.hcp_h.upload_folder("tests/data/aFolderOfFilesThatDoesNotExist") except: assert True else: # pragma: no cover assert False -def test_get_file(dynamic_config : DynamicConfig) -> None: - test_mount_bucket(dynamic_config) - assert dynamic_config.hcp_h.object_exists("a_sub_directory/a_file") - assert dynamic_config.hcp_h.get_object("a_sub_directory/a_file") +def test_get_file(custom_config : CustomConfig) -> None: + test_mount_bucket(custom_config) + assert custom_config.hcp_h.object_exists("a_sub_directory/a_file") + assert custom_config.hcp_h.get_object("a_sub_directory/a_file") -def test_get_folder_without_mounting(dynamic_config : DynamicConfig) -> None: - _hcp_h = dynamic_config.hcp_h +def test_get_folder_without_mounting(custom_config : CustomConfig) -> None: + _hcp_h = custom_config.hcp_h _without_mounting(_hcp_h.object_exists) _without_mounting(_hcp_h.get_object) -def test_get_file_in_sub_directory(dynamic_config : DynamicConfig) -> None: - test_mount_bucket(dynamic_config) - test_file = Path(dynamic_config.test_file_path).name - assert dynamic_config.hcp_h.object_exists(test_file) - assert dynamic_config.hcp_h.get_object(test_file) - -def test_download_file(dynamic_config : DynamicConfig) -> None: - test_mount_bucket(dynamic_config) - Path(dynamic_config.result_path).mkdir() - test_file = Path(dynamic_config.test_file_path).name - dynamic_config.hcp_h.download_file(test_file, dynamic_config.result_path + test_file) - assert cmp(dynamic_config.result_path + test_file, dynamic_config.test_file_path) - -def test_download_file_without_mounting(dynamic_config : DynamicConfig) -> None: - _hcp_h = dynamic_config.hcp_h +def test_get_file_in_sub_directory(custom_config : CustomConfig) -> None: + test_mount_bucket(custom_config) + test_file = Path(custom_config.test_file_path).name + assert custom_config.hcp_h.object_exists(test_file) + assert custom_config.hcp_h.get_object(test_file) + +def test_download_file(custom_config : CustomConfig) -> None: + test_mount_bucket(custom_config) + Path(custom_config.result_path).mkdir() + test_file = Path(custom_config.test_file_path).name + custom_config.hcp_h.download_file(test_file, custom_config.result_path + test_file) + assert cmp(custom_config.result_path + test_file, custom_config.test_file_path) + +def test_download_file_without_mounting(custom_config : CustomConfig) -> None: + _hcp_h = custom_config.hcp_h _without_mounting(_hcp_h.download_file) -def test_download_nonexistent_file(dynamic_config : DynamicConfig) -> None: - test_mount_bucket(dynamic_config) +def test_download_nonexistent_file(custom_config : CustomConfig) -> None: + test_mount_bucket(custom_config) try: - dynamic_config.hcp_h.download_file("aFileThatDoesNotExist", dynamic_config.result_path + "aFileThatDoesNotExist") + custom_config.hcp_h.download_file("aFileThatDoesNotExist", custom_config.result_path + "aFileThatDoesNotExist") except: assert True else: # pragma: no cover assert False -def test_download_folder(dynamic_config : DynamicConfig) -> None: - test_mount_bucket(dynamic_config) - dynamic_config.hcp_h.download_folder("a folder of data/", dynamic_config.result_path) +def test_download_folder(custom_config : CustomConfig) -> None: + test_mount_bucket(custom_config) + custom_config.hcp_h.download_folder("a folder of data/", custom_config.result_path) -def test_search_objects_in_bucket(dynamic_config : DynamicConfig) -> None: - test_mount_bucket(dynamic_config) - test_file = Path(dynamic_config.test_file_path).name - dynamic_config.hcp_h.search_objects_in_bucket(test_file) +def test_search_objects_in_bucket(custom_config : CustomConfig) -> None: + test_mount_bucket(custom_config) + test_file = Path(custom_config.test_file_path).name + custom_config.hcp_h.search_objects_in_bucket(test_file) -def test_search_objects_in_bucket_without_mounting(dynamic_config : DynamicConfig) -> None: - _hcp_h = dynamic_config.hcp_h +def test_search_objects_in_bucket_without_mounting(custom_config : CustomConfig) -> None: + _hcp_h = custom_config.hcp_h _without_mounting(_hcp_h.search_objects_in_bucket) -def test_get_object_acl(dynamic_config : DynamicConfig) -> None: - test_mount_bucket(dynamic_config) - test_file = Path(dynamic_config.test_file_path).name - dynamic_config.hcp_h.get_object_acl(test_file) +def test_get_object_acl(custom_config : CustomConfig) -> None: + test_mount_bucket(custom_config) + test_file = Path(custom_config.test_file_path).name + custom_config.hcp_h.get_object_acl(test_file) -def test_get_object_acl_without_mounting(dynamic_config : DynamicConfig) -> None: - _hcp_h = dynamic_config.hcp_h +def test_get_object_acl_without_mounting(custom_config : CustomConfig) -> None: + _hcp_h = custom_config.hcp_h _without_mounting(_hcp_h.get_object_acl) -def test_get_bucket_acl(dynamic_config : DynamicConfig) -> None: - test_mount_bucket(dynamic_config) - dynamic_config.hcp_h.get_bucket_acl() +def test_get_bucket_acl(custom_config : CustomConfig) -> None: + test_mount_bucket(custom_config) + custom_config.hcp_h.get_bucket_acl() -def test_get_bucket_acl_without_mounting(dynamic_config : DynamicConfig) -> None: - _hcp_h = dynamic_config.hcp_h +def test_get_bucket_acl_without_mounting(custom_config : CustomConfig) -> None: + _hcp_h = custom_config.hcp_h _without_mounting(_hcp_h.get_bucket_acl) -#def test_modify_single_object_acl(dynamic_config : DynamicConfig) -> None: -# test_mount_bucket(dynamic_config) -# dynamic_config.hcp_h.modify_single_object_acl() +#def test_modify_single_object_acl(custom_config : CustomConfig) -> None: +# test_mount_bucket(custom_config) +# custom_config.hcp_h.modify_single_object_acl() # -#def test_modify_single_bucket_acl(dynamic_config : DynamicConfig) -> None: -# test_mount_bucket(dynamic_config) -# dynamic_config.hcp_h.modify_single_bucket_acl() +#def test_modify_single_bucket_acl(custom_config : CustomConfig) -> None: +# test_mount_bucket(custom_config) +# custom_config.hcp_h.modify_single_bucket_acl() # -#def test_modify_object_acl(dynamic_config : DynamicConfig) -> None: -# test_mount_bucket(dynamic_config) -# dynamic_config.hcp_h.modify_object_acl() +#def test_modify_object_acl(custom_config : CustomConfig) -> None: +# test_mount_bucket(custom_config) +# custom_config.hcp_h.modify_object_acl() # -#def test_modify_bucket_acl(dynamic_config : DynamicConfig) -> None: -# test_mount_bucket(dynamic_config) -# dynamic_config.hcp_h.modify_bucket_acl() - -def test_delete_file(dynamic_config : DynamicConfig) -> None: - test_mount_bucket(dynamic_config) - test_file = Path(dynamic_config.test_file_path).name - dynamic_config.hcp_h.delete_object(test_file) - dynamic_config.hcp_h.delete_object("a_sub_directory/a_file") - dynamic_config.hcp_h.delete_object("a_sub_directory") - -def test_delete_file_without_mounting(dynamic_config : DynamicConfig) -> None: - _hcp_h = dynamic_config.hcp_h +#def test_modify_bucket_acl(custom_config : CustomConfig) -> None: +# test_mount_bucket(custom_config) +# custom_config.hcp_h.modify_bucket_acl() + +def test_delete_file(custom_config : CustomConfig) -> None: + test_mount_bucket(custom_config) + test_file = Path(custom_config.test_file_path).name + custom_config.hcp_h.delete_object(test_file) + custom_config.hcp_h.delete_object("a_sub_directory/a_file") + custom_config.hcp_h.delete_object("a_sub_directory") + +def test_delete_file_without_mounting(custom_config : CustomConfig) -> None: + _hcp_h = custom_config.hcp_h _without_mounting(_hcp_h.delete_object) -def test_delete_folder_with_sub_directory(dynamic_config : DynamicConfig) -> None: - test_mount_bucket(dynamic_config) - dynamic_config.hcp_h.upload_file(dynamic_config.test_file_path, "a folder of data/a sub dir/a file") +def test_delete_folder_with_sub_directory(custom_config : CustomConfig) -> None: + test_mount_bucket(custom_config) + custom_config.hcp_h.upload_file(custom_config.test_file_path, "a folder of data/a sub dir/a file") try: - dynamic_config.hcp_h.delete_folder("a folder of data/") + custom_config.hcp_h.delete_folder("a folder of data/") except: assert True else: # pragma: no cover assert False - dynamic_config.hcp_h.delete_folder("a folder of data/a sub dir/") + custom_config.hcp_h.delete_folder("a folder of data/a sub dir/") -def test_delete_folder(dynamic_config : DynamicConfig) -> None: - test_mount_bucket(dynamic_config) - dynamic_config.hcp_h.delete_folder("a folder of data/") +def test_delete_folder(custom_config : CustomConfig) -> None: + test_mount_bucket(custom_config) + custom_config.hcp_h.delete_folder("a folder of data/") -def test_delete_folder_without_mounting(dynamic_config : DynamicConfig) -> None: - _hcp_h = dynamic_config.hcp_h +def test_delete_folder_without_mounting(custom_config : CustomConfig) -> None: + _hcp_h = custom_config.hcp_h _without_mounting(_hcp_h.delete_folder) -def test_delete_nonexistent_files(dynamic_config : DynamicConfig) -> None: - dynamic_config.hcp_h.delete_objects(["some", "files", "that", "does", "not", "exist"]) +def test_delete_nonexistent_files(custom_config : CustomConfig) -> None: + custom_config.hcp_h.delete_objects(["some", "files", "that", "does", "not", "exist"]) -def test_clean_up(dynamic_config : DynamicConfig) -> None: - rmtree(dynamic_config.result_path) \ No newline at end of file +def test_clean_up(custom_config : CustomConfig) -> None: + rmtree(custom_config.result_path) \ No newline at end of file From 7e760fabe62d48038851cb48f0fecf34d8ecceef Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Wed, 8 Jan 2025 09:54:14 +0100 Subject: [PATCH 46/53] Update conftest.py --- tests/conftest.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9ddf6a1..84219f4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,8 @@ + from pytest import Config, fixture, UsageError from configparser import ConfigParser from typing import Any + from NGPIris.hcp import HCPHandler class CustomConfig: @@ -17,13 +19,11 @@ def __getattr__(self, name : str) -> Any: """Dynamically get attributes set during pytest configuration.""" return getattr(self._config, name) - def set_section(config : Config, parser : ConfigParser, section : str) -> None: parse_dict = dict(parser.items(section)) for k, v in parse_dict.items(): setattr(config, k, v) # Adds attributes dynamically to pytest.Config - def pytest_addoption(parser) -> None: parser.addoption( "--config", @@ -32,7 +32,6 @@ def pytest_addoption(parser) -> None: help="Path to the configuration file (e.g., path/to/config.ini)", ) - def pytest_configure(config : Config) -> None: config_path = config.getoption("--config") if not config_path: From 8a45c71d52be00fb676ad6f26aacd3023b3ac970 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Wed, 8 Jan 2025 09:56:13 +0100 Subject: [PATCH 47/53] Update test_hcp.py --- tests/test_hcp.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/tests/test_hcp.py b/tests/test_hcp.py index d5577ed..6080124 100644 --- a/tests/test_hcp.py +++ b/tests/test_hcp.py @@ -1,8 +1,6 @@ from typing import Callable -from pytest import Config -from configparser import ConfigParser from pathlib import Path from shutil import rmtree from filecmp import cmp @@ -11,18 +9,6 @@ # --------------------------- Helper fucntions --------------------------------- -#def _get_hcp_handler(config_parser : ConfigParser) -> HCPHandler: -# return HCPHandler(config_parser.get("General", "credentials_path")) - -def _get_all_config(config_parser : ConfigParser) -> dict: - return dict(config_parser.items("HCP_tests")) - -#def _get_test_bucket(config_parser : ConfigParser) -> str: -# return config_parser.get("hcp_tests", "bucket") -# -#def _get_test_file_path(config_parser : ConfigParser) -> str: -# return config_parser.get("hcp_tests","data_test_file") - def _without_mounting(test : Callable) -> None: try: test() From 26b0da6fd7f8ac68c6428b57c963ed7c9263b7de Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Wed, 8 Jan 2025 10:09:53 +0100 Subject: [PATCH 48/53] Update test_hcp.py --- tests/test_hcp.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/test_hcp.py b/tests/test_hcp.py index 6080124..ff3ce8c 100644 --- a/tests/test_hcp.py +++ b/tests/test_hcp.py @@ -42,12 +42,7 @@ def test_test_connection_with_bucket_name(custom_config : CustomConfig) -> None: def test_test_connection_without_mounting_bucket(custom_config : CustomConfig) -> None: _hcp_h = custom_config.hcp_h - try: - _hcp_h.test_connection() - except: - assert True - else: # pragma: no cover - assert False + _without_mounting(_hcp_h.test_connection) def test_list_objects(custom_config : CustomConfig) -> None: test_mount_bucket(custom_config) From 720d1ba688d758068c9e29a7df1fed45eff2767a Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Wed, 8 Jan 2025 10:58:18 +0100 Subject: [PATCH 49/53] Fix `_without_mounting ` --- tests/test_hcp.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/tests/test_hcp.py b/tests/test_hcp.py index ff3ce8c..a6c982a 100644 --- a/tests/test_hcp.py +++ b/tests/test_hcp.py @@ -1,17 +1,20 @@ -from typing import Callable from pathlib import Path from shutil import rmtree from filecmp import cmp +from typing import Any, Callable from conftest import CustomConfig +from NGPIris.hcp import HCPHandler + # --------------------------- Helper fucntions --------------------------------- -def _without_mounting(test : Callable) -> None: +def _without_mounting(hcp_h : HCPHandler, hcp_h_method : Callable[..., Any]) -> None: + hcp_h.bucket_name = None try: - test() + hcp_h_method(hcp_h) except: assert True else: # pragma: no cover @@ -42,7 +45,7 @@ def test_test_connection_with_bucket_name(custom_config : CustomConfig) -> None: def test_test_connection_without_mounting_bucket(custom_config : CustomConfig) -> None: _hcp_h = custom_config.hcp_h - _without_mounting(_hcp_h.test_connection) + _without_mounting(_hcp_h, HCPHandler.test_connection) def test_list_objects(custom_config : CustomConfig) -> None: test_mount_bucket(custom_config) @@ -50,7 +53,7 @@ def test_list_objects(custom_config : CustomConfig) -> None: def test_list_objects_without_mounting(custom_config : CustomConfig) -> None: _hcp_h = custom_config.hcp_h - _without_mounting(_hcp_h.list_objects) + _without_mounting(_hcp_h, HCPHandler.list_objects) def test_upload_file(custom_config : CustomConfig) -> None: test_mount_bucket(custom_config) @@ -58,7 +61,7 @@ def test_upload_file(custom_config : CustomConfig) -> None: def test_upload_file_without_mounting(custom_config : CustomConfig) -> None: _hcp_h = custom_config.hcp_h - _without_mounting(_hcp_h.upload_file) + _without_mounting(_hcp_h, HCPHandler.upload_file) def test_upload_file_in_sub_directory(custom_config : CustomConfig) -> None: test_mount_bucket(custom_config) @@ -79,7 +82,7 @@ def test_upload_folder(custom_config : CustomConfig) -> None: def test_upload_folder_without_mounting(custom_config : CustomConfig) -> None: _hcp_h = custom_config.hcp_h - _without_mounting(_hcp_h.upload_folder) + _without_mounting(_hcp_h, HCPHandler.upload_folder) def test_upload_nonexisting_folder(custom_config : CustomConfig) -> None: test_mount_bucket(custom_config) @@ -97,8 +100,8 @@ def test_get_file(custom_config : CustomConfig) -> None: def test_get_folder_without_mounting(custom_config : CustomConfig) -> None: _hcp_h = custom_config.hcp_h - _without_mounting(_hcp_h.object_exists) - _without_mounting(_hcp_h.get_object) + _without_mounting(_hcp_h, HCPHandler.object_exists) + _without_mounting(_hcp_h, HCPHandler.get_object) def test_get_file_in_sub_directory(custom_config : CustomConfig) -> None: test_mount_bucket(custom_config) @@ -115,7 +118,7 @@ def test_download_file(custom_config : CustomConfig) -> None: def test_download_file_without_mounting(custom_config : CustomConfig) -> None: _hcp_h = custom_config.hcp_h - _without_mounting(_hcp_h.download_file) + _without_mounting(_hcp_h, HCPHandler.download_file) def test_download_nonexistent_file(custom_config : CustomConfig) -> None: test_mount_bucket(custom_config) @@ -137,7 +140,7 @@ def test_search_objects_in_bucket(custom_config : CustomConfig) -> None: def test_search_objects_in_bucket_without_mounting(custom_config : CustomConfig) -> None: _hcp_h = custom_config.hcp_h - _without_mounting(_hcp_h.search_objects_in_bucket) + _without_mounting(_hcp_h, HCPHandler.search_objects_in_bucket) def test_get_object_acl(custom_config : CustomConfig) -> None: test_mount_bucket(custom_config) @@ -146,7 +149,7 @@ def test_get_object_acl(custom_config : CustomConfig) -> None: def test_get_object_acl_without_mounting(custom_config : CustomConfig) -> None: _hcp_h = custom_config.hcp_h - _without_mounting(_hcp_h.get_object_acl) + _without_mounting(_hcp_h, HCPHandler.get_object_acl) def test_get_bucket_acl(custom_config : CustomConfig) -> None: test_mount_bucket(custom_config) @@ -154,7 +157,7 @@ def test_get_bucket_acl(custom_config : CustomConfig) -> None: def test_get_bucket_acl_without_mounting(custom_config : CustomConfig) -> None: _hcp_h = custom_config.hcp_h - _without_mounting(_hcp_h.get_bucket_acl) + _without_mounting(_hcp_h, HCPHandler.get_bucket_acl) #def test_modify_single_object_acl(custom_config : CustomConfig) -> None: # test_mount_bucket(custom_config) @@ -181,7 +184,7 @@ def test_delete_file(custom_config : CustomConfig) -> None: def test_delete_file_without_mounting(custom_config : CustomConfig) -> None: _hcp_h = custom_config.hcp_h - _without_mounting(_hcp_h.delete_object) + _without_mounting(_hcp_h, HCPHandler.delete_object) def test_delete_folder_with_sub_directory(custom_config : CustomConfig) -> None: test_mount_bucket(custom_config) @@ -200,9 +203,10 @@ def test_delete_folder(custom_config : CustomConfig) -> None: def test_delete_folder_without_mounting(custom_config : CustomConfig) -> None: _hcp_h = custom_config.hcp_h - _without_mounting(_hcp_h.delete_folder) + _without_mounting(_hcp_h, HCPHandler.delete_folder) def test_delete_nonexistent_files(custom_config : CustomConfig) -> None: + test_mount_bucket(custom_config) custom_config.hcp_h.delete_objects(["some", "files", "that", "does", "not", "exist"]) def test_clean_up(custom_config : CustomConfig) -> None: From 7403711fa9ff5c89b19d770f0bc285bf118f8de2 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Thu, 9 Jan 2025 09:24:07 +0100 Subject: [PATCH 50/53] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b96f0f5..fe24afc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "NGPIris" -version = "5.1.1.dev2" +version = "5.2.0" readme = "README.md" dependencies = [ "requests >= 2.31.0", From eef3e86dce0ff5d24495f5a6c1698e0c938092ff Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Thu, 16 Jan 2025 10:01:30 +0100 Subject: [PATCH 51/53] Update test_hcp.py --- tests/test_hcp.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_hcp.py b/tests/test_hcp.py index a6c982a..f89d586 100644 --- a/tests/test_hcp.py +++ b/tests/test_hcp.py @@ -159,6 +159,8 @@ def test_get_bucket_acl_without_mounting(custom_config : CustomConfig) -> None: _hcp_h = custom_config.hcp_h _without_mounting(_hcp_h, HCPHandler.get_bucket_acl) +# ------------------ Possibly future ACL tests --------------------------------- + #def test_modify_single_object_acl(custom_config : CustomConfig) -> None: # test_mount_bucket(custom_config) # custom_config.hcp_h.modify_single_object_acl() @@ -175,6 +177,8 @@ def test_get_bucket_acl_without_mounting(custom_config : CustomConfig) -> None: # test_mount_bucket(custom_config) # custom_config.hcp_h.modify_bucket_acl() +# ------------------------------------------------------------------------------ + def test_delete_file(custom_config : CustomConfig) -> None: test_mount_bucket(custom_config) test_file = Path(custom_config.test_file_path).name From 95619eac710a15cab9e46f53aabb2d556ee73975 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Thu, 16 Jan 2025 10:43:47 +0100 Subject: [PATCH 52/53] Update test_conf_template.ini --- tests/test_conf_template.ini | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_conf_template.ini b/tests/test_conf_template.ini index 97b4b90..2d3933f 100644 --- a/tests/test_conf_template.ini +++ b/tests/test_conf_template.ini @@ -2,5 +2,6 @@ credentials_path = [HCP_tests] -bucket = -data_test_file = 80MB_test_file \ No newline at end of file +test_bucket = +test_file_path = tests/data/80MB_test_file +result_path = tests/data/results/ \ No newline at end of file From a6a2a63fc8bd8e4becfe3916cd9b173d14262f41 Mon Sep 17 00:00:00 2001 From: Erik Brink Date: Thu, 16 Jan 2025 11:01:40 +0100 Subject: [PATCH 53/53] Move teardown logic to `conftest.py` --- tests/conftest.py | 16 +++++++++++++++- tests/test_hcp.py | 4 ---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 84219f4..0c9f6aa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,8 @@ from pytest import Config, fixture, UsageError from configparser import ConfigParser -from typing import Any +from typing import Any, Generator +from shutil import rmtree from NGPIris.hcp import HCPHandler @@ -40,12 +41,25 @@ def pytest_configure(config : Config) -> None: parser = ConfigParser() parser.read(str(config_path)) + # Add the INI parser to config + setattr(config, "parser", parser) + # Dynamically add an HCPHandler instance to config setattr(config, "hcp_h", HCPHandler(parser.get("General", "credentials_path"))) # Dynamically add all key-value pairs from "HCP_tests" section set_section(config, parser, "HCP_tests") +@fixture(scope = "session") +def hcp_result_path(pytestconfig : Config) -> str: + return pytestconfig.parser.get("HCP_tests", "result_path") # type: ignore + +@fixture(scope = "session", autouse = True) +def clean_up_after_tests(hcp_result_path : str) -> Generator[None, Any, None]: + # Setup code can go here if needed + yield + # Teardown code + rmtree(hcp_result_path) @fixture def custom_config(pytestconfig : Config) -> CustomConfig: diff --git a/tests/test_hcp.py b/tests/test_hcp.py index f89d586..3802405 100644 --- a/tests/test_hcp.py +++ b/tests/test_hcp.py @@ -1,7 +1,6 @@ from pathlib import Path -from shutil import rmtree from filecmp import cmp from typing import Any, Callable @@ -212,6 +211,3 @@ def test_delete_folder_without_mounting(custom_config : CustomConfig) -> None: def test_delete_nonexistent_files(custom_config : CustomConfig) -> None: test_mount_bucket(custom_config) custom_config.hcp_h.delete_objects(["some", "files", "that", "does", "not", "exist"]) - -def test_clean_up(custom_config : CustomConfig) -> None: - rmtree(custom_config.result_path) \ No newline at end of file