From ba428104b3231d28e00ac3007f9fb23a241861d0 Mon Sep 17 00:00:00 2001 From: Bartosz Walkowicz Date: Wed, 19 Jun 2024 10:11:39 +0200 Subject: [PATCH 01/10] Add onedata file source template --- .../sample/file_sources_conf.yml.sample | 6 ++-- lib/galaxy/files/sources/onedata.py | 6 ++-- .../templates/examples/production_onedata.yml | 30 +++++++++++++++++++ lib/galaxy/files/templates/models.py | 23 +++++++++++++- 4 files changed, 58 insertions(+), 7 deletions(-) create mode 100644 lib/galaxy/files/templates/examples/production_onedata.yml diff --git a/lib/galaxy/config/sample/file_sources_conf.yml.sample b/lib/galaxy/config/sample/file_sources_conf.yml.sample index 534e77736541..0bc5e9e10aed 100644 --- a/lib/galaxy/config/sample/file_sources_conf.yml.sample +++ b/lib/galaxy/config/sample/file_sources_conf.yml.sample @@ -235,6 +235,6 @@ id: onedata1 label: Onedata doc: Your Onedata files - configure an access token via user preferences - accessToken: ${user.preferences['onedata|access_token']} - onezoneDomain: ${user.preferences['onedata|onezone_domain']} - disableTlsCertificateValidation: ${user.preferences['onedata|disable_tls_certificate_validation']} + access_token: ${user.preferences['onedata|access_token']} + onezone_domain: ${user.preferences['onedata|onezone_domain']} + disable_tls_certificate_validation: ${user.preferences['onedata|disable_tls_certificate_validation']} diff --git a/lib/galaxy/files/sources/onedata.py b/lib/galaxy/files/sources/onedata.py index af7c104c9eaf..a46bf761b1a1 100644 --- a/lib/galaxy/files/sources/onedata.py +++ b/lib/galaxy/files/sources/onedata.py @@ -22,10 +22,10 @@ class OnedataFilesSource(PyFilesystem2FilesSource): def _open_fs(self, user_context=None, opts: Optional[FilesSourceOptions] = None): props = self._serialization_props(user_context) - onezone_domain = props.pop("onezoneDomain", "") or "" + onezone_domain = props.pop("onezone_domain", "") or "" onezone_domain = remove_prefix("http://", remove_prefix("https://", onezone_domain)) - access_token = props.pop("accessToken", "") or "" - disable_tls_certificate_validation = props.pop("disableTlsCertificateValidation", False) or False + access_token = props.pop("access_token", "") or "" + disable_tls_certificate_validation = props.pop("disable_tls_certificate_validation", False) or False handle = OnedataRESTFS(onezone_domain, access_token, verify_ssl=not disable_tls_certificate_validation) return handle diff --git a/lib/galaxy/files/templates/examples/production_onedata.yml b/lib/galaxy/files/templates/examples/production_onedata.yml new file mode 100644 index 000000000000..d8d458398b49 --- /dev/null +++ b/lib/galaxy/files/templates/examples/production_onedata.yml @@ -0,0 +1,30 @@ +- id: onedata + version: 0 + name: Onedata Storage + description: | + This template allows connecting to Onedata servers. + + Onedata is a global data management system providing a unified data access + interface across different storage providers. More information on Onedata + can be found in [Onedata's website](https://onedata.org/#/home). + variables: + onezone_domain: + label: Onezone domain + type: string + help: Domain of the Onezone service (e.g. datahub.egi.eu) to connect to. + disable_tls_certificate_validation: + label: Disable tls certificate validation? + type: boolean + help: | + Allows connection to Onedata servers that do not present trusted SSL certificates. + SHOULD NOT be used unless you really know what you are doing. + default: False + secrets: + access_token: + label: REST Access Token + help: Your access token, suitable for REST API access in a Oneprovider service. + configuration: + type: onedata + access_token: "{{ secrets.access_token }}" + onezone_domain: "{{ variables.onezone_domain }}" + disable_tls_certificate_validation: "{{ variables.disable_tls_certificate_validation }}" diff --git a/lib/galaxy/files/templates/models.py b/lib/galaxy/files/templates/models.py index 2e22cc904c30..e4b7fcea49e8 100644 --- a/lib/galaxy/files/templates/models.py +++ b/lib/galaxy/files/templates/models.py @@ -25,7 +25,7 @@ UserDetailsDict, ) -FileSourceTemplateType = Literal["ftp", "posix", "s3fs", "azure"] +FileSourceTemplateType = Literal["ftp", "posix", "s3fs", "azure", "onedata"] class PosixFileSourceTemplateConfiguration(StrictModel): @@ -104,17 +104,37 @@ class AzureFileSourceConfiguration(StrictModel): writable: bool = False +class OnedataFileSourceTemplateConfiguration(StrictModel): + type: Literal["onedata"] + access_token: Union[str, TemplateExpansion] + onezone_domain: Union[str, TemplateExpansion] + disable_tls_certificate_validation: Union[bool, TemplateExpansion] = False + writable: Union[bool, TemplateExpansion] = False + template_start: Optional[str] = None + template_end: Optional[str] = None + + +class OnedataFileSourceConfiguration(StrictModel): + type: Literal["onedata"] + access_token: str + onezone_domain: str + disable_tls_certificate_validation: bool = False + writable: bool = False + + FileSourceTemplateConfiguration = Union[ PosixFileSourceTemplateConfiguration, S3FSFileSourceTemplateConfiguration, FtpFileSourceTemplateConfiguration, AzureFileSourceTemplateConfiguration, + OnedataFileSourceTemplateConfiguration, ] FileSourceConfiguration = Union[ PosixFileSourceConfiguration, S3FSFileSourceConfiguration, FtpFileSourceConfiguration, AzureFileSourceConfiguration, + OnedataFileSourceConfiguration, ] @@ -175,6 +195,7 @@ def template_to_configuration( "posix": PosixFileSourceConfiguration, "s3fs": S3FSFileSourceConfiguration, "azure": AzureFileSourceConfiguration, + "onedata": OnedataFileSourceConfiguration, } From ddf2a4891ab33b56dc4791840bdddfe6b41bf429 Mon Sep 17 00:00:00 2001 From: Bartosz Walkowicz Date: Wed, 19 Jun 2024 10:14:13 +0200 Subject: [PATCH 02/10] Add onedata object store template --- .../sample/object_store_conf.sample.yml | 2 +- lib/galaxy/objectstore/onedata.py | 4 +- .../templates/examples/production_onedata.yml | 46 +++++++++++++++++ lib/galaxy/objectstore/templates/models.py | 51 ++++++++++++++++++- 4 files changed, 99 insertions(+), 4 deletions(-) create mode 100644 lib/galaxy/objectstore/templates/examples/production_onedata.yml diff --git a/lib/galaxy/config/sample/object_store_conf.sample.yml b/lib/galaxy/config/sample/object_store_conf.sample.yml index afbcfc22323b..97d6413824bb 100644 --- a/lib/galaxy/config/sample/object_store_conf.sample.yml +++ b/lib/galaxy/config/sample/object_store_conf.sample.yml @@ -234,7 +234,7 @@ space: name: demo-space # the relative directory path in the space under which the Galaxy data will be stored. # Optional, if not provided, the data will be stored in the root of space. - path: galaxy-data + galaxy_root_dir: galaxy-data cache: path: database/object_store_cache_s3 size: 1000 diff --git a/lib/galaxy/objectstore/onedata.py b/lib/galaxy/objectstore/onedata.py index 7fc8d5b72c27..8a6aef16bf00 100644 --- a/lib/galaxy/objectstore/onedata.py +++ b/lib/galaxy/objectstore/onedata.py @@ -107,8 +107,8 @@ def __init__(self, config, config_dict): cache_dict = config_dict.get("cache") or {} self.enable_cache_monitor, self.cache_monitor_interval = enable_cache_monitor(config, config_dict) - self.cache_size = cache_dict["size"] or self.config.object_store_cache_size - self.staging_path = cache_dict["path"] or self.config.object_store_cache_path + self.cache_size = cache_dict.get("size") or self.config.object_store_cache_size + self.staging_path = cache_dict.get("path") or self.config.object_store_cache_path self.cache_updated_data = cache_dict.get("cache_updated_data", True) extra_dirs = {e["type"]: e["path"] for e in config_dict.get("extra_dirs", [])} diff --git a/lib/galaxy/objectstore/templates/examples/production_onedata.yml b/lib/galaxy/objectstore/templates/examples/production_onedata.yml new file mode 100644 index 000000000000..55f6ee2e8a73 --- /dev/null +++ b/lib/galaxy/objectstore/templates/examples/production_onedata.yml @@ -0,0 +1,46 @@ +- id: onedata + name: Onedata Storage + version: 0 + description: | + This template allows using Onedata storage. + + Onedata is a global data management system providing a unified data access + interface across different storage providers. More information on Onedata + can be found in [Onedata's website](https://onedata.org/#/home). + variables: + onezone_domain: + label: Onezone Domain + type: string + help: Domain of the Onezone service (e.g. datahub.egi.eu) to connect to. + disable_tls_certificate_validation: + label: Disable tls certificate validation? + type: boolean + help: | + Allows connection to Onedata servers that do not present trusted SSL certificates. + SHOULD NOT be used unless you really know what you are doing. + default: False + space_name: + label: Space Name + type: string + help: The name of the Onedata space where the Galaxy data will be stored. + galaxy_root_dir: + label: Galaxy root directory + type: string + help: | + The relative directory path in the space under which the Galaxy data + will be stored. + default: "" + secrets: + access_token: + label: REST Access Token + help: Your access token, suitable for REST API access in a Oneprovider service. + configuration: + type: onedata + auth: + access_token: '{{ secrets.access_token }}' + connection: + onezone_domain: "{{ variables.onezone_domain }}" + disable_tls_certificate_validation: "{{ variables.disable_tls_certificate_validation }}" + space: + name: "{{ variables.space_name }}" + galaxy_root_dir: "{{ variables.galaxy_root_dir }}" diff --git a/lib/galaxy/objectstore/templates/models.py b/lib/galaxy/objectstore/templates/models.py index dcbabb760434..a066163d5b37 100644 --- a/lib/galaxy/objectstore/templates/models.py +++ b/lib/galaxy/objectstore/templates/models.py @@ -32,7 +32,7 @@ ObjectStoreTemplateVariableType = TemplateVariableType ObjectStoreTemplateVariableValueType = TemplateVariableValueType -ObjectStoreTemplateType = Literal["aws_s3", "azure_blob", "boto3", "disk", "generic_s3"] +ObjectStoreTemplateType = Literal["aws_s3", "azure_blob", "boto3", "disk", "generic_s3", "onedata"] class S3AuthTemplate(StrictModel): @@ -266,12 +266,59 @@ class GenericS3ObjectStoreConfiguration(StrictModel): badges: BadgeList = None +class OnedataAuthTemplate(StrictModel): + access_token: Union[str, TemplateExpansion] + + +class OnedataAuth(StrictModel): + access_token: str + + +class OnedataConnectionTemplate(StrictModel): + onezone_domain: Union[str, TemplateExpansion] + disable_tls_certificate_validation: Union[bool, TemplateExpansion] = False + + +class OnedataConnection(StrictModel): + onezone_domain: str + disable_tls_certificate_validation: bool = False + + +class OnedataSpaceTemplate(StrictModel): + name: Union[str, TemplateExpansion] + galaxy_root_dir: Optional[Union[str, TemplateExpansion]] = "" + + +class OnedataSpace(StrictModel): + name: str + galaxy_root_dir: str + + +class OnedataObjectStoreTemplateConfiguration(StrictModel): + type: Literal["onedata"] + auth: OnedataAuthTemplate + connection: OnedataConnectionTemplate + space: OnedataSpaceTemplate + badges: BadgeList = None + template_start: Optional[str] = None + template_end: Optional[str] = None + + +class OnedataObjectStoreConfiguration(StrictModel): + type: Literal["onedata"] + auth: OnedataAuth + connection: OnedataConnection + space: OnedataSpace + badges: BadgeList = None + + ObjectStoreTemplateConfiguration = Union[ AwsS3ObjectStoreTemplateConfiguration, Boto3ObjectStoreTemplateConfiguration, GenericS3ObjectStoreTemplateConfiguration, DiskObjectStoreTemplateConfiguration, AzureObjectStoreTemplateConfiguration, + OnedataObjectStoreTemplateConfiguration, ] ObjectStoreConfiguration = Union[ AwsS3ObjectStoreConfiguration, @@ -279,6 +326,7 @@ class GenericS3ObjectStoreConfiguration(StrictModel): DiskObjectStoreConfiguration, AzureObjectStoreConfiguration, GenericS3ObjectStoreConfiguration, + OnedataObjectStoreConfiguration, ] @@ -345,6 +393,7 @@ def template_to_configuration( "generic_s3": GenericS3ObjectStoreConfiguration, "azure_blob": AzureObjectStoreConfiguration, "disk": DiskObjectStoreConfiguration, + "onedata": OnedataObjectStoreConfiguration, } From d2ccc07f1d58765b57196039e1b38b642db5f554 Mon Sep 17 00:00:00 2001 From: Bartosz Walkowicz Date: Mon, 24 Jun 2024 20:05:52 +0200 Subject: [PATCH 03/10] Unify onedata object store yml and xml configs --- lib/galaxy/config/sample/object_store_conf.sample.yml | 5 ++++- lib/galaxy/config/sample/object_store_conf.xml.sample | 9 ++++++--- lib/galaxy/objectstore/onedata.py | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/config/sample/object_store_conf.sample.yml b/lib/galaxy/config/sample/object_store_conf.sample.yml index 97d6413824bb..f597abd9a667 100644 --- a/lib/galaxy/config/sample/object_store_conf.sample.yml +++ b/lib/galaxy/config/sample/object_store_conf.sample.yml @@ -231,8 +231,11 @@ connection: disable_tls_certificate_validation: false space: # the name of the Onedata space where the Galaxy data will be stored. + # If there is more than one space with the same name, you can explicitly + # specify which one to select by using the format @ + # (e.g. demo@7285220ecc636075ae5759aec7ad65d3cha8f9). name: demo-space - # the relative directory path in the space under which the Galaxy data will be stored. + # the relative directory path in the space at which the Galaxy data will be stored. # Optional, if not provided, the data will be stored in the root of space. galaxy_root_dir: galaxy-data cache: diff --git a/lib/galaxy/config/sample/object_store_conf.xml.sample b/lib/galaxy/config/sample/object_store_conf.xml.sample index 1b9d997271e9..301b12463f9e 100644 --- a/lib/galaxy/config/sample/object_store_conf.xml.sample +++ b/lib/galaxy/config/sample/object_store_conf.xml.sample @@ -174,16 +174,19 @@ //space/@name - the name of the Onedata space where the Galaxy data will be stored. + If there is more than one space with the same name, you can explicitly + specify which one to select by using the format @ + (e.g. demo@7285220ecc636075ae5759aec7ad65d3cha8f9). - //space/@path - - the relative directory path in the space under which the Galaxy data will be stored. + //space/@galaxy_root_dir - + the relative directory path in the space at which the Galaxy data will be stored. Optional, if not provided, the data will be stored in the root of space. -->