Add new Azure Flat object store file source

client/src/api/fileSources.ts

@@ -16,6 +16,10 @@ export const templateTypes: FileSourceTypesDetail = {
         icon: faCloud,
         message: "This is a repository plugin based on the Azure service.",
     },
+    azureflat: {
+        icon: faCloud,
+        message: "This is a repository plugin based on the Azure flat namespace service.",
+    },
     dropbox: {
         icon: faDropbox,
         message: "This is a repository plugin that connects with the commercial Dropbox service.",

client/src/api/schema/schema.ts

@@ -11903,6 +11903,7 @@ export interface components {
                 | "posix"
                 | "s3fs"
                 | "azure"
+                | "azureflat"
                 | "onedata"
                 | "webdav"
                 | "dropbox"
@@ -23061,6 +23062,7 @@ export interface components {
                 | "posix"
                 | "s3fs"
                 | "azure"
+                | "azureflat"
                 | "onedata"
                 | "webdav"
                 | "dropbox"

lib/galaxy/dependencies/__init__.py

@@ -333,6 +333,9 @@ def is_redis_url(url: str) -> bool:
 
         return celery_enabled and is_redis_url(celery_result_backend) or is_redis_url(celery_broker_url)
 
+    def check_adlfs(self):
+        return "azureflat" in self.file_sources
+
     def check_huggingface_hub(self):
         return "huggingface" in self.file_sources
 

lib/galaxy/dependencies/conditional-requirements.txt

@@ -33,6 +33,7 @@ fs.onedatarestfs==21.2.5.2 # type: onedata, depends on onedatafilerestclient
 fs-basespace # type: basespace
 fs-azureblob # type: azure
 rspace-client>=2.6.1,<3  # type: rspace
+adlfs
 huggingface_hub
 
 # Vault backend

lib/galaxy/files/sources/azureflat.py

@@ -0,0 +1,166 @@
+import logging
+from typing import (
+    Optional,
+    Union,
+)
+
+from galaxy.files.models import (
+    AnyRemoteEntry,
+    FilesSourceRuntimeContext,
+    RemoteDirectory,
+)
+from galaxy.files.sources._fsspec import (
+    CacheOptionsDictType,
+    FsspecBaseFileSourceConfiguration,
+    FsspecBaseFileSourceTemplateConfiguration,
+    FsspecFilesSource,
+)
+from galaxy.util.config_templates import TemplateExpansion
+
+try:
+    from adlfs import AzureBlobFileSystem
+except ImportError:
+    AzureBlobFileSystem = None
+
+
+REQUIRED_PACKAGE = "adlfs"
+FS_PLUGIN_TYPE = "azureflat"
+
+log = logging.getLogger(__name__)
+
+
+class AzureFlatFileSourceTemplateConfiguration(FsspecBaseFileSourceTemplateConfiguration):
+    account_name: Union[str, TemplateExpansion]
+    container_name: Union[str, TemplateExpansion, None] = None
+    account_key: Union[str, TemplateExpansion]
+
+
+class AzureFlatFileSourceConfiguration(FsspecBaseFileSourceConfiguration):
+    account_name: str
+    container_name: Optional[str] = None
+    account_key: str
+
+
+class AzureFlatFilesSource(
+    FsspecFilesSource[AzureFlatFileSourceTemplateConfiguration, AzureFlatFileSourceConfiguration]
+):
+    plugin_type = FS_PLUGIN_TYPE
+    required_module = AzureBlobFileSystem
+    required_package = REQUIRED_PACKAGE
+    template_config_class = AzureFlatFileSourceTemplateConfiguration
+    resolved_config_class = AzureFlatFileSourceConfiguration
+
+    def _open_fs(
+        self, context: FilesSourceRuntimeContext[AzureFlatFileSourceConfiguration], cache_options: CacheOptionsDictType
+    ):
+        if AzureBlobFileSystem is None:
+            raise self.required_package_exception
+        else:
+            config = context.config
+            fs = AzureBlobFileSystem(account_name=config.account_name, credential=config.account_key, **cache_options)
+            return fs
+
+    def _to_container_path(self, path: str, config: AzureFlatFileSourceConfiguration) -> str:
+        result = ""
+        if path.startswith("az://"):
+            result = path.replace("az://", "", 1)
+        else:
+            container = config.container_name
+            if not container:
+                result = path.strip("/")
+            else:
+                result = self._container_path(container, path)
+        return result
+
+    def _adapt_entry_path(self, filesystem_path: str) -> str:
+        result = filesystem_path
+        container_name = self.template_config.container_name
+        if container_name:
+            prefix = f"{container_name}/"
+            if filesystem_path.startswith(prefix):
+                result = filesystem_path.replace(prefix, "", 1)
+            else:
+                result = filesystem_path
+        else:
+            result = filesystem_path
+        return result
+
+    def _list(
+        self,
+        context: FilesSourceRuntimeContext[AzureFlatFileSourceConfiguration],
+        path: str = "/",
+        recursive: bool = False,
+        write_intent: bool = False,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+        query: Optional[str] = None,
+        sort_by: Optional[str] = None,
+    ) -> tuple[list[AnyRemoteEntry], int]:
+        if context.config.container_name is None and path == "/":
+            fs = self._open_fs(context, {})
+            containers = fs.ls("", detail=True)
+            entries: list[AnyRemoteEntry] = []
+            for entry in containers:
+                name = entry["name"]
+                uri = f"az://{name}"
+                entries.append(
+                    RemoteDirectory(
+                        name=name,
+                        uri=uri,
+                        path=f"/{name}",
+                    )
+                )
+            return entries, len(entries)
+        container_path = self._to_container_path(path, context.config)
+        entries, count = super()._list(
+            context=context,
+            path=container_path,
+            recursive=recursive,
+            limit=limit,
+            offset=offset,
+            query=query,
+            sort_by=sort_by,
+        )
+        return entries, count
+
+    def _realize_to(
+        self, source_path: str, native_path: str, context: FilesSourceRuntimeContext[AzureFlatFileSourceConfiguration]
+    ):
+        container_path = self._to_container_path(source_path, context.config)
+        super()._realize_to(source_path=container_path, native_path=native_path, context=context)
+
+    def _write_from(
+        self, target_path: str, native_path: str, context: FilesSourceRuntimeContext[AzureFlatFileSourceConfiguration]
+    ):
+        container_path = self._to_container_path(target_path, context.config)
+        super()._write_from(target_path=container_path, native_path=native_path, context=context)
+
+    def _container_path(self, container_name: str, path: str) -> str:
+        adjusted_path = path
+        if path.startswith("az://"):
+            adjusted_path = path.replace("az://", "", 1)
+        else:
+            if not path.startswith("/"):
+                adjusted_path = f"/{path}"
+            else:
+                adjusted_path = path
+        return f"{container_name}{adjusted_path}"
+
+    def score_url_match(self, url: str):
+        container_name = self.template_config.container_name
+        result = 0
+        if container_name:
+            prefix = f"az://{container_name}"
+            if url.startswith(f"{prefix}/") or url == prefix:
+                result = len(prefix)
+            else:
+                result = super().score_url_match(url)
+        else:
+            if url.startswith("az://"):
+                result = len("az://")
+            else:
+                result = super().score_url_match(url)
+        return result
+
+
+__all__ = ("AzureFlatFilesSource",)

lib/galaxy/files/templates/examples/production_azureflat.yml

@@ -0,0 +1,35 @@
+- id: azureflat
+  version: 0
+  name: Azure Blob Storage Flat
+  description: |
+    Setup access to your [Azure Blob Storage](https://learn.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction).
+  configuration:
+    type: azure
+    container_name: "{{ variables.container_name }}"
+    account_name: "{{ variables.account_name }}"
+    account_key: "{{ secrets.account_key }}"
+    writable: "{{ variables.writable }}"
+  variables:
+    container_name:
+      label: Container Name
+      type: string
+      help: |
+        The name of your Azure Blob Storage container. More information on containers can be found
+        in the [Azure Storage documentation](https://learn.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction#containers).
+    account_name:
+      label: Storage Account Name
+      type: string
+      help: |
+        The name of your Azure Blob Storage account. More information on containers can be found in the
+        [Azure Storage documentation](https://learn.microsoft.com/en-us/azure/storage/common/storage-account-overview).
+    writable:
+      label: Writable?
+      type: boolean
+      default: false
+      help: Allow Galaxy to write data to this Azure Blob Storage container.
+  secrets:
+    account_key:
+      label: Account Key
+      help: |
+        The Azure Blob Storage account key to use to access your Azure Blob Storage data. More information
+        on account keys can be found in the [Azure Storage documentation](https://learn.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage).

lib/galaxy/files/templates/models.py

@@ -36,6 +36,7 @@
     "posix",
     "s3fs",
     "azure",
+    "azureflat",
     "onedata",
     "webdav",
     "dropbox",
@@ -173,6 +174,24 @@ class AzureFileSourceConfiguration(StrictModel):
     writable: bool = False
 
 
+class AzureFlatFileSourceTemplateConfiguration(StrictModel):
+    type: Literal["azureflat"]
+    account_name: Union[str, TemplateExpansion]
+    container_name: Union[str, TemplateExpansion, None] = None
+    account_key: Union[str, TemplateExpansion]
+    writable: Union[bool, TemplateExpansion] = False
+    template_start: Optional[str] = None
+    template_end: Optional[str] = None
+
+
+class AzureFlatFileSourceConfiguration(StrictModel):
+    type: Literal["azureflat"]
+    account_name: str
+    container_name: Optional[str] = None
+    account_key: str
+    writable: bool = False
+
+
 class OnedataFileSourceTemplateConfiguration(StrictModel):
     type: Literal["onedata"]
     access_token: Union[str, TemplateExpansion]
@@ -317,6 +336,7 @@ class HuggingFaceFileSourceConfiguration(StrictModel):
         S3FSFileSourceTemplateConfiguration,
         FtpFileSourceTemplateConfiguration,
         AzureFileSourceTemplateConfiguration,
+        AzureFlatFileSourceTemplateConfiguration,
         OnedataFileSourceTemplateConfiguration,
         WebdavFileSourceTemplateConfiguration,
         DropboxFileSourceTemplateConfiguration,
@@ -337,6 +357,7 @@ class HuggingFaceFileSourceConfiguration(StrictModel):
         S3FSFileSourceConfiguration,
         FtpFileSourceConfiguration,
         AzureFileSourceConfiguration,
+        AzureFlatFileSourceConfiguration,
         OnedataFileSourceConfiguration,
         WebdavFileSourceConfiguration,
         DropboxFileSourceConfiguration,
@@ -415,6 +436,7 @@ def template_to_configuration(
     "posix": PosixFileSourceConfiguration,
     "s3fs": S3FSFileSourceConfiguration,
     "azure": AzureFileSourceConfiguration,
+    "azureflat": AzureFlatFileSourceConfiguration,
     "onedata": OnedataFileSourceConfiguration,
     "webdav": WebdavFileSourceConfiguration,
     "dropbox": DropboxFileSourceConfiguration,