From e27221c839b161c64a4d6c2d5dc4df9d789aaff7 Mon Sep 17 00:00:00 2001 From: Seher Karakuzu Date: Fri, 12 Apr 2024 15:34:44 -0400 Subject: [PATCH] preliminary addressing to the comments --- tiled/adapters/array.py | 24 ++++------- tiled/adapters/awkward.py | 10 ++--- tiled/adapters/awkward_buffers.py | 11 +++-- tiled/adapters/awkward_directory_container.py | 5 +-- tiled/adapters/csv.py | 14 +++---- tiled/adapters/hdf5.py | 16 ++++---- tiled/adapters/mapping.py | 17 ++++---- tiled/adapters/parquet.py | 11 ++--- tiled/adapters/protocols.py | 41 +++++++++++-------- tiled/adapters/sparse.py | 18 ++++---- tiled/adapters/sparse_blocks_parquet.py | 19 ++++----- tiled/adapters/table.py | 8 ++-- tiled/adapters/tiff.py | 16 ++++---- tiled/adapters/type_alliases.py | 10 ++++- tiled/adapters/xarray.py | 8 ++-- tiled/adapters/zarr.py | 30 +++++--------- 16 files changed, 122 insertions(+), 136 deletions(-) diff --git a/tiled/adapters/array.py b/tiled/adapters/array.py index 1a7dc59cf..c849bc799 100644 --- a/tiled/adapters/array.py +++ b/tiled/adapters/array.py @@ -1,13 +1,12 @@ -from types import EllipsisType -from typing import Any, List, Optional, Tuple, Union +from typing import Any, List, Optional, Tuple import dask.array from numpy.typing import NDArray -from ..access_policies import DummyAccessPolicy, SimpleAccessPolicy from ..structures.array import ArrayStructure from ..structures.core import Spec, StructureFamily -from .type_alliases import JSON +from .protocols import AccessPolicy +from .type_alliases import JSON, NDSlice class ArrayAdapter: @@ -34,7 +33,7 @@ def __init__( *, metadata: Optional[JSON] = None, specs: Optional[List[Spec]] = None, - access_policy: Optional[Union[SimpleAccessPolicy, DummyAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, ) -> None: """ @@ -61,7 +60,7 @@ def from_array( dims: Optional[Tuple[str, ...]] = None, metadata: Optional[JSON] = None, specs: Optional[List[Spec]] = None, - access_policy: Optional[Union[SimpleAccessPolicy, DummyAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, ) -> "ArrayAdapter": """ @@ -129,9 +128,7 @@ def structure(self) -> ArrayStructure: def read( self, - slice: Union[ - int, slice, Tuple[Union[int, slice, EllipsisType], ...], EllipsisType - ] = ..., + slice: NDSlice = ..., ) -> NDArray[Any]: """ @@ -143,8 +140,7 @@ def read( ------- """ - array = self._array - array = array[slice] + array = self._array[slice] if isinstance(self._array, dask.array.Array): return array.compute() return array @@ -152,9 +148,7 @@ def read( def read_block( self, block: Tuple[int, ...], - slice: Union[ - int, slice, Tuple[Union[int, slice, EllipsisType], ...], EllipsisType - ] = ..., + slice: NDSlice = ..., ) -> NDArray[Any]: """ @@ -180,7 +174,7 @@ def read_block( def slice_and_shape_from_block_and_chunks( block: Tuple[int, ...], chunks: Tuple[Tuple[int, ...], ...] -) -> Tuple[Tuple[slice, ...], Tuple[int, ...]]: +) -> Tuple[NDSlice, Tuple[int, ...]]: """ Given dask-like chunks and block id, return slice and shape of the block. Parameters diff --git a/tiled/adapters/awkward.py b/tiled/adapters/awkward.py index a33b7975d..762fefcf0 100644 --- a/tiled/adapters/awkward.py +++ b/tiled/adapters/awkward.py @@ -1,13 +1,13 @@ -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional import awkward import awkward.forms from numpy.typing import NDArray -from ..access_policies import DummyAccessPolicy, SimpleAccessPolicy from ..structures.awkward import AwkwardStructure from ..structures.core import Spec, StructureFamily from .awkward_directory_container import DirectoryContainer +from .protocols import AccessPolicy from .type_alliases import JSON @@ -20,7 +20,7 @@ def __init__( structure: AwkwardStructure, metadata: Optional[JSON] = None, specs: Optional[List[Spec]] = None, - access_policy: Optional[Union[DummyAccessPolicy, SimpleAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, ) -> None: """ @@ -44,7 +44,7 @@ def from_array( array: NDArray[Any], metadata: Optional[JSON] = None, specs: Optional[List[Spec]] = None, - access_policy: Optional[Union[DummyAccessPolicy, SimpleAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, ) -> "AwkwardAdapter": """ @@ -78,7 +78,7 @@ def metadata(self) -> JSON: """ return self._metadata - def read_buffers(self, form_keys: Optional[List[str]] = None) -> Dict[str, Any]: + def read_buffers(self, form_keys: Optional[List[str]] = None) -> Dict[str, bytes]: """ Parameters diff --git a/tiled/adapters/awkward_buffers.py b/tiled/adapters/awkward_buffers.py index a418ef5c3..8a595e6d8 100644 --- a/tiled/adapters/awkward_buffers.py +++ b/tiled/adapters/awkward_buffers.py @@ -2,16 +2,17 @@ A directory containing awkward buffers, one file per form key. """ from pathlib import Path -from typing import Any, List, Optional, Union +from typing import List, Optional import awkward.forms -from ..access_policies import DummyAccessPolicy, SimpleAccessPolicy +from ..server.schemas import Asset from ..structures.awkward import AwkwardStructure from ..structures.core import Spec, StructureFamily from ..utils import path_from_uri from .awkward import AwkwardAdapter from .awkward_directory_container import DirectoryContainer +from .protocols import AccessPolicy from .type_alliases import JSON @@ -21,7 +22,7 @@ class AwkwardBuffersAdapter(AwkwardAdapter): structure_family = StructureFamily.awkward @classmethod - def init_storage(cls, data_uri: str, structure: AwkwardStructure) -> List[Any]: + def init_storage(cls, data_uri: str, structure: AwkwardStructure) -> List[Asset]: """ Parameters @@ -33,8 +34,6 @@ def init_storage(cls, data_uri: str, structure: AwkwardStructure) -> List[Any]: ------- """ - from ..server.schemas import Asset - directory: Path = path_from_uri(data_uri) directory.mkdir(parents=True, exist_ok=True) return [Asset(data_uri=data_uri, is_directory=True, parameter="data_uri")] @@ -46,7 +45,7 @@ def from_directory( structure: AwkwardStructure, metadata: Optional[JSON] = None, specs: Optional[List[Spec]] = None, - access_policy: Optional[Union[DummyAccessPolicy, SimpleAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, ) -> "AwkwardBuffersAdapter": """ diff --git a/tiled/adapters/awkward_directory_container.py b/tiled/adapters/awkward_directory_container.py index 1cc6103c9..fca412a81 100644 --- a/tiled/adapters/awkward_directory_container.py +++ b/tiled/adapters/awkward_directory_container.py @@ -1,7 +1,4 @@ -""" -A directory containing awkward buffers, one file per form key. -""" -import collections.abc +import collections from pathlib import Path from typing import Any, Iterator diff --git a/tiled/adapters/csv.py b/tiled/adapters/csv.py index 07772815a..c9c8c261c 100644 --- a/tiled/adapters/csv.py +++ b/tiled/adapters/csv.py @@ -4,13 +4,13 @@ import dask.dataframe import pandas -from ..access_policies import DummyAccessPolicy, SimpleAccessPolicy from ..structures.core import Spec, StructureFamily from ..structures.data_source import Asset, DataSource, Management from ..structures.table import TableStructure from ..utils import ensure_uri, path_from_uri from .array import ArrayAdapter from .dataframe import DataFrameAdapter +from .protocols import AccessPolicy from .table import TableAdapter from .type_alliases import JSON @@ -29,7 +29,7 @@ def read_csv( structure: Optional[TableStructure] = None, metadata: Optional[JSON] = None, specs: Optional[List[Spec]] = None, - access_policy: Optional[Union[DummyAccessPolicy, SimpleAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, **kwargs: Any, ) -> TableAdapter: """ @@ -85,7 +85,7 @@ def __init__( structure: Optional[TableStructure] = None, metadata: Optional[JSON] = None, specs: Optional[List[Spec]] = None, - access_policy: Optional[Union[DummyAccessPolicy, SimpleAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, ) -> None: """ @@ -134,7 +134,7 @@ def dataframe_adapter(self) -> TableAdapter: return DataFrameAdapter(partitions, self._structure) @classmethod - def init_storage(cls, data_uri: str, structure: TableStructure) -> Any: + def init_storage(cls, data_uri: str, structure: TableStructure) -> List[Asset]: """ Parameters @@ -146,8 +146,6 @@ def init_storage(cls, data_uri: str, structure: TableStructure) -> Any: ------- """ - from ..server.schemas import Asset - directory = path_from_uri(data_uri) directory.mkdir(parents=True, exist_ok=True) assets = [ @@ -267,7 +265,7 @@ def get(self, key: str) -> Union[ArrayAdapter, None]: def generate_data_sources( self, - mimetype: Any, + mimetype: str, dict_or_none: Callable[[TableStructure], Dict[str, str]], item: Union[str, Path], is_directory: bool, @@ -310,7 +308,7 @@ def from_single_file( structure: Optional[TableStructure] = None, metadata: Optional[JSON] = None, specs: Optional[List[Spec]] = None, - access_policy: Optional[Union[DummyAccessPolicy, SimpleAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, ) -> "CSVAdapter": """ diff --git a/tiled/adapters/hdf5.py b/tiled/adapters/hdf5.py index bd7f9ce5b..0428fff59 100644 --- a/tiled/adapters/hdf5.py +++ b/tiled/adapters/hdf5.py @@ -8,7 +8,6 @@ import numpy from numpy._typing import NDArray -from ..access_policies import DummyAccessPolicy, SimpleAccessPolicy from ..adapters.utils import IndexersMixin from ..iterviews import ItemsView, KeysView, ValuesView from ..structures.array import ArrayStructure @@ -16,6 +15,7 @@ from ..structures.table import TableStructure from ..utils import node_repr, path_from_uri from .array import ArrayAdapter +from .protocols import AccessPolicy from .resource_cache import with_resource_cache from .type_alliases import JSON @@ -76,7 +76,7 @@ def __init__( structure: Optional[ArrayStructure] = None, metadata: Optional[JSON] = None, specs: Optional[List[Spec]] = None, - access_policy: Optional[Union[SimpleAccessPolicy, DummyAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, ) -> None: """ @@ -104,7 +104,7 @@ def from_file( swmr: bool = SWMR_DEFAULT, libver: str = "latest", specs: Optional[List[Spec]] = None, - access_policy: Optional[Union[SimpleAccessPolicy, DummyAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, ) -> "HDF5Adapter": """ @@ -127,14 +127,14 @@ def from_file( @classmethod def from_uri( cls, - data_uri: Union[str, List[str]], + data_uri: str, *, structure: Optional[ArrayStructure] = None, metadata: Optional[JSON] = None, swmr: bool = SWMR_DEFAULT, libver: str = "latest", specs: Optional[list[Spec]] = None, - access_policy: Optional[Union[SimpleAccessPolicy, DummyAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, ) -> "HDF5Adapter": """ @@ -169,7 +169,7 @@ def __repr__(self) -> str: return node_repr(self, list(self)) @property - def access_policy(self) -> Optional[Union[SimpleAccessPolicy, DummyAccessPolicy]]: + def access_policy(self) -> Optional[AccessPolicy]: """ Returns @@ -367,14 +367,14 @@ def inlined_contents_enabled(self, depth: int) -> bool: def hdf5_lookup( - data_uri: Union[str, List[str]], + data_uri: str, *, structure: Optional[ArrayStructure] = None, metadata: Optional[JSON] = None, swmr: bool = SWMR_DEFAULT, libver: str = "latest", specs: Optional[List[Spec]] = None, - access_policy: Optional[Union[SimpleAccessPolicy, DummyAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, path: Optional[Union[List[Path], List[str]]] = None, ) -> Union[HDF5Adapter, ArrayAdapter]: """ diff --git a/tiled/adapters/mapping.py b/tiled/adapters/mapping.py index 0b4db9dae..a053ebe79 100644 --- a/tiled/adapters/mapping.py +++ b/tiled/adapters/mapping.py @@ -8,7 +8,6 @@ from fastapi import APIRouter -from ..access_policies import DummyAccessPolicy, SimpleAccessPolicy from ..iterviews import ItemsView, KeysView, ValuesView from ..queries import ( Comparison, @@ -28,12 +27,12 @@ from ..structures.core import Spec, StructureFamily from ..structures.table import TableStructure from ..utils import UNCHANGED, Sentinel -from .table import TableAdapter +from .protocols import AccessPolicy, AnyAdapter from .type_alliases import JSON from .utils import IndexersMixin -class MapAdapter(collections.abc.Mapping[str, TableAdapter], IndexersMixin): +class MapAdapter(collections.abc.Mapping[str, AnyAdapter], IndexersMixin): """ Adapt any mapping (dictionary-like object) to Tiled. """ @@ -60,13 +59,13 @@ class MapAdapter(collections.abc.Mapping[str, TableAdapter], IndexersMixin): def __init__( self, - mapping: dict[str, TableAdapter], + mapping: dict[str, Any], *, structure: Optional[TableStructure] = None, metadata: Optional[JSON] = None, sorting: Optional[List[SortingItem]] = None, specs: Optional[List[Spec]] = None, - access_policy: Optional[Union[SimpleAccessPolicy, DummyAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, entries_stale_after: Optional[timedelta] = None, metadata_stale_after: Optional[timedelta] = None, must_revalidate: bool = True, @@ -138,7 +137,7 @@ def must_revalidate(self, value: bool) -> None: self._must_revalidate = value @property - def access_policy(self) -> Optional[Union[SimpleAccessPolicy, DummyAccessPolicy]]: + def access_policy(self) -> Optional[AccessPolicy]: """ Returns @@ -148,9 +147,7 @@ def access_policy(self) -> Optional[Union[SimpleAccessPolicy, DummyAccessPolicy] return self._access_policy @access_policy.setter - def access_policy( - self, value: Union[SimpleAccessPolicy, DummyAccessPolicy] - ) -> None: + def access_policy(self, value: AccessPolicy) -> None: """ Parameters @@ -468,7 +465,7 @@ def _keys_slice( def _items_slice( self, start: int, stop: int, direction: int - ) -> Iterator[Tuple[str, TableAdapter]]: + ) -> Iterator[Tuple[str, Any]]: """ Parameters diff --git a/tiled/adapters/parquet.py b/tiled/adapters/parquet.py index a747b391f..cc7138c00 100644 --- a/tiled/adapters/parquet.py +++ b/tiled/adapters/parquet.py @@ -4,11 +4,12 @@ import dask.dataframe import pandas -from ..access_policies import DummyAccessPolicy, SimpleAccessPolicy +from ..server.schemas import Asset from ..structures.core import Spec, StructureFamily from ..structures.table import TableStructure from ..utils import path_from_uri from .dataframe import DataFrameAdapter +from .protocols import AccessPolicy from .type_alliases import JSON @@ -23,7 +24,7 @@ def __init__( structure: TableStructure, metadata: Optional[JSON] = None, specs: Optional[List[Spec]] = None, - access_policy: Optional[Union[DummyAccessPolicy, SimpleAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, ) -> None: """ @@ -69,9 +70,7 @@ def dataframe_adapter(self) -> DataFrameAdapter: return DataFrameAdapter(partitions, self._structure) @classmethod - def init_storage( - cls, data_uri: Union[str, List[str]], structure: TableStructure - ) -> Any: + def init_storage(cls, data_uri: str, structure: TableStructure) -> List[Asset]: """ Parameters @@ -83,8 +82,6 @@ def init_storage( ------- """ - from ..server.schemas import Asset - directory = path_from_uri(data_uri) directory.mkdir(parents=True, exist_ok=True) assets = [ diff --git a/tiled/adapters/protocols.py b/tiled/adapters/protocols.py index 9cb2cdbfa..0b9716e23 100644 --- a/tiled/adapters/protocols.py +++ b/tiled/adapters/protocols.py @@ -1,31 +1,28 @@ import collections.abc -from typing import Any, Dict, List, Literal, Optional, Set, Tuple, TypedDict, Union -from protcols import Protocol, abstractproperty +from abc import abstractmethod +from typing import Any, Dict, List, Literal, Optional, Protocol, Tuple, Union -from numpy.typing import NDArray import pandas import sparse +from numpy.typing import NDArray from ..server.schemas import Principal -from ..structures.core import StructureFamily from ..structures.array import ArrayStructure from ..structures.awkward import AwkwardStructure +from ..structures.core import Spec, StructureFamily from ..structures.sparse import SparseStructure from ..structures.table import TableStructure -from .type_aliases import JSON - - -Spec = TypedDict({"name": str, "version": str}) -Slice = Any # TODO Replace this with our Union for a slice/tuple/.../etc. +from .type_alliases import JSON, Filters, NDSlice, Scopes class BaseAdapter(Protocol): structure_family: StructureFamily + @abstractmethod def metadata(self) -> JSON: pass - @abstractproperty + @abstractmethod def specs(self) -> List[Spec]: pass @@ -33,6 +30,7 @@ def specs(self) -> List[Spec]: class ContainerAdapter(collections.abc.Mapping[str, "AnyAdapter"], BaseAdapter): structure_family = Literal[StructureFamily.container] + @abstractmethod def structure(self) -> None: pass @@ -40,14 +38,16 @@ def structure(self) -> None: class ArrayAdapter(BaseAdapter): structure_family = Literal[StructureFamily.array] + @abstractmethod def structure(self) -> ArrayStructure: pass - # TODO Fix slice - def read(self, slice: Slice) -> NDArray: + @abstractmethod + def read(self, slice: NDSlice) -> NDArray: pass # TODO Fix slice + @abstractmethod def read_block(self, block: Tuple[int, ...]) -> NDArray: pass @@ -55,12 +55,15 @@ def read_block(self, block: Tuple[int, ...]) -> NDArray: class AwkwardAdapter(BaseAdapter): structure_family = Literal[StructureFamily.awkward] + @abstractmethod def structure(self) -> AwkwardStructure: pass + @abstractmethod def read(self) -> NDArray: # Are Slice and Array defined by numpy somewhere? pass + @abstractmethod def read_buffers(self, form_keys: Optional[List[str]] = None) -> Dict[str, Any]: pass @@ -68,12 +71,13 @@ def read_buffers(self, form_keys: Optional[List[str]] = None) -> Dict[str, Any]: class SparseAdapter(BaseAdapter): structure_family = Literal[StructureFamily.sparse] + @abstractmethod def structure(self) -> SparseStructure: pass # TODO Fix slice (just like array) def read( - self, slice: Slice + self, slice: NDSlice ) -> sparse.COO: # Are Slice and Array defined by numpy somewhere? pass @@ -85,15 +89,19 @@ def read_block(self, block: Tuple[int, ...]) -> sparse.COO: class TableAdapter(BaseAdapter): structure_family = Literal[StructureFamily.table] + @abstractmethod def structure(self) -> TableStructure: pass + @abstractmethod def read(self, fields: list[str]) -> pandas.DataFrame: pass + @abstractmethod def read_partition(self, partition: int) -> pandas.DataFrame: pass + @abstractmethod def get(self, key: str) -> ArrayAdapter: pass @@ -103,15 +111,12 @@ def get(self, key: str) -> ArrayAdapter: ] -Scopes = Set[str] -Query = Any # for now... -Filters = List[Query] - - class AccessPolicy(Protocol): + @abstractmethod def allowed_scopes(self, node: BaseAdapter, principal: Principal) -> Scopes: pass + @abstractmethod def filters( self, node: BaseAdapter, principal: Principal, scopes: Scopes ) -> Filters: diff --git a/tiled/adapters/sparse.py b/tiled/adapters/sparse.py index e052117a9..60b122d5c 100644 --- a/tiled/adapters/sparse.py +++ b/tiled/adapters/sparse.py @@ -6,11 +6,11 @@ import sparse from numpy._typing import NDArray -from ..access_policies import DummyAccessPolicy, SimpleAccessPolicy from ..structures.core import Spec, StructureFamily from ..structures.sparse import COOStructure from .array import slice_and_shape_from_block_and_chunks -from .type_alliases import JSON +from .protocols import AccessPolicy +from .type_alliases import JSON, NDSlice class COOAdapter: @@ -26,7 +26,7 @@ def from_arrays( dims: Optional[Tuple[str, ...]] = None, metadata: Optional[JSON] = None, specs: Optional[List[Spec]] = None, - access_policy: Optional[Union[SimpleAccessPolicy, DummyAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, ) -> "COOAdapter": """ Simplest constructor. Single chunk from coords, data arrays. @@ -67,7 +67,7 @@ def from_coo( dims: Optional[Tuple[str, ...]] = None, metadata: Optional[JSON] = None, specs: Optional[List[Spec]] = None, - access_policy: Optional[Union[SimpleAccessPolicy, DummyAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, ) -> "COOAdapter": """ Construct from sparse.COO object. @@ -103,7 +103,7 @@ def from_global_ref( dims: Optional[Tuple[str, ...]] = None, metadata: Optional[JSON] = None, specs: Optional[List[Spec]] = None, - access_policy: Optional[Union[SimpleAccessPolicy, DummyAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, ) -> "COOAdapter": """ Construct from blocks with coords given in global reference frame. @@ -150,7 +150,7 @@ def __init__( *, metadata: Optional[JSON] = None, specs: Optional[List[Spec]] = None, - access_policy: Optional[Union[SimpleAccessPolicy, DummyAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, ) -> None: """ Construct from blocks with coords given in block-local reference frame. @@ -187,8 +187,8 @@ def structure(self) -> COOStructure: return self._structure def read_block( - self, block: Tuple[int, ...], slice: Optional[Union[int, slice]] = None - ) -> NDArray[Any]: + self, block: Tuple[int, ...], slice: Optional[NDSlice] = None + ) -> sparse.COO: """ Parameters @@ -207,7 +207,7 @@ def read_block( arr = arr[slice] return arr - def read(self, slice: Optional[Union[int, slice]] = None) -> NDArray[Any]: + def read(self, slice: Optional[NDSlice] = None) -> sparse.COO: """ Parameters diff --git a/tiled/adapters/sparse_blocks_parquet.py b/tiled/adapters/sparse_blocks_parquet.py index da4257aa7..7eb795838 100644 --- a/tiled/adapters/sparse_blocks_parquet.py +++ b/tiled/adapters/sparse_blocks_parquet.py @@ -8,12 +8,13 @@ import sparse from numpy._typing import NDArray -from ..access_policies import DummyAccessPolicy, SimpleAccessPolicy from ..adapters.array import slice_and_shape_from_block_and_chunks +from ..server.schemas import Asset from ..structures.core import Spec, StructureFamily from ..structures.sparse import COOStructure from ..utils import path_from_uri -from .type_alliases import JSON +from .protocols import AccessPolicy +from .type_alliases import JSON, NDSlice def load_block(uri: str) -> Tuple[List[int], Tuple[NDArray[Any], Any]]: @@ -46,7 +47,7 @@ def __init__( structure: COOStructure, metadata: Optional[JSON] = None, specs: Optional[List[Spec]] = None, - access_policy: Optional[Union[SimpleAccessPolicy, DummyAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, ) -> None: """ @@ -70,9 +71,9 @@ def __init__( @classmethod def init_storage( cls, - data_uri: Union[str, List[str]], + data_uri: str, structure: COOStructure, - ) -> Any: + ) -> List[Asset]: """ Parameters @@ -84,8 +85,6 @@ def init_storage( ------- """ - from ..server.schemas import Asset - directory = path_from_uri(data_uri) directory.mkdir(parents=True, exist_ok=True) @@ -145,7 +144,7 @@ def write(self, data: Union[dask.dataframe.DataFrame, pandas.DataFrame]) -> None uri = self.blocks[(0,) * len(self._structure.shape)] data.to_parquet(path_from_uri(uri)) - def read(self, slice: Optional[Union[int, slice]]) -> NDArray[Any]: + def read(self, slice: NDSlice) -> sparse.COO: """ Parameters @@ -175,8 +174,8 @@ def read(self, slice: Optional[Union[int, slice]]) -> NDArray[Any]: return arr[slice] def read_block( - self, block: Tuple[int, ...], slice: Optional[Union[int, slice]] - ) -> NDArray[Any]: + self, block: Tuple[int, ...], slice: Optional[NDSlice] + ) -> sparse.COO: """ Parameters diff --git a/tiled/adapters/table.py b/tiled/adapters/table.py index c347b2ee8..bb0260ef9 100644 --- a/tiled/adapters/table.py +++ b/tiled/adapters/table.py @@ -4,10 +4,10 @@ import dask.dataframe import pandas -from ..access_policies import DummyAccessPolicy, SimpleAccessPolicy from ..structures.core import Spec, StructureFamily from ..structures.table import TableStructure from .array import ArrayAdapter +from .protocols import AccessPolicy from .type_alliases import JSON @@ -31,7 +31,7 @@ def from_pandas( *args: Any, metadata: Optional[JSON] = None, specs: Optional[List[Spec]] = None, - access_policy: Optional[Union[SimpleAccessPolicy, DummyAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, npartitions: int = 1, **kwargs: Any, ) -> "TableAdapter": @@ -63,7 +63,7 @@ def from_dask_dataframe( ddf: dask.dataframe.DataFrame, metadata: Optional[JSON] = None, specs: Optional[List[Spec]] = None, - access_policy: Optional[Union[DummyAccessPolicy, SimpleAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, ) -> "TableAdapter": """ @@ -96,7 +96,7 @@ def __init__( *, metadata: Optional[JSON] = None, specs: Optional[List[Spec]] = None, - access_policy: Optional[Union[SimpleAccessPolicy, DummyAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, ) -> None: """ diff --git a/tiled/adapters/tiff.py b/tiled/adapters/tiff.py index 4ed793214..32fa76cc9 100644 --- a/tiled/adapters/tiff.py +++ b/tiled/adapters/tiff.py @@ -4,12 +4,12 @@ import tifffile from numpy._typing import NDArray -from ..access_policies import DummyAccessPolicy, SimpleAccessPolicy from ..structures.array import ArrayStructure, BuiltinDtype from ..structures.core import Spec, StructureFamily from ..utils import path_from_uri +from .protocols import AccessPolicy from .resource_cache import with_resource_cache -from .type_alliases import JSON +from .type_alliases import JSON, NDSlice class TiffAdapter: @@ -26,12 +26,12 @@ class TiffAdapter: def __init__( self, - data_uri: Union[str, List[str]], + data_uri: str, *, structure: Optional[ArrayStructure] = None, metadata: Optional[JSON] = None, specs: Optional[List[Spec]] = None, - access_policy: Optional[Union[DummyAccessPolicy, SimpleAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, ) -> None: """ @@ -146,7 +146,7 @@ def from_uris( structure: Optional[ArrayStructure] = None, metadata: Optional[JSON] = None, specs: Optional[List[Spec]] = None, - access_policy: Optional[Union[SimpleAccessPolicy, DummyAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, ) -> "TiffSequenceAdapter": """ @@ -179,7 +179,7 @@ def __init__( structure: Optional[ArrayStructure] = None, metadata: Optional[JSON] = None, specs: Optional[List[Spec]] = None, - access_policy: Optional[Union[SimpleAccessPolicy, DummyAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, ) -> None: """ @@ -217,7 +217,7 @@ def metadata(self) -> JSON: # TODO How to deal with the many headers? return self._provided_metadata - def read(self, slice: Optional[Union[int, slice]] = None) -> NDArray[Any]: + def read(self, slice: Optional[NDSlice] = None) -> NDArray[Any]: """ Receives a sequence of values to select from a collection of tiff files that were saved in a folder The input order is defined as files --> X slice --> Y slice @@ -283,7 +283,7 @@ def read(self, slice: Optional[Union[int, slice]] = None) -> NDArray[Any]: return arr def read_block( - self, block: Tuple[int, ...], slice: Optional[Union[int, slice]] = None + self, block: Tuple[int, ...], slice: Optional[NDSlice] = None ) -> NDArray[Any]: """ diff --git a/tiled/adapters/type_alliases.py b/tiled/adapters/type_alliases.py index 2547e41f2..aff318ddb 100644 --- a/tiled/adapters/type_alliases.py +++ b/tiled/adapters/type_alliases.py @@ -1,3 +1,11 @@ -from typing import Dict, List, Union +from types import EllipsisType +from typing import Any, Dict, List, Set, Tuple, Union JSON = Dict[str, Union[str, int, float, bool, Dict[str, "JSON"], List["JSON"]]] +NDSlice = Union[ + int, slice, Tuple[Union[int, slice, EllipsisType], ...], EllipsisType +] # TODO Replace this with our Union for a slice/tuple/.../etc. + +Scopes = Set[str] +Query = Any # for now... +Filters = List[Query] diff --git a/tiled/adapters/xarray.py b/tiled/adapters/xarray.py index de86cd5b6..c02605aa3 100644 --- a/tiled/adapters/xarray.py +++ b/tiled/adapters/xarray.py @@ -1,13 +1,13 @@ import collections.abc import itertools -from typing import Any, Iterator, List, Optional, Union +from typing import Any, Iterator, List, Optional import xarray -from ..access_policies import DummyAccessPolicy, SimpleAccessPolicy from ..structures.core import Spec from .array import ArrayAdapter from .mapping import MapAdapter +from .protocols import AccessPolicy class DatasetAdapter(MapAdapter): @@ -21,7 +21,7 @@ def from_dataset( dataset: Any, *, specs: Optional[List[Spec]] = None, - access_policy: Optional[Union[DummyAccessPolicy, SimpleAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, ) -> "DatasetAdapter": """ @@ -51,7 +51,7 @@ def __init__( mapping: Any, *args: Any, specs: Optional[List[Spec]] = None, - access_policy: Optional[Union[SimpleAccessPolicy, DummyAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, **kwargs: Any, ) -> None: """ diff --git a/tiled/adapters/zarr.py b/tiled/adapters/zarr.py index 480b5e7f9..dcf295252 100644 --- a/tiled/adapters/zarr.py +++ b/tiled/adapters/zarr.py @@ -1,7 +1,6 @@ import builtins import collections.abc import os -from types import EllipsisType from typing import Any, Iterator, List, Optional, Tuple, Union import dask @@ -11,20 +10,21 @@ import zarr.storage from numpy._typing import NDArray -from ..access_policies import DummyAccessPolicy, SimpleAccessPolicy from ..adapters.utils import IndexersMixin from ..iterviews import ItemsView, KeysView, ValuesView +from ..server.schemas import Asset from ..structures.array import ArrayStructure from ..structures.core import Spec, StructureFamily from ..utils import node_repr, path_from_uri from .array import ArrayAdapter, slice_and_shape_from_block_and_chunks -from .type_alliases import JSON +from .protocols import AccessPolicy +from .type_alliases import JSON, NDSlice INLINED_DEPTH = int(os.getenv("TILED_HDF5_INLINED_CONTENTS_MAX_DEPTH", "7")) def read_zarr( - data_uri: Union[str, List[str]], + data_uri: str, structure: Optional[ArrayStructure] = None, **kwargs: Any, ) -> Union["ZarrGroupAdapter", ArrayAdapter]: @@ -57,7 +57,7 @@ class ZarrArrayAdapter(ArrayAdapter): """ """ @classmethod - def init_storage(cls, data_uri: str, structure: ArrayStructure) -> Any: + def init_storage(cls, data_uri: str, structure: ArrayStructure) -> List[Asset]: """ Parameters @@ -69,8 +69,6 @@ def init_storage(cls, data_uri: str, structure: ArrayStructure) -> Any: ------- """ - from ..server.schemas import Asset - # Zarr requires evenly-sized chunks within each dimension. # Use the first chunk along each dimension. zarr_chunks = tuple(dim[0] for dim in structure.chunks) @@ -104,9 +102,7 @@ def _stencil(self) -> Tuple[slice, ...]: def read( self, - slice: Union[ - int, slice, Tuple[Union[int, slice, EllipsisType], ...], EllipsisType - ] = ..., + slice: NDSlice = ..., ) -> NDArray[Any]: """ @@ -123,9 +119,7 @@ def read( def read_block( self, block: Tuple[int, ...], - slice: Union[ - int, slice, Tuple[Union[int, slice, EllipsisType], ...], EllipsisType - ] = ..., + slice: NDSlice = ..., ) -> NDArray[Any]: """ @@ -148,9 +142,7 @@ def read_block( def write( self, data: Union[dask.dataframe.DataFrame, pandas.DataFrame], - slice: Union[ - int, slice, Tuple[Union[int, slice, EllipsisType], ...], EllipsisType - ] = ..., + slice: NDSlice = ..., ) -> None: """ @@ -171,7 +163,7 @@ async def write_block( self, data: Union[dask.dataframe.DataFrame, pandas.DataFrame], block: Tuple[int, ...], - slice: Optional[EllipsisType] = ..., + slice: Optional[NDSlice] = ..., ) -> None: """ @@ -208,7 +200,7 @@ def __init__( structure: Optional[ArrayStructure] = None, metadata: Optional[JSON] = None, specs: Optional[List[Spec]] = None, - access_policy: Optional[Union[SimpleAccessPolicy, DummyAccessPolicy]] = None, + access_policy: Optional[AccessPolicy] = None, ) -> None: """ @@ -240,7 +232,7 @@ def __repr__(self) -> str: return node_repr(self, list(self)) @property - def access_policy(self) -> Optional[Union[SimpleAccessPolicy, DummyAccessPolicy]]: + def access_policy(self) -> Optional[AccessPolicy]: """ Returns