diff --git a/src/common/tree/tree_node.py b/src/common/tree/tree_node.py index 10566fbc5..fdb51d3db 100644 --- a/src/common/tree/tree_node.py +++ b/src/common/tree/tree_node.py @@ -113,6 +113,23 @@ def path(self): path.reverse() return path + def fs_path(self): + """Path as it would look on a filesystem. Folders named by parent packages""" + path = [] + parent = self.parent + while parent: # Build the path from Node.key until the node has no key (root node) + if not parent.type == SIMOS.PACKAGE.value or parent.is_array(): + parent = parent.parent + continue + path += [parent.entity["name"]] # type: ignore + parent = parent.parent + if not path: + return "" + # Since we build the path "bottom-up", it needs to be revered. + # e.g. [parent, grand_parent, grand_grand_parent] + path.reverse() + return "/".join(path) + "/" + def traverse(self): """Iterate in pre-order depth-first search order (DFS)""" yield self diff --git a/src/features/export/use_cases/export_use_case.py b/src/features/export/use_cases/export_use_case.py index f241f2b60..9ae6c1063 100644 --- a/src/features/export/use_cases/export_use_case.py +++ b/src/features/export/use_cases/export_use_case.py @@ -25,11 +25,10 @@ def save_node_to_zipfile( with zipfile.ZipFile(archive_path, mode="w", compression=zipfile.ZIP_DEFLATED, compresslevel=5) as zip_file: if document_meta: document_node.entity["_meta_"] = document_meta - storage_client = ZipFileClient(zip_file) - path = "" # Path here is used to get the proper file structure in the zip file + storage_client = ZipFileClient(zip_file, document_service.get_data_source(data_source_id)) for node in document_node.traverse(): - if not node.storage_contained and not node.is_array(): - path = f"{path}/{node.entity.get('name', 'noname')}/" if path else node.entity.get("name", "noname") + if not node.storage_contained and not node.is_array() and not node.type == SIMOS.PACKAGE.value: + path = node.fs_path() document_service.save( node=node, data_source_id=data_source_id, diff --git a/src/services/document_service/document_service.py b/src/services/document_service/document_service.py index 6eab2b82e..dbfb002a6 100644 --- a/src/services/document_service/document_service.py +++ b/src/services/document_service/document_service.py @@ -156,6 +156,7 @@ def save( ref_dict, node.get_context_storage_attribute(), parent_id=parent_uid, + datasource=self.get_data_source(data_source_id), ) result = { "type": SIMOS.REFERENCE.value, diff --git a/src/storage/data_source_class.py b/src/storage/data_source_class.py index 245a0854c..c1b38bc95 100644 --- a/src/storage/data_source_class.py +++ b/src/storage/data_source_class.py @@ -127,10 +127,7 @@ def find(self, filter: dict) -> list[dict]: return documents_with_access def update( - self, - document: dict, - storage_attribute: StorageAttribute = None, - parent_id: str | None = None, + self, document: dict, storage_attribute: StorageAttribute = None, parent_id: str | None = None, **kwargs ) -> None: """ Create or update a document. diff --git a/src/storage/data_source_interface.py b/src/storage/data_source_interface.py index fa1155031..389a7e26c 100644 --- a/src/storage/data_source_interface.py +++ b/src/storage/data_source_interface.py @@ -57,10 +57,7 @@ def find(self, filter: dict) -> list[dict]: @abstractmethod def update( - self, - document: dict, - storage_attribute: StorageAttribute = None, - parent_id: str | None = None, + self, document: dict, storage_attribute: StorageAttribute = None, parent_id: str | None = None, **kwargs ) -> None: ... diff --git a/src/storage/repositories/azure_blob.py b/src/storage/repositories/azure_blob.py index 1352ea3ae..83f9459a4 100644 --- a/src/storage/repositories/azure_blob.py +++ b/src/storage/repositories/azure_blob.py @@ -29,7 +29,7 @@ def update_blob(self, uid: str, blob: bytearray): def get_blob(self, uid: str) -> bytearray: return self.blob_service_client.get_blob_to_bytes(self.container, uid).content - def update(self, uid: str, document: dict) -> bool: + def update(self, uid: str, document: dict, **kwargs) -> bool: output = json.dumps(document) self.blob_service_client.create_blob_from_text(self.container, uid, output) return True diff --git a/src/storage/repositories/file.py b/src/storage/repositories/file.py index a4e06dbad..6b4d85a8f 100644 --- a/src/storage/repositories/file.py +++ b/src/storage/repositories/file.py @@ -34,7 +34,7 @@ def add(self, document: dict) -> None: def delete(self, document: dict) -> None: raise NotImplementedError - def update(self, document: dict) -> None: + def update(self, document: dict, **kwargs) -> None: raise NotImplementedError def get_blob(self, uid): diff --git a/src/storage/repositories/mongo.py b/src/storage/repositories/mongo.py index c3d16fa16..bb275e14c 100644 --- a/src/storage/repositories/mongo.py +++ b/src/storage/repositories/mongo.py @@ -54,7 +54,7 @@ def add(self, uid: str, document: dict) -> bool: except DuplicateKeyError as ex: raise BadRequestException from ex - def update(self, uid: str, document: dict) -> bool: + def update(self, uid: str, document: dict, **kwargs) -> bool: attempts = 0 while attempts < 50: attempts += 1 diff --git a/src/storage/repositories/zip/zip_file_client.py b/src/storage/repositories/zip/zip_file_client.py index 7cc9079af..77ea0c662 100644 --- a/src/storage/repositories/zip/zip_file_client.py +++ b/src/storage/repositories/zip/zip_file_client.py @@ -6,6 +6,7 @@ from common.utils.logging import logger from domain_classes.dependency import Dependency from enums import SIMOS +from storage.data_source_interface import DataSource from storage.repositories.zip.replace_reference_with_alias import ( replace_absolute_references_in_entity_with_alias, ) @@ -13,8 +14,9 @@ class ZipFileClient(RepositoryInterface): - def __init__(self, zip_file: ZipFile): + def __init__(self, zip_file: ZipFile, datasource: DataSource): self.zip_file = zip_file + self.datasource = datasource def update(self, entity: dict, storage_recipe=None, **kwargs): """ @@ -23,13 +25,22 @@ def update(self, entity: dict, storage_recipe=None, **kwargs): By default, absolute references are resolved to aliases using the dependencies from entity["__combined_document_meta__"]. """ - entity.pop("_id", None) - entity.pop("uid", None) + + if entity["type"] == SIMOS.FILE.value: + write_to = f"{entity['__path__']}/{entity['name']}.{entity["filetype"]}" + blob_id = entity["content"]["address"] + if blob_id.startswith("$"): + blob_id = blob_id[1:] + blob = self.datasource.get_blob(blob_id) + self.zip_file.writestr(write_to, blob) + return + entity["__path__"] = entity["__path__"].rstrip("/") if "name" not in entity: - write_to = f"{entity['__path__']}/unnamed_document_{str(uuid4())[:8]}.json" + write_to = f"{entity['__path__']}/{str(uuid4())[:8]}.json" else: write_to = f"{entity['__path__']}/{entity['name']}.json" + entity.pop("__path__") combined_document_meta = entity.pop("__combined_document_meta__") logger.debug(f"Writing: {entity['type']} to {write_to}") diff --git a/src/storage/repository_interface.py b/src/storage/repository_interface.py index e73cfd41d..d5b7ad0be 100644 --- a/src/storage/repository_interface.py +++ b/src/storage/repository_interface.py @@ -14,7 +14,7 @@ def add(self, uid: str, document: dict) -> bool: """Get method to be implemented""" @abstractmethod - def update(self, uid: str, document: dict) -> bool: + def update(self, uid: str, document: dict, **kwargs) -> bool: """Update method to be implemented""" @abstractmethod