Skip to content

Commit

Permalink
Add _updatable_objects to Node
Browse files Browse the repository at this point in the history
All attributes in AiiDA are stored in the SQL database, which can become inefficient for handling **large data**. For example, the `checkpoint` data in WorkChain, or the `pickled` binary data in WorkGraph. To improve performance, it would be beneficial to store large data in the file repository (`base.repository`) instead.  However, unlike `attributes`, the repository does not support `updatable` objects in the same way as `updatable` attributes.  This PR introduces `_updatable_objects` to enable **mutable** repository storage for active processes while preserving AiiDA's immutability rules.
  • Loading branch information
superstar54 committed Feb 26, 2025
1 parent f4c55f5 commit 017d24a
Show file tree
Hide file tree
Showing 3 changed files with 42 additions and 8 deletions.
4 changes: 4 additions & 0 deletions src/aiida/orm/nodes/node.py
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,10 @@ def _query_type_string(cls) -> str: # noqa: N805
# Requires Sealable mixin, but needs empty tuple for base class
_updatable_attributes: Tuple[str, ...] = tuple()

# A tuple of object names that can be updated even after node is stored
# Requires Sealable mixin, but needs empty tuple for base class
_updatable_objects: Tuple[str, ...] = tuple()

# A tuple of attribute names that will be ignored when creating the hash.
_hash_ignored_attributes: Tuple[str, ...] = tuple()

Expand Down
34 changes: 26 additions & 8 deletions src/aiida/orm/nodes/repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,13 +61,31 @@ def _update_repository_metadata(self):
if self._node.is_stored:
self._node.backend_entity.repository_metadata = self.serialize()

def _check_mutability(self):
def _check_mutability(self, path: str | None = None) -> None:
"""Check if the node is mutable.
:raises `~aiida.common.exceptions.ModificationNotAllowed`: when the node is stored and therefore immutable.
- If the node is stored and sealed, modification is **not allowed**.
- If the node is stored but not sealed, check if `path` is updatable.
- If the node is stored but not sealable, modifications are **not allowed**.
:param path: The repository path being modified, if any.
:raises aiida.common.exceptions.ModificationNotAllowed: If modifications are not allowed.
"""
if self._node.is_stored:
raise exceptions.ModificationNotAllowed('the node is stored and therefore the repository is immutable.')
if not self._node.is_stored:
return

if hasattr(self._node, 'is_sealed'):
if self._node.is_sealed:
raise exceptions.ModificationNotAllowed('Modification not allowed: the node is sealed and immutable.')

if path and path in self._node._updatable_objects:
return

raise exceptions.ModificationNotAllowed(
f'Cannot modify non-updatable repository object of a stored+unsealed node: {path}'
)

raise exceptions.ModificationNotAllowed('Modification not allowed: the node is stored and immutable.')

@property
def _repository(self) -> Repository:
Expand Down Expand Up @@ -267,7 +285,7 @@ def put_object_from_bytes(self, content: bytes, path: str) -> None:
:raises TypeError: if the path is not a string and relative path.
:raises FileExistsError: if an object already exists at the given path.
"""
self._check_mutability()
self._check_mutability(path)
self._repository.put_object_from_filelike(io.BytesIO(content), path)
self._update_repository_metadata()

Expand All @@ -279,7 +297,7 @@ def put_object_from_filelike(self, handle: io.BufferedReader, path: str):
:raises TypeError: if the path is not a string and relative path.
:raises `~aiida.common.exceptions.ModificationNotAllowed`: when the node is stored and therefore immutable.
"""
self._check_mutability()
self._check_mutability(path)

if isinstance(handle, io.StringIO): # type: ignore[unreachable]
handle = io.BytesIO(handle.read().encode('utf-8')) # type: ignore[unreachable]
Expand All @@ -301,7 +319,7 @@ def put_object_from_file(self, filepath: str, path: str):
:raises TypeError: if the path is not a string and relative path, or the handle is not a byte stream.
:raises `~aiida.common.exceptions.ModificationNotAllowed`: when the node is stored and therefore immutable.
"""
self._check_mutability()
self._check_mutability(path)
self._repository.put_object_from_file(filepath, path)
self._update_repository_metadata()

Expand Down Expand Up @@ -356,7 +374,7 @@ def delete_object(self, path: str):
:raises OSError: if the file could not be deleted.
:raises `~aiida.common.exceptions.ModificationNotAllowed`: when the node is stored and therefore immutable.
"""
self._check_mutability()
self._check_mutability(path)
self._repository.delete_object(path)
self._update_repository_metadata()

Expand Down
12 changes: 12 additions & 0 deletions tests/orm/nodes/test_repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,18 @@ def test_sealed():
node.base.repository.put_object_from_bytes(b'content', 'path')


def test_updatable_objects():
node = CalcJobNode()
node._updatable_objects = ['not_raise']
node.store()
node.base.repository.put_object_from_bytes(b'content', 'not_raise')
with pytest.raises(exceptions.ModificationNotAllowed):
node.base.repository.put_object_from_bytes(b'content', 'raise')
node.seal()
with pytest.raises(exceptions.ModificationNotAllowed):
node.base.repository.put_object_from_bytes(b'content', 'not_raise')


def test_get_object_raises():
"""Test the ``NodeRepository.get_object`` method when it is supposed to raise."""
node = Data()
Expand Down

0 comments on commit 017d24a

Please sign in to comment.