diff --git a/environment.yml b/environment.yml index 62aa69f83f..380e8ba5c0 100644 --- a/environment.yml +++ b/environment.yml @@ -12,7 +12,7 @@ dependencies: - circus~=0.19.0 - click-spinner~=0.1.8 - click<8.3,>=8.1.0 -- disk-objectstore~=1.4.0 +- disk-objectstore@ git+https://github.com/GeigerJ2/disk-objectstore@4871edea58dd20fd62ec9f10fce0d53f63797aa4 - docstring_parser - get-annotations~=0.1 - python-graphviz~=0.19 diff --git a/pyproject.toml b/pyproject.toml index bfe48f5f2d..ecbdceb99a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,8 @@ dependencies = [ 'circus~=0.19.0', 'click-spinner~=0.1.8', 'click>=8.1.0,<8.3', - 'disk-objectstore~=1.4.0', + # 'disk-objectstore~=1.4.0', + 'disk-objectstore @ git+https://github.com/GeigerJ2/disk-objectstore@4871edea58dd20fd62ec9f10fce0d53f63797aa4', 'docstring-parser', 'get-annotations~=0.1;python_version<"3.10"', 'graphviz~=0.19', @@ -317,6 +318,9 @@ exclude = [ 'tests/' ] +[tool.hatch.metadata] +allow-direct-references = true + [tool.mypy] disallow_any_generics = false disallow_incomplete_defs = false diff --git a/src/aiida/cmdline/commands/cmd_storage.py b/src/aiida/cmdline/commands/cmd_storage.py index 743205e889..d9fe64b13c 100644 --- a/src/aiida/cmdline/commands/cmd_storage.py +++ b/src/aiida/cmdline/commands/cmd_storage.py @@ -152,9 +152,15 @@ def storage_info(detailed): @click.option( '--compress', is_flag=True, default=False, help='Use compression if possible when carrying out maintenance tasks.' ) +@click.option( + '--clean-loose-per-pack/--no-clean-loose-per-pack', + is_flag=True, + default=True, + help='Delete corresponding loose files immediately after each pack creation when running a `full maintenance`.', +) @decorators.with_dbenv() @click.pass_context -def storage_maintain(ctx, full, no_repack, force, dry_run, compress): +def storage_maintain(ctx, full, no_repack, force, dry_run, compress, clean_loose_per_pack): """Performs maintenance tasks on the repository.""" from aiida.common.exceptions import LockingProfileError from aiida.common.progress_reporter import set_progress_bar_tqdm, set_progress_reporter @@ -198,9 +204,15 @@ def storage_maintain(ctx, full, no_repack, force, dry_run, compress): try: if full and no_repack: - storage.maintain(full=full, dry_run=dry_run, do_repack=False, compress=compress) + storage.maintain( + full=full, + dry_run=dry_run, + do_repack=False, + compress=compress, + clean_loose_per_pack=clean_loose_per_pack, + ) else: - storage.maintain(full=full, dry_run=dry_run, compress=compress) + storage.maintain(full=full, dry_run=dry_run, compress=compress, clean_loose_per_pack=clean_loose_per_pack) except LockingProfileError as exception: echo.echo_critical(str(exception)) echo.echo_success('Requested maintenance procedures finished.') diff --git a/src/aiida/repository/backend/disk_object_store.py b/src/aiida/repository/backend/disk_object_store.py index f1bd22faaa..b2ad15278b 100644 --- a/src/aiida/repository/backend/disk_object_store.py +++ b/src/aiida/repository/backend/disk_object_store.py @@ -154,6 +154,7 @@ def maintain( # type: ignore[override] clean_storage: t.Optional[bool] = None, do_vacuum: t.Optional[bool] = None, compress: bool = False, + clean_loose_per_pack: bool = True, ) -> dict: """Performs maintenance operations. @@ -163,6 +164,8 @@ def maintain( # type: ignore[override] :param clean_storage: flag for forcing the cleaning of soft-deleted files from the repository. :param do_vacuum: flag for forcing the vacuuming of the internal database when cleaning the repository. :param compress: flag for compressing the data when packing loose files. Set to ``Compress.AUTO`` if ``True``. + :param clean_loose_per_pack: if True, the loose files that went into a `pack` are deleted immediately after this + `pack` is created. :return: a dictionary with information on the operations performed. """ from disk_objectstore import CompressMode @@ -196,7 +199,9 @@ def maintain( # type: ignore[override] if not dry_run: with get_progress_reporter()(total=1) as progress: callback = create_callback(progress) - container.pack_all_loose(compress=compress, callback=callback) + container.pack_all_loose( + compress=compress, callback=callback, clean_loose_per_pack=clean_loose_per_pack + ) if do_repack: files_numb = container.count_objects().packed diff --git a/uv.lock b/uv.lock index 4074414840..dddcab2490 100644 --- a/uv.lock +++ b/uv.lock @@ -206,7 +206,7 @@ requires-dist = [ { name = "circus", specifier = "~=0.19.0" }, { name = "click", specifier = ">=8.1.0,<8.3" }, { name = "click-spinner", specifier = "~=0.1.8" }, - { name = "disk-objectstore", specifier = "~=1.4.0" }, + { name = "disk-objectstore", git = "https://github.com/GeigerJ2/disk-objectstore?rev=4871edea58dd20fd62ec9f10fce0d53f63797aa4" }, { name = "docstring-parser" }, { name = "docutils", marker = "extra == 'tests'", specifier = "~=0.20" }, { name = "flask", marker = "extra == 'rest'", specifier = "~=2.3.3" }, @@ -1501,16 +1501,12 @@ wheels = [ [[package]] name = "disk-objectstore" version = "1.4.0" -source = { registry = "https://pypi.org/simple" } +source = { git = "https://github.com/GeigerJ2/disk-objectstore?rev=4871edea58dd20fd62ec9f10fce0d53f63797aa4#4871edea58dd20fd62ec9f10fce0d53f63797aa4" } dependencies = [ { name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "click", version = "8.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "sqlalchemy" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a6/06/2f76aef59917d47e372158580db988bae1d09f73f5a22721cf88cb5b1dd1/disk_objectstore-1.4.0.tar.gz", hash = "sha256:4d9ea7617dd2de1c817255d7c8123f69e106f6daa0d4c0bdd09ab919ddbc49c8", size = 7390473, upload-time = "2025-10-06T08:35:53.392Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fe/95/fc379912f1c2b336de2af95331da764b6f09e3f8fabc6050cd7b71a6b895/disk_objectstore-1.4.0-py3-none-any.whl", hash = "sha256:e29d33e51a1196468062f30a79126b3d82ef9fcca6a505f7984cdfc8bed72f5e", size = 70636, upload-time = "2025-10-06T08:35:51.915Z" }, -] [[package]] name = "distlib"