From a4c96be4f1214fb1bbf7532654454887ff330794 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Tue, 17 Sep 2024 13:41:08 +0200 Subject: [PATCH 01/42] make entities comparable by ID --- bfabric/entities/core/entity.py | 6 +++++ .../tests/unit/entities/core/test_entity.py | 22 +++++++++++++++++++ docs/changelog.md | 4 ++++ 3 files changed, 32 insertions(+) diff --git a/bfabric/entities/core/entity.py b/bfabric/entities/core/entity.py index 432f562b..468018be 100644 --- a/bfabric/entities/core/entity.py +++ b/bfabric/entities/core/entity.py @@ -67,6 +67,12 @@ def get(self, key: str, default: Any = None) -> Any: """Returns the value of a key in the data dictionary, or a default value if the key is not present.""" return self.__data_dict.get(key, default) + def __lt__(self, other: Entity) -> bool: + """Compares the entity with another entity based on their IDs.""" + if self.ENDPOINT != other.ENDPOINT: + return NotImplemented + return self.id < other.id + def __repr__(self) -> str: """Returns the string representation of the workunit.""" return f"{self.__class__.__name__}({repr(self.__data_dict)}, client={repr(self.__client)})" diff --git a/bfabric/tests/unit/entities/core/test_entity.py b/bfabric/tests/unit/entities/core/test_entity.py index 13fbf6c7..706214b1 100644 --- a/bfabric/tests/unit/entities/core/test_entity.py +++ b/bfabric/tests/unit/entities/core/test_entity.py @@ -112,5 +112,27 @@ def test_str(mock_entity, mock_data_dict) -> None: assert str(entity) == "Entity({'id': 1, 'name': 'Test Entity'}, client=None)" +def test_compare_when_possible(): + entity_1 = Entity({"id": 1, "name": "Test Entity"}, None) + entity_1.ENDPOINT = "X" + entity_10 = Entity({"id": 10, "name": "Test Entity"}, None) + entity_10.ENDPOINT = "X" + assert entity_1 == entity_1 + assert entity_1 < entity_10 + assert entity_10 > entity_1 + + +def test_compare_when_not_possible(): + entity_1 = Entity({"id": 1, "name": "Test Entity"}, None) + entity_1.ENDPOINT = "X" + entity_2 = Entity({"id": 2, "name": "Test Entity"}, None) + entity_2.ENDPOINT = "Y" + assert entity_1 != entity_2 + with pytest.raises(TypeError): + _ = entity_1 < entity_2 + with pytest.raises(TypeError): + _ = entity_1 > entity_2 + + if __name__ == "__main__": pytest.main() diff --git a/docs/changelog.md b/docs/changelog.md index 544a30ef..14e7c807 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -10,6 +10,10 @@ Versioning currently follows `X.Y.Z` where ## \[Unreleased\] +### Added + +- Entities can be compared and sorted by ID now. + ## \[1.13.7\] - 2024-09-17 ### Fixed From 4ce7ed6ed1044c4f9803038f58b50eb3ac804d4f Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Wed, 18 Sep 2024 11:30:03 +0200 Subject: [PATCH 02/42] set the workunit status in app_runner --- bfabric/experimental/app_interface/app_runner/runner.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/bfabric/experimental/app_interface/app_runner/runner.py b/bfabric/experimental/app_interface/app_runner/runner.py index 750bd816..72078f61 100644 --- a/bfabric/experimental/app_interface/app_runner/runner.py +++ b/bfabric/experimental/app_interface/app_runner/runner.py @@ -1,4 +1,5 @@ from __future__ import annotations + import shlex import subprocess from pathlib import Path @@ -58,6 +59,11 @@ def run_app( ssh_user: str | None = None, read_only: bool = False, ) -> None: + # TODO future: the workunit definition must be loaded from bfabric exactly once! this is quite inefficient right now + workunit_definition = WorkunitDefinition.from_ref(workunit_ref, client=client) + if not read_only: + client.save("workunit", {"id": workunit_definition.registration.workunit_id, "status": "processing"}) + runner = Runner(spec=app_spec, client=client, ssh_user=ssh_user) runner.run_dispatch(workunit_ref=workunit_ref, work_dir=work_dir) chunks_file = ChunksFile.model_validate(yaml.safe_load((work_dir / "chunks.yml").read_text())) @@ -67,3 +73,6 @@ def run_app( runner.run_process(chunk_dir=chunk) if not read_only: runner.run_register_outputs(chunk_dir=chunk, workunit_ref=workunit_ref) + + if not read_only: + client.save("workunit", {"id": workunit_definition.registration.workunit_id, "status": "available"}) From 7d68137b0e531ae1d7245ad1c67bc7998fc99823 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Thu, 19 Sep 2024 14:48:09 +0200 Subject: [PATCH 03/42] add version field --- bfabric/experimental/app_interface/app_runner/_spec.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bfabric/experimental/app_interface/app_runner/_spec.py b/bfabric/experimental/app_interface/app_runner/_spec.py index 393c116f..1fe1fbf0 100644 --- a/bfabric/experimental/app_interface/app_runner/_spec.py +++ b/bfabric/experimental/app_interface/app_runner/_spec.py @@ -9,4 +9,5 @@ class CommandsSpec(BaseModel): class AppSpec(BaseModel): + version: list[str] = [] commands: CommandsSpec From ea4c01d8bb319eb9a73924cba8e3d3c05d7a90fe Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Thu, 19 Sep 2024 14:52:29 +0200 Subject: [PATCH 04/42] convert to src layout --- bfabric/wrapper_creator/__init__.py | 0 docs/changelog.md | 4 ++++ pyproject.toml | 3 --- {bfabric => src/bfabric}/__init__.py | 0 {bfabric => src/bfabric}/bfabric.py | 0 {bfabric => src/bfabric}/bfabric2.py | 0 {bfabric => src/bfabric}/bfabric_config.py | 0 {bfabric => src/bfabric}/bfabric_legacy.py | 0 {bfabric => src/bfabric}/cli_formatting.py | 0 {bfabric => src/bfabric}/config/__init__.py | 0 {bfabric => src/bfabric}/config/bfabric_auth.py | 0 {bfabric => src/bfabric}/config/bfabric_client_config.py | 0 {bfabric => src/bfabric}/config/config_file.py | 0 {bfabric => src/bfabric}/engine/__init__.py | 0 {bfabric => src/bfabric}/engine/engine_suds.py | 0 {bfabric => src/bfabric}/engine/engine_zeep.py | 0 {bfabric => src/bfabric}/engine/response_format_suds.py | 0 {bfabric => src/bfabric}/entities/__init__.py | 0 {bfabric => src/bfabric}/entities/application.py | 0 {bfabric => src/bfabric}/entities/core/__init__.py | 0 {bfabric => src/bfabric}/entities/core/entity.py | 0 {bfabric => src/bfabric}/entities/core/has_many.py | 0 {bfabric => src/bfabric}/entities/core/has_one.py | 0 {bfabric => src/bfabric}/entities/core/relationship.py | 0 {bfabric => src/bfabric}/entities/dataset.py | 0 {bfabric => src/bfabric}/entities/executable.py | 0 {bfabric => src/bfabric}/entities/externaljob.py | 0 {bfabric => src/bfabric}/entities/multiplexid.py | 0 {bfabric => src/bfabric}/entities/multiplexkit.py | 0 {bfabric => src/bfabric}/entities/order.py | 0 {bfabric => src/bfabric}/entities/parameter.py | 0 {bfabric => src/bfabric}/entities/project.py | 0 {bfabric => src/bfabric}/entities/resource.py | 0 {bfabric => src/bfabric}/entities/storage.py | 0 {bfabric => src/bfabric}/entities/workunit.py | 0 {bfabric => src/bfabric}/errors.py | 0 .../bfabric}/examples/compare_zeep_suds_pagination.py | 0 {bfabric => src/bfabric}/examples/compare_zeep_suds_query.py | 0 {bfabric => src/bfabric}/examples/exists_multi.py | 0 {bfabric => src/bfabric}/examples/zeep_debug.py | 0 {bfabric => src/bfabric}/experimental/README.md | 0 {bfabric => src/bfabric}/experimental/__init__.py | 0 .../bfabric}/experimental/app_interface/__init__.py | 0 .../experimental/app_interface/app_runner/__init__.py | 0 .../experimental/app_interface/app_runner/__main__.py | 0 .../bfabric}/experimental/app_interface/app_runner/_spec.py | 0 .../bfabric}/experimental/app_interface/app_runner/runner.py | 0 .../experimental/app_interface/input_preparation/__init__.py | 0 .../experimental/app_interface/input_preparation/__main__.py | 0 .../experimental/app_interface/input_preparation/_spec.py | 0 .../experimental/app_interface/input_preparation/prepare.py | 0 .../app_interface/output_registration/__init__.py | 0 .../app_interface/output_registration/__main__.py | 0 .../experimental/app_interface/output_registration/_spec.py | 0 .../app_interface/output_registration/register.py | 0 .../bfabric}/experimental/app_interface/util/__init__.py | 0 .../bfabric}/experimental/app_interface/util/checksums.py | 0 .../bfabric}/experimental/app_interface/util/scp.py | 0 .../bfabric}/experimental/app_interface/workunit/__init__.py | 0 .../experimental/app_interface/workunit/definition.py | 0 {bfabric => src/bfabric}/experimental/multi_query.py | 0 {bfabric => src/bfabric}/results/__init__.py | 0 {bfabric => src/bfabric}/results/response_format_dict.py | 0 {bfabric => src/bfabric}/results/result_container.py | 0 {bfabric => src/bfabric}/scripts/__init__.py | 0 {bfabric => src/bfabric}/scripts/bfabric_delete.py | 0 .../scripts/bfabric_executable_submitter_functionalTest.py | 0 .../scripts/bfabric_executable_submitter_gridengine.py | 0 .../bfabric}/scripts/bfabric_executable_submitter_slurm.py | 0 .../bfabric}/scripts/bfabric_executable_wrappercreator.py | 0 {bfabric => src/bfabric}/scripts/bfabric_feeder_mascot.py | 0 .../bfabric}/scripts/bfabric_feeder_resource_autoQC.py | 0 {bfabric => src/bfabric}/scripts/bfabric_flask.py | 0 .../bfabric_list_not_available_proteomics_workunits.py | 0 .../scripts/bfabric_list_not_existing_storage_directories.py | 0 .../bfabric}/scripts/bfabric_list_workunit_parameters.py | 0 {bfabric => src/bfabric}/scripts/bfabric_logthis.py | 0 {bfabric => src/bfabric}/scripts/bfabric_read.py | 0 .../bfabric}/scripts/bfabric_read_samples_from_dataset.py | 0 .../bfabric}/scripts/bfabric_read_samples_of_workunit.py | 0 {bfabric => src/bfabric}/scripts/bfabric_save_csv2dataset.py | 0 {bfabric => src/bfabric}/scripts/bfabric_save_dataset2csv.py | 0 {bfabric => src/bfabric}/scripts/bfabric_save_fasta.py | 0 .../bfabric}/scripts/bfabric_save_importresource_sample.py | 0 .../bfabric}/scripts/bfabric_save_link_to_workunit.py | 0 {bfabric => src/bfabric}/scripts/bfabric_save_workflowstep.py | 0 .../bfabric}/scripts/bfabric_save_workunit_attribute.py | 0 .../bfabric}/scripts/bfabric_setExternalJobStatus_done.py | 0 .../bfabric}/scripts/bfabric_setResourceStatus_available.py | 0 {bfabric => src/bfabric}/scripts/bfabric_setWorkunitStatus.py | 0 .../bfabric}/scripts/bfabric_slurm_queue_status.py | 0 {bfabric => src/bfabric}/scripts/bfabric_upload_resource.py | 0 .../bfabric}/scripts/bfabric_upload_submitter_executable.py | 0 .../bfabric}/scripts/bfabric_wrapper_creator_yaml.py | 0 .../bfabric}/scripts/fgcz_maxquant_scaffold-wrapper.py | 0 {bfabric => src/bfabric}/scripts/fgcz_maxquant_wrapper.py | 0 {bfabric => src/bfabric}/scripts/fgcz_sge_maxquant_linux.bash | 0 {bfabric/tests => src/bfabric/utils}/__init__.py | 0 {bfabric => src/bfabric}/utils/math_helper.py | 0 {bfabric => src/bfabric}/utils/paginator.py | 0 .../tests/unit => src/bfabric/wrapper_creator}/__init__.py | 0 .../bfabric}/wrapper_creator/bfabric_external_job.py | 0 {bfabric => src/bfabric}/wrapper_creator/bfabric_feeder.py | 0 {bfabric => src/bfabric}/wrapper_creator/bfabric_submitter.py | 0 .../bfabric}/wrapper_creator/bfabric_wrapper_creator.py | 0 {bfabric => src/bfabric}/wrapper_creator/demo_config.yaml | 0 {bfabric => src/bfabric}/wrapper_creator/gridengine.py | 0 {bfabric => src/bfabric}/wrapper_creator/slurm.py | 0 {bfabric/tests/unit/config => tests/unit}/__init__.py | 0 {bfabric/tests/unit/engine => tests/unit/config}/__init__.py | 0 {bfabric/tests => tests}/unit/config/test_bfabric_auth.py | 0 .../tests => tests}/unit/config/test_bfabric_client_config.py | 0 {bfabric/tests => tests}/unit/config/test_config_file.py | 0 {bfabric/tests => tests}/unit/conftest.py | 0 .../tests/unit/entities => tests/unit/engine}/__init__.py | 0 {bfabric/tests => tests}/unit/engine/test_engine_suds.py | 0 {bfabric/tests => tests}/unit/engine/test_engine_zeep.py | 0 .../unit/entities/core => tests/unit/entities}/__init__.py | 0 .../unit/scripts => tests/unit/entities/core}/__init__.py | 0 {bfabric/tests => tests}/unit/entities/core/test_entity.py | 0 {bfabric/tests => tests}/unit/entities/core/test_has_many.py | 0 {bfabric/tests => tests}/unit/entities/core/test_has_one.py | 0 {bfabric/tests => tests}/unit/entities/test_dataset.py | 0 {bfabric/tests => tests}/unit/entities/test_workunit.py | 0 {bfabric/tests => tests}/unit/example_config.yml | 0 {bfabric/utils => tests/unit/scripts}/__init__.py | 0 .../unit/scripts/test_bfabric_slurm_queue_status.py | 0 .../scripts/test_list_not_existing_storage_directories.py | 0 .../tests => tests}/unit/scripts/test_save_csv2dataset.py | 0 {bfabric/tests => tests}/unit/test_bfabric.py | 0 {bfabric/tests => tests}/unit/test_bfabric_config.py | 0 {bfabric/tests => tests}/unit/test_dict_helper.py | 0 {bfabric/tests => tests}/unit/test_math_helper.py | 0 {bfabric/tests => tests}/unit/test_paginator.py | 0 {bfabric/tests => tests}/unit/test_response_format_dict.py | 0 {bfabric/tests => tests}/unit/test_result_container.py | 0 136 files changed, 4 insertions(+), 3 deletions(-) delete mode 100644 bfabric/wrapper_creator/__init__.py rename {bfabric => src/bfabric}/__init__.py (100%) rename {bfabric => src/bfabric}/bfabric.py (100%) rename {bfabric => src/bfabric}/bfabric2.py (100%) rename {bfabric => src/bfabric}/bfabric_config.py (100%) rename {bfabric => src/bfabric}/bfabric_legacy.py (100%) rename {bfabric => src/bfabric}/cli_formatting.py (100%) rename {bfabric => src/bfabric}/config/__init__.py (100%) rename {bfabric => src/bfabric}/config/bfabric_auth.py (100%) rename {bfabric => src/bfabric}/config/bfabric_client_config.py (100%) rename {bfabric => src/bfabric}/config/config_file.py (100%) rename {bfabric => src/bfabric}/engine/__init__.py (100%) rename {bfabric => src/bfabric}/engine/engine_suds.py (100%) rename {bfabric => src/bfabric}/engine/engine_zeep.py (100%) rename {bfabric => src/bfabric}/engine/response_format_suds.py (100%) rename {bfabric => src/bfabric}/entities/__init__.py (100%) rename {bfabric => src/bfabric}/entities/application.py (100%) rename {bfabric => src/bfabric}/entities/core/__init__.py (100%) rename {bfabric => src/bfabric}/entities/core/entity.py (100%) rename {bfabric => src/bfabric}/entities/core/has_many.py (100%) rename {bfabric => src/bfabric}/entities/core/has_one.py (100%) rename {bfabric => src/bfabric}/entities/core/relationship.py (100%) rename {bfabric => src/bfabric}/entities/dataset.py (100%) rename {bfabric => src/bfabric}/entities/executable.py (100%) rename {bfabric => src/bfabric}/entities/externaljob.py (100%) rename {bfabric => src/bfabric}/entities/multiplexid.py (100%) rename {bfabric => src/bfabric}/entities/multiplexkit.py (100%) rename {bfabric => src/bfabric}/entities/order.py (100%) rename {bfabric => src/bfabric}/entities/parameter.py (100%) rename {bfabric => src/bfabric}/entities/project.py (100%) rename {bfabric => src/bfabric}/entities/resource.py (100%) rename {bfabric => src/bfabric}/entities/storage.py (100%) rename {bfabric => src/bfabric}/entities/workunit.py (100%) rename {bfabric => src/bfabric}/errors.py (100%) rename {bfabric => src/bfabric}/examples/compare_zeep_suds_pagination.py (100%) rename {bfabric => src/bfabric}/examples/compare_zeep_suds_query.py (100%) rename {bfabric => src/bfabric}/examples/exists_multi.py (100%) rename {bfabric => src/bfabric}/examples/zeep_debug.py (100%) rename {bfabric => src/bfabric}/experimental/README.md (100%) rename {bfabric => src/bfabric}/experimental/__init__.py (100%) rename {bfabric => src/bfabric}/experimental/app_interface/__init__.py (100%) rename {bfabric => src/bfabric}/experimental/app_interface/app_runner/__init__.py (100%) rename {bfabric => src/bfabric}/experimental/app_interface/app_runner/__main__.py (100%) rename {bfabric => src/bfabric}/experimental/app_interface/app_runner/_spec.py (100%) rename {bfabric => src/bfabric}/experimental/app_interface/app_runner/runner.py (100%) rename {bfabric => src/bfabric}/experimental/app_interface/input_preparation/__init__.py (100%) rename {bfabric => src/bfabric}/experimental/app_interface/input_preparation/__main__.py (100%) rename {bfabric => src/bfabric}/experimental/app_interface/input_preparation/_spec.py (100%) rename {bfabric => src/bfabric}/experimental/app_interface/input_preparation/prepare.py (100%) rename {bfabric => src/bfabric}/experimental/app_interface/output_registration/__init__.py (100%) rename {bfabric => src/bfabric}/experimental/app_interface/output_registration/__main__.py (100%) rename {bfabric => src/bfabric}/experimental/app_interface/output_registration/_spec.py (100%) rename {bfabric => src/bfabric}/experimental/app_interface/output_registration/register.py (100%) rename {bfabric => src/bfabric}/experimental/app_interface/util/__init__.py (100%) rename {bfabric => src/bfabric}/experimental/app_interface/util/checksums.py (100%) rename {bfabric => src/bfabric}/experimental/app_interface/util/scp.py (100%) rename {bfabric => src/bfabric}/experimental/app_interface/workunit/__init__.py (100%) rename {bfabric => src/bfabric}/experimental/app_interface/workunit/definition.py (100%) rename {bfabric => src/bfabric}/experimental/multi_query.py (100%) rename {bfabric => src/bfabric}/results/__init__.py (100%) rename {bfabric => src/bfabric}/results/response_format_dict.py (100%) rename {bfabric => src/bfabric}/results/result_container.py (100%) rename {bfabric => src/bfabric}/scripts/__init__.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_delete.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_executable_submitter_functionalTest.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_executable_submitter_gridengine.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_executable_submitter_slurm.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_executable_wrappercreator.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_feeder_mascot.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_feeder_resource_autoQC.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_flask.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_list_not_available_proteomics_workunits.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_list_not_existing_storage_directories.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_list_workunit_parameters.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_logthis.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_read.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_read_samples_from_dataset.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_read_samples_of_workunit.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_save_csv2dataset.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_save_dataset2csv.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_save_fasta.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_save_importresource_sample.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_save_link_to_workunit.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_save_workflowstep.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_save_workunit_attribute.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_setExternalJobStatus_done.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_setResourceStatus_available.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_setWorkunitStatus.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_slurm_queue_status.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_upload_resource.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_upload_submitter_executable.py (100%) rename {bfabric => src/bfabric}/scripts/bfabric_wrapper_creator_yaml.py (100%) rename {bfabric => src/bfabric}/scripts/fgcz_maxquant_scaffold-wrapper.py (100%) rename {bfabric => src/bfabric}/scripts/fgcz_maxquant_wrapper.py (100%) rename {bfabric => src/bfabric}/scripts/fgcz_sge_maxquant_linux.bash (100%) rename {bfabric/tests => src/bfabric/utils}/__init__.py (100%) rename {bfabric => src/bfabric}/utils/math_helper.py (100%) rename {bfabric => src/bfabric}/utils/paginator.py (100%) rename {bfabric/tests/unit => src/bfabric/wrapper_creator}/__init__.py (100%) rename {bfabric => src/bfabric}/wrapper_creator/bfabric_external_job.py (100%) rename {bfabric => src/bfabric}/wrapper_creator/bfabric_feeder.py (100%) rename {bfabric => src/bfabric}/wrapper_creator/bfabric_submitter.py (100%) rename {bfabric => src/bfabric}/wrapper_creator/bfabric_wrapper_creator.py (100%) rename {bfabric => src/bfabric}/wrapper_creator/demo_config.yaml (100%) rename {bfabric => src/bfabric}/wrapper_creator/gridengine.py (100%) rename {bfabric => src/bfabric}/wrapper_creator/slurm.py (100%) rename {bfabric/tests/unit/config => tests/unit}/__init__.py (100%) rename {bfabric/tests/unit/engine => tests/unit/config}/__init__.py (100%) rename {bfabric/tests => tests}/unit/config/test_bfabric_auth.py (100%) rename {bfabric/tests => tests}/unit/config/test_bfabric_client_config.py (100%) rename {bfabric/tests => tests}/unit/config/test_config_file.py (100%) rename {bfabric/tests => tests}/unit/conftest.py (100%) rename {bfabric/tests/unit/entities => tests/unit/engine}/__init__.py (100%) rename {bfabric/tests => tests}/unit/engine/test_engine_suds.py (100%) rename {bfabric/tests => tests}/unit/engine/test_engine_zeep.py (100%) rename {bfabric/tests/unit/entities/core => tests/unit/entities}/__init__.py (100%) rename {bfabric/tests/unit/scripts => tests/unit/entities/core}/__init__.py (100%) rename {bfabric/tests => tests}/unit/entities/core/test_entity.py (100%) rename {bfabric/tests => tests}/unit/entities/core/test_has_many.py (100%) rename {bfabric/tests => tests}/unit/entities/core/test_has_one.py (100%) rename {bfabric/tests => tests}/unit/entities/test_dataset.py (100%) rename {bfabric/tests => tests}/unit/entities/test_workunit.py (100%) rename {bfabric/tests => tests}/unit/example_config.yml (100%) rename {bfabric/utils => tests/unit/scripts}/__init__.py (100%) rename {bfabric/tests => tests}/unit/scripts/test_bfabric_slurm_queue_status.py (100%) rename {bfabric/tests => tests}/unit/scripts/test_list_not_existing_storage_directories.py (100%) rename {bfabric/tests => tests}/unit/scripts/test_save_csv2dataset.py (100%) rename {bfabric/tests => tests}/unit/test_bfabric.py (100%) rename {bfabric/tests => tests}/unit/test_bfabric_config.py (100%) rename {bfabric/tests => tests}/unit/test_dict_helper.py (100%) rename {bfabric/tests => tests}/unit/test_math_helper.py (100%) rename {bfabric/tests => tests}/unit/test_paginator.py (100%) rename {bfabric/tests => tests}/unit/test_response_format_dict.py (100%) rename {bfabric/tests => tests}/unit/test_result_container.py (100%) diff --git a/bfabric/wrapper_creator/__init__.py b/bfabric/wrapper_creator/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/changelog.md b/docs/changelog.md index 14e7c807..cceb4b99 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -14,6 +14,10 @@ Versioning currently follows `X.Y.Z` where - Entities can be compared and sorted by ID now. +### Changed + +- (internal) migrate to src layout + ## \[1.13.7\] - 2024-09-17 ### Fixed diff --git a/pyproject.toml b/pyproject.toml index 906836fc..fc0edddb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,9 +2,6 @@ requires = ["setuptools >= 61.0"] build-backend = "setuptools.build_meta" -[tool.setuptools.packages.find] -include = ["bfabric*"] - [project] name = "bfabric" description = "Python client for the B-Fabric WSDL API" diff --git a/bfabric/__init__.py b/src/bfabric/__init__.py similarity index 100% rename from bfabric/__init__.py rename to src/bfabric/__init__.py diff --git a/bfabric/bfabric.py b/src/bfabric/bfabric.py similarity index 100% rename from bfabric/bfabric.py rename to src/bfabric/bfabric.py diff --git a/bfabric/bfabric2.py b/src/bfabric/bfabric2.py similarity index 100% rename from bfabric/bfabric2.py rename to src/bfabric/bfabric2.py diff --git a/bfabric/bfabric_config.py b/src/bfabric/bfabric_config.py similarity index 100% rename from bfabric/bfabric_config.py rename to src/bfabric/bfabric_config.py diff --git a/bfabric/bfabric_legacy.py b/src/bfabric/bfabric_legacy.py similarity index 100% rename from bfabric/bfabric_legacy.py rename to src/bfabric/bfabric_legacy.py diff --git a/bfabric/cli_formatting.py b/src/bfabric/cli_formatting.py similarity index 100% rename from bfabric/cli_formatting.py rename to src/bfabric/cli_formatting.py diff --git a/bfabric/config/__init__.py b/src/bfabric/config/__init__.py similarity index 100% rename from bfabric/config/__init__.py rename to src/bfabric/config/__init__.py diff --git a/bfabric/config/bfabric_auth.py b/src/bfabric/config/bfabric_auth.py similarity index 100% rename from bfabric/config/bfabric_auth.py rename to src/bfabric/config/bfabric_auth.py diff --git a/bfabric/config/bfabric_client_config.py b/src/bfabric/config/bfabric_client_config.py similarity index 100% rename from bfabric/config/bfabric_client_config.py rename to src/bfabric/config/bfabric_client_config.py diff --git a/bfabric/config/config_file.py b/src/bfabric/config/config_file.py similarity index 100% rename from bfabric/config/config_file.py rename to src/bfabric/config/config_file.py diff --git a/bfabric/engine/__init__.py b/src/bfabric/engine/__init__.py similarity index 100% rename from bfabric/engine/__init__.py rename to src/bfabric/engine/__init__.py diff --git a/bfabric/engine/engine_suds.py b/src/bfabric/engine/engine_suds.py similarity index 100% rename from bfabric/engine/engine_suds.py rename to src/bfabric/engine/engine_suds.py diff --git a/bfabric/engine/engine_zeep.py b/src/bfabric/engine/engine_zeep.py similarity index 100% rename from bfabric/engine/engine_zeep.py rename to src/bfabric/engine/engine_zeep.py diff --git a/bfabric/engine/response_format_suds.py b/src/bfabric/engine/response_format_suds.py similarity index 100% rename from bfabric/engine/response_format_suds.py rename to src/bfabric/engine/response_format_suds.py diff --git a/bfabric/entities/__init__.py b/src/bfabric/entities/__init__.py similarity index 100% rename from bfabric/entities/__init__.py rename to src/bfabric/entities/__init__.py diff --git a/bfabric/entities/application.py b/src/bfabric/entities/application.py similarity index 100% rename from bfabric/entities/application.py rename to src/bfabric/entities/application.py diff --git a/bfabric/entities/core/__init__.py b/src/bfabric/entities/core/__init__.py similarity index 100% rename from bfabric/entities/core/__init__.py rename to src/bfabric/entities/core/__init__.py diff --git a/bfabric/entities/core/entity.py b/src/bfabric/entities/core/entity.py similarity index 100% rename from bfabric/entities/core/entity.py rename to src/bfabric/entities/core/entity.py diff --git a/bfabric/entities/core/has_many.py b/src/bfabric/entities/core/has_many.py similarity index 100% rename from bfabric/entities/core/has_many.py rename to src/bfabric/entities/core/has_many.py diff --git a/bfabric/entities/core/has_one.py b/src/bfabric/entities/core/has_one.py similarity index 100% rename from bfabric/entities/core/has_one.py rename to src/bfabric/entities/core/has_one.py diff --git a/bfabric/entities/core/relationship.py b/src/bfabric/entities/core/relationship.py similarity index 100% rename from bfabric/entities/core/relationship.py rename to src/bfabric/entities/core/relationship.py diff --git a/bfabric/entities/dataset.py b/src/bfabric/entities/dataset.py similarity index 100% rename from bfabric/entities/dataset.py rename to src/bfabric/entities/dataset.py diff --git a/bfabric/entities/executable.py b/src/bfabric/entities/executable.py similarity index 100% rename from bfabric/entities/executable.py rename to src/bfabric/entities/executable.py diff --git a/bfabric/entities/externaljob.py b/src/bfabric/entities/externaljob.py similarity index 100% rename from bfabric/entities/externaljob.py rename to src/bfabric/entities/externaljob.py diff --git a/bfabric/entities/multiplexid.py b/src/bfabric/entities/multiplexid.py similarity index 100% rename from bfabric/entities/multiplexid.py rename to src/bfabric/entities/multiplexid.py diff --git a/bfabric/entities/multiplexkit.py b/src/bfabric/entities/multiplexkit.py similarity index 100% rename from bfabric/entities/multiplexkit.py rename to src/bfabric/entities/multiplexkit.py diff --git a/bfabric/entities/order.py b/src/bfabric/entities/order.py similarity index 100% rename from bfabric/entities/order.py rename to src/bfabric/entities/order.py diff --git a/bfabric/entities/parameter.py b/src/bfabric/entities/parameter.py similarity index 100% rename from bfabric/entities/parameter.py rename to src/bfabric/entities/parameter.py diff --git a/bfabric/entities/project.py b/src/bfabric/entities/project.py similarity index 100% rename from bfabric/entities/project.py rename to src/bfabric/entities/project.py diff --git a/bfabric/entities/resource.py b/src/bfabric/entities/resource.py similarity index 100% rename from bfabric/entities/resource.py rename to src/bfabric/entities/resource.py diff --git a/bfabric/entities/storage.py b/src/bfabric/entities/storage.py similarity index 100% rename from bfabric/entities/storage.py rename to src/bfabric/entities/storage.py diff --git a/bfabric/entities/workunit.py b/src/bfabric/entities/workunit.py similarity index 100% rename from bfabric/entities/workunit.py rename to src/bfabric/entities/workunit.py diff --git a/bfabric/errors.py b/src/bfabric/errors.py similarity index 100% rename from bfabric/errors.py rename to src/bfabric/errors.py diff --git a/bfabric/examples/compare_zeep_suds_pagination.py b/src/bfabric/examples/compare_zeep_suds_pagination.py similarity index 100% rename from bfabric/examples/compare_zeep_suds_pagination.py rename to src/bfabric/examples/compare_zeep_suds_pagination.py diff --git a/bfabric/examples/compare_zeep_suds_query.py b/src/bfabric/examples/compare_zeep_suds_query.py similarity index 100% rename from bfabric/examples/compare_zeep_suds_query.py rename to src/bfabric/examples/compare_zeep_suds_query.py diff --git a/bfabric/examples/exists_multi.py b/src/bfabric/examples/exists_multi.py similarity index 100% rename from bfabric/examples/exists_multi.py rename to src/bfabric/examples/exists_multi.py diff --git a/bfabric/examples/zeep_debug.py b/src/bfabric/examples/zeep_debug.py similarity index 100% rename from bfabric/examples/zeep_debug.py rename to src/bfabric/examples/zeep_debug.py diff --git a/bfabric/experimental/README.md b/src/bfabric/experimental/README.md similarity index 100% rename from bfabric/experimental/README.md rename to src/bfabric/experimental/README.md diff --git a/bfabric/experimental/__init__.py b/src/bfabric/experimental/__init__.py similarity index 100% rename from bfabric/experimental/__init__.py rename to src/bfabric/experimental/__init__.py diff --git a/bfabric/experimental/app_interface/__init__.py b/src/bfabric/experimental/app_interface/__init__.py similarity index 100% rename from bfabric/experimental/app_interface/__init__.py rename to src/bfabric/experimental/app_interface/__init__.py diff --git a/bfabric/experimental/app_interface/app_runner/__init__.py b/src/bfabric/experimental/app_interface/app_runner/__init__.py similarity index 100% rename from bfabric/experimental/app_interface/app_runner/__init__.py rename to src/bfabric/experimental/app_interface/app_runner/__init__.py diff --git a/bfabric/experimental/app_interface/app_runner/__main__.py b/src/bfabric/experimental/app_interface/app_runner/__main__.py similarity index 100% rename from bfabric/experimental/app_interface/app_runner/__main__.py rename to src/bfabric/experimental/app_interface/app_runner/__main__.py diff --git a/bfabric/experimental/app_interface/app_runner/_spec.py b/src/bfabric/experimental/app_interface/app_runner/_spec.py similarity index 100% rename from bfabric/experimental/app_interface/app_runner/_spec.py rename to src/bfabric/experimental/app_interface/app_runner/_spec.py diff --git a/bfabric/experimental/app_interface/app_runner/runner.py b/src/bfabric/experimental/app_interface/app_runner/runner.py similarity index 100% rename from bfabric/experimental/app_interface/app_runner/runner.py rename to src/bfabric/experimental/app_interface/app_runner/runner.py diff --git a/bfabric/experimental/app_interface/input_preparation/__init__.py b/src/bfabric/experimental/app_interface/input_preparation/__init__.py similarity index 100% rename from bfabric/experimental/app_interface/input_preparation/__init__.py rename to src/bfabric/experimental/app_interface/input_preparation/__init__.py diff --git a/bfabric/experimental/app_interface/input_preparation/__main__.py b/src/bfabric/experimental/app_interface/input_preparation/__main__.py similarity index 100% rename from bfabric/experimental/app_interface/input_preparation/__main__.py rename to src/bfabric/experimental/app_interface/input_preparation/__main__.py diff --git a/bfabric/experimental/app_interface/input_preparation/_spec.py b/src/bfabric/experimental/app_interface/input_preparation/_spec.py similarity index 100% rename from bfabric/experimental/app_interface/input_preparation/_spec.py rename to src/bfabric/experimental/app_interface/input_preparation/_spec.py diff --git a/bfabric/experimental/app_interface/input_preparation/prepare.py b/src/bfabric/experimental/app_interface/input_preparation/prepare.py similarity index 100% rename from bfabric/experimental/app_interface/input_preparation/prepare.py rename to src/bfabric/experimental/app_interface/input_preparation/prepare.py diff --git a/bfabric/experimental/app_interface/output_registration/__init__.py b/src/bfabric/experimental/app_interface/output_registration/__init__.py similarity index 100% rename from bfabric/experimental/app_interface/output_registration/__init__.py rename to src/bfabric/experimental/app_interface/output_registration/__init__.py diff --git a/bfabric/experimental/app_interface/output_registration/__main__.py b/src/bfabric/experimental/app_interface/output_registration/__main__.py similarity index 100% rename from bfabric/experimental/app_interface/output_registration/__main__.py rename to src/bfabric/experimental/app_interface/output_registration/__main__.py diff --git a/bfabric/experimental/app_interface/output_registration/_spec.py b/src/bfabric/experimental/app_interface/output_registration/_spec.py similarity index 100% rename from bfabric/experimental/app_interface/output_registration/_spec.py rename to src/bfabric/experimental/app_interface/output_registration/_spec.py diff --git a/bfabric/experimental/app_interface/output_registration/register.py b/src/bfabric/experimental/app_interface/output_registration/register.py similarity index 100% rename from bfabric/experimental/app_interface/output_registration/register.py rename to src/bfabric/experimental/app_interface/output_registration/register.py diff --git a/bfabric/experimental/app_interface/util/__init__.py b/src/bfabric/experimental/app_interface/util/__init__.py similarity index 100% rename from bfabric/experimental/app_interface/util/__init__.py rename to src/bfabric/experimental/app_interface/util/__init__.py diff --git a/bfabric/experimental/app_interface/util/checksums.py b/src/bfabric/experimental/app_interface/util/checksums.py similarity index 100% rename from bfabric/experimental/app_interface/util/checksums.py rename to src/bfabric/experimental/app_interface/util/checksums.py diff --git a/bfabric/experimental/app_interface/util/scp.py b/src/bfabric/experimental/app_interface/util/scp.py similarity index 100% rename from bfabric/experimental/app_interface/util/scp.py rename to src/bfabric/experimental/app_interface/util/scp.py diff --git a/bfabric/experimental/app_interface/workunit/__init__.py b/src/bfabric/experimental/app_interface/workunit/__init__.py similarity index 100% rename from bfabric/experimental/app_interface/workunit/__init__.py rename to src/bfabric/experimental/app_interface/workunit/__init__.py diff --git a/bfabric/experimental/app_interface/workunit/definition.py b/src/bfabric/experimental/app_interface/workunit/definition.py similarity index 100% rename from bfabric/experimental/app_interface/workunit/definition.py rename to src/bfabric/experimental/app_interface/workunit/definition.py diff --git a/bfabric/experimental/multi_query.py b/src/bfabric/experimental/multi_query.py similarity index 100% rename from bfabric/experimental/multi_query.py rename to src/bfabric/experimental/multi_query.py diff --git a/bfabric/results/__init__.py b/src/bfabric/results/__init__.py similarity index 100% rename from bfabric/results/__init__.py rename to src/bfabric/results/__init__.py diff --git a/bfabric/results/response_format_dict.py b/src/bfabric/results/response_format_dict.py similarity index 100% rename from bfabric/results/response_format_dict.py rename to src/bfabric/results/response_format_dict.py diff --git a/bfabric/results/result_container.py b/src/bfabric/results/result_container.py similarity index 100% rename from bfabric/results/result_container.py rename to src/bfabric/results/result_container.py diff --git a/bfabric/scripts/__init__.py b/src/bfabric/scripts/__init__.py similarity index 100% rename from bfabric/scripts/__init__.py rename to src/bfabric/scripts/__init__.py diff --git a/bfabric/scripts/bfabric_delete.py b/src/bfabric/scripts/bfabric_delete.py similarity index 100% rename from bfabric/scripts/bfabric_delete.py rename to src/bfabric/scripts/bfabric_delete.py diff --git a/bfabric/scripts/bfabric_executable_submitter_functionalTest.py b/src/bfabric/scripts/bfabric_executable_submitter_functionalTest.py similarity index 100% rename from bfabric/scripts/bfabric_executable_submitter_functionalTest.py rename to src/bfabric/scripts/bfabric_executable_submitter_functionalTest.py diff --git a/bfabric/scripts/bfabric_executable_submitter_gridengine.py b/src/bfabric/scripts/bfabric_executable_submitter_gridengine.py similarity index 100% rename from bfabric/scripts/bfabric_executable_submitter_gridengine.py rename to src/bfabric/scripts/bfabric_executable_submitter_gridengine.py diff --git a/bfabric/scripts/bfabric_executable_submitter_slurm.py b/src/bfabric/scripts/bfabric_executable_submitter_slurm.py similarity index 100% rename from bfabric/scripts/bfabric_executable_submitter_slurm.py rename to src/bfabric/scripts/bfabric_executable_submitter_slurm.py diff --git a/bfabric/scripts/bfabric_executable_wrappercreator.py b/src/bfabric/scripts/bfabric_executable_wrappercreator.py similarity index 100% rename from bfabric/scripts/bfabric_executable_wrappercreator.py rename to src/bfabric/scripts/bfabric_executable_wrappercreator.py diff --git a/bfabric/scripts/bfabric_feeder_mascot.py b/src/bfabric/scripts/bfabric_feeder_mascot.py similarity index 100% rename from bfabric/scripts/bfabric_feeder_mascot.py rename to src/bfabric/scripts/bfabric_feeder_mascot.py diff --git a/bfabric/scripts/bfabric_feeder_resource_autoQC.py b/src/bfabric/scripts/bfabric_feeder_resource_autoQC.py similarity index 100% rename from bfabric/scripts/bfabric_feeder_resource_autoQC.py rename to src/bfabric/scripts/bfabric_feeder_resource_autoQC.py diff --git a/bfabric/scripts/bfabric_flask.py b/src/bfabric/scripts/bfabric_flask.py similarity index 100% rename from bfabric/scripts/bfabric_flask.py rename to src/bfabric/scripts/bfabric_flask.py diff --git a/bfabric/scripts/bfabric_list_not_available_proteomics_workunits.py b/src/bfabric/scripts/bfabric_list_not_available_proteomics_workunits.py similarity index 100% rename from bfabric/scripts/bfabric_list_not_available_proteomics_workunits.py rename to src/bfabric/scripts/bfabric_list_not_available_proteomics_workunits.py diff --git a/bfabric/scripts/bfabric_list_not_existing_storage_directories.py b/src/bfabric/scripts/bfabric_list_not_existing_storage_directories.py similarity index 100% rename from bfabric/scripts/bfabric_list_not_existing_storage_directories.py rename to src/bfabric/scripts/bfabric_list_not_existing_storage_directories.py diff --git a/bfabric/scripts/bfabric_list_workunit_parameters.py b/src/bfabric/scripts/bfabric_list_workunit_parameters.py similarity index 100% rename from bfabric/scripts/bfabric_list_workunit_parameters.py rename to src/bfabric/scripts/bfabric_list_workunit_parameters.py diff --git a/bfabric/scripts/bfabric_logthis.py b/src/bfabric/scripts/bfabric_logthis.py similarity index 100% rename from bfabric/scripts/bfabric_logthis.py rename to src/bfabric/scripts/bfabric_logthis.py diff --git a/bfabric/scripts/bfabric_read.py b/src/bfabric/scripts/bfabric_read.py similarity index 100% rename from bfabric/scripts/bfabric_read.py rename to src/bfabric/scripts/bfabric_read.py diff --git a/bfabric/scripts/bfabric_read_samples_from_dataset.py b/src/bfabric/scripts/bfabric_read_samples_from_dataset.py similarity index 100% rename from bfabric/scripts/bfabric_read_samples_from_dataset.py rename to src/bfabric/scripts/bfabric_read_samples_from_dataset.py diff --git a/bfabric/scripts/bfabric_read_samples_of_workunit.py b/src/bfabric/scripts/bfabric_read_samples_of_workunit.py similarity index 100% rename from bfabric/scripts/bfabric_read_samples_of_workunit.py rename to src/bfabric/scripts/bfabric_read_samples_of_workunit.py diff --git a/bfabric/scripts/bfabric_save_csv2dataset.py b/src/bfabric/scripts/bfabric_save_csv2dataset.py similarity index 100% rename from bfabric/scripts/bfabric_save_csv2dataset.py rename to src/bfabric/scripts/bfabric_save_csv2dataset.py diff --git a/bfabric/scripts/bfabric_save_dataset2csv.py b/src/bfabric/scripts/bfabric_save_dataset2csv.py similarity index 100% rename from bfabric/scripts/bfabric_save_dataset2csv.py rename to src/bfabric/scripts/bfabric_save_dataset2csv.py diff --git a/bfabric/scripts/bfabric_save_fasta.py b/src/bfabric/scripts/bfabric_save_fasta.py similarity index 100% rename from bfabric/scripts/bfabric_save_fasta.py rename to src/bfabric/scripts/bfabric_save_fasta.py diff --git a/bfabric/scripts/bfabric_save_importresource_sample.py b/src/bfabric/scripts/bfabric_save_importresource_sample.py similarity index 100% rename from bfabric/scripts/bfabric_save_importresource_sample.py rename to src/bfabric/scripts/bfabric_save_importresource_sample.py diff --git a/bfabric/scripts/bfabric_save_link_to_workunit.py b/src/bfabric/scripts/bfabric_save_link_to_workunit.py similarity index 100% rename from bfabric/scripts/bfabric_save_link_to_workunit.py rename to src/bfabric/scripts/bfabric_save_link_to_workunit.py diff --git a/bfabric/scripts/bfabric_save_workflowstep.py b/src/bfabric/scripts/bfabric_save_workflowstep.py similarity index 100% rename from bfabric/scripts/bfabric_save_workflowstep.py rename to src/bfabric/scripts/bfabric_save_workflowstep.py diff --git a/bfabric/scripts/bfabric_save_workunit_attribute.py b/src/bfabric/scripts/bfabric_save_workunit_attribute.py similarity index 100% rename from bfabric/scripts/bfabric_save_workunit_attribute.py rename to src/bfabric/scripts/bfabric_save_workunit_attribute.py diff --git a/bfabric/scripts/bfabric_setExternalJobStatus_done.py b/src/bfabric/scripts/bfabric_setExternalJobStatus_done.py similarity index 100% rename from bfabric/scripts/bfabric_setExternalJobStatus_done.py rename to src/bfabric/scripts/bfabric_setExternalJobStatus_done.py diff --git a/bfabric/scripts/bfabric_setResourceStatus_available.py b/src/bfabric/scripts/bfabric_setResourceStatus_available.py similarity index 100% rename from bfabric/scripts/bfabric_setResourceStatus_available.py rename to src/bfabric/scripts/bfabric_setResourceStatus_available.py diff --git a/bfabric/scripts/bfabric_setWorkunitStatus.py b/src/bfabric/scripts/bfabric_setWorkunitStatus.py similarity index 100% rename from bfabric/scripts/bfabric_setWorkunitStatus.py rename to src/bfabric/scripts/bfabric_setWorkunitStatus.py diff --git a/bfabric/scripts/bfabric_slurm_queue_status.py b/src/bfabric/scripts/bfabric_slurm_queue_status.py similarity index 100% rename from bfabric/scripts/bfabric_slurm_queue_status.py rename to src/bfabric/scripts/bfabric_slurm_queue_status.py diff --git a/bfabric/scripts/bfabric_upload_resource.py b/src/bfabric/scripts/bfabric_upload_resource.py similarity index 100% rename from bfabric/scripts/bfabric_upload_resource.py rename to src/bfabric/scripts/bfabric_upload_resource.py diff --git a/bfabric/scripts/bfabric_upload_submitter_executable.py b/src/bfabric/scripts/bfabric_upload_submitter_executable.py similarity index 100% rename from bfabric/scripts/bfabric_upload_submitter_executable.py rename to src/bfabric/scripts/bfabric_upload_submitter_executable.py diff --git a/bfabric/scripts/bfabric_wrapper_creator_yaml.py b/src/bfabric/scripts/bfabric_wrapper_creator_yaml.py similarity index 100% rename from bfabric/scripts/bfabric_wrapper_creator_yaml.py rename to src/bfabric/scripts/bfabric_wrapper_creator_yaml.py diff --git a/bfabric/scripts/fgcz_maxquant_scaffold-wrapper.py b/src/bfabric/scripts/fgcz_maxquant_scaffold-wrapper.py similarity index 100% rename from bfabric/scripts/fgcz_maxquant_scaffold-wrapper.py rename to src/bfabric/scripts/fgcz_maxquant_scaffold-wrapper.py diff --git a/bfabric/scripts/fgcz_maxquant_wrapper.py b/src/bfabric/scripts/fgcz_maxquant_wrapper.py similarity index 100% rename from bfabric/scripts/fgcz_maxquant_wrapper.py rename to src/bfabric/scripts/fgcz_maxquant_wrapper.py diff --git a/bfabric/scripts/fgcz_sge_maxquant_linux.bash b/src/bfabric/scripts/fgcz_sge_maxquant_linux.bash similarity index 100% rename from bfabric/scripts/fgcz_sge_maxquant_linux.bash rename to src/bfabric/scripts/fgcz_sge_maxquant_linux.bash diff --git a/bfabric/tests/__init__.py b/src/bfabric/utils/__init__.py similarity index 100% rename from bfabric/tests/__init__.py rename to src/bfabric/utils/__init__.py diff --git a/bfabric/utils/math_helper.py b/src/bfabric/utils/math_helper.py similarity index 100% rename from bfabric/utils/math_helper.py rename to src/bfabric/utils/math_helper.py diff --git a/bfabric/utils/paginator.py b/src/bfabric/utils/paginator.py similarity index 100% rename from bfabric/utils/paginator.py rename to src/bfabric/utils/paginator.py diff --git a/bfabric/tests/unit/__init__.py b/src/bfabric/wrapper_creator/__init__.py similarity index 100% rename from bfabric/tests/unit/__init__.py rename to src/bfabric/wrapper_creator/__init__.py diff --git a/bfabric/wrapper_creator/bfabric_external_job.py b/src/bfabric/wrapper_creator/bfabric_external_job.py similarity index 100% rename from bfabric/wrapper_creator/bfabric_external_job.py rename to src/bfabric/wrapper_creator/bfabric_external_job.py diff --git a/bfabric/wrapper_creator/bfabric_feeder.py b/src/bfabric/wrapper_creator/bfabric_feeder.py similarity index 100% rename from bfabric/wrapper_creator/bfabric_feeder.py rename to src/bfabric/wrapper_creator/bfabric_feeder.py diff --git a/bfabric/wrapper_creator/bfabric_submitter.py b/src/bfabric/wrapper_creator/bfabric_submitter.py similarity index 100% rename from bfabric/wrapper_creator/bfabric_submitter.py rename to src/bfabric/wrapper_creator/bfabric_submitter.py diff --git a/bfabric/wrapper_creator/bfabric_wrapper_creator.py b/src/bfabric/wrapper_creator/bfabric_wrapper_creator.py similarity index 100% rename from bfabric/wrapper_creator/bfabric_wrapper_creator.py rename to src/bfabric/wrapper_creator/bfabric_wrapper_creator.py diff --git a/bfabric/wrapper_creator/demo_config.yaml b/src/bfabric/wrapper_creator/demo_config.yaml similarity index 100% rename from bfabric/wrapper_creator/demo_config.yaml rename to src/bfabric/wrapper_creator/demo_config.yaml diff --git a/bfabric/wrapper_creator/gridengine.py b/src/bfabric/wrapper_creator/gridengine.py similarity index 100% rename from bfabric/wrapper_creator/gridengine.py rename to src/bfabric/wrapper_creator/gridengine.py diff --git a/bfabric/wrapper_creator/slurm.py b/src/bfabric/wrapper_creator/slurm.py similarity index 100% rename from bfabric/wrapper_creator/slurm.py rename to src/bfabric/wrapper_creator/slurm.py diff --git a/bfabric/tests/unit/config/__init__.py b/tests/unit/__init__.py similarity index 100% rename from bfabric/tests/unit/config/__init__.py rename to tests/unit/__init__.py diff --git a/bfabric/tests/unit/engine/__init__.py b/tests/unit/config/__init__.py similarity index 100% rename from bfabric/tests/unit/engine/__init__.py rename to tests/unit/config/__init__.py diff --git a/bfabric/tests/unit/config/test_bfabric_auth.py b/tests/unit/config/test_bfabric_auth.py similarity index 100% rename from bfabric/tests/unit/config/test_bfabric_auth.py rename to tests/unit/config/test_bfabric_auth.py diff --git a/bfabric/tests/unit/config/test_bfabric_client_config.py b/tests/unit/config/test_bfabric_client_config.py similarity index 100% rename from bfabric/tests/unit/config/test_bfabric_client_config.py rename to tests/unit/config/test_bfabric_client_config.py diff --git a/bfabric/tests/unit/config/test_config_file.py b/tests/unit/config/test_config_file.py similarity index 100% rename from bfabric/tests/unit/config/test_config_file.py rename to tests/unit/config/test_config_file.py diff --git a/bfabric/tests/unit/conftest.py b/tests/unit/conftest.py similarity index 100% rename from bfabric/tests/unit/conftest.py rename to tests/unit/conftest.py diff --git a/bfabric/tests/unit/entities/__init__.py b/tests/unit/engine/__init__.py similarity index 100% rename from bfabric/tests/unit/entities/__init__.py rename to tests/unit/engine/__init__.py diff --git a/bfabric/tests/unit/engine/test_engine_suds.py b/tests/unit/engine/test_engine_suds.py similarity index 100% rename from bfabric/tests/unit/engine/test_engine_suds.py rename to tests/unit/engine/test_engine_suds.py diff --git a/bfabric/tests/unit/engine/test_engine_zeep.py b/tests/unit/engine/test_engine_zeep.py similarity index 100% rename from bfabric/tests/unit/engine/test_engine_zeep.py rename to tests/unit/engine/test_engine_zeep.py diff --git a/bfabric/tests/unit/entities/core/__init__.py b/tests/unit/entities/__init__.py similarity index 100% rename from bfabric/tests/unit/entities/core/__init__.py rename to tests/unit/entities/__init__.py diff --git a/bfabric/tests/unit/scripts/__init__.py b/tests/unit/entities/core/__init__.py similarity index 100% rename from bfabric/tests/unit/scripts/__init__.py rename to tests/unit/entities/core/__init__.py diff --git a/bfabric/tests/unit/entities/core/test_entity.py b/tests/unit/entities/core/test_entity.py similarity index 100% rename from bfabric/tests/unit/entities/core/test_entity.py rename to tests/unit/entities/core/test_entity.py diff --git a/bfabric/tests/unit/entities/core/test_has_many.py b/tests/unit/entities/core/test_has_many.py similarity index 100% rename from bfabric/tests/unit/entities/core/test_has_many.py rename to tests/unit/entities/core/test_has_many.py diff --git a/bfabric/tests/unit/entities/core/test_has_one.py b/tests/unit/entities/core/test_has_one.py similarity index 100% rename from bfabric/tests/unit/entities/core/test_has_one.py rename to tests/unit/entities/core/test_has_one.py diff --git a/bfabric/tests/unit/entities/test_dataset.py b/tests/unit/entities/test_dataset.py similarity index 100% rename from bfabric/tests/unit/entities/test_dataset.py rename to tests/unit/entities/test_dataset.py diff --git a/bfabric/tests/unit/entities/test_workunit.py b/tests/unit/entities/test_workunit.py similarity index 100% rename from bfabric/tests/unit/entities/test_workunit.py rename to tests/unit/entities/test_workunit.py diff --git a/bfabric/tests/unit/example_config.yml b/tests/unit/example_config.yml similarity index 100% rename from bfabric/tests/unit/example_config.yml rename to tests/unit/example_config.yml diff --git a/bfabric/utils/__init__.py b/tests/unit/scripts/__init__.py similarity index 100% rename from bfabric/utils/__init__.py rename to tests/unit/scripts/__init__.py diff --git a/bfabric/tests/unit/scripts/test_bfabric_slurm_queue_status.py b/tests/unit/scripts/test_bfabric_slurm_queue_status.py similarity index 100% rename from bfabric/tests/unit/scripts/test_bfabric_slurm_queue_status.py rename to tests/unit/scripts/test_bfabric_slurm_queue_status.py diff --git a/bfabric/tests/unit/scripts/test_list_not_existing_storage_directories.py b/tests/unit/scripts/test_list_not_existing_storage_directories.py similarity index 100% rename from bfabric/tests/unit/scripts/test_list_not_existing_storage_directories.py rename to tests/unit/scripts/test_list_not_existing_storage_directories.py diff --git a/bfabric/tests/unit/scripts/test_save_csv2dataset.py b/tests/unit/scripts/test_save_csv2dataset.py similarity index 100% rename from bfabric/tests/unit/scripts/test_save_csv2dataset.py rename to tests/unit/scripts/test_save_csv2dataset.py diff --git a/bfabric/tests/unit/test_bfabric.py b/tests/unit/test_bfabric.py similarity index 100% rename from bfabric/tests/unit/test_bfabric.py rename to tests/unit/test_bfabric.py diff --git a/bfabric/tests/unit/test_bfabric_config.py b/tests/unit/test_bfabric_config.py similarity index 100% rename from bfabric/tests/unit/test_bfabric_config.py rename to tests/unit/test_bfabric_config.py diff --git a/bfabric/tests/unit/test_dict_helper.py b/tests/unit/test_dict_helper.py similarity index 100% rename from bfabric/tests/unit/test_dict_helper.py rename to tests/unit/test_dict_helper.py diff --git a/bfabric/tests/unit/test_math_helper.py b/tests/unit/test_math_helper.py similarity index 100% rename from bfabric/tests/unit/test_math_helper.py rename to tests/unit/test_math_helper.py diff --git a/bfabric/tests/unit/test_paginator.py b/tests/unit/test_paginator.py similarity index 100% rename from bfabric/tests/unit/test_paginator.py rename to tests/unit/test_paginator.py diff --git a/bfabric/tests/unit/test_response_format_dict.py b/tests/unit/test_response_format_dict.py similarity index 100% rename from bfabric/tests/unit/test_response_format_dict.py rename to tests/unit/test_response_format_dict.py diff --git a/bfabric/tests/unit/test_result_container.py b/tests/unit/test_result_container.py similarity index 100% rename from bfabric/tests/unit/test_result_container.py rename to tests/unit/test_result_container.py From 3b0451d79b31f406ef24038ad2b7648e04474b41 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Thu, 19 Sep 2024 15:04:05 +0200 Subject: [PATCH 05/42] adapt to src-layout and use nox --- .github/workflows/run_unit_tests.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/run_unit_tests.yml b/.github/workflows/run_unit_tests.yml index 0f8b55e6..0e88d7db 100644 --- a/.github/workflows/run_unit_tests.yml +++ b/.github/workflows/run_unit_tests.yml @@ -11,12 +11,13 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/setup-bfabricpy + - uses: actions/setup-python@v2 with: python-version: 3.9 + - name: Install nox + run: pip install nox - name: Run unit tests - # Note: we use cd to double-check that the installation actually worked - run: cd bfabric/tests && pytest ./unit + run: nox -s tests code_style: name: Code Style runs-on: ubuntu-latest @@ -34,7 +35,7 @@ jobs: - uses: actions/checkout@v4 - run: sudo apt-get install -y ripgrep name: Install ripgrep - - run: rg -n TODO bfabric + - run: rg -n TODO . name: List TODOs license_check: name: License Check From 87b46ddf1adbbd7d20d59d0a69a95cd83ba751ad Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Thu, 19 Sep 2024 15:05:13 +0200 Subject: [PATCH 06/42] add uv --- .github/workflows/run_unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_unit_tests.yml b/.github/workflows/run_unit_tests.yml index 0e88d7db..01e61638 100644 --- a/.github/workflows/run_unit_tests.yml +++ b/.github/workflows/run_unit_tests.yml @@ -15,7 +15,7 @@ jobs: with: python-version: 3.9 - name: Install nox - run: pip install nox + run: pip install nox uv - name: Run unit tests run: nox -s tests code_style: From efbeca2b9f3133c4f354163ecd0193e1b3cd42a9 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Mon, 23 Sep 2024 13:33:59 +0200 Subject: [PATCH 07/42] split process into process and collect steps --- docs/changelog.md | 2 ++ .../experimental/app_interface/app_runner/_spec.py | 1 + .../experimental/app_interface/app_runner/runner.py | 8 ++++++++ 3 files changed, 11 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index cceb4b99..b69856c6 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -17,6 +17,8 @@ Versioning currently follows `X.Y.Z` where ### Changed - (internal) migrate to src layout +- (experimental) the former `process` step of the app runner has been split into a `process` and `collect` step where, + the collect step is responsible for generating the `output.yml` file that will then be used to register the results. ## \[1.13.7\] - 2024-09-17 diff --git a/src/bfabric/experimental/app_interface/app_runner/_spec.py b/src/bfabric/experimental/app_interface/app_runner/_spec.py index 1fe1fbf0..59d5da19 100644 --- a/src/bfabric/experimental/app_interface/app_runner/_spec.py +++ b/src/bfabric/experimental/app_interface/app_runner/_spec.py @@ -6,6 +6,7 @@ class CommandsSpec(BaseModel): dispatch: str process: str + collect: str class AppSpec(BaseModel): diff --git a/src/bfabric/experimental/app_interface/app_runner/runner.py b/src/bfabric/experimental/app_interface/app_runner/runner.py index 72078f61..157dfb76 100644 --- a/src/bfabric/experimental/app_interface/app_runner/runner.py +++ b/src/bfabric/experimental/app_interface/app_runner/runner.py @@ -33,6 +33,13 @@ def run_prepare_input(self, chunk_dir: Path) -> None: inputs_yaml=chunk_dir / "inputs.yml", target_folder=chunk_dir, client=self._client, ssh_user=self._ssh_user ) + def run_collect(self, workunit_ref: int | Path, chunk_dir: Path) -> None: + subprocess.run( + f"{self._app_spec.commands.collect} " f"{shlex.quote(str(workunit_ref))} " f"{shlex.quote(str(chunk_dir))}", + shell=True, + check=True, + ) + def run_process(self, chunk_dir: Path) -> None: subprocess.run(f"{self._app_spec.commands.process} {shlex.quote(str(chunk_dir))}", shell=True, check=True) @@ -71,6 +78,7 @@ def run_app( logger.info(f"Processing chunk {chunk}") runner.run_prepare_input(chunk_dir=chunk) runner.run_process(chunk_dir=chunk) + runner.run_collect(workunit_ref=workunit_ref, chunk_dir=chunk) if not read_only: runner.run_register_outputs(chunk_dir=chunk, workunit_ref=workunit_ref) From 3372b781afef998b1106e1c8ab785c5900661030 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Thu, 26 Sep 2024 11:55:10 +0200 Subject: [PATCH 08/42] add dispatch_individual_resources.py --- docs/changelog.md | 1 + .../app_interface/dispatch/__init__.py | 0 .../dispatch/dispatch_individual_resources.py | 109 ++++++++++++++++++ 3 files changed, 110 insertions(+) create mode 100644 src/bfabric/experimental/app_interface/dispatch/__init__.py create mode 100644 src/bfabric/experimental/app_interface/dispatch/dispatch_individual_resources.py diff --git a/docs/changelog.md b/docs/changelog.md index b69856c6..e59c82e7 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -13,6 +13,7 @@ Versioning currently follows `X.Y.Z` where ### Added - Entities can be compared and sorted by ID now. +- (experimental) add initial code for a resource based application dispatch ### Changed diff --git a/src/bfabric/experimental/app_interface/dispatch/__init__.py b/src/bfabric/experimental/app_interface/dispatch/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/bfabric/experimental/app_interface/dispatch/dispatch_individual_resources.py b/src/bfabric/experimental/app_interface/dispatch/dispatch_individual_resources.py new file mode 100644 index 00000000..5f4091f9 --- /dev/null +++ b/src/bfabric/experimental/app_interface/dispatch/dispatch_individual_resources.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any, Self + +import yaml +from loguru import logger +from pydantic import BaseModel, ConfigDict, model_validator + +from bfabric import Bfabric +from bfabric.entities import Resource, Dataset +from bfabric.experimental.app_interface.workunit.definition import WorkunitDefinition + + +class ConfigResourceFlow(BaseModel): + model_config = ConfigDict(extra="forbid") + filter_suffix: str | None = None + + +class ConfigDatasetFlow(BaseModel): + model_config = ConfigDict(extra="forbid") + resource_column: str = "Imzml" + param_columns: list[tuple[str, str]] = [("PanelDataset", "mass_list_id")] + + +class ConfigDispatchIndividualResources(BaseModel): + resource_flow: ConfigResourceFlow | None + dataset_flow: ConfigDatasetFlow | None + + @model_validator(mode="after") + def check_at_least_one_flow(self) -> Self: + if self.resource_flow is None and self.dataset_flow is None: + raise ValueError("either resource_flow or dataset_flow must be provided") + return self + + +def config_msi_imzml() -> ConfigDispatchIndividualResources: + return ConfigDispatchIndividualResources( + resource_flow=ConfigResourceFlow(filter_suffix=".imzML"), + dataset_flow=ConfigDatasetFlow(resource_column="Imzml", param_columns=[("PanelDataset", "mass_list_id")]), + ) + + +class DispatchIndividualResources: + """Dispatches jobs on individual resources specified in the workunit.""" + + def __init__(self, client: Bfabric, config: ConfigDispatchIndividualResources, out_dir: Path) -> None: + self._client = client + self._config = config + self._out_dir = out_dir + + def dispatch_job(self, resource: Resource, params: dict[str, Any]) -> Path: + # TODO it is not clear yet if this is the best approach to handle this by inheritance here, + # I would sort of prefer to decouple this but it's not fully clear how to do it and it would not always + # be possible to do this as generically + # -> maybe in an initial version we can handle it with python code, but in the future revisit if it might make + # more sense to be even more generic there + raise NotImplementedError + + def dispatch_workunit(self, definition: WorkunitDefinition) -> None: + params = definition.execution.raw_parameters + if definition.execution.resources: + paths = self._dispatch_jobs_resource_flow(definition, params) + elif definition.execution.dataset: + paths = self._dispatch_jobs_dataset_flow(definition, params) + else: + raise ValueError("either dataset or resources must be provided") + self._write_workunit_definition(definition=definition) + self._write_chunks(chunks=paths) + + def _write_workunit_definition(self, definition: WorkunitDefinition) -> None: + self._out_dir.mkdir(exist_ok=True, parents=True) + with (self._out_dir / "workunit_definition.yml").open("w") as f: + yaml.safe_dump(definition.model_dump(mode="json"), f) + + def _write_chunks(self, chunks: list[Path]) -> None: + self._out_dir.mkdir(exist_ok=True, parents=True) + with (self._out_dir / "chunks.yml").open("w") as f: + data = {"chunks": [str(chunk) for chunk in chunks]} + yaml.safe_dump(data, f) + + def _dispatch_jobs_resource_flow(self, definition: WorkunitDefinition, params: dict[str, Any]) -> list[Path]: + if self._config.resource_flow is None: + raise ValueError("resource_flow is not configured") + resources = Resource.find_all(ids=definition.execution.resources, client=self._client) + paths = [] + for resource in sorted(resources.values()): + if self._config.resource_flow.filter_suffix is not None and not resource["name"].endswith( + self._config.resource_flow.filter_suffix + ): + logger.info(f"Skipping resource {resource['name']!r} as it does not match the extension filter.") + continue + paths.append(self.dispatch_job(resource=resource, params=params)) + return paths + + def _dispatch_jobs_dataset_flow(self, definition: WorkunitDefinition, params: dict[str, Any]) -> list[Path]: + if self._config.dataset_flow is None: + raise ValueError("dataset_flow is not configured") + dataset = Dataset.find(id=definition.execution.dataset, client=self._client) + dataset_df = dataset.to_polars() + resources = Resource.find_all( + ids=dataset_df[self._config.dataset_flow.resource_column].unique().to_list(), client=self._client + ) + paths = [] + for row in dataset_df.iter_rows(named=True): + resource_id = int(row[self._config.dataset_flow.resource_column]) + row_params = {name: row[dataset_name] for dataset_name, name in self._config.dataset_flow.param_columns} + paths.append(self.dispatch_job(resource=resources[resource_id], params=params | row_params)) + return paths From 0a2317e39a1933fc6d9b2950c3be46f7fc302d52 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Thu, 26 Sep 2024 13:43:33 +0200 Subject: [PATCH 09/42] use relativepath for suffix identifcation --- .../bfabric_list_not_existing_storage_directories.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/bfabric/scripts/bfabric_list_not_existing_storage_directories.py b/src/bfabric/scripts/bfabric_list_not_existing_storage_directories.py index 075d53eb..b040ae0d 100755 --- a/src/bfabric/scripts/bfabric_list_not_existing_storage_directories.py +++ b/src/bfabric/scripts/bfabric_list_not_existing_storage_directories.py @@ -18,16 +18,20 @@ def list_not_existing_storage_dirs(client: Bfabric, root_dir: Path, technology_id: int | list[int]) -> None: """Lists not existing storage directories for a given technology id.""" - results = client.read(endpoint="container", obj={"technologyid": technology_id}, return_id_only=True) + results = client.read( + endpoint="container", + obj={"technologyid": technology_id}, + return_id_only=True, + max_results=500, + ) container_ids = sorted({x["id"] for x in results}) - for container_id in container_ids: if not (root_dir / f"p{container_id}").is_dir(): print(container_id) def main() -> None: - """Parses CLI arguments and calls `list_not_existing_storage_dirs`.""" + """Calls `list_not_existing_storage_dirs`.""" client = Bfabric.from_config() root_dir = Path("/srv/www/htdocs/") list_not_existing_storage_dirs(client=client, root_dir=root_dir, technology_id=[2, 4]) From 8288d3c0d2e4179927d08c703965b1422c7f60be Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Thu, 26 Sep 2024 13:55:38 +0200 Subject: [PATCH 10/42] adapt test --- .../scripts/test_list_not_existing_storage_directories.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/scripts/test_list_not_existing_storage_directories.py b/tests/unit/scripts/test_list_not_existing_storage_directories.py index 7f854037..ea425b10 100644 --- a/tests/unit/scripts/test_list_not_existing_storage_directories.py +++ b/tests/unit/scripts/test_list_not_existing_storage_directories.py @@ -28,7 +28,9 @@ def test_list_not_existing_storage_directories( assert err == "" assert out == "3050\n3300\n" - client.read.assert_called_once_with(endpoint="container", obj={"technologyid": [2, 4]}, return_id_only=True) + client.read.assert_called_once_with( + endpoint="container", obj={"technologyid": [2, 4]}, return_id_only=True, max_results=500 + ) if __name__ == "__main__": From dc658e1f89f2a81b867fe09f97cd305d0edf4703 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Thu, 26 Sep 2024 15:38:43 +0200 Subject: [PATCH 11/42] fix --- .../app_interface/dispatch/dispatch_individual_resources.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/bfabric/experimental/app_interface/dispatch/dispatch_individual_resources.py b/src/bfabric/experimental/app_interface/dispatch/dispatch_individual_resources.py index 5f4091f9..21560c20 100644 --- a/src/bfabric/experimental/app_interface/dispatch/dispatch_individual_resources.py +++ b/src/bfabric/experimental/app_interface/dispatch/dispatch_individual_resources.py @@ -85,10 +85,12 @@ def _dispatch_jobs_resource_flow(self, definition: WorkunitDefinition, params: d resources = Resource.find_all(ids=definition.execution.resources, client=self._client) paths = [] for resource in sorted(resources.values()): - if self._config.resource_flow.filter_suffix is not None and not resource["name"].endswith( + if self._config.resource_flow.filter_suffix is not None and not resource["relativepath"].endswith( self._config.resource_flow.filter_suffix ): - logger.info(f"Skipping resource {resource['name']!r} as it does not match the extension filter.") + logger.info( + f"Skipping resource {resource['relativepath']!r} as it does not match the extension filter." + ) continue paths.append(self.dispatch_job(resource=resource, params=params)) return paths From 08ce4d86123a00f6aeb62ee96d15e7093a59a001 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Fri, 27 Sep 2024 10:02:42 +0200 Subject: [PATCH 12/42] implement reuse default resource logic --- docs/changelog.md | 1 + .../app_interface/app_runner/_spec.py | 2 + .../app_interface/app_runner/runner.py | 7 ++- .../output_registration/register.py | 56 ++++++++++++++----- 4 files changed, 49 insertions(+), 17 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index e59c82e7..1adb7f87 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -20,6 +20,7 @@ Versioning currently follows `X.Y.Z` where - (internal) migrate to src layout - (experimental) the former `process` step of the app runner has been split into a `process` and `collect` step where, the collect step is responsible for generating the `output.yml` file that will then be used to register the results. +- (experimental) app runner apps by default reuse the default resource ## \[1.13.7\] - 2024-09-17 diff --git a/src/bfabric/experimental/app_interface/app_runner/_spec.py b/src/bfabric/experimental/app_interface/app_runner/_spec.py index 59d5da19..6653f148 100644 --- a/src/bfabric/experimental/app_interface/app_runner/_spec.py +++ b/src/bfabric/experimental/app_interface/app_runner/_spec.py @@ -12,3 +12,5 @@ class CommandsSpec(BaseModel): class AppSpec(BaseModel): version: list[str] = [] commands: CommandsSpec + # Note: While we use the old submitter, this is still necessary + reuse_default_resource: bool = True diff --git a/src/bfabric/experimental/app_interface/app_runner/runner.py b/src/bfabric/experimental/app_interface/app_runner/runner.py index 157dfb76..43890a7a 100644 --- a/src/bfabric/experimental/app_interface/app_runner/runner.py +++ b/src/bfabric/experimental/app_interface/app_runner/runner.py @@ -43,13 +43,14 @@ def run_collect(self, workunit_ref: int | Path, chunk_dir: Path) -> None: def run_process(self, chunk_dir: Path) -> None: subprocess.run(f"{self._app_spec.commands.process} {shlex.quote(str(chunk_dir))}", shell=True, check=True) - def run_register_outputs(self, chunk_dir: Path, workunit_ref: int | Path) -> None: + def run_register_outputs(self, chunk_dir: Path, workunit_ref: int | Path, reuse_default_resource: bool) -> None: workunit_definition = WorkunitDefinition.from_ref(workunit_ref, client=self._client) register_outputs( outputs_yaml=chunk_dir / "outputs.yml", workunit_id=workunit_definition.registration.workunit_id, client=self._client, ssh_user=self._ssh_user, + reuse_default_resource=reuse_default_resource, ) @@ -80,7 +81,9 @@ def run_app( runner.run_process(chunk_dir=chunk) runner.run_collect(workunit_ref=workunit_ref, chunk_dir=chunk) if not read_only: - runner.run_register_outputs(chunk_dir=chunk, workunit_ref=workunit_ref) + runner.run_register_outputs( + chunk_dir=chunk, workunit_ref=workunit_ref, reuse_default_resource=app_spec.reuse_default_resource + ) if not read_only: client.save("workunit", {"id": workunit_definition.registration.workunit_id, "status": "available"}) diff --git a/src/bfabric/experimental/app_interface/output_registration/register.py b/src/bfabric/experimental/app_interface/output_registration/register.py index 982da7b9..971b306a 100644 --- a/src/bfabric/experimental/app_interface/output_registration/register.py +++ b/src/bfabric/experimental/app_interface/output_registration/register.py @@ -30,24 +30,25 @@ def register_file_in_workunit( client: Bfabric, workunit: Workunit, storage: Storage, + resource_id: int | None = None, ): if spec.update_existing != UpdateExisting.NO: # TODO implement this functionality raise NotImplementedError("Update existing not implemented") checksum = md5sum(spec.local_path) output_folder = _get_output_folder(spec, workunit=workunit) - client.save( - "resource", - { - "name": spec.store_entry_path.name, - "workunitid": workunit.id, - "storageid": storage.id, - "relativepath": output_folder / spec.store_entry_path, - "filechecksum": checksum, - "status": "available", - "size": spec.local_path.stat().st_size, - }, - ) + resource_data = { + "name": spec.store_entry_path.name, + "workunitid": workunit.id, + "storageid": storage.id, + "relativepath": output_folder / spec.store_entry_path, + "filechecksum": checksum, + "status": "available", + "size": spec.local_path.stat().st_size, + } + if resource_id is not None: + resource_data["id"] = resource_id + client.save("resource", resource_data) def copy_file_to_storage(spec: CopyResourceSpec, workunit: Workunit, storage: Storage, ssh_user: str | None): @@ -70,13 +71,31 @@ def _save_dataset(spec: SaveDatasetSpec, client: Bfabric, workunit: Workunit): ) -def register_all(client: Bfabric, workunit: Workunit, specs_list: list[SpecType], ssh_user: str | None): +def find_default_resource_id(workunit: Workunit) -> int | None: + candidate_resources = [ + resource for resource in workunit.resources if resource["name"] not in ["slurm_stdout", "slurm_stderr"] + ] + if len(candidate_resources) != 1: + return None + else: + return candidate_resources[0].id + + +def register_all( + client: Bfabric, workunit: Workunit, specs_list: list[SpecType], ssh_user: str | None, reuse_default_resource: bool +): + default_resource_was_reused = not reuse_default_resource for spec in specs_list: logger.debug(f"Registering {spec}") if isinstance(spec, CopyResourceSpec): storage = workunit.application.storage copy_file_to_storage(spec, workunit=workunit, storage=storage, ssh_user=ssh_user) - register_file_in_workunit(spec, client=client, workunit=workunit, storage=storage) + if not default_resource_was_reused: + resource_id = find_default_resource_id(workunit=workunit) + default_resource_was_reused = True + else: + resource_id = None + register_file_in_workunit(spec, client=client, workunit=workunit, storage=storage, resource_id=resource_id) elif isinstance(spec, SaveDatasetSpec): _save_dataset(spec, client, workunit=workunit) else: @@ -88,10 +107,17 @@ def register_outputs( workunit_id: int, client: Bfabric, ssh_user: str | None, + reuse_default_resource: bool, ) -> None: # parse the specs specs_list = OutputsSpec.read_yaml(outputs_yaml) # register all specs workunit = Workunit.find(id=workunit_id, client=client) - register_all(client=client, workunit=workunit, specs_list=specs_list, ssh_user=ssh_user) + register_all( + client=client, + workunit=workunit, + specs_list=specs_list, + ssh_user=ssh_user, + reuse_default_resource=reuse_default_resource, + ) From 97809c7d928ff0db297a44ea0e52251ae7328a5b Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Fri, 27 Sep 2024 10:22:22 +0200 Subject: [PATCH 13/42] improve the reuse default resource logic --- .../app_interface/output_registration/register.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/bfabric/experimental/app_interface/output_registration/register.py b/src/bfabric/experimental/app_interface/output_registration/register.py index 971b306a..8816dff2 100644 --- a/src/bfabric/experimental/app_interface/output_registration/register.py +++ b/src/bfabric/experimental/app_interface/output_registration/register.py @@ -75,10 +75,10 @@ def find_default_resource_id(workunit: Workunit) -> int | None: candidate_resources = [ resource for resource in workunit.resources if resource["name"] not in ["slurm_stdout", "slurm_stderr"] ] - if len(candidate_resources) != 1: - return None - else: + # We also check that the resource is pending, as else we might re-use a resource that was created by the app... + if len(candidate_resources) == 1 and candidate_resources[0]["status"] == "pending": return candidate_resources[0].id + return None def register_all( From 2cfcdced7fb473c456d3001ec3d9cdd8f1c26e82 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Mon, 30 Sep 2024 16:23:47 +0200 Subject: [PATCH 14/42] move some tests to the correct location --- pyproject.toml | 5 +++ tests/__init__.py | 0 tests/unit/results/__init__.py | 0 .../test_response_format_dict.py} | 0 .../{ => results}/test_result_container.py | 0 ..._list_not_existing_storage_directories.py} | 0 ...et.py => test_bfabric_save_csv2dataset.py} | 0 tests/unit/test_response_format_dict.py | 42 ------------------- tests/unit/utils/__init__.py | 0 tests/unit/{ => utils}/test_math_helper.py | 0 tests/unit/{ => utils}/test_paginator.py | 0 11 files changed, 5 insertions(+), 42 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/unit/results/__init__.py rename tests/unit/{test_dict_helper.py => results/test_response_format_dict.py} (100%) rename tests/unit/{ => results}/test_result_container.py (100%) rename tests/unit/scripts/{test_list_not_existing_storage_directories.py => test_bfabric_list_not_existing_storage_directories.py} (100%) rename tests/unit/scripts/{test_save_csv2dataset.py => test_bfabric_save_csv2dataset.py} (100%) delete mode 100644 tests/unit/test_response_format_dict.py create mode 100644 tests/unit/utils/__init__.py rename tests/unit/{ => utils}/test_math_helper.py (100%) rename tests/unit/{ => utils}/test_paginator.py (100%) diff --git a/pyproject.toml b/pyproject.toml index fc0edddb..d03ee77f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,3 +96,8 @@ using = "PEP631" [tool.pytest.ini_options] logot_capturer = "logot.loguru.LoguruCapturer" + +[tool.check-tests-structure] +sources_path = "src/bfabric" +tests_path = "tests/unit" +allow_missing_tests = true diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/results/__init__.py b/tests/unit/results/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/test_dict_helper.py b/tests/unit/results/test_response_format_dict.py similarity index 100% rename from tests/unit/test_dict_helper.py rename to tests/unit/results/test_response_format_dict.py diff --git a/tests/unit/test_result_container.py b/tests/unit/results/test_result_container.py similarity index 100% rename from tests/unit/test_result_container.py rename to tests/unit/results/test_result_container.py diff --git a/tests/unit/scripts/test_list_not_existing_storage_directories.py b/tests/unit/scripts/test_bfabric_list_not_existing_storage_directories.py similarity index 100% rename from tests/unit/scripts/test_list_not_existing_storage_directories.py rename to tests/unit/scripts/test_bfabric_list_not_existing_storage_directories.py diff --git a/tests/unit/scripts/test_save_csv2dataset.py b/tests/unit/scripts/test_bfabric_save_csv2dataset.py similarity index 100% rename from tests/unit/scripts/test_save_csv2dataset.py rename to tests/unit/scripts/test_bfabric_save_csv2dataset.py diff --git a/tests/unit/test_response_format_dict.py b/tests/unit/test_response_format_dict.py deleted file mode 100644 index 9fe1765b..00000000 --- a/tests/unit/test_response_format_dict.py +++ /dev/null @@ -1,42 +0,0 @@ -import unittest - -import bfabric.results.response_format_dict as response_format_dict - - -class BfabricTestResponseFormatDict(unittest.TestCase): - def test_drop_empty_elements(self): - # Should delete all hierarchical instances of key-value pairs, where value is None or empty dict - input_list_dict = [{"a": [], "b": [1, {"aa": 14, "gg": None}], "c": []}, {"zz": None, "uu": "cat"}] - target_list_dict = [{"b": [1, {"aa": 14}]}, {"uu": "cat"}] - - output_list_dict = response_format_dict.drop_empty_elements(input_list_dict, inplace=False) - self.assertEqual(output_list_dict, target_list_dict) - - def test_map_element_keys(self): - # Main use is to delete underscores in specific keys - input_list_dict = [{"a": [], "b": [1, {"_aa": 14, "gg": None}], "c": []}, {"zz": None, "uu": "cat"}] - target_list_dict = [{"a": [], "b": [1, {"aa": 14, "gg": None}], "c": []}, {"zz": None, "uu": "cat"}] - - output_list_dict = response_format_dict.map_element_keys(input_list_dict, {"_aa": "aa"}, inplace=False) - self.assertEqual(output_list_dict, target_list_dict) - - def test_sort_dicts_by_key(self): - # NOTE: The main purpose of sorting is to ensure consistent string representation - input_list_dict = [{"b": 1, "a": 2, "c": 3}, {"dog": 25, "cat": [1, 2, 3]}] - target_list_dict = [{"a": 2, "b": 1, "c": 3}, {"cat": [1, 2, 3], "dog": 25}] - - output_list_dict = response_format_dict.sort_dicts_by_key(input_list_dict, inplace=False) - self.assertEqual(str(output_list_dict), str(target_list_dict)) - - def test_clean_result(self): - result_input = [{"b": 1, "a": 2, "_id": 3}, {"b": 4, "_id": 5, "a": 6}] - cleaned = response_format_dict.clean_result(result_input, drop_underscores_suds=True, sort_keys=True) - self.assertEqual(repr([{"a": 2, "b": 1, "id": 3}, {"a": 6, "b": 4, "id": 5}]), repr(cleaned)) - self.assertEqual( - repr([{"b": 1, "a": 2, "_id": 3}, {"b": 4, "_id": 5, "a": 6}]), - repr(result_input), - ) - - -if __name__ == "__main__": - unittest.main(verbosity=2) diff --git a/tests/unit/utils/__init__.py b/tests/unit/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/test_math_helper.py b/tests/unit/utils/test_math_helper.py similarity index 100% rename from tests/unit/test_math_helper.py rename to tests/unit/utils/test_math_helper.py diff --git a/tests/unit/test_paginator.py b/tests/unit/utils/test_paginator.py similarity index 100% rename from tests/unit/test_paginator.py rename to tests/unit/utils/test_paginator.py From 3f09ca006817d9f17ccc1db55f9ae4c389836559 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Tue, 1 Oct 2024 10:22:15 +0200 Subject: [PATCH 15/42] make more robust --- docs/changelog.md | 4 ++++ src/bfabric/scripts/bfabric_read.py | 19 +++++++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 1adb7f87..8c43da66 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -15,6 +15,10 @@ Versioning currently follows `X.Y.Z` where - Entities can be compared and sorted by ID now. - (experimental) add initial code for a resource based application dispatch +### Fixed + +- bfabric_read.py is a bit more robust if "name" misses and tabular output is requested. + ### Changed - (internal) migrate to src layout diff --git a/src/bfabric/scripts/bfabric_read.py b/src/bfabric/scripts/bfabric_read.py index 9743a7be..0cf8c07f 100755 --- a/src/bfabric/scripts/bfabric_read.py +++ b/src/bfabric/scripts/bfabric_read.py @@ -78,8 +78,8 @@ def _print_table_rich( f"[link={entry_url}]{x['id']}[/link]", str(x["createdby"]), str(x["modified"]), - str(x["name"]), - str(x.get("groupingvar", {}).get("name", "")), + x.get("name", ""), + x.get("groupingvar", {}).get("name", "") or "", ) console_out.print(table) @@ -87,10 +87,17 @@ def _print_table_rich( def _print_table_tsv(res: list[dict[str, Any]]) -> None: """Prints the results as a tab-separated table, using the original cols this script returned.""" for x in res: - try: - print(f'{x["id"]}\t{x["createdby"]}\t{x["modified"]}\t{x["name"]}\t{x["groupingvar"]["name"]}') - except (KeyError, TypeError): - print(f'{x["id"]}\t{x["createdby"]}\t{x["modified"]}') + print( + "\t".join( + [ + str(x["id"]), + str(x["createdby"]), + str(x["modified"]), + x.get("name", ""), + x.get("groupingvar", {}).get("name") or "", + ] + ) + ) def _determine_output_format(console_out: Console, output_format: str, n_results: int) -> str: From 0d1a267fc49133e7d46cb15451e9739a002797e6 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Tue, 1 Oct 2024 14:57:08 +0200 Subject: [PATCH 16/42] add caching for list_not_existing_storage_directories --- pyproject.toml | 1 + ...c_list_not_existing_storage_directories.py | 77 ++++++++++++++++--- 2 files changed, 68 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d03ee77f..d1693f1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ "pydantic", "eval_type_backport; python_version < '3.10'", "python-dateutil >= 2.9.0", + "platformdirs >= 4.3" ] [project.optional-dependencies] diff --git a/src/bfabric/scripts/bfabric_list_not_existing_storage_directories.py b/src/bfabric/scripts/bfabric_list_not_existing_storage_directories.py index b040ae0d..53179c3f 100755 --- a/src/bfabric/scripts/bfabric_list_not_existing_storage_directories.py +++ b/src/bfabric/scripts/bfabric_list_not_existing_storage_directories.py @@ -11,21 +11,77 @@ from __future__ import annotations +import datetime +import json +import os from pathlib import Path +from loguru import logger +from pydantic import BaseModel + from bfabric import Bfabric -def list_not_existing_storage_dirs(client: Bfabric, root_dir: Path, technology_id: int | list[int]) -> None: +class CacheData(BaseModel): + """Content of the cache file.""" + + container_ids: set[int] = set() + checked_at: datetime.datetime = datetime.datetime(2024, 1, 1) + query: dict[str, int | str | list[str | int]] = {"technologyid": [2, 4], "createdafter": "2024-01-01"} + + +class Cache: + """Caches retrieval of the list of all container ids.""" + + def __init__(self, path: Path, data: CacheData) -> None: + self._path = path + self._data = data + + @property + def container_ids(self) -> list[int]: + """Returns the container ids sorted.""" + return sorted(self._data.container_ids) + + @classmethod + def load(cls, path: Path) -> Cache: + """Returns the cache from the cache file if it exists, otherwise returns a new instance.""" + if path.exists(): + logger.info(f"Loading cache from {path}") + data = CacheData.model_validate_json(path.read_text()) + else: + logger.info(f"Cache file {path} does not exist. Using default values.") + data = CacheData() + return cls(path=path, data=data) + + def save(self) -> None: + """Saves the cache to the cache file.""" + logger.info(f"Saving cache to {self._path}") + with self._path.open("w") as f: + json.dump(self._data.model_dump(mode="json"), f) + + def update(self, client: Bfabric) -> None: + """Updates the cache with the container ids from B-Fabric.""" + now = datetime.datetime.now() + # add some buffer to deal e.g. with miss-configured clocks + timestamp = (self._data.checked_at - datetime.timedelta(days=1)).isoformat() + max_results = 300 if "BFABRICPY_DEBUG" in os.environ else None + logger.debug(f"Checking for new container ids since {timestamp} with limit {max_results}") + result = client.read( + "container", {**self._data.query, "createdafter": timestamp}, return_id_only=True, max_results=max_results + ) + if len(result): + ids = result.to_polars()["id"].unique().to_list() + self._data.container_ids.update(ids) + self._data.checked_at = now + + +def list_not_existing_storage_dirs(client: Bfabric, root_dir: Path, cache_path: Path) -> None: """Lists not existing storage directories for a given technology id.""" - results = client.read( - endpoint="container", - obj={"technologyid": technology_id}, - return_id_only=True, - max_results=500, - ) - container_ids = sorted({x["id"] for x in results}) - for container_id in container_ids: + cache = Cache.load(path=cache_path) + cache.update(client=client) + cache.save() + + for container_id in cache.container_ids: if not (root_dir / f"p{container_id}").is_dir(): print(container_id) @@ -34,7 +90,8 @@ def main() -> None: """Calls `list_not_existing_storage_dirs`.""" client = Bfabric.from_config() root_dir = Path("/srv/www/htdocs/") - list_not_existing_storage_dirs(client=client, root_dir=root_dir, technology_id=[2, 4]) + cache_path = Path("cache.json") + list_not_existing_storage_dirs(client=client, root_dir=root_dir, cache_path=cache_path) if __name__ == "__main__": From 567a78868ccba2706e61080c548432530a9d5638 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Wed, 2 Oct 2024 10:07:34 +0200 Subject: [PATCH 17/42] experimental command line interface for app runner --- docs/changelog.md | 1 + pyproject.toml | 3 +- src/bfabric/experimental/app_interface/cli.py | 205 ++++++++++++++++++ .../input_preparation/prepare.py | 24 ++ 4 files changed, 232 insertions(+), 1 deletion(-) create mode 100644 src/bfabric/experimental/app_interface/cli.py diff --git a/docs/changelog.md b/docs/changelog.md index 8c43da66..41d7740a 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -14,6 +14,7 @@ Versioning currently follows `X.Y.Z` where - Entities can be compared and sorted by ID now. - (experimental) add initial code for a resource based application dispatch +- (experimental) new app_runner cli that integrates all commands into a single interface ### Fixed diff --git a/pyproject.toml b/pyproject.toml index d1693f1a..747f1280 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,8 @@ dependencies = [ "pydantic", "eval_type_backport; python_version < '3.10'", "python-dateutil >= 2.9.0", - "platformdirs >= 4.3" + "cyclopts", + #"platformdirs >= 4.3", ] [project.optional-dependencies] diff --git a/src/bfabric/experimental/app_interface/cli.py b/src/bfabric/experimental/app_interface/cli.py new file mode 100644 index 00000000..3062f255 --- /dev/null +++ b/src/bfabric/experimental/app_interface/cli.py @@ -0,0 +1,205 @@ +from __future__ import annotations +from pathlib import Path +from venv import logger + +import cyclopts +import yaml + +from bfabric import Bfabric +from bfabric.cli_formatting import setup_script_logging +from bfabric.entities import Workunit +from bfabric.experimental.app_interface.app_runner._spec import AppSpec +from bfabric.experimental.app_interface.app_runner.runner import run_app, Runner, ChunksFile +from bfabric.experimental.app_interface.input_preparation import prepare_folder +from bfabric.experimental.app_interface.input_preparation.prepare import print_input_files_list +from bfabric.experimental.app_interface.output_registration._spec import OutputsSpec +from bfabric.experimental.app_interface.output_registration.register import register_all + +app = cyclopts.App() +app_inputs = cyclopts.App("inputs", help="Prepare input files for an app.") +app.command(app_inputs) +app_outputs = cyclopts.App("outputs", help="Register output files for an app.") +app.command(app_outputs) +app_app = cyclopts.App("app", help="Run an app.") +app.command(app_app) + + +@app_inputs.command() +def prepare( + inputs_yaml: Path, + target_folder: Path | None = None, + ssh_user: str | None = None, +) -> None: + """Prepare the input files by downloading them (if necessary). + + :param inputs_yaml: Path to the inputs.yml file. + :param target_folder: Path to the target folder where the input files should be downloaded. + :param ssh_user: Optional SSH user to use for downloading the input files, instead of the current user. + """ + setup_script_logging() + client = Bfabric.from_config() + prepare_folder( + inputs_yaml=inputs_yaml, + target_folder=target_folder, + ssh_user=ssh_user, + client=client, + action="prepare", + ) + + +@app_inputs.command() +def clean( + inputs_yaml: Path, + target_folder: Path | None = None, +) -> None: + """Removes all local copies of input files. + + :param inputs_yaml: Path to the inputs.yml file. + :param target_folder: Path to the target folder where the input files should be removed. + """ + setup_script_logging() + client = Bfabric.from_config() + # TODO clean shouldn't even need all these arguments, this could be refactored later + prepare_folder( + inputs_yaml=inputs_yaml, + target_folder=target_folder, + ssh_user=None, + action="clean", + client=client, + ) + + +@app_inputs.command() +def list( + inputs_yaml: Path, + target_folder: Path | None = None, +) -> None: + """Lists the input files for an app.""" + setup_script_logging() + print_input_files_list(inputs_yaml=inputs_yaml, target_folder=target_folder) + + +@app_outputs.command() +def register( + outputs_yaml: Path, + # TODO we should use the workunit definition instead + workunit_id: int, + ssh_user: str | None = None, + # TODO + reuse_default_resource: bool = True, +) -> None: + """Register the output files of a workunit.""" + setup_script_logging() + client = Bfabric.from_config() + + specs_list = OutputsSpec.read_yaml(outputs_yaml) + workunit = Workunit.find(id=workunit_id, client=client) + register_all( + client=client, + workunit=workunit, + specs_list=specs_list, + ssh_user=ssh_user, + reuse_default_resource=reuse_default_resource, + ) + + +@app_app.command() +def run( + app_spec: Path, + workunit_ref: int | Path, + target_folder: Path, + ssh_user: str | None = None, + read_only: bool = False, +) -> None: + """Runs all stages of an app.""" + setup_script_logging() + client = Bfabric.from_config() + app_spec_parsed = AppSpec.model_validate(yaml.safe_load(app_spec.read_text())) + run_app( + app_spec=app_spec_parsed, + workunit_ref=workunit_ref, + work_dir=target_folder, + client=client, + ssh_user=ssh_user, + read_only=read_only, + ) + + +@app_app.command() +def dispatch( + app_spec: Path, + workunit_ref: int | Path, + work_dir: Path, +) -> None: + setup_script_logging() + # TODO set workunit to processing? (i.e. add read-only option here) + client = Bfabric.from_config() + runner = Runner(spec=AppSpec.model_validate(yaml.safe_load(app_spec.read_text())), client=client, ssh_user=None) + runner.run_dispatch(workunit_ref=workunit_ref, work_dir=work_dir) + + +@app_app.command() +def process_all( + app_spec: Path, + workunit_ref: int | Path, + work_dir: Path, + ssh_user: str | None = None, + read_only: bool = False, +) -> None: + setup_script_logging() + client = Bfabric.from_config() + app_spec_parsed = AppSpec.model_validate(yaml.safe_load(app_spec.read_text())) + runner = Runner(spec=app_spec_parsed, client=client, ssh_user=ssh_user) + chunks_file = ChunksFile.model_validate(yaml.safe_load((work_dir / "chunks.yml").read_text())) + for chunk in chunks_file.chunks: + logger.info(f"Processing chunk {chunk}") + runner.run_prepare_input(chunk_dir=chunk) + runner.run_process(chunk_dir=chunk) + runner.run_collect(workunit_ref=workunit_ref, chunk_dir=chunk) + if not read_only: + runner.run_register_outputs( + chunk_dir=chunk, + workunit_ref=workunit_ref, + reuse_default_resource=app_spec_parsed.reuse_default_resource, + ) + + +@app_app.command() +def chunk_inputs( + chunk_dir: Path, + ssh_user: str | None = None, +) -> None: + setup_script_logging() + client = Bfabric.from_config() + runner = Runner(spec=AppSpec(), client=client, ssh_user=ssh_user) + runner.run_prepare_input(chunk_dir=chunk_dir) + + +@app_app.command() +def chunk_process(chunk_dir: Path) -> None: + setup_script_logging() + client = Bfabric.from_config() + runner = Runner(spec=AppSpec(), client=client, ssh_user=None) + runner.run_process(chunk_dir=chunk_dir) + + +@app_app.command() +def chunk_outputs( + chunk_dir: Path, + workunit_ref: int | Path, + ssh_user: str | None = None, + read_only: bool = False, + reuse_default_resource: bool = True, +) -> None: + setup_script_logging() + client = Bfabric.from_config() + runner = Runner(spec=AppSpec(), client=client, ssh_user=ssh_user) + runner.run_collect(workunit_ref=workunit_ref, chunk_dir=chunk_dir) + if not read_only: + runner.run_register_outputs( + chunk_dir=chunk_dir, workunit_ref=workunit_ref, reuse_default_resource=reuse_default_resource + ) + + +if __name__ == "__main__": + app() diff --git a/src/bfabric/experimental/app_interface/input_preparation/prepare.py b/src/bfabric/experimental/app_interface/input_preparation/prepare.py index 0789d03c..8876e9b0 100644 --- a/src/bfabric/experimental/app_interface/input_preparation/prepare.py +++ b/src/bfabric/experimental/app_interface/input_preparation/prepare.py @@ -4,6 +4,8 @@ from pathlib import Path from loguru import logger +from rich.console import Console +from rich.table import Table, Column from bfabric.bfabric import Bfabric from bfabric.entities import Resource, Dataset @@ -117,3 +119,25 @@ def prepare_folder( prepare.clean_all(specs=specs_list) else: raise ValueError(f"Unknown action: {action}") + + +def print_input_files_list( + inputs_yaml: Path, + target_folder: Path, +) -> None: + """Prints a list of inputs and whether they exist locally.""" + specs_list = InputsSpec.read_yaml(inputs_yaml) + table = Table( + Column("File"), + Column("Input Type"), + Column("Exists Locally"), + ) + for spec in specs_list: + path = target_folder / spec.filename if target_folder else Path(spec.filename) + table.add_row( + str(path), + "Resource" if isinstance(spec, ResourceSpec) else "Dataset", + "Yes" if path.exists() else "No", + ) + console = Console() + console.print(table) From 5aff195765c1d69747d7918459bfb8d355c963ec Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Wed, 2 Oct 2024 10:17:21 +0200 Subject: [PATCH 18/42] improve cli --- .../app_interface/app_runner/runner.py | 4 +- src/bfabric/experimental/app_interface/cli.py | 85 +++++++++++++------ 2 files changed, 64 insertions(+), 25 deletions(-) diff --git a/src/bfabric/experimental/app_interface/app_runner/runner.py b/src/bfabric/experimental/app_interface/app_runner/runner.py index 43890a7a..5e4543ef 100644 --- a/src/bfabric/experimental/app_interface/app_runner/runner.py +++ b/src/bfabric/experimental/app_interface/app_runner/runner.py @@ -66,6 +66,7 @@ def run_app( client: Bfabric, ssh_user: str | None = None, read_only: bool = False, + dispatch_active: bool = True, ) -> None: # TODO future: the workunit definition must be loaded from bfabric exactly once! this is quite inefficient right now workunit_definition = WorkunitDefinition.from_ref(workunit_ref, client=client) @@ -73,7 +74,8 @@ def run_app( client.save("workunit", {"id": workunit_definition.registration.workunit_id, "status": "processing"}) runner = Runner(spec=app_spec, client=client, ssh_user=ssh_user) - runner.run_dispatch(workunit_ref=workunit_ref, work_dir=work_dir) + if dispatch_active: + runner.run_dispatch(workunit_ref=workunit_ref, work_dir=work_dir) chunks_file = ChunksFile.model_validate(yaml.safe_load((work_dir / "chunks.yml").read_text())) for chunk in chunks_file.chunks: logger.info(f"Processing chunk {chunk}") diff --git a/src/bfabric/experimental/app_interface/cli.py b/src/bfabric/experimental/app_interface/cli.py index 3062f255..d5ba48fc 100644 --- a/src/bfabric/experimental/app_interface/cli.py +++ b/src/bfabric/experimental/app_interface/cli.py @@ -1,6 +1,6 @@ from __future__ import annotations + from pathlib import Path -from venv import logger import cyclopts import yaml @@ -9,7 +9,7 @@ from bfabric.cli_formatting import setup_script_logging from bfabric.entities import Workunit from bfabric.experimental.app_interface.app_runner._spec import AppSpec -from bfabric.experimental.app_interface.app_runner.runner import run_app, Runner, ChunksFile +from bfabric.experimental.app_interface.app_runner.runner import run_app, Runner from bfabric.experimental.app_interface.input_preparation import prepare_folder from bfabric.experimental.app_interface.input_preparation.prepare import print_input_files_list from bfabric.experimental.app_interface.output_registration._spec import OutputsSpec @@ -22,19 +22,22 @@ app.command(app_outputs) app_app = cyclopts.App("app", help="Run an app.") app.command(app_app) +app_chunk = cyclopts.App("chunk", help="Run an app on a chunk. You can create the chunks with `app dispatch`.") +app.command(app_chunk) @app_inputs.command() def prepare( inputs_yaml: Path, target_folder: Path | None = None, + *, ssh_user: str | None = None, ) -> None: """Prepare the input files by downloading them (if necessary). :param inputs_yaml: Path to the inputs.yml file. :param target_folder: Path to the target folder where the input files should be downloaded. - :param ssh_user: Optional SSH user to use for downloading the input files, instead of the current user. + :param ssh_user: SSH user to use for downloading the input files, instead of the current user. """ setup_script_logging() client = Bfabric.from_config() @@ -84,6 +87,7 @@ def register( outputs_yaml: Path, # TODO we should use the workunit definition instead workunit_id: int, + *, ssh_user: str | None = None, # TODO reuse_default_resource: bool = True, @@ -108,6 +112,7 @@ def run( app_spec: Path, workunit_ref: int | Path, target_folder: Path, + *, ssh_user: str | None = None, read_only: bool = False, ) -> None: @@ -131,6 +136,12 @@ def dispatch( workunit_ref: int | Path, work_dir: Path, ) -> None: + """Create chunks, which can be processed individually. + + :param app_spec: Path to the app spec file. + :param workunit_ref: Reference to the workunit (ID or YAML file path). + :param work_dir: Path to the work directory. + """ setup_script_logging() # TODO set workunit to processing? (i.e. add read-only option here) client = Bfabric.from_config() @@ -138,59 +149,85 @@ def dispatch( runner.run_dispatch(workunit_ref=workunit_ref, work_dir=work_dir) -@app_app.command() -def process_all( +@app_chunk.command() +def run_all( app_spec: Path, workunit_ref: int | Path, work_dir: Path, + *, ssh_user: str | None = None, read_only: bool = False, ) -> None: + """Run all chunks, including input preparation, processing, and output registration. + + :param app_spec: Path to the app spec file. + :param workunit_ref: Reference to the workunit (ID or YAML file path). + :param work_dir: Path to the work directory. + :param ssh_user: SSH user to use for downloading the input files, instead of the current user. + :param read_only: If True, results will not be registered and the workunit status will not be changed. + """ setup_script_logging() client = Bfabric.from_config() app_spec_parsed = AppSpec.model_validate(yaml.safe_load(app_spec.read_text())) - runner = Runner(spec=app_spec_parsed, client=client, ssh_user=ssh_user) - chunks_file = ChunksFile.model_validate(yaml.safe_load((work_dir / "chunks.yml").read_text())) - for chunk in chunks_file.chunks: - logger.info(f"Processing chunk {chunk}") - runner.run_prepare_input(chunk_dir=chunk) - runner.run_process(chunk_dir=chunk) - runner.run_collect(workunit_ref=workunit_ref, chunk_dir=chunk) - if not read_only: - runner.run_register_outputs( - chunk_dir=chunk, - workunit_ref=workunit_ref, - reuse_default_resource=app_spec_parsed.reuse_default_resource, - ) + run_app( + app_spec=app_spec_parsed, + workunit_ref=workunit_ref, + work_dir=work_dir, + client=client, + ssh_user=ssh_user, + read_only=read_only, + dispatch_active=False, + ) -@app_app.command() -def chunk_inputs( + +@app_chunk.command() +def inputs( chunk_dir: Path, + *, ssh_user: str | None = None, ) -> None: + """Prepare the input files for a chunk. + + :param chunk_dir: Path to the chunk directory. + :param ssh_user: SSH user to use for downloading the input files, instead of the current user. + """ setup_script_logging() client = Bfabric.from_config() runner = Runner(spec=AppSpec(), client=client, ssh_user=ssh_user) runner.run_prepare_input(chunk_dir=chunk_dir) -@app_app.command() -def chunk_process(chunk_dir: Path) -> None: +@app_chunk.command() +def process(chunk_dir: Path) -> None: + """Process a chunk. + + Note that the input files must be prepared before running this command. + :param chunk_dir: Path to the chunk directory. + """ setup_script_logging() client = Bfabric.from_config() runner = Runner(spec=AppSpec(), client=client, ssh_user=None) runner.run_process(chunk_dir=chunk_dir) -@app_app.command() -def chunk_outputs( +@app_chunk.command() +def outputs( chunk_dir: Path, workunit_ref: int | Path, + *, ssh_user: str | None = None, read_only: bool = False, reuse_default_resource: bool = True, ) -> None: + """Register the output files of a chunk. + + :param chunk_dir: Path to the chunk directory. + :param workunit_ref: Reference to the workunit (ID or YAML file path). + :param ssh_user: SSH user to use for downloading the input files, instead of the current user. + :param read_only: If True, the workunit will not be set to processing. + :param reuse_default_resource: If True, the default resource will be reused for the output files. (recommended) + """ setup_script_logging() client = Bfabric.from_config() runner = Runner(spec=AppSpec(), client=client, ssh_user=ssh_user) From bac89dd75c692450bcd1b2ed110d94d9bc21513b Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Wed, 2 Oct 2024 10:29:52 +0200 Subject: [PATCH 19/42] pass required AppSpec --- src/bfabric/experimental/app_interface/cli.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/bfabric/experimental/app_interface/cli.py b/src/bfabric/experimental/app_interface/cli.py index d5ba48fc..7398da59 100644 --- a/src/bfabric/experimental/app_interface/cli.py +++ b/src/bfabric/experimental/app_interface/cli.py @@ -183,36 +183,44 @@ def run_all( @app_chunk.command() def inputs( + app_spec: Path, chunk_dir: Path, *, ssh_user: str | None = None, ) -> None: """Prepare the input files for a chunk. + :param app_spec: Path to the app spec file. :param chunk_dir: Path to the chunk directory. :param ssh_user: SSH user to use for downloading the input files, instead of the current user. """ setup_script_logging() client = Bfabric.from_config() - runner = Runner(spec=AppSpec(), client=client, ssh_user=ssh_user) + app_spec_parsed = AppSpec.model_validate(yaml.safe_load(app_spec.read_text())) + + runner = Runner(spec=app_spec_parsed, client=client, ssh_user=ssh_user) runner.run_prepare_input(chunk_dir=chunk_dir) @app_chunk.command() -def process(chunk_dir: Path) -> None: +def process(app_spec: Path, chunk_dir: Path) -> None: """Process a chunk. Note that the input files must be prepared before running this command. + :param app_spec: Path to the app spec file. :param chunk_dir: Path to the chunk directory. """ setup_script_logging() client = Bfabric.from_config() - runner = Runner(spec=AppSpec(), client=client, ssh_user=None) + app_spec_parsed = AppSpec.model_validate(yaml.safe_load(app_spec.read_text())) + + runner = Runner(spec=app_spec_parsed, client=client, ssh_user=None) runner.run_process(chunk_dir=chunk_dir) @app_chunk.command() def outputs( + app_spec: Path, chunk_dir: Path, workunit_ref: int | Path, *, @@ -222,6 +230,7 @@ def outputs( ) -> None: """Register the output files of a chunk. + :param app_spec: Path to the app spec file. :param chunk_dir: Path to the chunk directory. :param workunit_ref: Reference to the workunit (ID or YAML file path). :param ssh_user: SSH user to use for downloading the input files, instead of the current user. @@ -230,7 +239,9 @@ def outputs( """ setup_script_logging() client = Bfabric.from_config() - runner = Runner(spec=AppSpec(), client=client, ssh_user=ssh_user) + app_spec_parsed = AppSpec.model_validate(yaml.safe_load(app_spec.read_text())) + + runner = Runner(spec=app_spec_parsed, client=client, ssh_user=ssh_user) runner.run_collect(workunit_ref=workunit_ref, chunk_dir=chunk_dir) if not read_only: runner.run_register_outputs( From 0ec060f44d1b0843f6448b6c2d9f520f6cab001f Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Wed, 2 Oct 2024 10:34:03 +0200 Subject: [PATCH 20/42] more consistent param ordering --- src/bfabric/experimental/app_interface/cli.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/bfabric/experimental/app_interface/cli.py b/src/bfabric/experimental/app_interface/cli.py index 7398da59..810111cd 100644 --- a/src/bfabric/experimental/app_interface/cli.py +++ b/src/bfabric/experimental/app_interface/cli.py @@ -110,13 +110,14 @@ def register( @app_app.command() def run( app_spec: Path, - workunit_ref: int | Path, target_folder: Path, + workunit_ref: int | Path, *, ssh_user: str | None = None, read_only: bool = False, ) -> None: """Runs all stages of an app.""" + # TODO doc setup_script_logging() client = Bfabric.from_config() app_spec_parsed = AppSpec.model_validate(yaml.safe_load(app_spec.read_text())) @@ -133,14 +134,14 @@ def run( @app_app.command() def dispatch( app_spec: Path, - workunit_ref: int | Path, work_dir: Path, + workunit_ref: int | Path, ) -> None: """Create chunks, which can be processed individually. :param app_spec: Path to the app spec file. - :param workunit_ref: Reference to the workunit (ID or YAML file path). :param work_dir: Path to the work directory. + :param workunit_ref: Reference to the workunit (ID or YAML file path). """ setup_script_logging() # TODO set workunit to processing? (i.e. add read-only option here) @@ -152,8 +153,8 @@ def dispatch( @app_chunk.command() def run_all( app_spec: Path, - workunit_ref: int | Path, work_dir: Path, + workunit_ref: int | Path, *, ssh_user: str | None = None, read_only: bool = False, @@ -161,8 +162,8 @@ def run_all( """Run all chunks, including input preparation, processing, and output registration. :param app_spec: Path to the app spec file. - :param workunit_ref: Reference to the workunit (ID or YAML file path). :param work_dir: Path to the work directory. + :param workunit_ref: Reference to the workunit (ID or YAML file path). :param ssh_user: SSH user to use for downloading the input files, instead of the current user. :param read_only: If True, results will not be registered and the workunit status will not be changed. """ From d240be5a0b96eb52c48a7dc61136832a3eba0629 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Wed, 2 Oct 2024 10:40:24 +0200 Subject: [PATCH 21/42] remove version field this will be handled differently e.g. with separate yaml files --- src/bfabric/experimental/app_interface/app_runner/_spec.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/bfabric/experimental/app_interface/app_runner/_spec.py b/src/bfabric/experimental/app_interface/app_runner/_spec.py index 6653f148..f7b5d58e 100644 --- a/src/bfabric/experimental/app_interface/app_runner/_spec.py +++ b/src/bfabric/experimental/app_interface/app_runner/_spec.py @@ -10,7 +10,6 @@ class CommandsSpec(BaseModel): class AppSpec(BaseModel): - version: list[str] = [] commands: CommandsSpec # Note: While we use the old submitter, this is still necessary reuse_default_resource: bool = True From c65ad1fc6a47c155570e3a4d5dc6a05788352ef5 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Wed, 2 Oct 2024 10:54:09 +0200 Subject: [PATCH 22/42] add validation command --- src/bfabric/experimental/app_interface/cli.py | 26 +++++++++++++++++++ .../app_interface/input_preparation/_spec.py | 6 ++--- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/bfabric/experimental/app_interface/cli.py b/src/bfabric/experimental/app_interface/cli.py index 810111cd..95148f3f 100644 --- a/src/bfabric/experimental/app_interface/cli.py +++ b/src/bfabric/experimental/app_interface/cli.py @@ -3,6 +3,8 @@ from pathlib import Path import cyclopts +import rich +import rich.pretty import yaml from bfabric import Bfabric @@ -11,6 +13,7 @@ from bfabric.experimental.app_interface.app_runner._spec import AppSpec from bfabric.experimental.app_interface.app_runner.runner import run_app, Runner from bfabric.experimental.app_interface.input_preparation import prepare_folder +from bfabric.experimental.app_interface.input_preparation._spec import InputsSpec from bfabric.experimental.app_interface.input_preparation.prepare import print_input_files_list from bfabric.experimental.app_interface.output_registration._spec import OutputsSpec from bfabric.experimental.app_interface.output_registration.register import register_all @@ -24,6 +27,8 @@ app.command(app_app) app_chunk = cyclopts.App("chunk", help="Run an app on a chunk. You can create the chunks with `app dispatch`.") app.command(app_chunk) +app_validate = cyclopts.App("validate", help="Validate yaml files.") +app.command(app_validate) @app_inputs.command() @@ -250,5 +255,26 @@ def outputs( ) +@app_validate.command() +def app_spec(yaml_file: Path) -> None: + """Validate an app spec file.""" + app_spec = AppSpec.model_validate(yaml.safe_load(yaml_file.read_text())) + rich.pretty.pprint(app_spec) + + +@app_validate.command() +def inputs_spec(yaml_file: Path) -> None: + """Validate an inputs spec file.""" + inputs_spec = InputsSpec.model_validate(yaml.safe_load(yaml_file.read_text())) + rich.pretty.pprint(inputs_spec) + + +@app_validate.command() +def outputs_spec(yaml_file: Path) -> None: + """Validate an outputs spec file.""" + outputs_spec = OutputsSpec.model_validate(yaml.safe_load(yaml_file.read_text())) + rich.pretty.pprint(outputs_spec) + + if __name__ == "__main__": app() diff --git a/src/bfabric/experimental/app_interface/input_preparation/_spec.py b/src/bfabric/experimental/app_interface/input_preparation/_spec.py index 059bddaf..0a546648 100644 --- a/src/bfabric/experimental/app_interface/input_preparation/_spec.py +++ b/src/bfabric/experimental/app_interface/input_preparation/_spec.py @@ -4,7 +4,7 @@ from typing import Annotated, Literal, Union import yaml -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, Discriminator # ":" are not allowed, as well as absolute paths (starting with "/") RelativeFilePath = Annotated[str, Field(pattern=r"^[^/][^:]*$")] @@ -30,12 +30,12 @@ class DatasetSpec(BaseModel): # invalid_characters: str = "" -InputSpecType = Union[ResourceSpec, DatasetSpec] +InputSpecType = Annotated[Union[ResourceSpec, DatasetSpec], Discriminator("type")] class InputsSpec(BaseModel): model_config = ConfigDict(extra="forbid") - inputs: list[Annotated[InputSpecType, Field(..., discriminator="type")]] + inputs: list[InputSpecType] @classmethod def read_yaml(cls, path: Path) -> list[InputSpecType]: From 179da899b4d77f60b7883cdea99fe3469228ee7f Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Wed, 2 Oct 2024 11:05:17 +0200 Subject: [PATCH 23/42] new app config format that supports docker containers --- .../app_interface/app_runner/_spec.py | 43 +++++++++++++++++-- .../app_interface/app_runner/runner.py | 15 ++----- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/src/bfabric/experimental/app_interface/app_runner/_spec.py b/src/bfabric/experimental/app_interface/app_runner/_spec.py index f7b5d58e..b179089f 100644 --- a/src/bfabric/experimental/app_interface/app_runner/_spec.py +++ b/src/bfabric/experimental/app_interface/app_runner/_spec.py @@ -1,12 +1,47 @@ -from pydantic import BaseModel +import shlex +from typing import Literal, Annotated, Union + +from pydantic import BaseModel, Discriminator + # TODO: This is kept very simple for now, so that it could be easily extended in the future. +class CommandShell(BaseModel): + type: Literal["shell"] = "shell" + command: str + + def to_shell(self) -> list[str]: + return shlex.split(self.command) + + +class CommandDocker(BaseModel): + type: Literal["docker"] = "docker" + image: str + command: str + mount_dirs_readonly: list[tuple[str, str]] = [] + + def to_shell(self) -> list[str]: + return [ + "docker", + "run", + "--rm", + *[ + f"--mount type=bind,source={host},target={container},readonly" + for host, container in self.mount_dirs_readonly + ], + self.image, + *shlex.split(self.command), + ] + + +Command = Annotated[Union[CommandShell, CommandDocker], Discriminator("type")] + + class CommandsSpec(BaseModel): - dispatch: str - process: str - collect: str + dispatch: Command + process: Command + collect: Command class AppSpec(BaseModel): diff --git a/src/bfabric/experimental/app_interface/app_runner/runner.py b/src/bfabric/experimental/app_interface/app_runner/runner.py index 5e4543ef..c9bdfa74 100644 --- a/src/bfabric/experimental/app_interface/app_runner/runner.py +++ b/src/bfabric/experimental/app_interface/app_runner/runner.py @@ -1,6 +1,5 @@ from __future__ import annotations -import shlex import subprocess from pathlib import Path from venv import logger @@ -22,11 +21,7 @@ def __init__(self, spec: AppSpec, client: Bfabric, ssh_user: str | None = None) self._ssh_user = ssh_user def run_dispatch(self, workunit_ref: int | Path, work_dir: Path) -> None: - subprocess.run( - f"{self._app_spec.commands.dispatch} {shlex.quote(str(workunit_ref))} {shlex.quote(str(work_dir))}", - shell=True, - check=True, - ) + subprocess.run([*self._app_spec.commands.dispatch.to_shell(), str(workunit_ref), str(work_dir)], check=True) def run_prepare_input(self, chunk_dir: Path) -> None: prepare_folder( @@ -34,14 +29,10 @@ def run_prepare_input(self, chunk_dir: Path) -> None: ) def run_collect(self, workunit_ref: int | Path, chunk_dir: Path) -> None: - subprocess.run( - f"{self._app_spec.commands.collect} " f"{shlex.quote(str(workunit_ref))} " f"{shlex.quote(str(chunk_dir))}", - shell=True, - check=True, - ) + subprocess.run([*self._app_spec.commands.collect.to_shell(), str(workunit_ref), str(chunk_dir)], check=True) def run_process(self, chunk_dir: Path) -> None: - subprocess.run(f"{self._app_spec.commands.process} {shlex.quote(str(chunk_dir))}", shell=True, check=True) + subprocess.run([*self._app_spec.commands.process.to_shell(), str(chunk_dir)], check=True) def run_register_outputs(self, chunk_dir: Path, workunit_ref: int | Path, reuse_default_resource: bool) -> None: workunit_definition = WorkunitDefinition.from_ref(workunit_ref, client=self._client) From 343691612cf05bab7967b2e07e07ddd397b81580 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Wed, 2 Oct 2024 11:11:28 +0200 Subject: [PATCH 24/42] mount the working directory into the container --- .../experimental/app_interface/app_runner/_spec.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/bfabric/experimental/app_interface/app_runner/_spec.py b/src/bfabric/experimental/app_interface/app_runner/_spec.py index b179089f..593e91e2 100644 --- a/src/bfabric/experimental/app_interface/app_runner/_spec.py +++ b/src/bfabric/experimental/app_interface/app_runner/_spec.py @@ -1,4 +1,6 @@ +from __future__ import annotations import shlex +from pathlib import Path from typing import Literal, Annotated, Union from pydantic import BaseModel, Discriminator @@ -19,15 +21,18 @@ class CommandDocker(BaseModel): type: Literal["docker"] = "docker" image: str command: str + work_dir_mount: str = "/work" mount_dirs_readonly: list[tuple[str, str]] = [] - def to_shell(self) -> list[str]: + def to_shell(self, work_dir: Path | None = None) -> list[str]: + work_dir = (work_dir or Path(".")).absolute() return [ "docker", "run", "--rm", + f"--mount type=bind,source={shlex.quote(str(work_dir))},target={shlex.quote(self.work_dir_mount)}", *[ - f"--mount type=bind,source={host},target={container},readonly" + f"--mount type=bind,source={shlex.quote(host)},target={shlex.quote(container)},readonly" for host, container in self.mount_dirs_readonly ], self.image, From 3799a8280caec6edbf8d8f5974ffc736b95a6e50 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Wed, 2 Oct 2024 11:39:17 +0200 Subject: [PATCH 25/42] correctly implement mounts --- .../experimental/app_interface/app_runner/_spec.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/bfabric/experimental/app_interface/app_runner/_spec.py b/src/bfabric/experimental/app_interface/app_runner/_spec.py index 593e91e2..c3dc6a4e 100644 --- a/src/bfabric/experimental/app_interface/app_runner/_spec.py +++ b/src/bfabric/experimental/app_interface/app_runner/_spec.py @@ -1,4 +1,5 @@ from __future__ import annotations + import shlex from pathlib import Path from typing import Literal, Annotated, Union @@ -26,15 +27,18 @@ class CommandDocker(BaseModel): def to_shell(self, work_dir: Path | None = None) -> list[str]: work_dir = (work_dir or Path(".")).absolute() + mount_args = [] + for host, container in self.mount_dirs_readonly: + mount_args.extend( + ["--mount", f"type=bind,source={shlex.quote(host)},target={shlex.quote(container)},readonly"] + ) return [ "docker", "run", "--rm", - f"--mount type=bind,source={shlex.quote(str(work_dir))},target={shlex.quote(self.work_dir_mount)}", - *[ - f"--mount type=bind,source={shlex.quote(host)},target={shlex.quote(container)},readonly" - for host, container in self.mount_dirs_readonly - ], + "--mount", + f"type=bind,source={shlex.quote(str(work_dir))},target={shlex.quote(self.work_dir_mount)}", + *mount_args, self.image, *shlex.split(self.command), ] From 35b87a65c716aa53e302161dbcef6b3b4c7f61b6 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Wed, 2 Oct 2024 11:40:20 +0200 Subject: [PATCH 26/42] log the commands for diagnostic purposes --- .../app_interface/app_runner/runner.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/bfabric/experimental/app_interface/app_runner/runner.py b/src/bfabric/experimental/app_interface/app_runner/runner.py index c9bdfa74..03f9514a 100644 --- a/src/bfabric/experimental/app_interface/app_runner/runner.py +++ b/src/bfabric/experimental/app_interface/app_runner/runner.py @@ -1,8 +1,9 @@ from __future__ import annotations +import shlex import subprocess from pathlib import Path -from venv import logger +from loguru import logger import yaml from pydantic import BaseModel @@ -21,7 +22,9 @@ def __init__(self, spec: AppSpec, client: Bfabric, ssh_user: str | None = None) self._ssh_user = ssh_user def run_dispatch(self, workunit_ref: int | Path, work_dir: Path) -> None: - subprocess.run([*self._app_spec.commands.dispatch.to_shell(), str(workunit_ref), str(work_dir)], check=True) + command = [*self._app_spec.commands.dispatch.to_shell(), str(workunit_ref), str(work_dir)] + logger.info(f"Running dispatch command: {shlex.join(command)}") + subprocess.run(command, check=True) def run_prepare_input(self, chunk_dir: Path) -> None: prepare_folder( @@ -29,10 +32,14 @@ def run_prepare_input(self, chunk_dir: Path) -> None: ) def run_collect(self, workunit_ref: int | Path, chunk_dir: Path) -> None: - subprocess.run([*self._app_spec.commands.collect.to_shell(), str(workunit_ref), str(chunk_dir)], check=True) + command = [*self._app_spec.commands.collect.to_shell(), str(workunit_ref), str(chunk_dir)] + logger.info(f"Running collect command: {shlex.join(command)}") + subprocess.run(command, check=True) def run_process(self, chunk_dir: Path) -> None: - subprocess.run([*self._app_spec.commands.process.to_shell(), str(chunk_dir)], check=True) + command = [*self._app_spec.commands.process.to_shell(), str(chunk_dir)] + logger.info(f"Running process command: {shlex.join(command)}") + subprocess.run(command, check=True) def run_register_outputs(self, chunk_dir: Path, workunit_ref: int | Path, reuse_default_resource: bool) -> None: workunit_definition = WorkunitDefinition.from_ref(workunit_ref, client=self._client) From f146569d7e27d7e337d825e5b9b8a53a862b1753 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Wed, 2 Oct 2024 12:39:27 +0200 Subject: [PATCH 27/42] improve definition of mounts --- .../app_interface/app_runner/_spec.py | 39 ++++++++++++++----- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/src/bfabric/experimental/app_interface/app_runner/_spec.py b/src/bfabric/experimental/app_interface/app_runner/_spec.py index c3dc6a4e..e8312678 100644 --- a/src/bfabric/experimental/app_interface/app_runner/_spec.py +++ b/src/bfabric/experimental/app_interface/app_runner/_spec.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import shlex from pathlib import Path from typing import Literal, Annotated, Union @@ -18,26 +19,44 @@ def to_shell(self) -> list[str]: return shlex.split(self.command) +class MountOptions(BaseModel): + work_dir_target: Path = "/work" + read_only: list[tuple[Path, Path]] = [] + share_bfabric_config: bool = True + + def collect(self, work_dir: Path): + mounts = [] + if self.share_bfabric_config: + mounts.append((Path("~/.bfabricpy.yml"), Path("/home/user/.bfabricpy.yml"), True)) + mounts.append((work_dir, self.work_dir_target, False)) + for source, target in self.read_only: + mounts.append((source, target, True)) + return [(source.expanduser().absolute(), target, read_only) for source, target, read_only in mounts] + + class CommandDocker(BaseModel): + # TODO not sure if to call this "docker", since "docker-compatible" would be appropriate type: Literal["docker"] = "docker" image: str command: str - work_dir_mount: str = "/work" - mount_dirs_readonly: list[tuple[str, str]] = [] + engine: str = "docker" + mounts: MountOptions = MountOptions() def to_shell(self, work_dir: Path | None = None) -> list[str]: - work_dir = (work_dir or Path(".")).absolute() + work_dir = (work_dir or Path(".")).expanduser().absolute() + mounts = self.mounts.collect(work_dir=work_dir) mount_args = [] - for host, container in self.mount_dirs_readonly: - mount_args.extend( - ["--mount", f"type=bind,source={shlex.quote(host)},target={shlex.quote(container)},readonly"] - ) + for host, container, read_only in mounts: + source = shlex.quote(str(host)) + target = shlex.quote(str(container)) + mount_args.append("--mount") + mount_args.append(f"type=bind,source={source},target={target}" + (",readonly" if read_only else "")) return [ - "docker", + self.engine, "run", + "--user", + f"{os.getuid()}:{os.getgid()}", "--rm", - "--mount", - f"type=bind,source={shlex.quote(str(work_dir))},target={shlex.quote(self.work_dir_mount)}", *mount_args, self.image, *shlex.split(self.command), From ca6bf1c2da94e6e6cd8f878ba942e78dde294d9d Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Wed, 2 Oct 2024 15:15:50 +0200 Subject: [PATCH 28/42] changelog entry --- docs/changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.md b/docs/changelog.md index 41d7740a..d9d9334f 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -13,6 +13,7 @@ Versioning currently follows `X.Y.Z` where ### Added - Entities can be compared and sorted by ID now. +- Caching for list_not_existing_storage_directories. - (experimental) add initial code for a resource based application dispatch - (experimental) new app_runner cli that integrates all commands into a single interface From 6fd956ede3d988e68c6aeafd380b09225b2a35b4 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Wed, 2 Oct 2024 15:23:26 +0200 Subject: [PATCH 29/42] adapt test --- .../scripts/bfabric_list_not_existing_storage_directories.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/bfabric/scripts/bfabric_list_not_existing_storage_directories.py b/src/bfabric/scripts/bfabric_list_not_existing_storage_directories.py index 53179c3f..65d23d2f 100755 --- a/src/bfabric/scripts/bfabric_list_not_existing_storage_directories.py +++ b/src/bfabric/scripts/bfabric_list_not_existing_storage_directories.py @@ -67,7 +67,10 @@ def update(self, client: Bfabric) -> None: max_results = 300 if "BFABRICPY_DEBUG" in os.environ else None logger.debug(f"Checking for new container ids since {timestamp} with limit {max_results}") result = client.read( - "container", {**self._data.query, "createdafter": timestamp}, return_id_only=True, max_results=max_results + endpoint="container", + obj={**self._data.query, "createdafter": timestamp}, + return_id_only=True, + max_results=max_results, ) if len(result): ids = result.to_polars()["id"].unique().to_list() From 67824731afaf804f5905b872fbefc381068c2048 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Wed, 2 Oct 2024 15:24:48 +0200 Subject: [PATCH 30/42] adapt test --- ...st_bfabric_list_not_existing_storage_directories.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/unit/scripts/test_bfabric_list_not_existing_storage_directories.py b/tests/unit/scripts/test_bfabric_list_not_existing_storage_directories.py index ea425b10..5258841f 100644 --- a/tests/unit/scripts/test_bfabric_list_not_existing_storage_directories.py +++ b/tests/unit/scripts/test_bfabric_list_not_existing_storage_directories.py @@ -5,6 +5,7 @@ import pytest from pytest_mock import MockerFixture +from bfabric.results.result_container import ResultContainer from bfabric.scripts.bfabric_list_not_existing_storage_directories import list_not_existing_storage_dirs @@ -18,10 +19,10 @@ def test_list_not_existing_storage_directories( # mock a client client = mocker.MagicMock() - client.read.return_value = [{"id": 3000}, {"id": 3050}, {"id": 3100}, {"id": 3200}, {"id": 3300}] + client.read.return_value = ResultContainer([{"id": 3000}, {"id": 3050}, {"id": 3100}, {"id": 3200}, {"id": 3300}]) # call the function - list_not_existing_storage_dirs(client, tmp_path, [2, 4]) + list_not_existing_storage_dirs(client, tmp_path, cache_path=tmp_path / "cache.json") # check output out, err = capfd.readouterr() @@ -29,7 +30,10 @@ def test_list_not_existing_storage_directories( assert out == "3050\n3300\n" client.read.assert_called_once_with( - endpoint="container", obj={"technologyid": [2, 4]}, return_id_only=True, max_results=500 + endpoint="container", + obj={"technologyid": [2, 4], "createdafter": "2023-12-31T00:00:00"}, + return_id_only=True, + max_results=None, ) From b6d71fe2231110f94b160b17e99652b715ccd51e Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Wed, 2 Oct 2024 16:09:57 +0200 Subject: [PATCH 31/42] print the python version --- docs/changelog.md | 3 ++- src/bfabric/bfabric.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index d9d9334f..95fd0a67 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -13,7 +13,8 @@ Versioning currently follows `X.Y.Z` where ### Added - Entities can be compared and sorted by ID now. -- Caching for list_not_existing_storage_directories. +- Show Python version in version info string. +- Caching for bfabric_list_not_existing_storage_directories.py. - (experimental) add initial code for a resource based application dispatch - (experimental) new app_runner cli that integrates all commands into a single interface diff --git a/src/bfabric/bfabric.py b/src/bfabric/bfabric.py index 4b7f68a2..72693604 100644 --- a/src/bfabric/bfabric.py +++ b/src/bfabric/bfabric.py @@ -15,6 +15,7 @@ import base64 import importlib.metadata +import sys from contextlib import AbstractContextManager from contextlib import contextmanager from datetime import datetime @@ -257,8 +258,9 @@ def _get_version_message(self) -> tuple[str, str]: engine_name = self._engine.__class__.__name__ base_url = self.config.base_url user_name = f"U={self._auth.login if self._auth else None}" + python_version = f"PY={sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" return ( - f"bfabricPy v{package_version} ({engine_name}, {base_url}, {user_name})", + f"bfabricPy v{package_version} ({engine_name}, {base_url}, {user_name}, {python_version})", f"Copyright (C) 2014-{year} Functional Genomics Center Zurich", ) From d70797db9f6337206c07673bb208597a87019383 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Wed, 2 Oct 2024 16:14:29 +0200 Subject: [PATCH 32/42] adapt test --- tests/unit/test_bfabric.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_bfabric.py b/tests/unit/test_bfabric.py index c52d6ff0..48d3d709 100644 --- a/tests/unit/test_bfabric.py +++ b/tests/unit/test_bfabric.py @@ -350,7 +350,7 @@ def test_upload_resource(bfabric_instance, mocker): def test_get_version_message(mock_config, bfabric_instance): mock_config.base_url = "dummy_url" line1, line2 = bfabric_instance._get_version_message() - pattern = r"bfabricPy v\d+\.\d+\.\d+ \(EngineSUDS, dummy_url, U=None\)" + pattern = r"bfabricPy v\d+\.\d+\.\d+ \(EngineSUDS, dummy_url, U=None, PY=\d\.\d+\.\d+\)" assert re.match(pattern, line1) year = datetime.datetime.now().year assert line2 == f"Copyright (C) 2014-{year} Functional Genomics Center Zurich" From 06567976cb2dc71e34cb92c9fbb86ba572a0bc31 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Wed, 2 Oct 2024 17:20:01 +0200 Subject: [PATCH 33/42] split cli.py into indvidual files for each subcommand --- src/bfabric/experimental/app_interface/cli.py | 280 ------------------ .../app_interface/cli/__init__.py | 0 .../app_interface/cli/__main__.py | 19 ++ .../experimental/app_interface/cli/app.py | 56 ++++ .../experimental/app_interface/cli/chunk.py | 113 +++++++ .../experimental/app_interface/cli/inputs.py | 68 +++++ .../experimental/app_interface/cli/outputs.py | 38 +++ .../app_interface/cli/validate.py | 35 +++ 8 files changed, 329 insertions(+), 280 deletions(-) delete mode 100644 src/bfabric/experimental/app_interface/cli.py create mode 100644 src/bfabric/experimental/app_interface/cli/__init__.py create mode 100644 src/bfabric/experimental/app_interface/cli/__main__.py create mode 100644 src/bfabric/experimental/app_interface/cli/app.py create mode 100644 src/bfabric/experimental/app_interface/cli/chunk.py create mode 100644 src/bfabric/experimental/app_interface/cli/inputs.py create mode 100644 src/bfabric/experimental/app_interface/cli/outputs.py create mode 100644 src/bfabric/experimental/app_interface/cli/validate.py diff --git a/src/bfabric/experimental/app_interface/cli.py b/src/bfabric/experimental/app_interface/cli.py deleted file mode 100644 index 95148f3f..00000000 --- a/src/bfabric/experimental/app_interface/cli.py +++ /dev/null @@ -1,280 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import cyclopts -import rich -import rich.pretty -import yaml - -from bfabric import Bfabric -from bfabric.cli_formatting import setup_script_logging -from bfabric.entities import Workunit -from bfabric.experimental.app_interface.app_runner._spec import AppSpec -from bfabric.experimental.app_interface.app_runner.runner import run_app, Runner -from bfabric.experimental.app_interface.input_preparation import prepare_folder -from bfabric.experimental.app_interface.input_preparation._spec import InputsSpec -from bfabric.experimental.app_interface.input_preparation.prepare import print_input_files_list -from bfabric.experimental.app_interface.output_registration._spec import OutputsSpec -from bfabric.experimental.app_interface.output_registration.register import register_all - -app = cyclopts.App() -app_inputs = cyclopts.App("inputs", help="Prepare input files for an app.") -app.command(app_inputs) -app_outputs = cyclopts.App("outputs", help="Register output files for an app.") -app.command(app_outputs) -app_app = cyclopts.App("app", help="Run an app.") -app.command(app_app) -app_chunk = cyclopts.App("chunk", help="Run an app on a chunk. You can create the chunks with `app dispatch`.") -app.command(app_chunk) -app_validate = cyclopts.App("validate", help="Validate yaml files.") -app.command(app_validate) - - -@app_inputs.command() -def prepare( - inputs_yaml: Path, - target_folder: Path | None = None, - *, - ssh_user: str | None = None, -) -> None: - """Prepare the input files by downloading them (if necessary). - - :param inputs_yaml: Path to the inputs.yml file. - :param target_folder: Path to the target folder where the input files should be downloaded. - :param ssh_user: SSH user to use for downloading the input files, instead of the current user. - """ - setup_script_logging() - client = Bfabric.from_config() - prepare_folder( - inputs_yaml=inputs_yaml, - target_folder=target_folder, - ssh_user=ssh_user, - client=client, - action="prepare", - ) - - -@app_inputs.command() -def clean( - inputs_yaml: Path, - target_folder: Path | None = None, -) -> None: - """Removes all local copies of input files. - - :param inputs_yaml: Path to the inputs.yml file. - :param target_folder: Path to the target folder where the input files should be removed. - """ - setup_script_logging() - client = Bfabric.from_config() - # TODO clean shouldn't even need all these arguments, this could be refactored later - prepare_folder( - inputs_yaml=inputs_yaml, - target_folder=target_folder, - ssh_user=None, - action="clean", - client=client, - ) - - -@app_inputs.command() -def list( - inputs_yaml: Path, - target_folder: Path | None = None, -) -> None: - """Lists the input files for an app.""" - setup_script_logging() - print_input_files_list(inputs_yaml=inputs_yaml, target_folder=target_folder) - - -@app_outputs.command() -def register( - outputs_yaml: Path, - # TODO we should use the workunit definition instead - workunit_id: int, - *, - ssh_user: str | None = None, - # TODO - reuse_default_resource: bool = True, -) -> None: - """Register the output files of a workunit.""" - setup_script_logging() - client = Bfabric.from_config() - - specs_list = OutputsSpec.read_yaml(outputs_yaml) - workunit = Workunit.find(id=workunit_id, client=client) - register_all( - client=client, - workunit=workunit, - specs_list=specs_list, - ssh_user=ssh_user, - reuse_default_resource=reuse_default_resource, - ) - - -@app_app.command() -def run( - app_spec: Path, - target_folder: Path, - workunit_ref: int | Path, - *, - ssh_user: str | None = None, - read_only: bool = False, -) -> None: - """Runs all stages of an app.""" - # TODO doc - setup_script_logging() - client = Bfabric.from_config() - app_spec_parsed = AppSpec.model_validate(yaml.safe_load(app_spec.read_text())) - run_app( - app_spec=app_spec_parsed, - workunit_ref=workunit_ref, - work_dir=target_folder, - client=client, - ssh_user=ssh_user, - read_only=read_only, - ) - - -@app_app.command() -def dispatch( - app_spec: Path, - work_dir: Path, - workunit_ref: int | Path, -) -> None: - """Create chunks, which can be processed individually. - - :param app_spec: Path to the app spec file. - :param work_dir: Path to the work directory. - :param workunit_ref: Reference to the workunit (ID or YAML file path). - """ - setup_script_logging() - # TODO set workunit to processing? (i.e. add read-only option here) - client = Bfabric.from_config() - runner = Runner(spec=AppSpec.model_validate(yaml.safe_load(app_spec.read_text())), client=client, ssh_user=None) - runner.run_dispatch(workunit_ref=workunit_ref, work_dir=work_dir) - - -@app_chunk.command() -def run_all( - app_spec: Path, - work_dir: Path, - workunit_ref: int | Path, - *, - ssh_user: str | None = None, - read_only: bool = False, -) -> None: - """Run all chunks, including input preparation, processing, and output registration. - - :param app_spec: Path to the app spec file. - :param work_dir: Path to the work directory. - :param workunit_ref: Reference to the workunit (ID or YAML file path). - :param ssh_user: SSH user to use for downloading the input files, instead of the current user. - :param read_only: If True, results will not be registered and the workunit status will not be changed. - """ - setup_script_logging() - client = Bfabric.from_config() - app_spec_parsed = AppSpec.model_validate(yaml.safe_load(app_spec.read_text())) - - run_app( - app_spec=app_spec_parsed, - workunit_ref=workunit_ref, - work_dir=work_dir, - client=client, - ssh_user=ssh_user, - read_only=read_only, - dispatch_active=False, - ) - - -@app_chunk.command() -def inputs( - app_spec: Path, - chunk_dir: Path, - *, - ssh_user: str | None = None, -) -> None: - """Prepare the input files for a chunk. - - :param app_spec: Path to the app spec file. - :param chunk_dir: Path to the chunk directory. - :param ssh_user: SSH user to use for downloading the input files, instead of the current user. - """ - setup_script_logging() - client = Bfabric.from_config() - app_spec_parsed = AppSpec.model_validate(yaml.safe_load(app_spec.read_text())) - - runner = Runner(spec=app_spec_parsed, client=client, ssh_user=ssh_user) - runner.run_prepare_input(chunk_dir=chunk_dir) - - -@app_chunk.command() -def process(app_spec: Path, chunk_dir: Path) -> None: - """Process a chunk. - - Note that the input files must be prepared before running this command. - :param app_spec: Path to the app spec file. - :param chunk_dir: Path to the chunk directory. - """ - setup_script_logging() - client = Bfabric.from_config() - app_spec_parsed = AppSpec.model_validate(yaml.safe_load(app_spec.read_text())) - - runner = Runner(spec=app_spec_parsed, client=client, ssh_user=None) - runner.run_process(chunk_dir=chunk_dir) - - -@app_chunk.command() -def outputs( - app_spec: Path, - chunk_dir: Path, - workunit_ref: int | Path, - *, - ssh_user: str | None = None, - read_only: bool = False, - reuse_default_resource: bool = True, -) -> None: - """Register the output files of a chunk. - - :param app_spec: Path to the app spec file. - :param chunk_dir: Path to the chunk directory. - :param workunit_ref: Reference to the workunit (ID or YAML file path). - :param ssh_user: SSH user to use for downloading the input files, instead of the current user. - :param read_only: If True, the workunit will not be set to processing. - :param reuse_default_resource: If True, the default resource will be reused for the output files. (recommended) - """ - setup_script_logging() - client = Bfabric.from_config() - app_spec_parsed = AppSpec.model_validate(yaml.safe_load(app_spec.read_text())) - - runner = Runner(spec=app_spec_parsed, client=client, ssh_user=ssh_user) - runner.run_collect(workunit_ref=workunit_ref, chunk_dir=chunk_dir) - if not read_only: - runner.run_register_outputs( - chunk_dir=chunk_dir, workunit_ref=workunit_ref, reuse_default_resource=reuse_default_resource - ) - - -@app_validate.command() -def app_spec(yaml_file: Path) -> None: - """Validate an app spec file.""" - app_spec = AppSpec.model_validate(yaml.safe_load(yaml_file.read_text())) - rich.pretty.pprint(app_spec) - - -@app_validate.command() -def inputs_spec(yaml_file: Path) -> None: - """Validate an inputs spec file.""" - inputs_spec = InputsSpec.model_validate(yaml.safe_load(yaml_file.read_text())) - rich.pretty.pprint(inputs_spec) - - -@app_validate.command() -def outputs_spec(yaml_file: Path) -> None: - """Validate an outputs spec file.""" - outputs_spec = OutputsSpec.model_validate(yaml.safe_load(yaml_file.read_text())) - rich.pretty.pprint(outputs_spec) - - -if __name__ == "__main__": - app() diff --git a/src/bfabric/experimental/app_interface/cli/__init__.py b/src/bfabric/experimental/app_interface/cli/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/bfabric/experimental/app_interface/cli/__main__.py b/src/bfabric/experimental/app_interface/cli/__main__.py new file mode 100644 index 00000000..435743e4 --- /dev/null +++ b/src/bfabric/experimental/app_interface/cli/__main__.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +import cyclopts + +from bfabric.experimental.app_interface.cli.app import app_app +from bfabric.experimental.app_interface.cli.chunk import app_chunk +from bfabric.experimental.app_interface.cli.inputs import app_inputs +from bfabric.experimental.app_interface.cli.outputs import app_outputs +from bfabric.experimental.app_interface.cli.validate import app_validate + +app = cyclopts.App() +app.command(app_inputs) +app.command(app_outputs) +app.command(app_app) +app.command(app_chunk) +app.command(app_validate) + +if __name__ == "__main__": + app() diff --git a/src/bfabric/experimental/app_interface/cli/app.py b/src/bfabric/experimental/app_interface/cli/app.py new file mode 100644 index 00000000..d81d953c --- /dev/null +++ b/src/bfabric/experimental/app_interface/cli/app.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from pathlib import Path + +import cyclopts +import yaml + +from bfabric import Bfabric +from bfabric.cli_formatting import setup_script_logging +from bfabric.experimental.app_interface.app_runner._spec import AppSpec +from bfabric.experimental.app_interface.app_runner.runner import run_app, Runner + +app_app = cyclopts.App("app", help="Run an app.") + + +@app_app.command() +def run( + app_spec: Path, + target_folder: Path, + workunit_ref: int | Path, + *, + ssh_user: str | None = None, + read_only: bool = False, +) -> None: + """Runs all stages of an app.""" + # TODO doc + setup_script_logging() + client = Bfabric.from_config() + app_spec_parsed = AppSpec.model_validate(yaml.safe_load(app_spec.read_text())) + run_app( + app_spec=app_spec_parsed, + workunit_ref=workunit_ref, + work_dir=target_folder, + client=client, + ssh_user=ssh_user, + read_only=read_only, + ) + + +@app_app.command() +def dispatch( + app_spec: Path, + work_dir: Path, + workunit_ref: int | Path, +) -> None: + """Create chunks, which can be processed individually. + + :param app_spec: Path to the app spec file. + :param work_dir: Path to the work directory. + :param workunit_ref: Reference to the workunit (ID or YAML file path). + """ + setup_script_logging() + # TODO set workunit to processing? (i.e. add read-only option here) + client = Bfabric.from_config() + runner = Runner(spec=AppSpec.model_validate(yaml.safe_load(app_spec.read_text())), client=client, ssh_user=None) + runner.run_dispatch(workunit_ref=workunit_ref, work_dir=work_dir) diff --git a/src/bfabric/experimental/app_interface/cli/chunk.py b/src/bfabric/experimental/app_interface/cli/chunk.py new file mode 100644 index 00000000..5392dd97 --- /dev/null +++ b/src/bfabric/experimental/app_interface/cli/chunk.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +from pathlib import Path + +import cyclopts +import yaml + +from bfabric import Bfabric +from bfabric.cli_formatting import setup_script_logging +from bfabric.experimental.app_interface.app_runner._spec import AppSpec +from bfabric.experimental.app_interface.app_runner.runner import run_app, Runner + +app_chunk = cyclopts.App("chunk", help="Run an app on a chunk. You can create the chunks with `app dispatch`.") + + +@app_chunk.command() +def run_all( + app_spec: Path, + work_dir: Path, + workunit_ref: int | Path, + *, + ssh_user: str | None = None, + read_only: bool = False, +) -> None: + """Run all chunks, including input preparation, processing, and output registration. + + :param app_spec: Path to the app spec file. + :param work_dir: Path to the work directory. + :param workunit_ref: Reference to the workunit (ID or YAML file path). + :param ssh_user: SSH user to use for downloading the input files, instead of the current user. + :param read_only: If True, results will not be registered and the workunit status will not be changed. + """ + setup_script_logging() + client = Bfabric.from_config() + app_spec_parsed = AppSpec.model_validate(yaml.safe_load(app_spec.read_text())) + + run_app( + app_spec=app_spec_parsed, + workunit_ref=workunit_ref, + work_dir=work_dir, + client=client, + ssh_user=ssh_user, + read_only=read_only, + dispatch_active=False, + ) + + +@app_chunk.command() +def inputs( + app_spec: Path, + chunk_dir: Path, + *, + ssh_user: str | None = None, +) -> None: + """Prepare the input files for a chunk. + + :param app_spec: Path to the app spec file. + :param chunk_dir: Path to the chunk directory. + :param ssh_user: SSH user to use for downloading the input files, instead of the current user. + """ + setup_script_logging() + client = Bfabric.from_config() + app_spec_parsed = AppSpec.model_validate(yaml.safe_load(app_spec.read_text())) + + runner = Runner(spec=app_spec_parsed, client=client, ssh_user=ssh_user) + runner.run_prepare_input(chunk_dir=chunk_dir) + + +@app_chunk.command() +def process(app_spec: Path, chunk_dir: Path) -> None: + """Process a chunk. + + Note that the input files must be prepared before running this command. + :param app_spec: Path to the app spec file. + :param chunk_dir: Path to the chunk directory. + """ + setup_script_logging() + client = Bfabric.from_config() + app_spec_parsed = AppSpec.model_validate(yaml.safe_load(app_spec.read_text())) + + runner = Runner(spec=app_spec_parsed, client=client, ssh_user=None) + runner.run_process(chunk_dir=chunk_dir) + + +@app_chunk.command() +def outputs( + app_spec: Path, + chunk_dir: Path, + workunit_ref: int | Path, + *, + ssh_user: str | None = None, + read_only: bool = False, + reuse_default_resource: bool = True, +) -> None: + """Register the output files of a chunk. + + :param app_spec: Path to the app spec file. + :param chunk_dir: Path to the chunk directory. + :param workunit_ref: Reference to the workunit (ID or YAML file path). + :param ssh_user: SSH user to use for downloading the input files, instead of the current user. + :param read_only: If True, the workunit will not be set to processing. + :param reuse_default_resource: If True, the default resource will be reused for the output files. (recommended) + """ + setup_script_logging() + client = Bfabric.from_config() + app_spec_parsed = AppSpec.model_validate(yaml.safe_load(app_spec.read_text())) + + runner = Runner(spec=app_spec_parsed, client=client, ssh_user=ssh_user) + runner.run_collect(workunit_ref=workunit_ref, chunk_dir=chunk_dir) + if not read_only: + runner.run_register_outputs( + chunk_dir=chunk_dir, workunit_ref=workunit_ref, reuse_default_resource=reuse_default_resource + ) diff --git a/src/bfabric/experimental/app_interface/cli/inputs.py b/src/bfabric/experimental/app_interface/cli/inputs.py new file mode 100644 index 00000000..83c37851 --- /dev/null +++ b/src/bfabric/experimental/app_interface/cli/inputs.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +from pathlib import Path + +import cyclopts + +from bfabric import Bfabric +from bfabric.cli_formatting import setup_script_logging +from bfabric.experimental.app_interface.input_preparation import prepare_folder +from bfabric.experimental.app_interface.input_preparation.prepare import print_input_files_list + +app_inputs = cyclopts.App("inputs", help="Prepare input files for an app.") + + +@app_inputs.command() +def prepare( + inputs_yaml: Path, + target_folder: Path | None = None, + *, + ssh_user: str | None = None, +) -> None: + """Prepare the input files by downloading them (if necessary). + + :param inputs_yaml: Path to the inputs.yml file. + :param target_folder: Path to the target folder where the input files should be downloaded. + :param ssh_user: SSH user to use for downloading the input files, instead of the current user. + """ + setup_script_logging() + client = Bfabric.from_config() + prepare_folder( + inputs_yaml=inputs_yaml, + target_folder=target_folder, + ssh_user=ssh_user, + client=client, + action="prepare", + ) + + +@app_inputs.command() +def clean( + inputs_yaml: Path, + target_folder: Path | None = None, +) -> None: + """Removes all local copies of input files. + + :param inputs_yaml: Path to the inputs.yml file. + :param target_folder: Path to the target folder where the input files should be removed. + """ + setup_script_logging() + client = Bfabric.from_config() + # TODO clean shouldn't even need all these arguments, this could be refactored later + prepare_folder( + inputs_yaml=inputs_yaml, + target_folder=target_folder, + ssh_user=None, + action="clean", + client=client, + ) + + +@app_inputs.command() +def list( + inputs_yaml: Path, + target_folder: Path | None = None, +) -> None: + """Lists the input files for an app.""" + setup_script_logging() + print_input_files_list(inputs_yaml=inputs_yaml, target_folder=target_folder) diff --git a/src/bfabric/experimental/app_interface/cli/outputs.py b/src/bfabric/experimental/app_interface/cli/outputs.py new file mode 100644 index 00000000..2a656d54 --- /dev/null +++ b/src/bfabric/experimental/app_interface/cli/outputs.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from pathlib import Path + +import cyclopts + +from bfabric import Bfabric +from bfabric.cli_formatting import setup_script_logging +from bfabric.entities import Workunit +from bfabric.experimental.app_interface.output_registration._spec import OutputsSpec +from bfabric.experimental.app_interface.output_registration.register import register_all + +app_outputs = cyclopts.App("outputs", help="Register output files for an app.") + + +@app_outputs.command() +def register( + outputs_yaml: Path, + # TODO we should use the workunit definition instead + workunit_id: int, + *, + ssh_user: str | None = None, + # TODO + reuse_default_resource: bool = True, +) -> None: + """Register the output files of a workunit.""" + setup_script_logging() + client = Bfabric.from_config() + + specs_list = OutputsSpec.read_yaml(outputs_yaml) + workunit = Workunit.find(id=workunit_id, client=client) + register_all( + client=client, + workunit=workunit, + specs_list=specs_list, + ssh_user=ssh_user, + reuse_default_resource=reuse_default_resource, + ) diff --git a/src/bfabric/experimental/app_interface/cli/validate.py b/src/bfabric/experimental/app_interface/cli/validate.py new file mode 100644 index 00000000..ecb6e527 --- /dev/null +++ b/src/bfabric/experimental/app_interface/cli/validate.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from pathlib import Path + +import cyclopts +import rich +import rich.pretty +import yaml + +from bfabric.experimental.app_interface.app_runner._spec import AppSpec +from bfabric.experimental.app_interface.input_preparation._spec import InputsSpec +from bfabric.experimental.app_interface.output_registration._spec import OutputsSpec + +app_validate = cyclopts.App("validate", help="Validate yaml files.") + + +@app_validate.command() +def app_spec(yaml_file: Path) -> None: + """Validate an app spec file.""" + app_spec = AppSpec.model_validate(yaml.safe_load(yaml_file.read_text())) + rich.pretty.pprint(app_spec) + + +@app_validate.command() +def inputs_spec(yaml_file: Path) -> None: + """Validate an inputs spec file.""" + inputs_spec = InputsSpec.model_validate(yaml.safe_load(yaml_file.read_text())) + rich.pretty.pprint(inputs_spec) + + +@app_validate.command() +def outputs_spec(yaml_file: Path) -> None: + """Validate an outputs spec file.""" + outputs_spec = OutputsSpec.model_validate(yaml.safe_load(yaml_file.read_text())) + rich.pretty.pprint(outputs_spec) From 6479df11100437174a28c6a5c53bbdb57cdbb337 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Thu, 3 Oct 2024 10:49:14 +0200 Subject: [PATCH 34/42] move scripts into bfabric_scripts namespace package --- pyproject.toml | 50 +++++++++---------- .../scripts => bfabric_scripts}/__init__.py | 0 .../bfabric_delete.py | 0 ...ric_executable_submitter_functionalTest.py | 0 ...bfabric_executable_submitter_gridengine.py | 0 .../bfabric_executable_submitter_slurm.py | 0 .../bfabric_executable_wrappercreator.py | 0 .../bfabric_feeder_mascot.py | 0 .../bfabric_feeder_resource_autoQC.py | 0 .../bfabric_flask.py | 0 ...list_not_available_proteomics_workunits.py | 0 ...c_list_not_existing_storage_directories.py | 0 .../bfabric_list_workunit_parameters.py | 0 .../bfabric_logthis.py | 0 .../bfabric_read.py | 0 .../bfabric_read_samples_from_dataset.py | 0 .../bfabric_read_samples_of_workunit.py | 0 .../bfabric_save_csv2dataset.py | 0 .../bfabric_save_dataset2csv.py | 0 .../bfabric_save_fasta.py | 0 .../bfabric_save_importresource_sample.py | 0 .../bfabric_save_link_to_workunit.py | 0 .../bfabric_save_workflowstep.py | 0 .../bfabric_save_workunit_attribute.py | 0 .../bfabric_setExternalJobStatus_done.py | 0 .../bfabric_setResourceStatus_available.py | 0 .../bfabric_setWorkunitStatus.py | 0 .../bfabric_slurm_queue_status.py | 0 .../bfabric_upload_resource.py | 0 .../bfabric_upload_submitter_executable.py | 0 .../bfabric_wrapper_creator_yaml.py | 0 .../fgcz_maxquant_scaffold-wrapper.py | 0 .../fgcz_maxquant_wrapper.py | 0 .../fgcz_sge_maxquant_linux.bash | 0 34 files changed, 25 insertions(+), 25 deletions(-) rename src/{bfabric/scripts => bfabric_scripts}/__init__.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_delete.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_executable_submitter_functionalTest.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_executable_submitter_gridengine.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_executable_submitter_slurm.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_executable_wrappercreator.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_feeder_mascot.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_feeder_resource_autoQC.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_flask.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_list_not_available_proteomics_workunits.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_list_not_existing_storage_directories.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_list_workunit_parameters.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_logthis.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_read.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_read_samples_from_dataset.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_read_samples_of_workunit.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_save_csv2dataset.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_save_dataset2csv.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_save_fasta.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_save_importresource_sample.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_save_link_to_workunit.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_save_workflowstep.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_save_workunit_attribute.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_setExternalJobStatus_done.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_setResourceStatus_available.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_setWorkunitStatus.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_slurm_queue_status.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_upload_resource.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_upload_submitter_executable.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/bfabric_wrapper_creator_yaml.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/fgcz_maxquant_scaffold-wrapper.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/fgcz_maxquant_wrapper.py (100%) rename src/{bfabric/scripts => bfabric_scripts}/fgcz_sge_maxquant_linux.bash (100%) diff --git a/pyproject.toml b/pyproject.toml index 747f1280..3e802008 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,31 +53,31 @@ Homepage = "https://github.com/fgcz/bfabricPy" Repository = "https://github.com/fgcz/bfabricPy" [project.scripts] -"bfabric_flask.py"="bfabric.scripts.bfabric_flask:main" -#bfabric_feeder_resource_autoQC="bfabric.scripts.bfabric_feeder_resource_autoQC:main" -"bfabric_list_not_existing_storage_directories.py"="bfabric.scripts.bfabric_list_not_existing_storage_directories:main" -"bfabric_list_not_available_proteomics_workunits.py"="bfabric.scripts.bfabric_list_not_available_proteomics_workunits:main" -"bfabric_list_workunit_parameters.py"="bfabric.scripts.bfabric_list_workunit_parameters:main" -"bfabric_upload_resource.py"="bfabric.scripts.bfabric_upload_resource:main" -"bfabric_logthis.py"="bfabric.scripts.bfabric_logthis:main" -"bfabric_setResourceStatus_available.py"="bfabric.scripts.bfabric_setResourceStatus_available:main" -"bfabric_setExternalJobStatus_done.py"="bfabric.scripts.bfabric_setExternalJobStatus_done:main" -"bfabric_setWorkunitStatus_available.py"="bfabric.scripts.bfabric_setWorkunitStatus:main_available" -"bfabric_setWorkunitStatus_processing.py"="bfabric.scripts.bfabric_setWorkunitStatus:main_processing" -"bfabric_setWorkunitStatus_failed.py"="bfabric.scripts.bfabric_setWorkunitStatus:main_failed" -"bfabric_delete.py"="bfabric.scripts.bfabric_delete:main" -"bfabric_read.py"="bfabric.scripts.bfabric_read:main" -"bfabric_read_samples_of_workunit.py"="bfabric.scripts.bfabric_read_samples_of_workunit:main" -"bfabric_read_samples_from_dataset.py"="bfabric.scripts.bfabric_read_samples_from_dataset:main" -"bfabric_save_csv2dataset.py"="bfabric.scripts.bfabric_save_csv2dataset:main" -"bfabric_save_dataset2csv.py"="bfabric.scripts.bfabric_save_dataset2csv:main" -"bfabric_save_fasta.py"="bfabric.scripts.bfabric_save_fasta:main" -"bfabric_save_importresource_sample.py"="bfabric.scripts.bfabric_save_importresource_sample:main" -"bfabric_save_link_to_workunit.py"="bfabric.scripts.bfabric_save_link_to_workunit:main" -#bfabric_save_resource="bfabric.scripts.bfabric_save_resource:main" -"bfabric_save_workunit_attribute.py"="bfabric.scripts.bfabric_save_workunit_attribute:main" -"bfabric_save_workflowstep.py"="bfabric.scripts.bfabric_save_workflowstep:main" -"bfabric_slurm_queue_status.py"="bfabric.scripts.bfabric_slurm_queue_status:main" +"bfabric_flask.py"="bfabric_scripts.bfabric_flask:main" +#bfabric_feeder_resource_autoQC="bfabric_scripts.bfabric_feeder_resource_autoQC:main" +"bfabric_list_not_existing_storage_directories.py"="bfabric_scripts.bfabric_list_not_existing_storage_directories:main" +"bfabric_list_not_available_proteomics_workunits.py"="bfabric_scripts.bfabric_list_not_available_proteomics_workunits:main" +"bfabric_list_workunit_parameters.py"="bfabric_scripts.bfabric_list_workunit_parameters:main" +"bfabric_upload_resource.py"="bfabric_scripts.bfabric_upload_resource:main" +"bfabric_logthis.py"="bfabric_scripts.bfabric_logthis:main" +"bfabric_setResourceStatus_available.py"="bfabric_scripts.bfabric_setResourceStatus_available:main" +"bfabric_setExternalJobStatus_done.py"="bfabric_scripts.bfabric_setExternalJobStatus_done:main" +"bfabric_setWorkunitStatus_available.py"="bfabric_scripts.bfabric_setWorkunitStatus:main_available" +"bfabric_setWorkunitStatus_processing.py"="bfabric_scripts.bfabric_setWorkunitStatus:main_processing" +"bfabric_setWorkunitStatus_failed.py"="bfabric_scripts.bfabric_setWorkunitStatus:main_failed" +"bfabric_delete.py"="bfabric_scripts.bfabric_delete:main" +"bfabric_read.py"="bfabric_scripts.bfabric_read:main" +"bfabric_read_samples_of_workunit.py"="bfabric_scripts.bfabric_read_samples_of_workunit:main" +"bfabric_read_samples_from_dataset.py"="bfabric_scripts.bfabric_read_samples_from_dataset:main" +"bfabric_save_csv2dataset.py"="bfabric_scripts.bfabric_save_csv2dataset:main" +"bfabric_save_dataset2csv.py"="bfabric_scripts.bfabric_save_dataset2csv:main" +"bfabric_save_fasta.py"="bfabric_scripts.bfabric_save_fasta:main" +"bfabric_save_importresource_sample.py"="bfabric_scripts.bfabric_save_importresource_sample:main" +"bfabric_save_link_to_workunit.py"="bfabric_scripts.bfabric_save_link_to_workunit:main" +#bfabric_save_resource="bfabric_scripts.bfabric_save_resource:main" +"bfabric_save_workunit_attribute.py"="bfabric_scripts.bfabric_save_workunit_attribute:main" +"bfabric_save_workflowstep.py"="bfabric_scripts.bfabric_save_workflowstep:main" +"bfabric_slurm_queue_status.py"="bfabric_scripts.bfabric_slurm_queue_status:main" [tool.black] line-length = 120 diff --git a/src/bfabric/scripts/__init__.py b/src/bfabric_scripts/__init__.py similarity index 100% rename from src/bfabric/scripts/__init__.py rename to src/bfabric_scripts/__init__.py diff --git a/src/bfabric/scripts/bfabric_delete.py b/src/bfabric_scripts/bfabric_delete.py similarity index 100% rename from src/bfabric/scripts/bfabric_delete.py rename to src/bfabric_scripts/bfabric_delete.py diff --git a/src/bfabric/scripts/bfabric_executable_submitter_functionalTest.py b/src/bfabric_scripts/bfabric_executable_submitter_functionalTest.py similarity index 100% rename from src/bfabric/scripts/bfabric_executable_submitter_functionalTest.py rename to src/bfabric_scripts/bfabric_executable_submitter_functionalTest.py diff --git a/src/bfabric/scripts/bfabric_executable_submitter_gridengine.py b/src/bfabric_scripts/bfabric_executable_submitter_gridengine.py similarity index 100% rename from src/bfabric/scripts/bfabric_executable_submitter_gridengine.py rename to src/bfabric_scripts/bfabric_executable_submitter_gridengine.py diff --git a/src/bfabric/scripts/bfabric_executable_submitter_slurm.py b/src/bfabric_scripts/bfabric_executable_submitter_slurm.py similarity index 100% rename from src/bfabric/scripts/bfabric_executable_submitter_slurm.py rename to src/bfabric_scripts/bfabric_executable_submitter_slurm.py diff --git a/src/bfabric/scripts/bfabric_executable_wrappercreator.py b/src/bfabric_scripts/bfabric_executable_wrappercreator.py similarity index 100% rename from src/bfabric/scripts/bfabric_executable_wrappercreator.py rename to src/bfabric_scripts/bfabric_executable_wrappercreator.py diff --git a/src/bfabric/scripts/bfabric_feeder_mascot.py b/src/bfabric_scripts/bfabric_feeder_mascot.py similarity index 100% rename from src/bfabric/scripts/bfabric_feeder_mascot.py rename to src/bfabric_scripts/bfabric_feeder_mascot.py diff --git a/src/bfabric/scripts/bfabric_feeder_resource_autoQC.py b/src/bfabric_scripts/bfabric_feeder_resource_autoQC.py similarity index 100% rename from src/bfabric/scripts/bfabric_feeder_resource_autoQC.py rename to src/bfabric_scripts/bfabric_feeder_resource_autoQC.py diff --git a/src/bfabric/scripts/bfabric_flask.py b/src/bfabric_scripts/bfabric_flask.py similarity index 100% rename from src/bfabric/scripts/bfabric_flask.py rename to src/bfabric_scripts/bfabric_flask.py diff --git a/src/bfabric/scripts/bfabric_list_not_available_proteomics_workunits.py b/src/bfabric_scripts/bfabric_list_not_available_proteomics_workunits.py similarity index 100% rename from src/bfabric/scripts/bfabric_list_not_available_proteomics_workunits.py rename to src/bfabric_scripts/bfabric_list_not_available_proteomics_workunits.py diff --git a/src/bfabric/scripts/bfabric_list_not_existing_storage_directories.py b/src/bfabric_scripts/bfabric_list_not_existing_storage_directories.py similarity index 100% rename from src/bfabric/scripts/bfabric_list_not_existing_storage_directories.py rename to src/bfabric_scripts/bfabric_list_not_existing_storage_directories.py diff --git a/src/bfabric/scripts/bfabric_list_workunit_parameters.py b/src/bfabric_scripts/bfabric_list_workunit_parameters.py similarity index 100% rename from src/bfabric/scripts/bfabric_list_workunit_parameters.py rename to src/bfabric_scripts/bfabric_list_workunit_parameters.py diff --git a/src/bfabric/scripts/bfabric_logthis.py b/src/bfabric_scripts/bfabric_logthis.py similarity index 100% rename from src/bfabric/scripts/bfabric_logthis.py rename to src/bfabric_scripts/bfabric_logthis.py diff --git a/src/bfabric/scripts/bfabric_read.py b/src/bfabric_scripts/bfabric_read.py similarity index 100% rename from src/bfabric/scripts/bfabric_read.py rename to src/bfabric_scripts/bfabric_read.py diff --git a/src/bfabric/scripts/bfabric_read_samples_from_dataset.py b/src/bfabric_scripts/bfabric_read_samples_from_dataset.py similarity index 100% rename from src/bfabric/scripts/bfabric_read_samples_from_dataset.py rename to src/bfabric_scripts/bfabric_read_samples_from_dataset.py diff --git a/src/bfabric/scripts/bfabric_read_samples_of_workunit.py b/src/bfabric_scripts/bfabric_read_samples_of_workunit.py similarity index 100% rename from src/bfabric/scripts/bfabric_read_samples_of_workunit.py rename to src/bfabric_scripts/bfabric_read_samples_of_workunit.py diff --git a/src/bfabric/scripts/bfabric_save_csv2dataset.py b/src/bfabric_scripts/bfabric_save_csv2dataset.py similarity index 100% rename from src/bfabric/scripts/bfabric_save_csv2dataset.py rename to src/bfabric_scripts/bfabric_save_csv2dataset.py diff --git a/src/bfabric/scripts/bfabric_save_dataset2csv.py b/src/bfabric_scripts/bfabric_save_dataset2csv.py similarity index 100% rename from src/bfabric/scripts/bfabric_save_dataset2csv.py rename to src/bfabric_scripts/bfabric_save_dataset2csv.py diff --git a/src/bfabric/scripts/bfabric_save_fasta.py b/src/bfabric_scripts/bfabric_save_fasta.py similarity index 100% rename from src/bfabric/scripts/bfabric_save_fasta.py rename to src/bfabric_scripts/bfabric_save_fasta.py diff --git a/src/bfabric/scripts/bfabric_save_importresource_sample.py b/src/bfabric_scripts/bfabric_save_importresource_sample.py similarity index 100% rename from src/bfabric/scripts/bfabric_save_importresource_sample.py rename to src/bfabric_scripts/bfabric_save_importresource_sample.py diff --git a/src/bfabric/scripts/bfabric_save_link_to_workunit.py b/src/bfabric_scripts/bfabric_save_link_to_workunit.py similarity index 100% rename from src/bfabric/scripts/bfabric_save_link_to_workunit.py rename to src/bfabric_scripts/bfabric_save_link_to_workunit.py diff --git a/src/bfabric/scripts/bfabric_save_workflowstep.py b/src/bfabric_scripts/bfabric_save_workflowstep.py similarity index 100% rename from src/bfabric/scripts/bfabric_save_workflowstep.py rename to src/bfabric_scripts/bfabric_save_workflowstep.py diff --git a/src/bfabric/scripts/bfabric_save_workunit_attribute.py b/src/bfabric_scripts/bfabric_save_workunit_attribute.py similarity index 100% rename from src/bfabric/scripts/bfabric_save_workunit_attribute.py rename to src/bfabric_scripts/bfabric_save_workunit_attribute.py diff --git a/src/bfabric/scripts/bfabric_setExternalJobStatus_done.py b/src/bfabric_scripts/bfabric_setExternalJobStatus_done.py similarity index 100% rename from src/bfabric/scripts/bfabric_setExternalJobStatus_done.py rename to src/bfabric_scripts/bfabric_setExternalJobStatus_done.py diff --git a/src/bfabric/scripts/bfabric_setResourceStatus_available.py b/src/bfabric_scripts/bfabric_setResourceStatus_available.py similarity index 100% rename from src/bfabric/scripts/bfabric_setResourceStatus_available.py rename to src/bfabric_scripts/bfabric_setResourceStatus_available.py diff --git a/src/bfabric/scripts/bfabric_setWorkunitStatus.py b/src/bfabric_scripts/bfabric_setWorkunitStatus.py similarity index 100% rename from src/bfabric/scripts/bfabric_setWorkunitStatus.py rename to src/bfabric_scripts/bfabric_setWorkunitStatus.py diff --git a/src/bfabric/scripts/bfabric_slurm_queue_status.py b/src/bfabric_scripts/bfabric_slurm_queue_status.py similarity index 100% rename from src/bfabric/scripts/bfabric_slurm_queue_status.py rename to src/bfabric_scripts/bfabric_slurm_queue_status.py diff --git a/src/bfabric/scripts/bfabric_upload_resource.py b/src/bfabric_scripts/bfabric_upload_resource.py similarity index 100% rename from src/bfabric/scripts/bfabric_upload_resource.py rename to src/bfabric_scripts/bfabric_upload_resource.py diff --git a/src/bfabric/scripts/bfabric_upload_submitter_executable.py b/src/bfabric_scripts/bfabric_upload_submitter_executable.py similarity index 100% rename from src/bfabric/scripts/bfabric_upload_submitter_executable.py rename to src/bfabric_scripts/bfabric_upload_submitter_executable.py diff --git a/src/bfabric/scripts/bfabric_wrapper_creator_yaml.py b/src/bfabric_scripts/bfabric_wrapper_creator_yaml.py similarity index 100% rename from src/bfabric/scripts/bfabric_wrapper_creator_yaml.py rename to src/bfabric_scripts/bfabric_wrapper_creator_yaml.py diff --git a/src/bfabric/scripts/fgcz_maxquant_scaffold-wrapper.py b/src/bfabric_scripts/fgcz_maxquant_scaffold-wrapper.py similarity index 100% rename from src/bfabric/scripts/fgcz_maxquant_scaffold-wrapper.py rename to src/bfabric_scripts/fgcz_maxquant_scaffold-wrapper.py diff --git a/src/bfabric/scripts/fgcz_maxquant_wrapper.py b/src/bfabric_scripts/fgcz_maxquant_wrapper.py similarity index 100% rename from src/bfabric/scripts/fgcz_maxquant_wrapper.py rename to src/bfabric_scripts/fgcz_maxquant_wrapper.py diff --git a/src/bfabric/scripts/fgcz_sge_maxquant_linux.bash b/src/bfabric_scripts/fgcz_sge_maxquant_linux.bash similarity index 100% rename from src/bfabric/scripts/fgcz_sge_maxquant_linux.bash rename to src/bfabric_scripts/fgcz_sge_maxquant_linux.bash From d42ebc7aff6df01ef450993d52b5e618b1cd0976 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Thu, 3 Oct 2024 10:54:49 +0200 Subject: [PATCH 35/42] changelog --- docs/changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.md b/docs/changelog.md index 95fd0a67..7b5a9a2e 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -24,6 +24,7 @@ Versioning currently follows `X.Y.Z` where ### Changed +- `bfabric.scripts` has been moved into a namespace package `bfabric_scripts` so we can later split it off. - (internal) migrate to src layout - (experimental) the former `process` step of the app runner has been split into a `process` and `collect` step where, the collect step is responsible for generating the `output.yml` file that will then be used to register the results. From 169d20d4980d2da8a0efded228ff34068fdae86d Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Thu, 3 Oct 2024 10:59:43 +0200 Subject: [PATCH 36/42] changelog --- docs/changelog.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index 7b5a9a2e..9a55ee85 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -10,6 +10,8 @@ Versioning currently follows `X.Y.Z` where ## \[Unreleased\] +This release contains mainly internal changes and ongoing development on the experimental app interface functionality. + ### Added - Entities can be compared and sorted by ID now. From 5328e4e986701bf36c1b403c302d573251520c46 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Thu, 3 Oct 2024 11:00:27 +0200 Subject: [PATCH 37/42] v1.13.8 --- docs/changelog.md | 2 ++ pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index 9a55ee85..580e3aad 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -10,6 +10,8 @@ Versioning currently follows `X.Y.Z` where ## \[Unreleased\] +## \[1.13.8\] - 2024-10-03 + This release contains mainly internal changes and ongoing development on the experimental app interface functionality. ### Added diff --git a/pyproject.toml b/pyproject.toml index 3e802008..9fb47bec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "bfabric" description = "Python client for the B-Fabric WSDL API" -version = "1.13.7" +version = "1.13.8" license = { text = "GPL-3.0" } authors = [ {name = "Christian Panse", email = "cp@fgcz.ethz.ch"}, From 8884281cc02ee8e10f612284698687ec77d63a63 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Thu, 3 Oct 2024 11:03:00 +0200 Subject: [PATCH 38/42] reorganize tests directory --- tests/{unit => bfabric}/__init__.py | 0 tests/{unit => bfabric}/config/__init__.py | 0 tests/{unit => bfabric}/config/test_bfabric_auth.py | 0 tests/{unit => bfabric}/config/test_bfabric_client_config.py | 0 tests/{unit => bfabric}/config/test_config_file.py | 0 tests/{unit => bfabric}/conftest.py | 0 tests/{unit => bfabric}/engine/__init__.py | 0 tests/{unit => bfabric}/engine/test_engine_suds.py | 0 tests/{unit => bfabric}/engine/test_engine_zeep.py | 0 tests/{unit => bfabric}/entities/__init__.py | 0 tests/{unit => bfabric}/entities/core/__init__.py | 0 tests/{unit => bfabric}/entities/core/test_entity.py | 0 tests/{unit => bfabric}/entities/core/test_has_many.py | 0 tests/{unit => bfabric}/entities/core/test_has_one.py | 0 tests/{unit => bfabric}/entities/test_dataset.py | 0 tests/{unit => bfabric}/entities/test_workunit.py | 0 tests/{unit => bfabric}/example_config.yml | 0 tests/{unit => bfabric}/results/__init__.py | 0 tests/{unit => bfabric}/results/test_response_format_dict.py | 0 tests/{unit => bfabric}/results/test_result_container.py | 0 tests/{unit => bfabric}/test_bfabric.py | 0 tests/{unit => bfabric}/test_bfabric_config.py | 0 tests/{unit/scripts => bfabric/utils}/__init__.py | 0 tests/{unit => bfabric}/utils/test_math_helper.py | 0 tests/{unit => bfabric}/utils/test_paginator.py | 0 tests/{unit/utils => bfabric_scripts}/__init__.py | 0 .../test_bfabric_list_not_existing_storage_directories.py | 2 +- .../test_bfabric_save_csv2dataset.py | 2 +- .../test_bfabric_slurm_queue_status.py | 2 +- 29 files changed, 3 insertions(+), 3 deletions(-) rename tests/{unit => bfabric}/__init__.py (100%) rename tests/{unit => bfabric}/config/__init__.py (100%) rename tests/{unit => bfabric}/config/test_bfabric_auth.py (100%) rename tests/{unit => bfabric}/config/test_bfabric_client_config.py (100%) rename tests/{unit => bfabric}/config/test_config_file.py (100%) rename tests/{unit => bfabric}/conftest.py (100%) rename tests/{unit => bfabric}/engine/__init__.py (100%) rename tests/{unit => bfabric}/engine/test_engine_suds.py (100%) rename tests/{unit => bfabric}/engine/test_engine_zeep.py (100%) rename tests/{unit => bfabric}/entities/__init__.py (100%) rename tests/{unit => bfabric}/entities/core/__init__.py (100%) rename tests/{unit => bfabric}/entities/core/test_entity.py (100%) rename tests/{unit => bfabric}/entities/core/test_has_many.py (100%) rename tests/{unit => bfabric}/entities/core/test_has_one.py (100%) rename tests/{unit => bfabric}/entities/test_dataset.py (100%) rename tests/{unit => bfabric}/entities/test_workunit.py (100%) rename tests/{unit => bfabric}/example_config.yml (100%) rename tests/{unit => bfabric}/results/__init__.py (100%) rename tests/{unit => bfabric}/results/test_response_format_dict.py (100%) rename tests/{unit => bfabric}/results/test_result_container.py (100%) rename tests/{unit => bfabric}/test_bfabric.py (100%) rename tests/{unit => bfabric}/test_bfabric_config.py (100%) rename tests/{unit/scripts => bfabric/utils}/__init__.py (100%) rename tests/{unit => bfabric}/utils/test_math_helper.py (100%) rename tests/{unit => bfabric}/utils/test_paginator.py (100%) rename tests/{unit/utils => bfabric_scripts}/__init__.py (100%) rename tests/{unit/scripts => bfabric_scripts}/test_bfabric_list_not_existing_storage_directories.py (94%) rename tests/{unit/scripts => bfabric_scripts}/test_bfabric_save_csv2dataset.py (96%) rename tests/{unit/scripts => bfabric_scripts}/test_bfabric_slurm_queue_status.py (95%) diff --git a/tests/unit/__init__.py b/tests/bfabric/__init__.py similarity index 100% rename from tests/unit/__init__.py rename to tests/bfabric/__init__.py diff --git a/tests/unit/config/__init__.py b/tests/bfabric/config/__init__.py similarity index 100% rename from tests/unit/config/__init__.py rename to tests/bfabric/config/__init__.py diff --git a/tests/unit/config/test_bfabric_auth.py b/tests/bfabric/config/test_bfabric_auth.py similarity index 100% rename from tests/unit/config/test_bfabric_auth.py rename to tests/bfabric/config/test_bfabric_auth.py diff --git a/tests/unit/config/test_bfabric_client_config.py b/tests/bfabric/config/test_bfabric_client_config.py similarity index 100% rename from tests/unit/config/test_bfabric_client_config.py rename to tests/bfabric/config/test_bfabric_client_config.py diff --git a/tests/unit/config/test_config_file.py b/tests/bfabric/config/test_config_file.py similarity index 100% rename from tests/unit/config/test_config_file.py rename to tests/bfabric/config/test_config_file.py diff --git a/tests/unit/conftest.py b/tests/bfabric/conftest.py similarity index 100% rename from tests/unit/conftest.py rename to tests/bfabric/conftest.py diff --git a/tests/unit/engine/__init__.py b/tests/bfabric/engine/__init__.py similarity index 100% rename from tests/unit/engine/__init__.py rename to tests/bfabric/engine/__init__.py diff --git a/tests/unit/engine/test_engine_suds.py b/tests/bfabric/engine/test_engine_suds.py similarity index 100% rename from tests/unit/engine/test_engine_suds.py rename to tests/bfabric/engine/test_engine_suds.py diff --git a/tests/unit/engine/test_engine_zeep.py b/tests/bfabric/engine/test_engine_zeep.py similarity index 100% rename from tests/unit/engine/test_engine_zeep.py rename to tests/bfabric/engine/test_engine_zeep.py diff --git a/tests/unit/entities/__init__.py b/tests/bfabric/entities/__init__.py similarity index 100% rename from tests/unit/entities/__init__.py rename to tests/bfabric/entities/__init__.py diff --git a/tests/unit/entities/core/__init__.py b/tests/bfabric/entities/core/__init__.py similarity index 100% rename from tests/unit/entities/core/__init__.py rename to tests/bfabric/entities/core/__init__.py diff --git a/tests/unit/entities/core/test_entity.py b/tests/bfabric/entities/core/test_entity.py similarity index 100% rename from tests/unit/entities/core/test_entity.py rename to tests/bfabric/entities/core/test_entity.py diff --git a/tests/unit/entities/core/test_has_many.py b/tests/bfabric/entities/core/test_has_many.py similarity index 100% rename from tests/unit/entities/core/test_has_many.py rename to tests/bfabric/entities/core/test_has_many.py diff --git a/tests/unit/entities/core/test_has_one.py b/tests/bfabric/entities/core/test_has_one.py similarity index 100% rename from tests/unit/entities/core/test_has_one.py rename to tests/bfabric/entities/core/test_has_one.py diff --git a/tests/unit/entities/test_dataset.py b/tests/bfabric/entities/test_dataset.py similarity index 100% rename from tests/unit/entities/test_dataset.py rename to tests/bfabric/entities/test_dataset.py diff --git a/tests/unit/entities/test_workunit.py b/tests/bfabric/entities/test_workunit.py similarity index 100% rename from tests/unit/entities/test_workunit.py rename to tests/bfabric/entities/test_workunit.py diff --git a/tests/unit/example_config.yml b/tests/bfabric/example_config.yml similarity index 100% rename from tests/unit/example_config.yml rename to tests/bfabric/example_config.yml diff --git a/tests/unit/results/__init__.py b/tests/bfabric/results/__init__.py similarity index 100% rename from tests/unit/results/__init__.py rename to tests/bfabric/results/__init__.py diff --git a/tests/unit/results/test_response_format_dict.py b/tests/bfabric/results/test_response_format_dict.py similarity index 100% rename from tests/unit/results/test_response_format_dict.py rename to tests/bfabric/results/test_response_format_dict.py diff --git a/tests/unit/results/test_result_container.py b/tests/bfabric/results/test_result_container.py similarity index 100% rename from tests/unit/results/test_result_container.py rename to tests/bfabric/results/test_result_container.py diff --git a/tests/unit/test_bfabric.py b/tests/bfabric/test_bfabric.py similarity index 100% rename from tests/unit/test_bfabric.py rename to tests/bfabric/test_bfabric.py diff --git a/tests/unit/test_bfabric_config.py b/tests/bfabric/test_bfabric_config.py similarity index 100% rename from tests/unit/test_bfabric_config.py rename to tests/bfabric/test_bfabric_config.py diff --git a/tests/unit/scripts/__init__.py b/tests/bfabric/utils/__init__.py similarity index 100% rename from tests/unit/scripts/__init__.py rename to tests/bfabric/utils/__init__.py diff --git a/tests/unit/utils/test_math_helper.py b/tests/bfabric/utils/test_math_helper.py similarity index 100% rename from tests/unit/utils/test_math_helper.py rename to tests/bfabric/utils/test_math_helper.py diff --git a/tests/unit/utils/test_paginator.py b/tests/bfabric/utils/test_paginator.py similarity index 100% rename from tests/unit/utils/test_paginator.py rename to tests/bfabric/utils/test_paginator.py diff --git a/tests/unit/utils/__init__.py b/tests/bfabric_scripts/__init__.py similarity index 100% rename from tests/unit/utils/__init__.py rename to tests/bfabric_scripts/__init__.py diff --git a/tests/unit/scripts/test_bfabric_list_not_existing_storage_directories.py b/tests/bfabric_scripts/test_bfabric_list_not_existing_storage_directories.py similarity index 94% rename from tests/unit/scripts/test_bfabric_list_not_existing_storage_directories.py rename to tests/bfabric_scripts/test_bfabric_list_not_existing_storage_directories.py index 5258841f..2ccf6910 100644 --- a/tests/unit/scripts/test_bfabric_list_not_existing_storage_directories.py +++ b/tests/bfabric_scripts/test_bfabric_list_not_existing_storage_directories.py @@ -6,7 +6,7 @@ from pytest_mock import MockerFixture from bfabric.results.result_container import ResultContainer -from bfabric.scripts.bfabric_list_not_existing_storage_directories import list_not_existing_storage_dirs +from bfabric_scripts.bfabric_list_not_existing_storage_directories import list_not_existing_storage_dirs def test_list_not_existing_storage_directories( diff --git a/tests/unit/scripts/test_bfabric_save_csv2dataset.py b/tests/bfabric_scripts/test_bfabric_save_csv2dataset.py similarity index 96% rename from tests/unit/scripts/test_bfabric_save_csv2dataset.py rename to tests/bfabric_scripts/test_bfabric_save_csv2dataset.py index 9b357cff..9e5a6873 100644 --- a/tests/unit/scripts/test_bfabric_save_csv2dataset.py +++ b/tests/bfabric_scripts/test_bfabric_save_csv2dataset.py @@ -1,7 +1,7 @@ import pytest import polars as pl -from bfabric.scripts.bfabric_save_csv2dataset import check_for_invalid_characters +from bfabric_scripts.bfabric_save_csv2dataset import check_for_invalid_characters def test_check_for_invalid_characters_no_invalid_chars(): diff --git a/tests/unit/scripts/test_bfabric_slurm_queue_status.py b/tests/bfabric_scripts/test_bfabric_slurm_queue_status.py similarity index 95% rename from tests/unit/scripts/test_bfabric_slurm_queue_status.py rename to tests/bfabric_scripts/test_bfabric_slurm_queue_status.py index 886c6ce5..dac45445 100644 --- a/tests/unit/scripts/test_bfabric_slurm_queue_status.py +++ b/tests/bfabric_scripts/test_bfabric_slurm_queue_status.py @@ -3,7 +3,7 @@ import pytest from bfabric.entities import Workunit -from bfabric.scripts.bfabric_slurm_queue_status import get_slurm_jobs, get_workunit_status +from bfabric_scripts.bfabric_slurm_queue_status import get_slurm_jobs, get_workunit_status @pytest.fixture From c88431a2e4be6ac16fefcfdfef7aa108e24fd25d Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Thu, 3 Oct 2024 12:08:20 +0200 Subject: [PATCH 39/42] add new workflow --- .github/workflows/complete_release.yml | 40 ++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/complete_release.yml diff --git a/.github/workflows/complete_release.yml b/.github/workflows/complete_release.yml new file mode 100644 index 00000000..20c12a90 --- /dev/null +++ b/.github/workflows/complete_release.yml @@ -0,0 +1,40 @@ +name: Complete Release +on: + push: + branches: [stable] + # TODO remove after testing + workflow_dispatch: +jobs: + create_tag: + name: Create Tag + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + - name: Create tag (with Python) + run: | + # Find the version + VERSION=$(python3 -c "import tomllib; from pathlib import Path; print(tomllib.loads(Path('pyproject.toml').read_text())['project']['version'], end='')") + # Check if tag exists + if git rev-parse $VERSION >/dev/null 2>&1; then + echo "Tag $VERSION already exists" + exit 1 + fi + # Create tag + git tag -a $VERSION -m "Release $VERSION" + # Push tag + git push origin $VERSION + publish_documentation: + name: Publish Docs + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-bfabricpy + with: + python-version: 3.12 + - name: Publish documentation + run: | + uv run mkdocs gh-deploy From 217a0024e470d575fc2a269bd97bf05a9063ac55 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Thu, 3 Oct 2024 12:10:14 +0200 Subject: [PATCH 40/42] git config --- .github/workflows/complete_release.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/complete_release.yml b/.github/workflows/complete_release.yml index 20c12a90..c89a94a2 100644 --- a/.github/workflows/complete_release.yml +++ b/.github/workflows/complete_release.yml @@ -14,6 +14,10 @@ jobs: uses: actions/setup-python@v5 with: python-version: 3.12 + - name: Git Config + run: | + git config --global user.name "bfabricPy Bot" + git config --global user.email "bfabricpy-bot-noreply@fgcz.ethz.ch" - name: Create tag (with Python) run: | # Find the version From 6ed44b3ec0d95091b483eeaeeeca1ebff08ec5b8 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Thu, 3 Oct 2024 12:12:13 +0200 Subject: [PATCH 41/42] pull branch before deploy --- .github/workflows/complete_release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/complete_release.yml b/.github/workflows/complete_release.yml index c89a94a2..f93f8948 100644 --- a/.github/workflows/complete_release.yml +++ b/.github/workflows/complete_release.yml @@ -41,4 +41,5 @@ jobs: python-version: 3.12 - name: Publish documentation run: | + git checkout gh-pages && git pull && git checkout - uv run mkdocs gh-deploy From ee917139469b61347499e0036e3d635566bdf7be Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Thu, 3 Oct 2024 12:15:52 +0200 Subject: [PATCH 42/42] unshallow gh-pages --- .github/workflows/complete_release.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/complete_release.yml b/.github/workflows/complete_release.yml index f93f8948..cdb9fb93 100644 --- a/.github/workflows/complete_release.yml +++ b/.github/workflows/complete_release.yml @@ -20,6 +20,7 @@ jobs: git config --global user.email "bfabricpy-bot-noreply@fgcz.ethz.ch" - name: Create tag (with Python) run: | + set -euxo pipefail # Find the version VERSION=$(python3 -c "import tomllib; from pathlib import Path; print(tomllib.loads(Path('pyproject.toml').read_text())['project']['version'], end='')") # Check if tag exists @@ -41,5 +42,7 @@ jobs: python-version: 3.12 - name: Publish documentation run: | + set -euxo pipefail + git fetch --unshallow origin gh-pages git checkout gh-pages && git pull && git checkout - uv run mkdocs gh-deploy