move app runner code into separate package

.gitignore

@@ -1,6 +1,6 @@
 .idea/
 __pycache__
-bfabric.egg-info/
+*.egg-info/
 bfabric/scripts/query_result.txt
 build/
 dist/

app_runner/.gitignore

@@ -0,0 +1,2 @@
+builds
+app_runner.spec

app_runner/deploy/build.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -euxo pipefail
+# Parse arguments
+TARGET_DIR=$(readlink -f "${1:-./dist}")
+TARGET_NAME="${2:-app_runner}"
+DOCKER=docker
+
+DEPLOY_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+APP_RUNNER_PROJECT_DIR=$(realpath "$DEPLOY_DIR/..")
+BUILDER_IMAGE=local-build_app_runner:0.0.1
+$DOCKER build -t $BUILDER_IMAGE "$DEPLOY_DIR/builder"
+
+mkdir -p "$TARGET_DIR"
+$DOCKER run \
+    --user "$(id -u):$(id -g)" \
+    --rm \
+    --mount type=bind,source="$APP_RUNNER_PROJECT_DIR",target=/work/app_runner \
+    --mount type=bind,source="$DEPLOY_DIR"/build_steps.sh,target=/work/build_steps.sh,readonly \
+    --mount type=bind,source="$TARGET_DIR",target=/work/dist \
+    --workdir /work/app_runner \
+    "$BUILDER_IMAGE" \
+    bash /work/build_steps.sh /work/dist "$TARGET_NAME"

app_runner/deploy/build_steps.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -euxo pipefail
+TARGET_DIR="${1:-dist}"
+TARGET_NAME="${2:-app_runner}"
+rm -rf /work/venv
+python -m venv /work/venv
+source /work/venv/bin/activate
+uv pip install .
+uv pip install pyinstaller
+pyinstaller -y --onedir --name "${TARGET_NAME}" --distpath "${TARGET_DIR}" src/app_runner/cli/__main__.py
+deactivate

app_runner/deploy/builder/Dockerfile

@@ -0,0 +1,25 @@
+ARG DEBIAN_VERSION=buster
+FROM debian:${DEBIAN_VERSION}
+ARG PYTHON_VERSION=3.13.0
+
+LABEL org.opencontainers.image.authors="Leonardo Schwarz"
+
+RUN apt-get update \
+    && apt-get upgrade -y \
+    && apt-get install -y curl git bash build-essential ccache \
+    && apt-get install -y libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev curl git libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev
+
+RUN curl https://pyenv.run | bash
+
+ENV PATH=$PATH:/root/.pyenv/bin
+RUN pyenv install $PYTHON_VERSION
+RUN pyenv global $PYTHON_VERSION
+ENV PATH=/root/.pyenv/versions/${PYTHON_VERSION}/bin:$PATH
+
+RUN pip install --root-user-action ignore uv pyinstaller
+RUN chmod -R 0777 /root
+RUN mkdir /work && chmod 0777 /work
+RUN mkdir /home/user && chmod 0777 /home/user
+
+ENV HOME=/home/user
+WORKDIR /work

app_runner/pyproject.toml

@@ -0,0 +1,19 @@
+[build-system]
+requires = ["setuptools >= 61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "app_runner"
+description = "Application runner for B-Fabric apps"
+version = "0.1.0"
+license = { text = "GPL-3.0" }
+authors = [
+    {name = "Leonardo Schwarz", email = "leonardo.schwarz@fgcz.ethz.ch"},
+]
+requires-python = ">=3.12"
+dependencies = [
+    "bfabric @ git+https://github.com/fgcz/bfabricPy.git@main",
+]
+
+[project.scripts]
+"bfabric-app-runner"="app_runner.cli.__main__:app"

src/bfabric/experimental/app_interface/app_runner/__main__.py → app_runner/src/app_runner/app_runner/__main__.py

@@ -9,8 +9,8 @@
 
 from bfabric.bfabric import Bfabric
 from bfabric.cli_formatting import setup_script_logging
-from bfabric.experimental.app_interface.app_runner._spec import AppSpec
-from bfabric.experimental.app_interface.app_runner.runner import run_app
+from app_runner.app_runner._spec import AppSpec
+from app_runner.app_runner.runner import run_app
 
 
 def main() -> None:

src/bfabric/experimental/app_interface/app_runner/_spec.py → app_runner/src/app_runner/app_runner/_spec.py

@@ -46,7 +46,7 @@ class CommandDocker(BaseModel):
     mounts: MountOptions = MountOptions()
 
     def to_shell(self, work_dir: Path | None = None) -> list[str]:
-        work_dir = (work_dir or Path(".")).expanduser().absolute()
+        work_dir = (work_dir or Path("")).expanduser().absolute()
         mounts = self.mounts.collect(work_dir=work_dir)
         mount_args = []
         for host, container, read_only in mounts:

src/bfabric/experimental/app_interface/app_runner/runner.py → app_runner/src/app_runner/app_runner/runner.py

@@ -9,10 +9,10 @@
 from pydantic import BaseModel
 
 from bfabric import Bfabric
-from bfabric.experimental.app_interface.app_runner._spec import AppSpec
-from bfabric.experimental.app_interface.input_preparation import prepare_folder
-from bfabric.experimental.app_interface.output_registration import register_outputs
-from bfabric.experimental.app_interface.workunit.definition import WorkunitDefinition
+from app_runner.app_runner._spec import AppSpec
+from app_runner.input_preparation import prepare_folder
+from app_runner.output_registration import register_outputs
+from bfabric.experimental.workunit_definition import WorkunitDefinition
 
 
 class Runner:

app_runner/src/app_runner/cli/__main__.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+import cyclopts
+
+from app_runner.cli.app import app_app
+from app_runner.cli.chunk import app_chunk
+from app_runner.cli.inputs import app_inputs
+from app_runner.cli.outputs import app_outputs
+from app_runner.cli.validate import app_validate
+
+app = cyclopts.App(help="Provides an entrypoint to app execution.\n\nFunctionality/API under active development!")
+app.command(app_inputs)
+app.command(app_outputs)
+app.command(app_app)
+app.command(app_chunk)
+app.command(app_validate)
+
+if __name__ == "__main__":
+    app()

src/bfabric/experimental/app_interface/cli/app.py → app_runner/src/app_runner/cli/app.py

@@ -7,8 +7,8 @@
 
 from bfabric import Bfabric
 from bfabric.cli_formatting import setup_script_logging
-from bfabric.experimental.app_interface.app_runner._spec import AppSpec
-from bfabric.experimental.app_interface.app_runner.runner import run_app, Runner
+from app_runner.app_runner._spec import AppSpec
+from app_runner.app_runner.runner import run_app, Runner
 from bfabric.experimental.entity_lookup_cache import EntityLookupCache
 
 app_app = cyclopts.App("app", help="Run an app.")

src/bfabric/experimental/app_interface/cli/chunk.py → app_runner/src/app_runner/cli/chunk.py

@@ -7,8 +7,8 @@
 
 from bfabric import Bfabric
 from bfabric.cli_formatting import setup_script_logging
-from bfabric.experimental.app_interface.app_runner._spec import AppSpec
-from bfabric.experimental.app_interface.app_runner.runner import run_app, Runner
+from app_runner.app_runner._spec import AppSpec
+from app_runner.app_runner.runner import run_app, Runner
 from bfabric.experimental.entity_lookup_cache import EntityLookupCache
 
 app_chunk = cyclopts.App("chunk", help="Run an app on a chunk. You can create the chunks with `app dispatch`.")

src/bfabric/experimental/app_interface/cli/inputs.py → app_runner/src/app_runner/cli/inputs.py

@@ -6,10 +6,10 @@
 
 from bfabric import Bfabric
 from bfabric.cli_formatting import setup_script_logging
-from bfabric.experimental.app_interface.input_preparation import prepare_folder
-from bfabric.experimental.app_interface.input_preparation._spec import InputsSpec
-from bfabric.experimental.app_interface.input_preparation.integrity import IntegrityState
-from bfabric.experimental.app_interface.input_preparation.list_inputs import (
+from app_runner.input_preparation import prepare_folder
+from app_runner.input_preparation._spec import InputsSpec
+from app_runner.input_preparation.integrity import IntegrityState
+from app_runner.input_preparation.list_inputs import (
     list_input_states,
     print_input_states,
     FileState,
@@ -72,7 +72,7 @@ def get_inputs_and_print(
     client = Bfabric.from_config()
     input_states = list_input_states(
         specs=InputsSpec.read_yaml(inputs_yaml),
-        target_folder=target_folder or Path("."),
+        target_folder=target_folder or Path(""),
         client=client,
         check_files=check,
     )

src/bfabric/experimental/app_interface/cli/outputs.py → app_runner/src/app_runner/cli/outputs.py

@@ -7,8 +7,8 @@
 from bfabric import Bfabric
 from bfabric.cli_formatting import setup_script_logging
 from bfabric.entities import Workunit
-from bfabric.experimental.app_interface.output_registration._spec import OutputsSpec
-from bfabric.experimental.app_interface.output_registration.register import register_all
+from app_runner.output_registration._spec import OutputsSpec
+from app_runner.output_registration.register import register_all
 
 app_outputs = cyclopts.App("outputs", help="Register output files for an app.")
 

src/bfabric/experimental/app_interface/cli/validate.py → app_runner/src/app_runner/cli/validate.py

@@ -7,9 +7,9 @@
 import rich.pretty
 import yaml
 
-from bfabric.experimental.app_interface.app_runner._spec import AppSpec
-from bfabric.experimental.app_interface.input_preparation._spec import InputsSpec
-from bfabric.experimental.app_interface.output_registration._spec import OutputsSpec
+from app_runner.app_runner._spec import AppSpec
+from app_runner.input_preparation._spec import InputsSpec
+from app_runner.output_registration._spec import OutputsSpec
 
 app_validate = cyclopts.App("validate", help="Validate yaml files.")
 

src/bfabric/experimental/app_interface/dispatch/dispatch_individual_resources.py → app_runner/src/app_runner/dispatch/dispatch_individual_resources.py

@@ -9,7 +9,7 @@
 
 from bfabric import Bfabric
 from bfabric.entities import Resource, Dataset
-from bfabric.experimental.app_interface.workunit.definition import WorkunitDefinition
+from bfabric.experimental.workunit_definition import WorkunitDefinition
 
 
 class ConfigResourceFlow(BaseModel):

src/bfabric/experimental/app_interface/input_preparation/__main__.py → app_runner/src/app_runner/input_preparation/__main__.py

@@ -5,7 +5,7 @@
 
 from bfabric.bfabric import Bfabric
 from bfabric.cli_formatting import setup_script_logging
-from bfabric.experimental.app_interface.input_preparation.prepare import prepare_folder
+from app_runner.input_preparation.prepare import prepare_folder
 
 
 def main() -> None:

src/bfabric/experimental/app_interface/input_preparation/integrity.py → app_runner/src/app_runner/input_preparation/integrity.py

@@ -5,8 +5,8 @@
 
 from bfabric.bfabric import Bfabric
 from bfabric.entities import Resource, Dataset
-from bfabric.experimental.app_interface.input_preparation._spec import InputSpecType, ResourceSpec, DatasetSpec
-from bfabric.experimental.app_interface.util.checksums import md5sum
+from app_runner.input_preparation._spec import InputSpecType, ResourceSpec, DatasetSpec
+from app_runner.util.checksums import md5sum
 
 
 class IntegrityState(Enum):

src/bfabric/experimental/app_interface/input_preparation/list_inputs.py → app_runner/src/app_runner/input_preparation/list_inputs.py

@@ -7,8 +7,8 @@
 from rich.table import Table, Column
 
 from bfabric.bfabric import Bfabric
-from bfabric.experimental.app_interface.input_preparation._spec import InputSpecType
-from bfabric.experimental.app_interface.input_preparation.integrity import check_integrity, IntegrityState
+from app_runner.input_preparation._spec import InputSpecType
+from app_runner.input_preparation.integrity import check_integrity, IntegrityState
 
 
 @dataclass

src/bfabric/experimental/app_interface/input_preparation/prepare.py → app_runner/src/app_runner/input_preparation/prepare.py

@@ -6,16 +6,16 @@
 
 from bfabric.bfabric import Bfabric
 from bfabric.entities import Resource, Dataset
-from bfabric.experimental.app_interface.input_preparation._spec import (
+from app_runner.input_preparation._spec import (
     ResourceSpec,
     DatasetSpec,
     InputSpecType,
     InputsSpec,
 )
-from bfabric.experimental.app_interface.input_preparation.integrity import IntegrityState
-from bfabric.experimental.app_interface.input_preparation.list_inputs import list_input_states
-from bfabric.experimental.app_interface.util.checksums import md5sum
-from bfabric.experimental.app_interface.util.scp import scp
+from app_runner.input_preparation.integrity import IntegrityState
+from app_runner.input_preparation.list_inputs import list_input_states
+from app_runner.util.checksums import md5sum
+from app_runner.util.scp import scp
 
 
 class PrepareInputs:

src/bfabric/experimental/app_interface/output_registration/__main__.py → app_runner/src/app_runner/output_registration/__main__.py

@@ -3,7 +3,7 @@
 
 from bfabric import Bfabric
 from bfabric.cli_formatting import setup_script_logging
-from bfabric.experimental.app_interface.output_registration import register_outputs
+from app_runner.output_registration import register_outputs
 
 
 def main() -> None:

src/bfabric/experimental/app_interface/output_registration/register.py → app_runner/src/app_runner/output_registration/register.py

@@ -6,15 +6,15 @@
 
 from bfabric import Bfabric
 from bfabric.entities import Storage, Workunit
-from bfabric.experimental.app_interface.output_registration._spec import (
+from app_runner.output_registration._spec import (
     CopyResourceSpec,
     UpdateExisting,
     OutputsSpec,
     SpecType,
     SaveDatasetSpec,
 )
-from bfabric.experimental.app_interface.util.checksums import md5sum
-from bfabric.experimental.app_interface.util.scp import scp
+from app_runner.util.checksums import md5sum
+from app_runner.util.scp import scp
 from bfabric_scripts.bfabric_save_csv2dataset import bfabric_save_csv2dataset
 
 

src/bfabric/wrapper_creator/bfabric_wrapper_creator.py

@@ -10,7 +10,7 @@
 
 from bfabric import Bfabric
 from bfabric.entities import Workunit, ExternalJob, Application, Resource, Storage, Order, Project
-from bfabric.experimental.app_interface.workunit.definition import WorkunitDefinition
+from bfabric.experimental.workunit_definition import WorkunitDefinition
 
 
 class BfabricWrapperCreator: