diff --git a/CITATION.cff b/CITATION.cff index 6b0cdf09..4ee17472 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -9,7 +9,7 @@ message: >- If you use this software, please cite it using the metadata from this file. type: software -version: 0.5.0 +version: 0.5.1 url: https://github.com/rdnfn/beobench authors: - given-names: Arduin diff --git a/HISTORY.rst b/HISTORY.rst index 4b3300f0..6774585e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,22 @@ History ======= +0.5.1 (2022-06-28) +------------------ + +* Features: + + * Add pretty logging based on loguru package. Now all Beobench output is clearly marked as such. + +* Improvements + + * Enable adding wrapper without setting config. + * Add ``demo.yaml`` simple example config. + +* Fixes + + * Update Sinergym integration to latest Sinergym version. + 0.5.0 (2022-05-26) ------------------ @@ -11,15 +27,16 @@ History * Support for automatically running multiple samples/trials of same experiment via ``num_samples`` config parameter. * Configs named `.beobench.yml` will be automatically parsed when Beobench is run in directory containing such a config. This allows users to set e.g. wandb API keys without referring to the config in every Beobench command call. * Configs from experiments now specify the Beobench version used. When trying to rerun an experiment this version will be checked, and an error thrown if there is a mismatch between installed and requested version. - * Add improved high-level API for getting started. This uses the CLI arguments ``--method``, ``--gym`` and ``--env``. Example usage: ``beobench run --method ppo --gym sinergym --env Eplus-5Zone-hot-continuous-v1``. + * Add improved high-level API for getting started. This uses the CLI arguments ``--method``, ``--gym`` and ``--env``. Example usage: ``beobench run --method ppo --gym sinergym --env Eplus-5Zone-hot-continuous-v1`` (#55). * Improvements - * Add ``CITATION.cff`` file to citing software easier. + * Add ``CITATION.cff`` file to make citing software easier. * By default, docker builds of experiment images are now skipped if an image with tag corresponding to installed Beobench version already exists. - * Remove outdated guides and add yaml configuration description from docs. + * Remove outdated guides and add yaml configuration description from docs (#38, #76, #78). * Add support for logging multidimensional actions to wandb. * Add support for logging summary metrics on every env reset to wandb. + * Energym config now uses ``name`` argument like other integrations (#34). * Fixes diff --git a/beobench/__init__.py b/beobench/__init__.py index 07ea095f..8008eba6 100644 --- a/beobench/__init__.py +++ b/beobench/__init__.py @@ -2,7 +2,7 @@ __author__ = """Beobench authors""" __email__ = "-" -__version__ = "0.5.0" +__version__ = "0.5.1" from beobench.utils import restart from beobench.experiment.scheduler import run diff --git a/beobench/beobench_contrib b/beobench/beobench_contrib index 172d5622..857092d1 160000 --- a/beobench/beobench_contrib +++ b/beobench/beobench_contrib @@ -1 +1 @@ -Subproject commit 172d5622e17db61cfc3d7c9a9fe5002120b57de6 +Subproject commit 857092d14eb6950f016fac2f746e97a0a5e578e1 diff --git a/beobench/data/configs/default.yaml b/beobench/data/configs/default.yaml index 30aab120..5d0677e9 100644 --- a/beobench/data/configs/default.yaml +++ b/beobench/data/configs/default.yaml @@ -109,4 +109,4 @@ general: # experiment once. num_samples: 1 # Beobench version - version: 0.5.0 + version: 0.5.1 diff --git a/beobench/data/configs/demo.yaml b/beobench/data/configs/demo.yaml new file mode 100644 index 00000000..f6676a92 --- /dev/null +++ b/beobench/data/configs/demo.yaml @@ -0,0 +1,16 @@ +agent: + origin: random_action + config: + stop: + timesteps_total: 10 +env: + gym: sinergym + config: + name: Eplus-5Zone-hot-continuous-v1 +wrappers: + - origin: general + class: WandbLogger +general: + wandb_entity: beobench + wandb_project: demo + # wandb_api_key: HIDDEN \ No newline at end of file diff --git a/beobench/experiment/config_parser.py b/beobench/experiment/config_parser.py index 92672c11..4abf844b 100644 --- a/beobench/experiment/config_parser.py +++ b/beobench/experiment/config_parser.py @@ -8,6 +8,7 @@ import sys import random import os +from beobench.logging import logger import beobench import beobench.utils @@ -138,7 +139,7 @@ def get_user() -> dict: """ if os.path.isfile(USER_CONFIG_PATH): - print(f"Beobench: recognised user config at '{USER_CONFIG_PATH}'.") + logger.info(f"Recognised user config at '{USER_CONFIG_PATH}'.") user_config = parse(USER_CONFIG_PATH) else: user_config = {} diff --git a/beobench/experiment/containers.py b/beobench/experiment/containers.py index 88d86640..d0723c61 100644 --- a/beobench/experiment/containers.py +++ b/beobench/experiment/containers.py @@ -4,6 +4,7 @@ import subprocess import os import docker +from loguru import logger import beobench from beobench.constants import AVAILABLE_INTEGRATIONS @@ -74,7 +75,7 @@ def build_experiment_container( ) package_build_context = True - print(f"Beobench: recognised integration named {gym_name}.") + logger.info(f"Recognised integration named {gym_name}.") else: # get alphanumeric name from context context_name = "".join(e for e in build_context if e.isalnum()) @@ -88,15 +89,14 @@ def build_experiment_container( # skip build if image already exists. if not force_build and check_image_exists(stage2_image_tag): - print(f"Beobench: existing image found ({stage2_image_tag}). Skipping build.") + logger.info(f"Existing image found ({stage2_image_tag}). Skipping build.") return stage2_image_tag - print( - f"Beobench: image not found ({stage2_image_tag}) or forced", - "rebuild. Building image.", + logger.warning( + f"Image not found ({stage2_image_tag}) or forced rebuild. Building image.", ) - print(f"Building experiment base image `{stage0_image_tag}`...") + logger.info(f"Building experiment base image `{stage0_image_tag}`...") with contextlib.ExitStack() as stack: # if using build context from beobench package, get (potentially temp.) build @@ -114,7 +114,7 @@ def build_experiment_container( build_context, ] env = os.environ.copy() - print("Running command: " + " ".join(stage0_build_args)) + logger.info("Running command: " + " ".join(stage0_build_args)) subprocess.check_call( stage0_build_args, env=env, # this enables accessing dockerfile in subdir @@ -146,7 +146,7 @@ def build_experiment_container( with subprocess.Popen( ["cat", stage1_dockerfile], stdout=subprocess.PIPE ) as proc: - print("Running command: " + " ".join(stage1_build_args)) + logger.info("Running command: " + " ".join(stage1_build_args)) subprocess.check_call( stage1_build_args, stdin=proc.stdout, @@ -191,14 +191,14 @@ def build_experiment_container( with subprocess.Popen( ["cat", stage2_dockerfile], stdout=subprocess.PIPE ) as proc: - print("Running command: " + " ".join(stage2_build_args)) + logger.info("Running command: " + " ".join(stage2_build_args)) subprocess.check_call( stage2_build_args, stdin=proc.stdout, env=env, # this enables accessing dockerfile in subdir ) - print("Experiment gym image build finished.") + logger.info("Experiment gym image build finished.") return stage2_image_tag @@ -213,10 +213,10 @@ def create_docker_network(network_name: str) -> None: network_name (str): name of docker network. """ - print("Creating docker network ...") + logger.info("Creating docker network ...") try: args = ["docker", "network", "create", network_name] subprocess.check_call(args) - print("Docker network created.") + logger.info("Docker network created.") except subprocess.CalledProcessError: - print("No new network created. Network may already exist.") + logger.info("No new network created. Network may already exist.") diff --git a/beobench/experiment/provider.py b/beobench/experiment/provider.py index c69f786e..a938b06c 100644 --- a/beobench/experiment/provider.py +++ b/beobench/experiment/provider.py @@ -39,7 +39,11 @@ def create_env(env_config: dict = None) -> object: for wrapper_dict in config["wrappers"]: wrapper = _get_wrapper(wrapper_dict) - env = wrapper(env, **wrapper_dict["config"]) + if "config" in wrapper_dict.keys(): + wrapper_config = wrapper_dict["config"] + else: + wrapper_config = {} + env = wrapper(env, **wrapper_config) return env diff --git a/beobench/experiment/scheduler.py b/beobench/experiment/scheduler.py index c198b0d9..500369a5 100644 --- a/beobench/experiment/scheduler.py +++ b/beobench/experiment/scheduler.py @@ -23,8 +23,12 @@ import beobench.experiment.containers import beobench.experiment.config_parser import beobench.utils +import beobench.logging +from beobench.logging import logger from beobench.constants import CONTAINER_DATA_DIR, CONTAINER_RO_DIR, AVAILABLE_AGENTS +beobench.logging.setup() + def run( config: Union[str, dict, pathlib.Path, list] = None, @@ -94,7 +98,7 @@ def run( num_samples (int, optional): number of experiment samples to run. This defaults to a single sample, i.e. just running the experiment once. """ - print("Beobench: starting experiment run ...") + logger.info("Starting experiment run ...") # parsing relevant kwargs and adding them to config kwarg_config = _create_config_from_kwargs( local_dir=local_dir, @@ -134,9 +138,9 @@ def run( for i in range(1, num_samples + 1): # TODO: enable checking whether something is run in container # and do not print the statement below if inside experiment container. - print( + logger.info( ( - f"Beobench: running experiment in container with environment " + f"Running experiment in container with environment " f"{config['env']['name']}" f" and agent from {config['agent']['origin']}. Sample {i} of" f" {num_samples}." @@ -152,6 +156,8 @@ def run( # Execute experiment # (this is usually reached from inside an experiment container) + logger.info("Running agent script.") + container_ro_dir_abs = CONTAINER_RO_DIR.absolute() args = [ "python", @@ -301,9 +307,10 @@ def _build_and_run_in_container(config: dict) -> None: arg_str = " ".join(args) if wandb_api_key: arg_str = arg_str.replace(wandb_api_key, "") - print(f"Executing docker command: {arg_str}") + logger.info(f"Executing docker command: {arg_str}") - subprocess.check_call(args) + # subprocess.check_call(args) + beobench.utils.run_command(args, process_name="container") def _create_config_from_kwargs(**kwargs) -> dict: diff --git a/beobench/logging.py b/beobench/logging.py new file mode 100644 index 00000000..9599ea98 --- /dev/null +++ b/beobench/logging.py @@ -0,0 +1,37 @@ +"""Logging utilities for Beobench.""" + +from loguru import logger +import sys + + +def setup(include_time=False) -> None: + """Setup Beobench loguru logging setup.""" + if include_time: + time_str = "[{time:YYYY-MM-DD, HH:mm:ss.SSSS}] " + else: + time_str = "" + logger.remove() + logger.level("INFO", color="") + logger.add( + sys.stdout, + colorize=True, + format=( + "Beobench " + "⚡️" + f"{time_str}" + "{message}" + ), + ) + + +def log_subprocess(pipe, process_name="subprocess"): + """Log subprocess pipe. + + Adapted from from https://stackoverflow.com/a/21978778. + + Color setting of context is described in https://stackoverflow.com/a/33206814. + """ + for line in iter(pipe.readline, b""): # b'\n'-separated lines + context = f"\033[34m{process_name}:\033[0m" # .decode("ascii") + line = line.decode("utf-8").rstrip() + logger.info(f"{context} {line}") diff --git a/beobench/utils.py b/beobench/utils.py index 0c3f036c..72623a7e 100644 --- a/beobench/utils.py +++ b/beobench/utils.py @@ -1,6 +1,10 @@ """Module with a number of utility functions.""" import docker +import subprocess + +import beobench.logging +from beobench.logging import logger def check_if_in_notebook() -> bool: @@ -87,17 +91,19 @@ def merge_dicts( def shutdown() -> None: """Shut down all beobench and BOPTEST containers.""" - print("Stopping any remaining beobench and BOPTEST docker containers...") + beobench.logging.setup() + + logger.info("Stopping any remaining beobench and BOPTEST docker containers...") client = docker.from_env() container_num = 0 for container in client.containers.list(): if "auto_beobench" in container.name or "auto_boptest" in container.name: - print(f"Stopping container {container.name}") + logger.info(f"Stopping container {container.name}") container.stop(timeout=0) container_num += 1 - print(f"Stopped {container_num} container(s).") + logger.info(f"Stopped {container_num} container(s).") def restart() -> None: @@ -110,3 +116,19 @@ def restart() -> None: """ shutdown() + + +def run_command(cmd_line_args, process_name): + """Run command and log its output.""" + + process = subprocess.Popen( # pylint: disable=consider-using-with + cmd_line_args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + with process.stdout: + beobench.logging.log_subprocess( + process.stdout, + process_name=process_name, + ) + _ = process.wait() # 0 means success diff --git a/beobench/wrappers/energym.py b/beobench/wrappers/energym.py index 5cf6ca41..e718319c 100644 --- a/beobench/wrappers/energym.py +++ b/beobench/wrappers/energym.py @@ -26,7 +26,7 @@ def __init__(self, env: gym.Env, info_obs_weights: dict): def step(self, action): obs, _, done, info = self.env.step(action) - reward = sum( + reward = sum( # pylint: disable=consider-using-generator [info["obs"][key] * value for key, value in self.info_obs_weights.items()] ) return obs, reward, done, info diff --git a/pylintrc b/pylintrc index 2de9082d..d3a1c93a 100644 --- a/pylintrc +++ b/pylintrc @@ -155,12 +155,6 @@ disable=abstract-method, # mypackage.mymodule.MyReporterClass. output-format=text -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". This option is deprecated -# and it will be removed in Pylint 2.0. -files-output=no - # Tells whether to display a full report or only the messages reports=no @@ -283,7 +277,7 @@ single-line-if-stmt=yes # separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. # `trailing-comma` allows a space between comma and closing bracket: (a, ). # `empty-line` allows space-only lines. -no-space-check= +# no-space-check= # Maximum number of lines in a module max-module-lines=99999 diff --git a/requirements/dev_requirements.txt b/requirements/dev_requirements.txt index 13abc6a9..c2fa301b 100644 --- a/requirements/dev_requirements.txt +++ b/requirements/dev_requirements.txt @@ -19,15 +19,9 @@ build # PyPA build tool # Jupyter notebooks jupyterlab -# ML & RL tools -# gym -# torch -# ray[rllib] -# wandb - -# Convex solver tools (OPTIONAL) -# cvxpy - # for rst development (used by vscode) doc8 -rstcheck \ No newline at end of file +rstcheck + +# logging +loguru \ No newline at end of file diff --git a/requirements/doc_requirements.txt b/requirements/doc_requirements.txt index c443cb46..75d4176c 100644 --- a/requirements/doc_requirements.txt +++ b/requirements/doc_requirements.txt @@ -17,4 +17,7 @@ docutils==0.16 Jinja2<3.1 # For allowing type hints -sphinx-autodoc-typehints \ No newline at end of file +sphinx-autodoc-typehints + +# logging +loguru \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index e3abeb36..85aec9ae 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.5.0 +current_version = 0.5.1 commit = True tag = True diff --git a/setup.py b/setup.py index aca89ad4..f6b49640 100644 --- a/setup.py +++ b/setup.py @@ -10,13 +10,14 @@ with open("HISTORY.rst", encoding="UTF-8") as history_file: history = history_file.read() -version = "0.5.0" # pylint: disable=invalid-name +version = "0.5.1" # pylint: disable=invalid-name requirements = [ "docker", "click", "pyyaml", "importlib-resources", # backport of importlib.resources, required for Python<=3.8 + "loguru", ] # The extended requirements are only used inside experiment/gym containers