diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 38bdee6..4349b9d 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,9 +1,9 @@ version: 2 build: - os: "ubuntu-20.04" + os: "ubuntu-lts-latest" tools: - python: "mambaforge-4.10" + python: "mambaforge-latest" jobs: post_install: - conda list diff --git a/CITATION.cff b/CITATION.cff index 5111c5f..2d91c6c 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -9,7 +9,7 @@ type: software authors: - given-names: Madeline family-names: Scyphers - email: scyphers.3@osu.edu + email: mescyphers@gmail.com orcid: 'https://orcid.org/0000-0003-3733-4330' - given-names: Justine family-names: Missik @@ -54,4 +54,4 @@ references: title: 'BoTorch: A Framework for Efficient Monte-Carlo Bayesian Optimization' journal: 'Advances in Neural Information Processing Systems 33' year: 2020 - url: 'http://arxiv.org/abs/1910.06403' \ No newline at end of file + url: 'http://arxiv.org/abs/1910.06403' diff --git a/boa/__main__.py b/boa/__main__.py index 8914c09..fd5d664 100644 --- a/boa/__main__.py +++ b/boa/__main__.py @@ -1,218 +1,10 @@ -import os -import sys -import tempfile -from pathlib import Path +def main(): + from boa.cli import main as _main -import click -from attrs import fields_dict -from ax.storage.json_store.decoder import object_from_json - -from boa.config import BOAScriptOptions -from boa.controller import Controller -from boa.storage import scheduler_from_json_file -from boa.wrappers.script_wrapper import ScriptWrapper -from boa.wrappers.wrapper_utils import cd_and_cd_back, load_jsonlike - - -@click.command() -@click.option( - "-c", - "--config-path", - type=click.Path(exists=True, dir_okay=False, path_type=Path), - help="Path to configuration YAML file.", -) -@click.option( - "-sp", - "--scheduler-path", - type=click.Path(), - default="", - help="Path to scheduler json file.", -) -@click.option( - "-wp", - "--wrapper-path", - type=click.Path(), - default="", - help="Path to where file where your wrapper is located. Used when loaded from scheduler json file," - " and the path to your wrapper has changed (such as when loading on a different computer then" - " originally ran from).", -) -@click.option( - "-wn", - "--wrapper-name", - type=str, - default="", - help="Name of the wrapper class to use. Used when loaded from scheduler json file,", -) -@click.option( - "-td", - "--temporary-dir", - is_flag=True, - show_default=True, - default=False, - help="Modify/add to the config file a temporary directory as the experiment_dir that will get deleted after running" - " (useful for testing)." - " This requires your Wrapper to have the ability to take experiment_dir as an argument" - " to ``load_config``. The default ``load_config`` does support this." - " This is also only done for initial run, not for reloading from scheduler json file.", -) -@click.option( - "--rel-to-config/--rel-to-here", # more cli friendly name for config option of rel_to_launch - default=None, - help="Define all path and dir options in your config file relative to where boa is launched from" - " instead of relative to the config file location (the default)" - " ex:" - " given working_dir=path/to/dir" - " if you don't pass --rel-to-here then path/to/dir is defined in terms of where your config file is" - " if you do pass --rel-to-here then path/to/dir is defined in terms of where you launch boa from", -) -def main(config_path, scheduler_path, wrapper_path, wrapper_name, temporary_dir, rel_to_config): - """Run experiment run from config path or scheduler path""" - - if temporary_dir: - with tempfile.TemporaryDirectory() as temp_dir: - experiment_dir = Path(temp_dir) - return run( - config_path, - scheduler_path=scheduler_path, - wrapper_path=wrapper_path, - wrapper_name=wrapper_name, - rel_to_config=rel_to_config, - experiment_dir=experiment_dir, - ) - return run(config_path, scheduler_path=scheduler_path, wrapper_path=wrapper_path, rel_to_config=rel_to_config) - - -def run(config_path, scheduler_path, rel_to_config, wrapper_path=None, wrapper_name=None, experiment_dir=None): - """Run experiment run from config path or scheduler path - - Parameters - ---------- - config_path - Path to configuration YAML file. - scheduler_path - Path to scheduler json file. - wrapper_path - Path to where file where your wrapper is located. Used when loaded from scheduler json file, - and the path to your wrapper has changed (such as when loading on a different computer then - originally ran from). - rel_to_config - Define all path and dir options in your config file relative to to your config file location - or rel_to_here (relative to cli launch) - experiment_dir - experiment output directory to save BOA run to, can only be specified during an initial run - (when passing in a config_path, not a scheduler_path) - - Returns - ------- - Scheduler - """ - config = {} - script_options = {} - if config_path: - config_path = Path(config_path).resolve() - config = load_jsonlike(config_path) - script_options = config.get("script_options", {}) - if rel_to_config is None: - rel_to_config = get_rel_from_script_options(script_options) - if scheduler_path: - scheduler_path = Path(scheduler_path).resolve() - if not config: - sch_jsn = load_jsonlike(scheduler_path) - config = object_from_json(sch_jsn["wrapper"]["config"]) - config_path = object_from_json(sch_jsn["wrapper"]["config_path"]) - script_options = config.get("script_options", {}) - if rel_to_config is None: - rel_to_config = get_rel_from_script_options(script_options) - - if experiment_dir: - experiment_dir = Path(experiment_dir).resolve() - wrapper_path = Path(wrapper_path).resolve() if wrapper_path else None - - if config_path and rel_to_config: - rel_path = config_path.parent - else: - rel_path = os.getcwd() - - options = get_config_options( - experiment_dir=experiment_dir, rel_path=rel_path, script_options=script_options, wrapper_path=wrapper_path - ) - - if wrapper_name: - options["wrapper_name"] = wrapper_name - - with cd_and_cd_back(options["working_dir"]): - if scheduler_path: - scheduler = scheduler_from_json_file(filepath=scheduler_path, wrapper_path=options["wrapper_path"]) - controller = Controller.from_scheduler(scheduler=scheduler, **options) - else: - if options["wrapper_path"] and Path(options["wrapper_path"]).exists(): - options["wrapper"] = options["wrapper_path"] - else: - options["wrapper"] = ScriptWrapper - controller = Controller( - config_path=config_path, - **options, - ) - controller.initialize_scheduler() - - scheduler = controller.run() - return scheduler - - -def get_rel_from_script_options(script_options): - rel_to_config = script_options.get("rel_to_config", None) or not script_options.get("rel_to_launch", None) - if rel_to_config is None: - rel_to_config = ( - fields_dict(BOAScriptOptions)["rel_to_config"].default - or not fields_dict(BOAScriptOptions)["rel_to_launch"].default - ) - return rel_to_config - - -def get_config_options(experiment_dir, rel_path, script_options: dict = None, wrapper_path=None): - script_options = script_options if script_options is not None else {} - wrapper_name = script_options.get("wrapper_name", fields_dict(BOAScriptOptions)["wrapper_name"].default) - append_timestamp = ( - script_options.get("append_timestamp", None) - if script_options.get("append_timestamp", None) is not None - else fields_dict(BOAScriptOptions)["append_timestamp"].default - ) - - wrapper_path = ( - wrapper_path - if wrapper_path is not None - else script_options.get("wrapper_path", fields_dict(BOAScriptOptions)["wrapper_path"].default) - ) - wrapper_path = _prepend_rel_path(rel_path, wrapper_path) if wrapper_path else wrapper_path - - working_dir = script_options.get("working_dir", fields_dict(BOAScriptOptions)["working_dir"].default) - working_dir = _prepend_rel_path(rel_path, working_dir) - - experiment_dir = experiment_dir or script_options.get( - "experiment_dir", fields_dict(BOAScriptOptions)["experiment_dir"].default - ) - experiment_dir = _prepend_rel_path(rel_path, experiment_dir) if experiment_dir else experiment_dir - - if working_dir: - sys.path.append(str(working_dir)) - - return dict( - append_timestamp=append_timestamp, - experiment_dir=experiment_dir, - working_dir=working_dir, - wrapper_name=wrapper_name, - wrapper_path=wrapper_path, - ) - - -def _prepend_rel_path(rel_path, path): - if not path: - return path - path = Path(path) - if not path.is_absolute(): - path = rel_path / path - return path.resolve() + # Main entry point + # Can be invoked with `python -m boa` + # or just `boa` (see pypoject.toml`) + _main() if __name__ == "__main__": diff --git a/boa/cli/__init__.py b/boa/cli/__init__.py new file mode 100644 index 0000000..8914c09 --- /dev/null +++ b/boa/cli/__init__.py @@ -0,0 +1,219 @@ +import os +import sys +import tempfile +from pathlib import Path + +import click +from attrs import fields_dict +from ax.storage.json_store.decoder import object_from_json + +from boa.config import BOAScriptOptions +from boa.controller import Controller +from boa.storage import scheduler_from_json_file +from boa.wrappers.script_wrapper import ScriptWrapper +from boa.wrappers.wrapper_utils import cd_and_cd_back, load_jsonlike + + +@click.command() +@click.option( + "-c", + "--config-path", + type=click.Path(exists=True, dir_okay=False, path_type=Path), + help="Path to configuration YAML file.", +) +@click.option( + "-sp", + "--scheduler-path", + type=click.Path(), + default="", + help="Path to scheduler json file.", +) +@click.option( + "-wp", + "--wrapper-path", + type=click.Path(), + default="", + help="Path to where file where your wrapper is located. Used when loaded from scheduler json file," + " and the path to your wrapper has changed (such as when loading on a different computer then" + " originally ran from).", +) +@click.option( + "-wn", + "--wrapper-name", + type=str, + default="", + help="Name of the wrapper class to use. Used when loaded from scheduler json file,", +) +@click.option( + "-td", + "--temporary-dir", + is_flag=True, + show_default=True, + default=False, + help="Modify/add to the config file a temporary directory as the experiment_dir that will get deleted after running" + " (useful for testing)." + " This requires your Wrapper to have the ability to take experiment_dir as an argument" + " to ``load_config``. The default ``load_config`` does support this." + " This is also only done for initial run, not for reloading from scheduler json file.", +) +@click.option( + "--rel-to-config/--rel-to-here", # more cli friendly name for config option of rel_to_launch + default=None, + help="Define all path and dir options in your config file relative to where boa is launched from" + " instead of relative to the config file location (the default)" + " ex:" + " given working_dir=path/to/dir" + " if you don't pass --rel-to-here then path/to/dir is defined in terms of where your config file is" + " if you do pass --rel-to-here then path/to/dir is defined in terms of where you launch boa from", +) +def main(config_path, scheduler_path, wrapper_path, wrapper_name, temporary_dir, rel_to_config): + """Run experiment run from config path or scheduler path""" + + if temporary_dir: + with tempfile.TemporaryDirectory() as temp_dir: + experiment_dir = Path(temp_dir) + return run( + config_path, + scheduler_path=scheduler_path, + wrapper_path=wrapper_path, + wrapper_name=wrapper_name, + rel_to_config=rel_to_config, + experiment_dir=experiment_dir, + ) + return run(config_path, scheduler_path=scheduler_path, wrapper_path=wrapper_path, rel_to_config=rel_to_config) + + +def run(config_path, scheduler_path, rel_to_config, wrapper_path=None, wrapper_name=None, experiment_dir=None): + """Run experiment run from config path or scheduler path + + Parameters + ---------- + config_path + Path to configuration YAML file. + scheduler_path + Path to scheduler json file. + wrapper_path + Path to where file where your wrapper is located. Used when loaded from scheduler json file, + and the path to your wrapper has changed (such as when loading on a different computer then + originally ran from). + rel_to_config + Define all path and dir options in your config file relative to to your config file location + or rel_to_here (relative to cli launch) + experiment_dir + experiment output directory to save BOA run to, can only be specified during an initial run + (when passing in a config_path, not a scheduler_path) + + Returns + ------- + Scheduler + """ + config = {} + script_options = {} + if config_path: + config_path = Path(config_path).resolve() + config = load_jsonlike(config_path) + script_options = config.get("script_options", {}) + if rel_to_config is None: + rel_to_config = get_rel_from_script_options(script_options) + if scheduler_path: + scheduler_path = Path(scheduler_path).resolve() + if not config: + sch_jsn = load_jsonlike(scheduler_path) + config = object_from_json(sch_jsn["wrapper"]["config"]) + config_path = object_from_json(sch_jsn["wrapper"]["config_path"]) + script_options = config.get("script_options", {}) + if rel_to_config is None: + rel_to_config = get_rel_from_script_options(script_options) + + if experiment_dir: + experiment_dir = Path(experiment_dir).resolve() + wrapper_path = Path(wrapper_path).resolve() if wrapper_path else None + + if config_path and rel_to_config: + rel_path = config_path.parent + else: + rel_path = os.getcwd() + + options = get_config_options( + experiment_dir=experiment_dir, rel_path=rel_path, script_options=script_options, wrapper_path=wrapper_path + ) + + if wrapper_name: + options["wrapper_name"] = wrapper_name + + with cd_and_cd_back(options["working_dir"]): + if scheduler_path: + scheduler = scheduler_from_json_file(filepath=scheduler_path, wrapper_path=options["wrapper_path"]) + controller = Controller.from_scheduler(scheduler=scheduler, **options) + else: + if options["wrapper_path"] and Path(options["wrapper_path"]).exists(): + options["wrapper"] = options["wrapper_path"] + else: + options["wrapper"] = ScriptWrapper + controller = Controller( + config_path=config_path, + **options, + ) + controller.initialize_scheduler() + + scheduler = controller.run() + return scheduler + + +def get_rel_from_script_options(script_options): + rel_to_config = script_options.get("rel_to_config", None) or not script_options.get("rel_to_launch", None) + if rel_to_config is None: + rel_to_config = ( + fields_dict(BOAScriptOptions)["rel_to_config"].default + or not fields_dict(BOAScriptOptions)["rel_to_launch"].default + ) + return rel_to_config + + +def get_config_options(experiment_dir, rel_path, script_options: dict = None, wrapper_path=None): + script_options = script_options if script_options is not None else {} + wrapper_name = script_options.get("wrapper_name", fields_dict(BOAScriptOptions)["wrapper_name"].default) + append_timestamp = ( + script_options.get("append_timestamp", None) + if script_options.get("append_timestamp", None) is not None + else fields_dict(BOAScriptOptions)["append_timestamp"].default + ) + + wrapper_path = ( + wrapper_path + if wrapper_path is not None + else script_options.get("wrapper_path", fields_dict(BOAScriptOptions)["wrapper_path"].default) + ) + wrapper_path = _prepend_rel_path(rel_path, wrapper_path) if wrapper_path else wrapper_path + + working_dir = script_options.get("working_dir", fields_dict(BOAScriptOptions)["working_dir"].default) + working_dir = _prepend_rel_path(rel_path, working_dir) + + experiment_dir = experiment_dir or script_options.get( + "experiment_dir", fields_dict(BOAScriptOptions)["experiment_dir"].default + ) + experiment_dir = _prepend_rel_path(rel_path, experiment_dir) if experiment_dir else experiment_dir + + if working_dir: + sys.path.append(str(working_dir)) + + return dict( + append_timestamp=append_timestamp, + experiment_dir=experiment_dir, + working_dir=working_dir, + wrapper_name=wrapper_name, + wrapper_path=wrapper_path, + ) + + +def _prepend_rel_path(rel_path, path): + if not path: + return path + path = Path(path) + if not path.is_absolute(): + path = rel_path / path + return path.resolve() + + +if __name__ == "__main__": + main() diff --git a/boa/config/converters.py b/boa/config/converters.py index 875c3fa..0b42b9d 100644 --- a/boa/config/converters.py +++ b/boa/config/converters.py @@ -5,11 +5,18 @@ import ax.early_stopping.strategies as early_stopping_strats import ax.global_stopping.strategies as global_stopping_strats +import botorch.acquisition +import botorch.models +import gpytorch.kernels +import gpytorch.mlls from ax.modelbridge.generation_node import GenerationStep from ax.modelbridge.registry import Models +from ax.models.torch.botorch_modular.surrogate import Surrogate from ax.service.utils.instantiation import TParameterRepresentation from ax.service.utils.scheduler_options import SchedulerOptions +from boa.utils import check_min_package_version + if TYPE_CHECKING: from .config import BOAMetric @@ -49,6 +56,42 @@ def _gen_strat_converter(gs: Optional[dict] = None) -> dict: gs["steps"][i] = step steps.append(step) continue + if step["model"] == "BOTORCH_MODULAR" and not check_min_package_version("ax-platform", "0.3.5"): + raise ValueError( + "BOTORCH_MODULAR model is not available in BOA with Ax version < 0.3.5. " + "Please upgrade to a newer version of Ax." + ) + + if "model_kwargs" in step: + if "botorch_acqf_class" in step["model_kwargs"] and not isinstance( + step["model_kwargs"]["botorch_acqf_class"], botorch.acquisition.AcquisitionFunction + ): + step["model_kwargs"]["botorch_acqf_class"] = getattr( + botorch.acquisition, step["model_kwargs"]["botorch_acqf_class"] + ) + + if "surrogate" in step["model_kwargs"]: + if "mll_class" in step["model_kwargs"]["surrogate"] and not isinstance( + step["model_kwargs"]["surrogate"]["mll_class"], gpytorch.mlls.MarginalLogLikelihood + ): + step["model_kwargs"]["surrogate"]["mll_class"] = getattr( + gpytorch.mlls, step["model_kwargs"]["surrogate"]["mll_class"] + ) + if "botorch_model_class" in step["model_kwargs"]["surrogate"] and not isinstance( + step["model_kwargs"]["surrogate"]["botorch_model_class"], botorch.models.model.Model + ): + step["model_kwargs"]["surrogate"]["botorch_model_class"] = getattr( + botorch.models, step["model_kwargs"]["surrogate"]["botorch_model_class"] + ) + if "covar_module_class" in step["model_kwargs"]["surrogate"] and not isinstance( + step["model_kwargs"]["surrogate"]["covar_module_class"], gpytorch.kernels.Kernel + ): + step["model_kwargs"]["surrogate"]["covar_module_class"] = getattr( + gpytorch.kernels, step["model_kwargs"]["surrogate"]["covar_module_class"] + ) + + step["model_kwargs"]["surrogate"] = Surrogate(**step["model_kwargs"]["surrogate"]) + try: step["model"] = Models[step["model"]] except KeyError: diff --git a/boa/logger.py b/boa/logger.py index 078e369..7d4d5d7 100644 --- a/boa/logger.py +++ b/boa/logger.py @@ -1,15 +1,12 @@ import logging import logging.config import logging.handlers -import multiprocessing from boa.definitions import PathLike DEFAULT_LOG_LEVEL: int = logging.INFO ROOT_LOGGER_NAME = "boa" -queue = multiprocessing.Manager().Queue() - def get_logger(name: str = ROOT_LOGGER_NAME, level: int = DEFAULT_LOG_LEVEL, filename=None) -> logging.Logger: """Get a logger. diff --git a/boa/plot.py b/boa/plot.py index 8c71ea8..ab892e7 100644 --- a/boa/plot.py +++ b/boa/plot.py @@ -3,9 +3,11 @@ Plotting & EDA CLI ################################### -You can launch a basic EDA plot view +You can launch a basic EDA plot dashboard view of your optimization with:: + boa.plot path/to/scheduler.json + or python -m boa.plot path/to/scheduler.json diff --git a/boa/registry.py b/boa/registry.py index e7b4773..1b083b4 100644 --- a/boa/registry.py +++ b/boa/registry.py @@ -1,4 +1,22 @@ -from ax.storage.json_store.registry import CORE_DECODER_REGISTRY, CORE_ENCODER_REGISTRY +from __future__ import annotations + +from typing import Type + +import botorch.acquisition +import gpytorch.kernels +from ax.storage.botorch_modular_registry import ( + ACQUISITION_FUNCTION_REGISTRY, + CLASS_TO_REGISTRY, + CLASS_TO_REVERSE_REGISTRY, +) +from ax.storage.json_store.registry import ( + CORE_CLASS_DECODER_REGISTRY, + CORE_CLASS_ENCODER_REGISTRY, + CORE_DECODER_REGISTRY, + CORE_ENCODER_REGISTRY, + botorch_modular_to_dict, + class_from_json, +) def config_to_dict(inst): @@ -15,3 +33,25 @@ def _add_common_encodes_and_decodes(): CORE_ENCODER_REGISTRY[BOAConfig] = config_to_dict # CORE_DECODER_REGISTRY[BOAConfig.__name__] = BOAConfig CORE_DECODER_REGISTRY[MetricType.__name__] = MetricType + + CORE_CLASS_DECODER_REGISTRY["Type[Kernel]"] = class_from_json + CORE_CLASS_ENCODER_REGISTRY[gpytorch.kernels.Kernel] = botorch_modular_to_dict + + KERNEL_REGISTRY = {getattr(gpytorch.kernels, kernel): kernel for kernel in gpytorch.kernels.__all__} + + REVERSE_KERNEL_REGISTRY: dict[str, Type[gpytorch.kernels.Kernel]] = {v: k for k, v in KERNEL_REGISTRY.items()} + + CLASS_TO_REGISTRY[gpytorch.kernels.Kernel] = KERNEL_REGISTRY + CLASS_TO_REVERSE_REGISTRY[gpytorch.kernels.Kernel] = REVERSE_KERNEL_REGISTRY + + for acq_func_name in botorch.acquisition.__all__: + acq_func = getattr(botorch.acquisition, acq_func_name) + if acq_func not in ACQUISITION_FUNCTION_REGISTRY: + ACQUISITION_FUNCTION_REGISTRY[acq_func] = acq_func_name + + REVERSE_ACQUISITION_FUNCTION_REGISTRY: dict[str, Type[botorch.acquisition.AcquisitionFunction]] = { + v: k for k, v in ACQUISITION_FUNCTION_REGISTRY.items() + } + + CLASS_TO_REGISTRY[botorch.acquisition.AcquisitionFunction] = ACQUISITION_FUNCTION_REGISTRY + CLASS_TO_REVERSE_REGISTRY[botorch.acquisition.AcquisitionFunction] = REVERSE_ACQUISITION_FUNCTION_REGISTRY diff --git a/boa/runner.py b/boa/runner.py index 7e26547..bbd46d1 100644 --- a/boa/runner.py +++ b/boa/runner.py @@ -9,6 +9,7 @@ import concurrent.futures import logging +import multiprocessing from collections import defaultdict from typing import Any, Dict, Iterable, Set @@ -16,7 +17,7 @@ from ax.core.runner import Runner from ax.core.trial import Trial -from boa.logger import get_logger, queue +from boa.logger import get_logger from boa.metaclasses import RunnerRegister from boa.utils import serialize_init_args from boa.wrappers.base_wrapper import BaseWrapper @@ -28,6 +29,7 @@ class WrappedJobRunner(Runner, metaclass=RunnerRegister): def __init__(self, wrapper: BaseWrapper = None, *args, **kwargs): self.wrapper = wrapper or BaseWrapper() + self.queue = multiprocessing.Manager().Queue() super().__init__(*args, **kwargs) def run(self, trial: Trial) -> Dict[str, Any]: @@ -42,7 +44,7 @@ def run(self, trial: Trial) -> Dict[str, Any]: Returns: Dict of run metadata from the deployment process. """ - qh = logging.handlers.QueueHandler(queue) + qh = logging.handlers.QueueHandler(self.queue) logger = logging.getLogger() logger.addHandler(qh) ax_logger = logging.getLogger("ax") @@ -119,7 +121,7 @@ def to_dict(self) -> dict: parents = self.__class__.mro()[1:] # index 0 is the class itself - properties = serialize_init_args(self, parents=parents, match_private=True, exclude_fields=["wrapper"]) + properties = serialize_init_args(self, parents=parents, match_private=True, exclude_fields=["wrapper", "queue"]) properties["__type"] = self.__class__.__name__ return properties diff --git a/docs/assets/BO_workflow_diagram.png b/docs/assets/BO_workflow_diagram.png new file mode 100644 index 0000000..f6d7163 Binary files /dev/null and b/docs/assets/BO_workflow_diagram.png differ diff --git a/docs/examples/1run_r_streamlined.ipynb b/docs/examples/1run_r_streamlined.ipynb index 32802a5..bfaab34 100644 --- a/docs/examples/1run_r_streamlined.ipynb +++ b/docs/examples/1run_r_streamlined.ipynb @@ -1507,13 +1507,13 @@ "To run our script we just need to path the config file to BOA's CLI\n", "\n", "```python\n", - "python -m boa --config-file path/to/config.yaml\n", + "boa --config-file path/to/config.yaml\n", "```\n", "\n", "or\n", "\n", "```python\n", - "python -m boa -c path/to/config.yaml\n", + "boa -c path/to/config.yaml\n", "```" ] }, @@ -1891,7 +1891,7 @@ } ], "source": [ - "output = !python -m boa -c {config_path} # we capture ipython terminal output to python variable\n", + "output = !boa -c {config_path} # we capture ipython terminal output to python variable\n", "o = \"\\n\".join(ln for ln in output) # it comes in as a list, we convert to string\n", "o = o.replace(str(r_dir), \"[/path/to/your/dir/]\") # replace the actual dir with a stand in for privacy reasons\n", "Code(o)" diff --git a/docs/examples/2EDA_from_r_run.ipynb b/docs/examples/2EDA_from_r_run.ipynb index 16d146d..1819583 100644 --- a/docs/examples/2EDA_from_r_run.ipynb +++ b/docs/examples/2EDA_from_r_run.ipynb @@ -24,11 +24,11 @@ "From the command line, we can issue\n", "\n", "```python\n", - "python -m boa.plot --scheduler-path path/to/scheduler.json\n", + "boa.plot --scheduler-path path/to/scheduler.json\n", "\n", "or\n", "\n", - "python -m boa.plot -sp path/to/scheduler.json\n", + "boa.plot -sp path/to/scheduler.json\n", "```\n", "\n", "```{attention} \n", diff --git a/docs/examples/example_py_run.rst b/docs/examples/example_py_run.rst index bde2faa..16865d9 100644 --- a/docs/examples/example_py_run.rst +++ b/docs/examples/example_py_run.rst @@ -30,7 +30,7 @@ You can start and run your optimization like this: .. code-block:: console - $ python -m boa -c config.json + $ boa -c config.json Start time: 20221026T210522 [INFO 10-26 21:05:22] ax.service.utils.instantiation: Inferred value type of ParameterType.FLOAT for parameter x0. If that is not the expected value type, you can explicity specify 'value_type' ('int', 'float', 'bool' or 'str') in parameter dict. [INFO 10-26 21:05:22] ax.service.utils.instantiation: Inferred value type of ParameterType.FLOAT for parameter x1. If that is not the expected value type, you can explicity specify 'value_type' ('int', 'float', 'bool' or 'str') in parameter dict. diff --git a/docs/examples/index.rst b/docs/examples/index.rst index 9ab6c1c..6d146cb 100644 --- a/docs/examples/index.rst +++ b/docs/examples/index.rst @@ -2,6 +2,23 @@ Examples ######## +.. toctree:: + :caption: Examples with Synthetic Functions in R on the BOA Paper Examples Page (https://boa-paper.readthedocs.io/en/latest/) + :maxdepth: 2 + + Branin + Branin Saasbo - High Dimensional Optimization + Hartmann6 + Hartmann6 with Constraints + Hartmann6 with Global Stopping Strategy + +.. toctree:: + :caption: Examples with Environmental Models on the BOA Paper Examples Page (https://boa-paper.readthedocs.io/en/latest/) + :maxdepth: 2 + + SWAT+ + FERTCH3.14 + .. toctree:: :caption: Running Through the CLI (Command Line Interface) :maxdepth: 2 @@ -22,7 +39,7 @@ Examples .. toctree:: - :caption: Examples from Models Using BOA + :caption: Processing Model Outputs from a Optimization of a Real Model :maxdepth: 2 - cached_notebooks/example_optimization_results \ No newline at end of file + cached_notebooks/example_optimization_results diff --git a/docs/index.rst b/docs/index.rst index 2723a82..25ff280 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,32 +1,20 @@ Welcome to boa's documentation! ==================================== -:doc:`BOA's ` is a high-level Bayesian optimization framework and model wrapping tool. It provides an easy-to-use interface -between models and the python libraries `Ax `_ and `BoTorch `_. +BOA is a high-level Bayesian optimization framework and model-wrapping toolkit. It is designed to be highly flexible and easy-to-use. BOA is built upon the lower-level packages `Ax `_ (Adaptive Experimentation Platform, https://ax.dev//index.html) and `BoTorch `_ to do the heavy lifting of the BO process and subsequent analysis. It supplements these lower-level packages with model-wrapping tools, language-agnostic features, and a flexible interface framework. + Key features ------------ -- **Model agnostic** - - - Can be used for models in any language (not just python) - - Can be used for Wrappers in any language (You don't even need to write any python!) See :mod:`Script Wrapper <.script_wrapper>` for details on how to do that. - - Simple to implement for new models, with minimal coding required - -- **Scalable** - - - Can be used for simple models or complex models that require a lot of computational resources - - Scheduler to manage individual model runs - - Supports parallelization - -- **Modular & customizable** - - - Can take advantages of the many features of Ax/BoTorch - - Customizable objective functions, multi-objective optimization, acquisition functions, etc - - Choice of built-in evaluation metrics, but it’s also easy to implement custom metrics - -.. important:: - - This site is still under construction. More content will be added soon! +- **Language-Agnostic**: Although BOA itself is written in Python, users do not need to write any Python code in order to use it. The user’s model, as well as the model wrapper, can be written in any programming language. Users can configure and run an optimization, save outputs, and view results entirely without writing any Python code. This allows the user to write their code in any language they want, or even reuse processing code they already have, and still have access to two of the most full-featured BO (BoTorch) and GP (GPyTorch) libraries available today. +- **Scalability and Parallelization**: BOA handles optimization tasks of any size, from small problems to large, complex models. It supports parallel evaluations, allowing multiple optimization trials to run at the same time. This greatly reduces optimization time, especially when using powerful computing resources like supercomputing clusters. In many other BO packages, even if batched trial evaluation is supported, the actual parallelization implementation is left as an exercise to the user. +- **Reducing Boilerplate Code**: BOA aims to reduce the amount of boilerplate code often needed to set up and launch optimizations. BOA does this by providing an application programming interface (API) to the lower-level BO libraries BoTorch and Ax that it is built upon. This API is responsible for initializing, starting, and controlling the user’s optimization. The BOA API can be accessed and controlled almost entirely through a human readable, text based, YAML configuration file, reducing the need to write boilerplate setup code. +- **Automatic Saving and Resuming**: BOA automatically saves the state of an optimization process, allowing users to pause and resume optimizations easily. This ensures continuous progress and makes it easy to recover and retrieve results, even if there are interruptions or system crashes, making the workflow more resilient and user-friendly. Users can also add additional trials to a completed optimization or explore incoming results as the optimization is still running. +- **Support for Multi-Objective Optimization**: Streamlined and customizable support for multi-objective optimization. +- **Handling High-Dimensional and Complex Models**: Support for high-dimensional problems. +- **Customizability**: BOA allows customization of the optimization process as needed, including adding constraints, adjusting the kernel or acquisition function, or incorporating an early stopping criterion. + +Head over to the :doc:`Bayesian Optimization overview page ` to read about Bayesian Optimization and how it works. Contents -------- diff --git a/docs/user_guide/bo_overview.md b/docs/user_guide/bo_overview.md new file mode 100644 index 0000000..8377ef0 --- /dev/null +++ b/docs/user_guide/bo_overview.md @@ -0,0 +1,100 @@ +# Basics of Bayesian Optimization + +## Fundamental concepts + +Bayesian Optimization (BO) is a statistical method to optimize an objective function f over some feasible search space 𝕏. For example, f could be the difference between model predictions and observed values of a particular variable. BO relies on constructing a probabilistic surrogate model for f that is leveraged to make decisions about where to query from 𝕏 for future evaluations (Brochu et al., 2010; Frazier 2018). BO builds the surrogate model using all previous evaluations, resulting in a process that can find optima of non-convex problems in relatively few evaluations compared to methods that rely on more local information like gradients or more exhaustive approaches like grid search (Snoek et al. 2012). With this approach, the trade-off is that building the surrogate model is more computationally expensive than other optimization methods, resulting in the time between evaluations being larger (Snoek et al. 2012). However, when evaluation time of f is large, which is the case with many environmental models, the trade-off of some extra computation time to build the surrogate model is well worth it for the benefit of fewer overall evaluations (Snoek et al. 2012). + +BO is particularly well-suited to the following application challenges: + +- Black-Box Functions: Functions without a known closed-form expression or where the derivative is not readily available (Phan-Trong et al., 2023). +- Expensive Functions: Functions where each evaluation incurs a high computational or financial cost, typically due to long computation time and large requirement of computational resources (Snoek et al., 2012). +- Noisy Functions: Functions where evaluations are noisy and non-deterministic (Daulton et al., 2021). +- Multi-modal Functions: Functions that are non-convex with potentially a large number of local optima where traditional gradient-based methods can get stuck (Riche and Picheny, 2021). +- Limited Budget for Evaluations: When the number of times the function can be evaluated is limited (Frazier 2018). +- High-Dimensional Spaces: Although more challenging, recent advancement have allowed BO to be applied to problems with in high-dimensional spaces (Eriksson and Jankowiak 2021; Moriconi et al. 2020). + +Given a finite computational resource, we can only afford a limited number of evaluations in the process of optimizing f. Therefore, we want to evaluate f a relatively small number of times. Because f is expensive (as is the case with many environmental models), we cannot use an evaluation “heavy” method like grid search or genetic algorithms. We need a way to extrapolate relatively few evaluations to a prediction about f as a whole. To do this, we build a surrogate model, a simpler, cheaper to evaluate approximation of the actual function f (Brochu et al., 2010; Frazier 2018; Riche and Picheny, 2021; Snoek et al. 2012). + +In BO, the surrogate model is most typically modeled as a Gaussian Process (GP) (Gardner et al., 2018; Rasmussen and Williams, 2006). A GP is a powerful and flexible tool which uses a multivariate normal distribution to quantify the prediction uncertainty for any finite set of observations, and is specified by: + +1. A mean function, m(x), that represents the expected value of the f at any given point in the domain (Frazier 2018; Snoek et al. 2012).. +1. A covariance function, usually called a kernel, which defines the covariance (or similarity) between any given input pairs in the search space. The kernel encodes assumptions we have about f, such as smoothness and periodicity (Frazier 2018; Snoek et al. 2012). + +Typically, an optimization starts with a small number of sampling trials representing a good initial spread across the input space. A commonly used method for this initial sampling is Sobol sequences (Sobol’, 1967), a quasi-random, low-discrepancy sequence generating algorithm. Sobol generates points to fill the search space more randomly than a grid but more evenly than random sampling, which guarantees coverage of the search space while avoiding sampling biases that can occur with grids. These initial samples provide a basis for building the surrogate model, which then predicts f’s output and provides an estimate of the uncertainty of those predictions, resulting in a posterior distribution (Balandat et al., 2020; Frazier 2018; Snoek et al. 2012). + +Next, we need to select a “best” point from our posterior distribution to query from f next. There is no universally correct method for selecting this point; options include picking the point most likely to improve, the point expected to provide the largest average improvement, or a point that balances potential improvement with uncertainty about that improvement. We use heuristics called acquisition functions to make this selection. An acquisition function defines the search strategy and must balance the trade-off between exploration and exploitation trade-off. Exploration refers to querying points with high uncertainty, which often means higher risk but higher reward and more information gathered (Frazier 2018; Snoek et al. 2012). Exploitation refers to sampling points in regions the surrogate model already has high confidence in improving the objective function f (Balandat et al., 2020; Frazier 2018; Snoek et al. 2012). The three most common acquisition functions used in BO are Probability of Improvement (PI), Expected Improvement (EI), and Upper Confidence Bound (UCB). + + +![BO_workflow_diagram.png](../assets/BO_workflow_diagram.png) + +This showcases an example of BO on a 1D toy problem. The figure shows 3 iterations of an in-progress BO experiment (potentially after an initial Sobol sampling). It shows a GP approximation of the objective function (the posterior mean and posterior uncertainty) as well as the acquisition function. We can see that the acquisition function is high near current high points (exploitation) as well as where there is a high degree of uncertainty (exploration). + +## BO Algorithm Outline +To optimize f using BO: + +1. Query initial Points: Evaluate f at a set of initial points {x1, x2, …,xn} +1. Build Surrogate Model: Build a GP model using the initial points and fit a posterior distribution +1. Calculate Acquisition Function: Calculate the acquisition function over the posterior to find the next evaluation point xn+1 +1. Evaluate: Evaluate the objective function at xn+1, obtaining f(xn+1) +1. Update the Surrogate Model: Update the surrogate model with the new data point (xn+1, f(xn+1)) +1. Repeat: Repeat steps 3-5 until the stopping criteria are met. Stopping criteria could be a convergence stopping condition or a specified number of trials. + +## Constraints + +Constraints are often used to ensure that solutions meet specific criteria or stay within feasible regions defined by the problem's requirements. The set 𝕏 can be easily restricted by applying linear constraints to the parameters. The problem can be further generalized by adding additional constraining functions, gi ≥ 0 for i = 1, …, n, where gi are constraints that can be as expensive to evaluate as f (Frazier 2018). These additional constraints are known as black-box constraints (Balandat et al., 2020). + +## Multi-objective optimization + +Many problems suitable for optimization (e.g., complex physical models) lack an obvious single objective to optimize. In single-objective optimization, the goal is to find the best solution based on one criterion, such as maximizing profit or minimizing cost. However, real-world scenarios often require balancing multiple, conflicting objectives. For example, designing a car involves considering speed, fuel efficiency, and safety, where improving one aspect can worsen another. + +In multi-objective optimization, there isn't a single 'best' solution. Instead, the aim is to find a set of Pareto-optimal solutions, where no improvement can be made in one objective without degrading another (Emmerich et al., 2006). The collection of all Pareto-optimal solutions constitutes the Pareto front. + +A common metric used to estimate the quality of a Pareto set is the hypervolume (Balandat et al., 2020; Chugh, 2020; Emmerich et al., 2006). The hypervolume indicator calculates the n-dimensional volume (with n being the number of objective functions) of the region in the objective space that is dominated by the solution set and bounded by a reference point. This reference point is typically chosen to be worse than the worst possible values for each objective, but if unknown, can be inferred (Balandat et al., 2020). The hypervolume measure approaches its maximum if the set it covers is the true Pareto set (Fleischer 2003). + +Due to the challenges in ranking and evaluating the optimal point from the Pareto front (Rao and Lakshmi, 2021; Wang and Rangaiah, 2017), which involves assessing the feasibility region, the reasonableness of parameters, and often employing techniques like multicriteria decision analysis (MCDA), some users opt to scalarize their multi-objective optimization into a single-objective optimization (Chugh, 2020; Rasmussen and Williams, 2006) by creating a scalarized (linear) combination of the multiple objectives (e.g., by using the hypervolume). This scalarization simplifies the decision-making process by reducing the multi-dimensional objective space into a single dimension, making it easier to identify and select a single optimal solution. + +## High Dimensionality + +BO is often limited to 10-20 parameters, which in the optimization space are the equivalent of dimensions (Frazier 2018; Moriconi et al. 2020). However, with recent algorithms such as Sparse Axis-Aligned Subspace Bayesian Optimization (SAASBO) (Eriksson and Jankowiak 2021), that number can be pushed into the hundreds of dimensions. SAASBO is a high-dimensional BO algorithm that acts as structured priors over the kernel hyperparameters, expressly assuming a hierarchy of feature relevance. That is, given the dimensions d∈D, some subset of dimensions impacts the objective significantly, while others moderately, and the rest negligibly. + +SAASBO’s length scales are proportional to the half-Cauchy distribution, an inverse squared distribution concentrating around 0. Though the half-Cauchy distribution favors near 0 values, it is heavy-tailed. Because the length scales dictate how correlated values in the search space are, the pooling around 0 has the effect of most parameters being “turned off”, while because of the heavy tails the few most sensitive parameters can escape and become “turned-on” (Eriksson and Jankowiak 2021). + +## Additional Resources + +For more tutorials on Bayesian Optimization, please refer to the following resources: + +- Botorch has a comprehensive tutorial on Bayesian Optimization: https://botorch.org/docs/overview +- This is a great interactive tutorial on Gaussian Processes and kernels: https://distill.pub/2019/visual-exploration-gaussian-processes/ +- University of Toronto has a good slide show tutorial on Bayesian Optimization: https://www.cs.toronto.edu/~rgrosse/courses/csc411_f18/tutorials/tut8_adams_slides.pdf + +Additionally, the following papers provide a more in-depth look at Bayesian Optimization: + +## References + +Balandat, M., Karrer, B., Jiang, D.R., Daulton, S., Letham, B., Gordon Wilson, A., Bakshy, E., 2020. BOTORCH: a framework for efficient Monte-Carlo Bayesian optimization, Proceedings of the 34th International Conference on Neural Information Processing Systems. Curran Associates Inc.: Vancouver, BC, Canada, pp. 21524–21538. + +Brochu, E., Cora, V.M., de Freitas, N., 2010. A Tutorial on Bayesian Optimization of Expensive Cost Functions, with Application to Active User Modeling and Hierarchical Reinforcement Learning. + +Chugh, T., 2020. Scalarizing functions in Bayesian multiobjective optimization, 2020 IEEE Congress on Evolutionary Computation (CEC). IEEE Press, pp. 1-8. + +Daulton, S., Balandat, M., Bakshy, E., 2021. Parallel Bayesian optimization of multiple noisy objectives with expected hypervolume improvement, 35th Conference on Neural Information Processing Systems. + +Emmerich, M.T.M., Giannakoglou, K.C., Naujoks, B., 2006. Single- and multiobjective evolutionary optimization assisted by Gaussian random field metamodels. IEEE Transactions on Evolutionary Computation 10(4) 421-439. + +Eriksson, D., Jankowiak, M., 2021. High-dimensional Bayesian optimization with sparse axis-aligned subspaces, In: Cassio de, C., Marloes, H.M. (Eds.), Proceedings of the Thirty-Seventh Conference on Uncertainty in Artificial Intelligence. PMLR: Proceedings of Machine Learning Research, pp. 493-503. + +Fleischer, M., 2003. The measure of Pareto optima applications to multi-objective metaheuristics. Springer Berlin Heidelberg: Berlin, Heidelberg, pp. 519-533. + +Frazier, P.I., 2018. Bayesian Optimization, Recent Advances in Optimization and Modeling of Contemporary Problems, pp. 255-278. + +Gardner, J.R., Pleiss, G., Bindel, D., Weinberger, K.Q., Wilson, A.G., 2018. GPyTorch: blackbox matrix-matrix Gaussian process inference with GPU acceleration, Proceedings of the 32nd International Conference on Neural Information Processing Systems. Curran Associates Inc.: Montréal, Canada, pp. 7587–7597. + +Moriconi, R., Deisenroth, M.P., Sesh Kumar, K.S., 2020. High-dimensional Bayesian optimization using low-dimensional feature spaces. Machine Learning 109(9) 1925-1943. + +Phan-Trong, D., Tran-The, H., Gupta, S., 2023. NeuralBO: A black-box optimization algorithm using deep neural networks. Neurocomputing 559 126776. + +Rasmussen, C.E., Williams, C.K.I., 2006. Gaussian Processes for Machine Learning. MIT Press, Cambridge, MA. + +Riche, R.L., Picheny, V., 2021. Revisiting Bayesian Optimization in the light of the COCO benchmark. Struct Multidisc Optim 64, 3063–3087. https://doi.org/10.1007/s00158-021-02977-1 + +Snoek, J., Larochelle, H., Adams, R.P., 2012. Practical Bayesian optimization of machine learning algorithms. Advances in Neural Information Processing Systems 25 2960-2968. diff --git a/docs/user_guide/customizing_gp_acq.md b/docs/user_guide/customizing_gp_acq.md new file mode 100644 index 0000000..6064990 --- /dev/null +++ b/docs/user_guide/customizing_gp_acq.md @@ -0,0 +1,115 @@ +# Customizing Your Gaussian Process and Acquisition Function + +BOA is designed with flexibility of options for selecting the acquisition function and +kernel. BOA could be used by advanced BO users that want to control detailed aspects of the +optimization. However, for non-domain experts of BO, BOA defaults to common sensible +choices. BOA defers to BoTorch's defaults of a Matern 5/2 kernel, one of the most widely +used and flexible choices for BO and GPs (Frazier, 2018; Riche and Picheny, 2021; Jakeman, +2023). This is considered to be a flexible and broadly applicable kernel (Riche and Picheny,2021) and it is used as the default by many other BO and GP toolkits (Akiba et al., 2019; +Balandat et al., 2020; Brea, 2023; Nogueira, 2014; Jakeman, 2023). Similarly, BOA defaults +to Expected Improvement (EI) for single-objective functions and Expected Hypervolume +Improvement (EHVI) for multi-objective problems, which are well-regarded for their +performance across a broad range of applications (Balandat et al., 2020; Daulton et al., 2021; +Frazier, 2018). Users do have the option to explicitly control steps in the optimization process +in the ‘generation strategy’ section of the configuration, for example, to control the number of +Sobol trials, if SAASBO should be utilized, etc. As an advanced use case, users can also +explicitly set up different kernels and acquisition functions for different stages in the +optimization if they so choose. On this page, we will go over both ways to customize, as well as link to some examples of how to do so. + +## General Control Options + +The `generation_strategy` section of the configuration file is where you can control the optimization process. This includes the number of Sobol trials, the number of initial points, the number of iterations, and whether to use SAASBO. The `generation_strategy` section is shown below with common options: + +```yaml +generation_strategy: + # Specific number of initialization trials, + # Typically, initialization trials are generated quasi-randomly. + num_initialization_trials: 50 + # Integer, with which to override the default max parallelism setting for all + # steps in the generation strategy returned from this function. Each generation + # step has a ``max_parallelism`` value, which restricts how many trials can run + # simultaneously during a given generation step. + max_parallelism_override: 10 + # Whether to use SAAS prior for any GPEI generation steps. + # See, BO overview page, high dimensionality section for more details. + use_saasbo: true + random_seed: 42 # Fixed random seed for the Sobol generator. +``` + +## Utilizing Ax's Predefined Kernels and Acquisition Functions + +Ax has a number of predefined kernels and acquisition function combos that can be used in the optimization process. Each of these sit inside a "step" inside the generation strategy, where your optimization is broken into a number of "steps" and each step can have its own kernel and acquisition function. For example, the first step is usually a Sobol step that does a quasi-random initialization of the optimization process. The second step could be a "GPEI" step (GPEI is the Ax model class name, and is the default used for single objective optimization) that uses the Matern 5/2 kernel and the batched noisy Expected Improvement acquisition function. + +```yaml + +generation_strategy: + steps: + - model: Sobol + num_trials: 50 + # specify the maximum number of trials to run in parallel + # + max_parallelism: 10 + - model: GPEI + num_trials: -1 # -1 means the rest of the trials + max_parallelism: 10 +``` + +Ax does not have a good spot in their docs currently that lists all the available kernels and acquisition functions combo models, but you can find them listed on their [api docs here](https://ax.dev/api/modelbridge.html#ax.modelbridge.registry.Models) and you can see the source code for the models by clicking the source link on the api docs page. Some of the available models are: + +- `GPEI`: Gaussian Process Expected Improvement, the default for single objective optimization, uses the Matern 5/2 kernel +- `GPKG`: Gaussian Process Knowledge Gradient, uses the Matern 5/2 kernel +- `SAASBO`: Sparse Axis-Aligned Subspace Bayesian Optimization, see [BO Overview High Dimensionality](bo_overview.md#high-dimensionality) for more details, uses the Matern 5/2 kernel and the batched noisy Expected Improvement acquisition function +- `Sobol`: Sobol initialization +- `MOO`: Gaussian Process Expected Hypervolume Improvement, uses the Matern 5/2 kernel + +If you want to specify your kernel and acquisition function, you can do so by creating a custom model. The way to do that is with the `BOTORCH_MODULAR` model. This model allows you to specify the kernel and acquisition function you want to use. Here is an example of how to use the `BOTORCH_MODULAR` model: + +```yaml +generation_strategy: + steps: + - model: SOBOL + num_trials: 5 + - model: BOTORCH_MODULAR + num_trials: -1 # No limitation on how many trials should be produced from this step + model_kwargs: + surrogate: + botorch_model_class: SingleTaskGP # BoTorch model class name + covar_module_class: RBFKernel # GPyTorch kernel class name + mll_class: LeaveOneOutPseudoLikelihood # GPyTorch MarginalLogLikelihood class name + botorch_acqf_class: qUpperConfidenceBound # BoTorch acquisition function class name + acquisition_options: + beta: 0.5 +``` + +In the above example, the `BOTORCH_MODULAR` model is used to specify the `SingleTaskGP` model class, the `RBFKernel` kernel class, and the `qUpperConfidenceBound` acquisition function class. The `qUpperConfidenceBound` acquisition function is a batched version of UpperConfidenceBound. The `beta` parameter is a hyperparameter of the acquisition function that controls the trade-off between exploration and exploitation. + +BoTorch model classes can be found in the [BoTorch model api documentation](https://botorch.org/docs/models) and the BoTorch acquisition functions can be found in the [BoTorch acquisition api documentation](https://botorch.org/api/acquisition.html). + +GPyTorch kernel classes can be found in the [GPyTorch kernel api documentation](https://gpytorch.readthedocs.io/en/latest/kernels.html). + +The GPyTorch MarginalLogLikelihood classes can be found in the [GPyTorch MarginalLogLikelihood api documentation](https://gpytorch.readthedocs.io/en/latest/marginal_log_likelihoods.html). But the only MLL class that for sure work currently are `ExactMarginalLogLikelihood` and `LeaveOneOutPseudoLikelihood`. Other MLL classes may work, but they have not been tested and are depended on some other implementation details in Ax. + + +```{caution} +The `BOTORCH_MODULAR` class is an area of Ax's code that is still under active development and a lot of components of it are very dependent on the current implementation of Ax, BoTorch, and GPyTorch, and therefore it is impossible to test every possible combination of kernel and acquisition function. Therefore, it is recommended to use when possible the predefined models that Ax provides. +``` + + + +## References + +Akiba, T., Sano, S., Yanase, T., Ohta, T., Koyama, M., 2019. Optuna: A next-generation hyperparameter optimization framework, Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining. Association for Computing Machinery: Anchorage, AK, USA, pp. 2623–2631. + +Balandat, M., Karrer, B., Jiang, D.R., Daulton, S., Letham, B., Gordon Wilson, A., Bakshy, E., 2020. BOTORCH: a framework for efficient Monte-Carlo Bayesian optimization, Proceedings of the 34th International Conference on Neural Information Processing Systems. Curran Associates Inc.: Vancouver, BC, Canada, pp. 21524–21538. + +Brea, J., 2023. BayesianOptimization.jl. https://github.com/jbrea/BayesianOptimization.jl (accessed 17 July 2024) + +Daulton, S., Balandat, M., Bakshy, E., 2021. Parallel Bayesian optimization of multiple noisy objectives with expected hypervolume improvement, 35th Conference on Neural Information Processing Systems. + +Frazier, P.I., 2018. Bayesian Optimization, Recent Advances in Optimization and Modeling of Contemporary Problems, pp. 255-278. + +Jakeman, J.D., 2023. PyApprox: A software package for sensitivity analysis, Bayesian inference, optimal experimental design, and multi-fidelity uncertainty quantification and surrogate modeling. Environmental Modelling & Software 170 105825. + +Nogueira, F., 2014. Bayesian Optimization: Open source constrained global optimization tool for Python. https://github.com/bayesian-optimization/BayesianOptimization (accessed 17 July 2024) + +Riche, R.L., Picheny, V., 2021. Revisiting Bayesian Optimization in the light of the COCO benchmark. Struct Multidisc Optim 64, 3063–3087. https://doi.org/10.1007/s00158-021-02977-1 diff --git a/docs/user_guide/getting_started.rst b/docs/user_guide/getting_started.rst index 926b726..9c3de50 100644 --- a/docs/user_guide/getting_started.rst +++ b/docs/user_guide/getting_started.rst @@ -30,7 +30,7 @@ Here are instructions on how to install python through Anaconda or miniconda: Install boa =========== -If using conda and you don't already have a dedicated conda environment for your model:: +If using conda and you don't already have a dedicated conda environment for your model (if using mamba, replace all conda calls with mamba):: conda create -n boa conda activate boa @@ -42,11 +42,11 @@ or if not using Python, make sure you have a virtual environment create:: and activate the virtual environment, on windows:: - venv\Scripts\activate.bat + . venv\Scripts\activate.bat on linux and mac:: - source tutorial-env/bin/activate + . venv/bin/activate Once your environment is activated, if using conda, run:: @@ -78,6 +78,10 @@ so if on either of those, it should install pytorch>2 by default but if not and something doesn't work, upgrade pytorch, torchvision, and torchaudio +and then activate this environment:: + + conda activate boa-dev + :doc:`/contributing` ******** @@ -86,10 +90,16 @@ Test run Once everything is installed, run the test script to ensure everything is install properly:: - python -m boa.scripts.run_branin + boa.scripts.run_branin If this test case runs successfully, you can move on to the next steps. +If you encounter a problem, make sure you environment that you install boa into is activated (`conda activate boa`, `. venv/bin/activate`, or `. venv\Scripts\activate.bat` for conda or pip (unix or windows), see above)), and then try again. If it still doesn't work, you can try:: + + python -m boa.scripts.run_branin + +If this works, it may mean it installed an older version of BOA or that the boa command is not found on your path. If this works, you should be able to run boa commands by using the `python -m boa` instead of just `boa` and the plot command with `python -m boa.plot`. + ********** Update BOA ********** @@ -105,3 +115,10 @@ if using pip to install BOA, run:: pip install -U boa-framework If you have errors, see the :doc:`/troubleshooting` section. + + +****************************** +Bayesian Optimization Overview +****************************** + +Head over to the :doc:`/user_guide/bo_overview` to read about Bayesian Optimization and how it works. diff --git a/docs/user_guide/index.rst b/docs/user_guide/index.rst index 1848a53..240078d 100644 --- a/docs/user_guide/index.rst +++ b/docs/user_guide/index.rst @@ -6,5 +6,7 @@ User guide :maxdepth: 2 getting_started + bo_overview package_overview + customizing_gp_acq /api/boa.wrappers diff --git a/docs/user_guide/package_overview.rst b/docs/user_guide/package_overview.rst index e55849e..6ec641c 100644 --- a/docs/user_guide/package_overview.rst +++ b/docs/user_guide/package_overview.rst @@ -26,8 +26,6 @@ Objective functions When specifying your objective function to minimize or maximize, :doc:`BOA ` comes with a number of metrics you can use with your model output, such as MSE, :math:`R^2`, and others. For a list of current list of premade available of metrics, see See :mod:`.metrics.metrics` - - ************************************************************************ Creating a model wrapper (Language Agnostic or Python API) ************************************************************************ @@ -39,13 +37,24 @@ and there is a standard interface to follow. See the :mod:`instructions for creating a model wrapper <.boa.wrappers>` for details. +See the :doc:`examples of model wrappers <.boa.wrappers>` for examples. +See :doc:`tutorials ` for a number of examples of model wrappers in both Python and R. + + +************************************************************************ +Choosing a Custom Kernel and Acquisition Function +************************************************************************ + +BOA tries to make it easy to use the default kernel and acquisition function, but if you need to specify a different kernel or acquisition function, you can do so in the configuration file. + +See :doc:`details on how to specify kernel and acquisition function ` for details. **************************************************** Creating a Python launch script (Usually Not Needed) **************************************************** Most of the time you won't need to write a launch script because BOA has an built-in launch script in -its :mod:`.controller` that is called when calling `python -m boa`. But if you do need more control over your launch script than the default +its :mod:`.controller` that is called when calling `boa`. But if you do need more control over your launch script than the default provides, you can either subclass :class:`.Controller` or write your own launch script. Subclassing :class:`.Controller` might be easier if you just need to modify :meth:`.Controller.run` or :meth:`.Controller.initialize_wrapper` or :meth:`.Controller.initialize_scheduler` but can utilize the rest of the functions. If you need a lot of customization, writing your own script might be @@ -63,24 +72,24 @@ you can start your run easily from the command line. With your conda environment for boa activated, run:: - python -m boa --config-path path/to/your/config/file + boa --config-path path/to/your/config/file or:: - python -m boa -c path/to/your/config/file + boa -c path/to/your/config/file :doc:`BOA's ` will save the its current state automatically to a `scheduler.json` file in your output experiment directory every 1-few trials (depending on parallelism settings). It will also save a optimization.csv at the end of your run with the trial information as well in the same directory as scheduler.json. The console will output the Output directory at the start and end of your runs to the console, it will also throughout the run, whenever it saves the `scheduler.json` file, output to the console the location where the file is being saved. You can resume a stopped run from a scheduler file:: - python -m boa --scheduler-path path/to/your/scheduler.json + boa --scheduler-path path/to/your/scheduler.json or:: - python -m boa -sp path/to/your/scheduler.json + boa -sp path/to/your/scheduler.json For a list of options and descriptions, type:: - python -m boa --help + boa --help A fuller example using the command line interface can be found :doc:`here ` diff --git a/pyproject.toml b/pyproject.toml index 775de7a..f1813f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,10 @@ dynamic = ["version", "dependencies", "optional-dependencies"] "Bug Tracker" = "https://github.com/madeline-scyphers/boa/issues" Documentation = "http://boa-framework.readthedocs.io" +[project.scripts] +boa = "boa.cli:main" +"boa-plot" = "boa.plot:main" + [tool.setuptools.packages.find] include = ["boa*"] diff --git a/tests/1unit_tests/test_generation_strategy.py b/tests/1unit_tests/test_generation_strategy.py index 1ac1e57..84a8610 100644 --- a/tests/1unit_tests/test_generation_strategy.py +++ b/tests/1unit_tests/test_generation_strategy.py @@ -1,3 +1,8 @@ +import botorch.acquisition +import botorch.models +import gpytorch.kernels +import gpytorch.mlls +import pytest from ax.modelbridge.generation_strategy import GenerationStep, GenerationStrategy from ax.modelbridge.registry import Models @@ -41,3 +46,35 @@ def test_auto_gen_use_saasbo(saasbo_config, tmp_path): assert "SAASBO" in gs.name else: assert "FullyBayesian" in gs.name + + +@pytest.importorskip( + "ax-platform", minversion="0.3.5", reason="BOTORCH_MODULAR model is not available in BOA with Ax version < 0.3.5." +) +def test_modular_botorch(gen_strat_modular_botorch_config, tmp_path): + controller = Controller( + config=gen_strat_modular_botorch_config, + wrapper=ScriptWrapper(config=gen_strat_modular_botorch_config, experiment_dir=tmp_path), + ) + exp = get_experiment( + config=controller.config, runner=WrappedJobRunner(wrapper=controller.wrapper), wrapper=controller.wrapper + ) + gs = get_generation_strategy(config=controller.config, experiment=exp) + cfg_botorch_modular = gen_strat_modular_botorch_config.orig_config["generation_strategy"]["steps"][-1] + step = gs._steps[-1] + assert step.model == Models.BOTORCH_MODULAR + mdl_kw = step.model_kwargs + assert mdl_kw["botorch_acqf_class"] == getattr( + botorch.acquisition, cfg_botorch_modular["model_kwargs"]["botorch_acqf_class"] + ) + assert mdl_kw["acquisition_options"] == cfg_botorch_modular["model_kwargs"]["acquisition_options"] + + assert mdl_kw["surrogate"].mll_class == getattr( + gpytorch.mlls, cfg_botorch_modular["model_kwargs"]["surrogate"]["mll_class"] + ) + assert mdl_kw["surrogate"].botorch_model_class == getattr( + botorch.models, cfg_botorch_modular["model_kwargs"]["surrogate"]["botorch_model_class"] + ) + assert mdl_kw["surrogate"].covar_module_class == getattr( + gpytorch.kernels, cfg_botorch_modular["model_kwargs"]["surrogate"]["covar_module_class"] + ) diff --git a/tests/conftest.py b/tests/conftest.py index 6983ab7..c59e7f2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,10 +5,10 @@ import pytest -import boa.__main__ as dunder_main import boa.scripts.moo as run_moo import boa.scripts.run_branin as run_branin from boa import BOAConfig, cd_and_cd_back, split_shell_command +from boa.cli import main as cli_main from boa.definitions import ROOT, TEST_SCRIPTS_DIR logger = logging.getLogger(__file__) @@ -85,6 +85,12 @@ def gen_strat1_config(): return BOAConfig.from_jsonlike(file=config_path) +@pytest.fixture +def gen_strat_modular_botorch_config(): + config_path = TEST_DIR / f"scripts/other_langs/r_package_streamlined/config_modular_botorch.yaml" + return BOAConfig.from_jsonlike(file=config_path) + + @pytest.fixture def synth_config(): config_path = TEST_CONFIG_DIR / "test_config_synth.yaml" @@ -192,7 +198,7 @@ def denormed_custom_wrapper_run(tmp_path_factory, cd_to_root_and_back_session): config_path = temp_dir / "different_name_config.json" with open(Path(config_path), "w") as file: json.dump(config, file) - scheduler = dunder_main.main(split_shell_command(f"--config-path {config_path} -td"), standalone_mode=False) + scheduler = cli_main(split_shell_command(f"--config-path {config_path} -td"), standalone_mode=False) os.remove(config_path) yield scheduler @@ -211,25 +217,31 @@ def moo_main_run(tmp_path_factory, cd_to_root_and_back_session): def stand_alone_opt_package_run(tmp_path_factory, cd_to_root_and_back_session): config_path = TEST_DIR / "scripts/stand_alone_opt_package/stand_alone_pkg_config.yaml" args = f"--config-path {config_path} -td" - yield dunder_main.main(split_shell_command(args), standalone_mode=False) + yield cli_main(split_shell_command(args), standalone_mode=False) @pytest.fixture(scope="session") def r_full(tmp_path_factory, cd_to_root_and_back_session): config_path = TEST_DIR / f"scripts/other_langs/r_package_full/config.yaml" - yield dunder_main.main(split_shell_command(f"--config-path {config_path} -td"), standalone_mode=False) + yield cli_main(split_shell_command(f"--config-path {config_path} -td"), standalone_mode=False) @pytest.fixture(scope="session") def r_light(tmp_path_factory, cd_to_root_and_back_session): config_path = TEST_DIR / f"scripts/other_langs/r_package_light/config.yaml" - yield dunder_main.main(split_shell_command(f"--config-path {config_path} -td"), standalone_mode=False) + yield cli_main(split_shell_command(f"--config-path {config_path} -td"), standalone_mode=False) @pytest.fixture(scope="session") def r_streamlined(tmp_path_factory, cd_to_root_and_back_session): config_path = TEST_DIR / f"scripts/other_langs/r_package_streamlined/config.yaml" - yield dunder_main.main(split_shell_command(f"--config-path {config_path} -td"), standalone_mode=False) + yield cli_main(split_shell_command(f"--config-path {config_path} -td"), standalone_mode=False) + + +@pytest.fixture(scope="session") +def r_streamlined_botorch_modular(tmp_path_factory, cd_to_root_and_back_session): + config_path = TEST_DIR / f"scripts/other_langs/r_package_streamlined/config_modular_botorch.yaml" + return cli_main(split_shell_command(f"--config-path {config_path} -td"), standalone_mode=False) diff --git a/tests/integration_tests/test_dunder_main.py b/tests/integration_tests/test_cli.py similarity index 82% rename from tests/integration_tests/test_dunder_main.py rename to tests/integration_tests/test_cli.py index 009bd3f..824005d 100644 --- a/tests/integration_tests/test_dunder_main.py +++ b/tests/integration_tests/test_cli.py @@ -5,7 +5,6 @@ import pytest from ax.service.scheduler import FailureRateExceededError -import boa.__main__ as dunder_main from boa import ( BaseWrapper, BOAConfig, @@ -13,9 +12,9 @@ get_trial_dir, load_jsonlike, scheduler_from_json_file, - scheduler_to_json_file, split_shell_command, ) +from boa.cli import main as cli_main from boa.definitions import ROOT try: @@ -75,9 +74,25 @@ def test_calling_command_line_test_script_doesnt_error_out_and_produces_correct_ # parametrize the test to use the full version (all scripts) or the light version (only run_model.R) # or parametrize the test to use the streamlined version (doesn't use trial_status.json, only use output.json) +# the botorch modular version is the same as the streamlined version, but also uses botorch modular +# which uses a custom kernel, acquisition function, mll and botorch model class +# (which can customize the GP process even more) @pytest.mark.parametrize( "r_scripts_run", - ["r_full", "r_light", "r_streamlined"], + [ + "r_full", + "r_light", + "r_streamlined", + "r_streamlined_botorch_modular", + pytest.param( + "r_streamlined_botorch_modular", + marks=pytest.importorskip( + "ax-platform", + minversion="0.3.5", + reason="BOTORCH_MODULAR model is not available in BOA with Ax version < 0.3.5.", + ), + ), + ], ) @pytest.mark.skipif(not R_INSTALLED, reason="requires R to be installed") def test_calling_command_line_r_test_scripts(r_scripts_run, request): @@ -92,7 +107,7 @@ def test_calling_command_line_r_test_scripts(r_scripts_run, request): assert "param_names" in data assert "metric_properties" in data - if "r_streamlined" == r_scripts_run: + if r_scripts_run in ("r_streamlined", "r_streamlined_botorch_modular"): with cd_and_cd_back(scheduler.wrapper.config_path.parent): pre_num_trials = len(scheduler.experiment.trials) @@ -110,14 +125,14 @@ def test_calling_command_line_r_test_scripts(r_scripts_run, request): def test_cli_interface_with_failing_test_that_sends_back_failed_trial_status(): with pytest.raises(FailureRateExceededError): config_path = ROOT / "tests" / f"scripts/other_langs/r_package_streamlined/config_fail.yaml" - dunder_main.main(split_shell_command(f"--config-path {config_path} -td"), standalone_mode=False) + cli_main(split_shell_command(f"--config-path {config_path} -td"), standalone_mode=False) @pytest.mark.skipif(not R_INSTALLED, reason="requires R to be installed") def test_cli_interface_with_failing_test_that_sends_back_failed_trial_status(): with pytest.raises(FailureRateExceededError): config_path = ROOT / "tests" / f"scripts/other_langs/r_pass_back_fail_trial_status/config.yaml" - dunder_main.main(split_shell_command(f"--config-path {config_path} -td"), standalone_mode=False) + cli_main(split_shell_command(f"--config-path {config_path} -td"), standalone_mode=False) def test_wrapper_with_custom_load_config(): @@ -125,7 +140,7 @@ def test_wrapper_with_custom_load_config(): config_path = ROOT / "tests" / f"scripts/other_langs/r_package_streamlined/config_fail.yaml" # But we override the failing config in our wrapper with a working one in a custom load_config wrapper_path = Path(__file__) - dunder_main.main( + cli_main( split_shell_command( f"--config-path {config_path}" f" --wrapper-path {wrapper_path}" @@ -153,10 +168,10 @@ def test_parallelism(r_light, caplog): def test_non_zero_exit_code_fails_trial(): with pytest.raises(FailureRateExceededError): config_path = ROOT / "tests" / f"scripts/other_langs/r_failure_exit_code/config.yaml" - dunder_main.main(split_shell_command(f"--config-path {config_path} -td"), standalone_mode=False) + cli_main(split_shell_command(f"--config-path {config_path} -td"), standalone_mode=False) def test_return_nan_fails_trial(): with pytest.raises(FailureRateExceededError): config_path = ROOT / "tests" / f"scripts/other_langs/r_failure_nan/config.yaml" - dunder_main.main(split_shell_command(f"--config-path {config_path} -td"), standalone_mode=False) + cli_main(split_shell_command(f"--config-path {config_path} -td"), standalone_mode=False) diff --git a/tests/integration_tests/test_storage.py b/tests/integration_tests/test_storage.py index 265c5ad..9e23f72 100644 --- a/tests/integration_tests/test_storage.py +++ b/tests/integration_tests/test_storage.py @@ -15,7 +15,6 @@ CORE_ENCODER_REGISTRY, ) -import boa.__main__ as dunder_main from boa import ( BaseWrapper, BOAConfig, @@ -31,6 +30,7 @@ split_shell_command, ) from boa.__version__ import __version__ +from boa.cli import main as cli_main from boa.definitions import ROOT TEST_DIR = ROOT / "tests" @@ -200,7 +200,7 @@ def test_can_pass_custom_wrapper_path_when_loading_scheduler_from_cli(stand_alon pre_num_trials = len(scheduler.experiment.trials) - scheduler = dunder_main.main( + scheduler = cli_main( split_shell_command(f"--scheduler-path {file_out} --wrapper-path {orig_wrapper_path} -td"), standalone_mode=False, ) diff --git a/tests/scripts/other_langs/r_package_streamlined/config_modular_botorch.yaml b/tests/scripts/other_langs/r_package_streamlined/config_modular_botorch.yaml new file mode 100644 index 0000000..42808ae --- /dev/null +++ b/tests/scripts/other_langs/r_package_streamlined/config_modular_botorch.yaml @@ -0,0 +1,60 @@ +objective: + metrics: + - name: metric +scheduler: + n_trials: 15 + +parameters: + x0: + 'bounds': [ 0, 1 ] + 'type': 'range' + 'value_type': 'float' + x1: + 'bounds': [ 0, 1] + 'type': 'range' + 'value_type': 'float' + x2: + 'bounds': [ 0, 1 ] + 'type': 'range' + 'value_type': 'float' + x3: + 'bounds': [ 0, 1] + 'type': 'range' + 'value_type': 'float' + x4: + 'bounds': [ 0, 1 ] + 'type': 'range' + 'value_type': 'float' + x5: + 'bounds': [ 0, 1] + 'type': 'range' + 'value_type': 'float' + +script_options: + # notice here that this is a shell command + # this is what BOA will do to launch your script + # it will also pass as a command line argument the current trial directory + # that is being parameterized + + # This can either be a relative path or absolute path + # (by default when BOA launches from a config file + # it uses the config file directory as your working directory) + # here config.yaml and run_model.R are in the same directory + run_model: Rscript run_model.R + exp_name: "r_streamlined_botorch_modular" + +generation_strategy: + steps: + - model: SOBOL + num_trials: 5 + - model: BOTORCH_MODULAR + num_trials: -1 # No limitation on how many trials should be produced from this step + model_kwargs: + surrogate: + botorch_model_class: SingleTaskGP # BoTorch model class name + + covar_module_class: RBFKernel # GPyTorch kernel class name + mll_class: LeaveOneOutPseudoLikelihood # GPyTorch MarginalLogLikelihood class name + botorch_acqf_class: qUpperConfidenceBound # BoTorch acquisition function class name + acquisition_options: + beta: 0.5