Add ifbo (#115)

Co-authored-by: Timur M. Carstensen <timurcarstensen@gmail.com>
Co-authored-by: karibbov <karibbov@gmail.com>
Co-authored-by: eddiebergman <eddiebergmanhs@gmail.com>

.github/workflows/pre-commit.yaml

@@ -27,8 +27,7 @@ jobs:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
-        python-version: 3.8
+        python-version: '3.10'
     - run: pip install pre-commit
     - run: pre-commit install
     - run: pre-commit run --all-files
-

.github/workflows/tests.yaml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.10', '3.11']
         os: [ubuntu-latest, macos-latest, windows-latest]
     defaults:
       run:

.gitignore

@@ -47,3 +47,6 @@ jahs_bench_data/
 
 # Yaml tests
 path
+
+# From example that uses MNIST
+.data

docs/_code/api_generator.py

@@ -2,7 +2,7 @@
 
 # https://mkdocstrings.github.io/recipes/
 """
-from __future__ import annotations
+
 
 import logging
 from pathlib import Path

docs/_code/example_generator.py

@@ -2,7 +2,6 @@
 
 # https://mkdocstrings.github.io/recipes/
 """
-from __future__ import annotations
 
 import logging
 from pathlib import Path
@@ -16,7 +15,6 @@
 EXAMPLE_FOLDER = ROOT / "neps_examples"
 TAB = "    "
 
-
 if not SRCDIR.exists():
     raise FileNotFoundError(
         f"{SRCDIR} does not exist, make sure you are running this from the root of the repository."

docs/doc_yamls/architecture_search_space.py

@@ -1,4 +1,4 @@
-from __future__ import annotations
+
 from torch import nn
 import neps
 from neps.search_spaces.architecture import primitives as ops

neps/__init__.py

@@ -1,5 +1,6 @@
 from neps.api import run
 from neps.plot.plot import plot
+from neps.plot.tensorboard_eval import tblogger
 from neps.search_spaces import (
     ArchitectureParameter,
     CategoricalParameter,
@@ -38,4 +39,5 @@
     "GraphGrammar",
     "GraphGrammarCell",
     "GraphGrammarRepetitive",
+    "tblogger",
 ]

neps/api.py

@@ -1,6 +1,6 @@
 """API for the neps package."""
 
-from __future__ import annotations
+
 
 import inspect
 import logging
@@ -80,7 +80,7 @@ def run(
         root_directory: The directory to save progress to. This is also used to
             synchronize multiple calls to run(.) for parallelization.
         run_args: An option for providing the optimization settings e.g.
-            max_evaluation_total in a YAML file.
+            max_evaluations_total in a YAML file.
         overwrite_working_directory: If true, delete the working directory at the start of
             the run. This is, e.g., useful when debugging a run_pipeline function.
         post_run_summary: If True, creates a csv file after each worker is done,

neps/env.py

@@ -3,7 +3,8 @@
 from __future__ import annotations
 
 import os
-from typing import Any, Callable, TypeVar
+from collections.abc import Callable
+from typing import Any, TypeVar
 
 T = TypeVar("T")
 V = TypeVar("V")

neps/optimizers/__init__.py

@@ -1,14 +1,13 @@
-from __future__ import annotations
+
 
 from functools import partial
 from typing import Callable, Mapping
 
 from .base_optimizer import BaseOptimizer
 from .bayesian_optimization.cost_cooling import CostCooling
-from .bayesian_optimization.mf_tpe import MultiFidelityPriorWeightedTreeParzenEstimator
 from .bayesian_optimization.optimizer import BayesianOptimization
 from .grid_search.optimizer import GridSearch
-from .multi_fidelity.dyhpo import MFEIBO
+from .multi_fidelity.ifbo import IFBO
 from .multi_fidelity.hyperband import (
     MOBSTER,
     AsynchronousHyperband,
@@ -41,9 +40,11 @@
     "asha": AsynchronousSuccessiveHalving,
     "hyperband": Hyperband,
     "asha_prior": AsynchronousSuccessiveHalvingWithPriors,
-    "multifidelity_tpe": MultiFidelityPriorWeightedTreeParzenEstimator,
     "hyperband_custom_default": HyperbandCustomDefault,
     "priorband": PriorBand,
+    "priorband_bo": partial(PriorBand, model_based=True),
+    "priorband_asha": PriorBandAsha,
+    "priorband_asha_hyperband": PriorBandAshaHB,
     "mobster": MOBSTER,
-    "mf_ei_bo": MFEIBO,
+    "ifbo": IFBO,
 }

neps/optimizers/base_optimizer.py

@@ -1,4 +1,4 @@
-from __future__ import annotations
+
 
 import logging
 from abc import abstractmethod

neps/optimizers/bayesian_optimization/acquisition_functions/__init__.py

@@ -1,15 +1,14 @@
-from __future__ import annotations
+
 
 from functools import partial
 from typing import Callable
 
 from neps.optimizers.bayesian_optimization.acquisition_functions.ei import (
     ComprehensiveExpectedImprovement,
 )
-from neps.optimizers.bayesian_optimization.acquisition_functions.mf_ei import MFEI
+from neps.optimizers.bayesian_optimization.acquisition_functions.mf_pi import MFPI_Random
 from neps.optimizers.bayesian_optimization.acquisition_functions.ucb import (
     UpperConfidenceBound,
-    MF_UCB,
 )
 from neps.optimizers.bayesian_optimization.acquisition_functions.prior_weighted import (
     DecayingPriorWeightedAcquisition,
@@ -28,33 +27,28 @@
         augmented_ei=False,
         log_ei=True,
     ),
-    #     # Uses the augmented EI heuristic and changed the in-fill criterion to the best test location with
-    #     # the highest *posterior mean*, which are preferred when the optimisation is noisy.
+    ## Uses the augmented EI heuristic and changed the in-fill criterion to the best test location with
+    ## the highest *posterior mean*, which are preferred when the optimisation is noisy.
     "AEI": partial(
         ComprehensiveExpectedImprovement,
         in_fill="posterior",
         augmented_ei=True,
     ),
-    "MFEI": partial(
-        MFEI,
-        in_fill="best",
-        augmented_ei=False,
+    "MFPI-random": partial(
+        MFPI_Random,
+        threshold="random",
+        horizon="random",
     ),
     "UCB": partial(
         UpperConfidenceBound,
         maximize=False,
     ),
-    "MF-UCB": partial(
-        MF_UCB,
-        maximize=False,
-    ),
 }
 
 __all__ = [
     "AcquisitionMapping",
     "ComprehensiveExpectedImprovement",
-    "MFEI",
     "UpperConfidenceBound",
-    "MF_UCB",
     "DecayingPriorWeightedAcquisition",
+    "MFPI_Random",
 ]

neps/optimizers/bayesian_optimization/acquisition_functions/ei.py

@@ -10,6 +10,7 @@
 if TYPE_CHECKING:
     from neps.search_spaces import SearchSpace
 
+
 class ComprehensiveExpectedImprovement(BaseAcquisition):
     def __init__(
         self,