From 50ce038f4f2ec5c59646e5ffaee8d5805463d22a Mon Sep 17 00:00:00 2001 From: vizier-team Date: Tue, 28 Jan 2025 12:10:34 -0800 Subject: [PATCH] Implements metric normalization in the HV Curve Converter PiperOrigin-RevId: 720664003 --- .../algorithms/ensemble/ensemble_designer.py | 1 + .../benchmarks/analyzers/convergence_curve.py | 40 ++++++++++----- .../analyzers/convergence_curve_test.py | 51 ++++++++++--------- 3 files changed, 56 insertions(+), 36 deletions(-) diff --git a/vizier/_src/algorithms/ensemble/ensemble_designer.py b/vizier/_src/algorithms/ensemble/ensemble_designer.py index 1dc8fb759..362c6b19e 100644 --- a/vizier/_src/algorithms/ensemble/ensemble_designer.py +++ b/vizier/_src/algorithms/ensemble/ensemble_designer.py @@ -74,6 +74,7 @@ def curve_generator() -> analyzers.StatefulCurveConverter: reference_value=self.reference_value, num_vectors=self.num_vectors, infer_origin_factor=0.1, + disable_metric_normalization=True, ) stateful_curve_generator = analyzers.RestartingCurveConverter( diff --git a/vizier/_src/benchmarks/analyzers/convergence_curve.py b/vizier/_src/benchmarks/analyzers/convergence_curve.py index 86ef1df39..b4d44505e 100644 --- a/vizier/_src/benchmarks/analyzers/convergence_curve.py +++ b/vizier/_src/benchmarks/analyzers/convergence_curve.py @@ -349,6 +349,7 @@ def __init__( reference_value: Optional[np.ndarray] = None, num_vectors: int = 10000, infer_origin_factor: float = 0.0, + disable_metric_normalization: bool = False, ): """Init. @@ -361,6 +362,7 @@ def __init__( num_vectors: Number of vectors from which hypervolume is computed. infer_origin_factor: When inferring the reference point, set origin to be minimum value - factor * (range). + disable_metric_normalization: If True, disable metric normalization. """ if len(metric_informations) < 2: raise ValueError( @@ -386,8 +388,9 @@ def create_metric_converter(mc): self._origin_value = reference_value # TODO: Speed this up with hypervolume vector tracking. self._min_trial_idx = 1 - self._pareto_frontier = np.empty(shape=(0, len(metric_informations))) self._infer_origin_factor = infer_origin_factor + self._disable_metric_normalization = disable_metric_normalization + self._all_metrics = np.empty(shape=(0, len(metric_informations))) def convert(self, trials: Sequence[pyvizier.Trial]) -> ConvergenceCurve: """Returns ConvergenceCurve with a curve of shape 1 x len(trials).""" @@ -396,6 +399,28 @@ def convert(self, trials: Sequence[pyvizier.Trial]) -> ConvergenceCurve: raise ValueError(f'No trials provided {trials}') metrics = self._converter.to_labels_array(trials) + self._all_metrics = np.vstack([self._all_metrics, metrics]) + if self._disable_metric_normalization: + normalizer = 1.0 + else: + normalizer = np.nanmedian( + np.absolute( + self._all_metrics + - np.nanmedian(self._all_metrics, axis=0, keepdims=True) + ), + axis=0, + keepdims=True, + ) + if np.any(normalizer <= 0): + raise ValueError( + 'Some metric normalizers are zero. This is likely due to some' + ' metrics being identical or contain a lot of duplicates across' + f' trials. Normalizers: {normalizer}' + ) + metrics = metrics / normalizer + # shape is [num_existing_points + num_new_points, num_metrics] + all_metrics = self._all_metrics / normalizer + if self._origin_value is None: # Set origin to the minimum of finite values. origin = np.zeros(shape=(metrics.shape[1],)) @@ -428,10 +453,6 @@ def convert(self, trials: Sequence[pyvizier.Trial]) -> ConvergenceCurve: ) origin = self._origin_value - # Calculate cumulative hypervolume with the Pareto frontier. - all_metrics = np.vstack( - [self._pareto_frontier, metrics] - ) # shape is [num_pareto_points + num_points, feature dimension] front = multimetric.ParetoFrontier( points=all_metrics, origin=origin, @@ -440,17 +461,12 @@ def convert(self, trials: Sequence[pyvizier.Trial]) -> ConvergenceCurve: ) all_hv_curve = front.hypervolume(is_cumulative=True) - # Remove the Pareto frontier add-in and update state. - hv_curve = all_hv_curve[len(self._pareto_frontier) :] + # Extracts the hv points for the new trials. + hv_curve = all_hv_curve[-metrics.shape[0] :] xs = np.asarray( range(self._min_trial_idx, len(hv_curve) + self._min_trial_idx) ) self._min_trial_idx += len(hv_curve) - algo = multimetric.FastParetoOptimalAlgorithm( - xla_pareto.JaxParetoOptimalAlgorithm() - ) - pareto_points = algo.is_pareto_optimal(points=all_metrics) - self._pareto_frontier = all_metrics[pareto_points] return ConvergenceCurve( xs=xs, diff --git a/vizier/_src/benchmarks/analyzers/convergence_curve_test.py b/vizier/_src/benchmarks/analyzers/convergence_curve_test.py index a65667d90..eff3fa09f 100644 --- a/vizier/_src/benchmarks/analyzers/convergence_curve_test.py +++ b/vizier/_src/benchmarks/analyzers/convergence_curve_test.py @@ -278,25 +278,28 @@ def test_convert_with_origin_reference(self): pytrials = [] pytrials.append( pyvizier.Trial().complete( - pyvizier.Measurement(metrics={'max': 4.0, 'min': 2.0}) + pyvizier.Measurement(metrics={'max': 8.0, 'min': 0.0}) ) ) pytrials.append( pyvizier.Trial().complete( - pyvizier.Measurement(metrics={'max': 3.0, 'min': -1.0}) + pyvizier.Measurement(metrics={'max': 6.0, 'min': -2.0}) ) ) pytrials.append( pyvizier.Trial().complete( - pyvizier.Measurement(metrics={'max': 4.0, 'min': -2.0}) + pyvizier.Measurement(metrics={'max': 4.0, 'min': -4.0}) ) ) curve = generator.convert(pytrials) np.testing.assert_array_equal(curve.xs, [1, 2, 3]) - np.testing.assert_array_almost_equal( - curve.ys, [[0.0, 3.0, 8.0]], decimal=0.5 - ) + # After the sign of the minimization metric is flipped, the three metric + # points are (8, 0), (6, 2), (4, 4). After normalization, they become + # (4, 0), (3, 1), (2, 2). The origin is (0, 0). The sequence of hypervolume + # is expected to be (0, 3.5, 6.0). However, we allow a large error because + # the hypervolume computation is approximate. + np.testing.assert_array_almost_equal(curve.ys, [[0.0, 3.5, 6.0]], decimal=0) def test_convert_with_reference(self): generator = convergence.HypervolumeCurveConverter( @@ -313,25 +316,29 @@ def test_convert_with_reference(self): pytrials = [] pytrials.append( pyvizier.Trial().complete( - pyvizier.Measurement(metrics={'max': 4.0, 'min': 2.0}) + pyvizier.Measurement(metrics={'max': 8.0, 'min': 0.0}) ) ) pytrials.append( pyvizier.Trial().complete( - pyvizier.Measurement(metrics={'max': 3.0, 'min': -1.0}) + pyvizier.Measurement(metrics={'max': 6.0, 'min': -2.0}) ) ) pytrials.append( pyvizier.Trial().complete( - pyvizier.Measurement(metrics={'max': 4.0, 'min': -2.0}) + pyvizier.Measurement(metrics={'max': 4.0, 'min': -4.0}) ) ) curve = generator.convert(pytrials) np.testing.assert_array_equal(curve.xs, [1, 2, 3]) - np.testing.assert_array_almost_equal( - curve.ys, [[0.0, 0.0, 2.0]], decimal=0.5 - ) + # After the sign of the minimization metric is flipped, the three metric + # points are (8, 0), (6, 2), (4, 4). After normalization, they become + # (4, 0), (3, 1), (2, 2). After accounting for the reference point (3, 0), + # they are (1, 0), (0, 1), (-1, 2). The sequence of hypervolume + # is expected to be (0, 0.5, 0.5). However, we allow a large error + # because the hypervolume computation is approximate. + np.testing.assert_array_almost_equal(curve.ys, [[0.0, 0.5, 0.5]], decimal=0) def test_convert_with_none_reference(self): generator = convergence.HypervolumeCurveConverter([ @@ -361,9 +368,7 @@ def test_convert_with_none_reference(self): curve = generator.convert(pytrials) np.testing.assert_array_equal(curve.xs, [1, 2, 3]) - np.testing.assert_array_almost_equal( - curve.ys, [[0.0, 0.0, 1.0]], decimal=0.5 - ) + np.testing.assert_array_almost_equal(curve.ys, [[0.0, 0.0, 1.0]], decimal=0) def test_convert_with_inf_none_reference(self): generator = convergence.HypervolumeCurveConverter([ @@ -401,6 +406,7 @@ def test_convert_with_state(self): ), ], reference_value=np.array([0.0]), + disable_metric_normalization=True, ) pytrials = [] pytrials.append( @@ -415,15 +421,13 @@ def test_convert_with_state(self): ) pytrials.append( pyvizier.Trial().complete( - pyvizier.Measurement(metrics={'max': 4.0, 'min': -2.0}) + pyvizier.Measurement(metrics={'max': 3.0, 'min': -2.0}) ) ) curve = generator.convert(pytrials) np.testing.assert_array_equal(curve.xs, [1, 2, 3]) - np.testing.assert_array_almost_equal( - curve.ys, [[0.0, 5.0, 9.0]], decimal=0.5 - ) + np.testing.assert_array_almost_equal(curve.ys, [[0.0, 5.0, 9.0]], decimal=0) pytrials = [] pytrials.append( @@ -439,7 +443,7 @@ def test_convert_with_state(self): curve = generator.convert(pytrials) np.testing.assert_array_equal(curve.xs, [4, 5]) - np.testing.assert_array_almost_equal(curve.ys, [[9.0, 10.0]], decimal=0.5) + np.testing.assert_array_almost_equal(curve.ys, [[9.0, 10.0]], decimal=0) def test_convert_factor_with_inf(self): generator = convergence.HypervolumeCurveConverter( @@ -534,7 +538,7 @@ def test_convert_multiobjective(self): pytrials = [] pytrials.append( pyvizier.Trial().complete( - pyvizier.Measurement(metrics={'max': 4.0, 'min': -1.0, 'safe': 1.0}) + pyvizier.Measurement(metrics={'max': 2.0, 'min': 0.0, 'safe': 1.0}) ) ) pytrials.append( @@ -552,9 +556,7 @@ def test_convert_multiobjective(self): curve = generator.convert(pytrials) np.testing.assert_array_equal(curve.xs, [1, 2, 3]) - np.testing.assert_array_almost_equal( - curve.ys, [[4.0, 4.0, 8.0]], decimal=0.5 - ) + np.testing.assert_array_almost_equal(curve.ys, [[0.0, 0.0, 2.0]], decimal=0) class RestartingCurveConverterTest(absltest.TestCase): @@ -570,6 +572,7 @@ def converter_factory(): name='min', goal=pyvizier.ObjectiveMetricGoal.MINIMIZE ), ], + disable_metric_normalization=True, ) restart_converter = convergence.RestartingCurveConverter(