Source code for ax.benchmark.benchmark_method
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
from dataclasses import dataclass
from ax.modelbridge.generation_strategy import GenerationStrategy
from ax.service.utils.scheduler_options import SchedulerOptions
from ax.utils.common.base import Base
[docs]@dataclass(frozen=True)
class BenchmarkMethod(Base):
"""Benchmark method, represented in terms of Ax generation strategy (which tells us
which models to use when) and scheduler options (which tell us extra execution
information like maximum parallelism, early stopping configuration, etc.). Note:
if BenchmarkMethod.scheduler_optionss.total_trials is lower than
BenchmarkProblem.num_trials only the number of trials specified in the former will
be run.
"""
name: str
generation_strategy: GenerationStrategy
scheduler_options: SchedulerOptions
distribute_replications: bool = False
[docs]def get_sequential_optimization_scheduler_options(
timeout_hours: int = 4,
) -> SchedulerOptions:
"""The typical SchedulerOptions used in benchmarking.
Args:
timeout_hours: The maximum amount of time (in hours) to run each
benchmark replication. Defaults to 4 hours.
"""
return SchedulerOptions(
# Enforce sequential trials by default
max_pending_trials=1,
# Do not throttle, as is often necessary when polling real endpoints
init_seconds_between_polls=0,
min_seconds_before_poll=0,
timeout_hours=timeout_hours,
)