Skip to content

add verbosity parameter to tuners (more user friendly) #771

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

add verbosity parameter to tuners (more user friendly) #771

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 52 additions & 6 deletions cpmpy/tools/tune_solver.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,33 +32,42 @@ def __init__(self, solvername, model, all_params=None, defaults=None):
:param solvername: Name of solver to tune
:param model: CPMpy model to tune parameters on
:param all_params: optional, dictionary with parameter names and values to tune. If None, use predefined parameter set.
:param defaults: required ONLY IF all_params is set, dictionary with corresponding default parameter values.
"""
self.solvername = solvername
self.model = model
self.all_params = all_params
self.best_params = defaults
if self.all_params is None:
if all_params is None:
self.all_params = SolverLookup.lookup(solvername).tunable_params()
self.best_params = SolverLookup.lookup(solvername).default_params()
else:
self.all_params = all_params
assert defaults is not None, "ParameterTuner: if 'all_params' is set then 'defaults' must too"
self.best_params = defaults

self._param_order = list(self.all_params.keys())
self._best_config = self._params_to_np([self.best_params])

def tune(self, time_limit=None, max_tries=None, fix_params={}):
def tune(self, time_limit=None, max_tries=None, fix_params={}, verbose=1):
"""
:param time_limit: Time budget to run tuner in seconds. Solver will be interrupted when time budget is exceeded
:param max_tries: Maximum number of configurations to test
:param fix_params: Non-default parameters to run solvers with.
:param verbose: how much information to print (0=none)
"""
if time_limit is not None:
start_time = time.time()

# Init solver
if verbose >= 1:
print(f"Running {self.solvername} with default parameters")
solver = SolverLookup.get(self.solvername, self.model)
solver.solve(**self.best_params)

self.base_runtime = solver.status().runtime
self.best_runtime = self.base_runtime
self._best_config = {}
if verbose >= 1:
print(f" - took {self.best_runtime:.1f} seconds")

# Get all possible hyperparameter configurations
combos = list(param_combinations(self.all_params))
Expand Down Expand Up @@ -88,19 +97,29 @@ def tune(self, time_limit=None, max_tries=None, fix_params={}):
# set timeout depending on time budget
if time_limit is not None:
timeout = min(timeout, time_limit - (time.time() - start_time))

if verbose >= 1:
print(f"Starting trial {i+1}/{max_tries}, cap: {timeout:.1f}s -- remaining configs: {len(combos_np)}" + f" budget: {time_limit-(time.time()-start_time):.1f}s" if time_limit else "")
# run solver
solver.solve(**params_dict, time_limit=timeout)
if solver.status().exitstatus == ExitStatus.OPTIMAL and solver.status().runtime < self.best_runtime:
self.best_runtime = solver.status().runtime
# update surrogate
self._best_config = params_np
if verbose >= 1:
print(f" - new best runtime {self.best_runtime}")
if verbose >= 2:
print(f" - new best params {self._np_to_params(self._best_config)}")

if time_limit is not None and (time.time() - start_time) >= time_limit:
break
i += 1

self.best_params = self._np_to_params(self._best_config)
self.best_params.update(fix_params)
if verbose >= 2:
print("Best runtime: {self.best_runtime} for params {self.best_params}")

return self.best_params

def _get_score(self, combos):
Expand All @@ -120,26 +139,43 @@ def _np_to_params(self,arr):


class GridSearchTuner(ParameterTuner):
"""
Grid search parameter tuner that exhaustively tests all parameter combinations.
Inherits from ParameterTuner but uses a simple grid search strategy
"""

def __init__(self, solvername, model, all_params=None, defaults=None):
"""
Initialize a grid search parameter tuner.

:param solvername: Name of solver to tune
:param model: CPMpy model to tune parameters on
:param all_params: optional, dictionary with parameter names and values to tune. If None, use predefined parameter set.
:param defaults: required ONLY IF all_params is set, dictionary with corresponding default parameter values.
"""
super().__init__(solvername, model, all_params, defaults)

def tune(self, time_limit=None, max_tries=None, fix_params={}):
def tune(self, time_limit=None, max_tries=None, fix_params={}, verbose=1):
"""
:param time_limit: Time budget to run tuner in seconds. Solver will be interrupted when time budget is exceeded
:param max_tries: Maximum number of configurations to test
:param fix_params: Non-default parameters to run solvers with.
:param verbose: how much information to print (0=none)
"""
if time_limit is not None:
start_time = time.time()

# Init solver
if verbose >= 1:
print(f"Running {self.solvername} with default parameters")
solver = SolverLookup.get(self.solvername, self.model)
solver.solve(**self.best_params)


self.base_runtime = solver.status().runtime
self.best_runtime = self.base_runtime
if verbose >= 1:
print(f" - took {self.best_runtime:.1f} seconds")

# Get all possible hyperparameter configurations
combos = list(param_combinations(self.all_params))
Expand All @@ -148,7 +184,7 @@ def tune(self, time_limit=None, max_tries=None, fix_params={}):
if max_tries is not None:
combos = combos[:max_tries]

for params_dict in combos:
for i, params_dict in enumerate(combos):
# Make new solver
solver = SolverLookup.get(self.solvername, self.model)
# set fixed params
Expand All @@ -157,16 +193,26 @@ def tune(self, time_limit=None, max_tries=None, fix_params={}):
# set timeout depending on time budget
if time_limit is not None:
timeout = min(timeout, time_limit - (time.time() - start_time))

if verbose >= 1:
print(f"Starting trial {i+1}/{len(combos)}, cap: {timeout:.1f}s" + f" budget: {time_limit-(time.time()-start_time):.1f}s" if time_limit else "")
# run solver
solver.solve(**params_dict, time_limit=timeout)
if solver.status().exitstatus == ExitStatus.OPTIMAL and solver.status().runtime < self.best_runtime:
self.best_runtime = solver.status().runtime
# update surrogate
self.best_params = params_dict
if verbose >= 1:
print(f" - new best runtime {self.best_runtime:.1f}")
if verbose >= 2:
print(f" - new best params {self.best_params}")

if time_limit is not None and (time.time() - start_time) >= time_limit:
break

if verbose >= 2:
print(f"Best runtime: {self.best_runtime:.1f} for params {self.best_params}")

return self.best_params


Expand Down
Loading