Merge pull request #218 from nabenabe0928/code-fix/refactor-simple-ex…

…amples

Modify simple examples based on the Optuna code conventions

multi_objective/quadratic_simple.py

@@ -1,30 +1,34 @@
 """
 Optuna example that optimizes simple quadratic functions.
 
-In this example, we optimize simple quadratic functions.
+In this example, we optimize two objective values.
+Unlike single-objective optimization, an optimization gives a trade-off between two objectives.
+As a result, we get the best trade-offs between two objectives, a.k.a Pareto solutions.
 
 """
 
 import optuna
 
 
-# Define simple 2-dimensional objective functions.
+# Define two objective functions.
+# We would like to minimize f1 and maximize f2.
 def objective(trial):
     x = trial.suggest_float("x", -100, 100)
     y = trial.suggest_categorical("y", [-1, 0, 1])
-    obj1 = x**2 + y
-    obj2 = -((x - 2) ** 2 + y)
-    return [obj1, obj2]
+    f1 = x**2 + y
+    f2 = -((x - 2) ** 2 + y)
+    return f1, f2
 
 
 if __name__ == "__main__":
-    # We minimize obj1 and maximize obj2.
+    # We minimize the first objective value and maximize the second objective value.
     study = optuna.create_study(directions=["minimize", "maximize"])
     study.optimize(objective, n_trials=500, timeout=1)
 
-    pareto_front = [t.values for t in study.best_trials]
-    pareto_sols = [t.params for t in study.best_trials]
+    print("Number of finished trials: ", len(study.trials))
 
-    for i, (params, values) in enumerate(zip(pareto_sols, pareto_front)):
-        print(f"The {i}-th Pareto solution and its objective values")
-        print("\t", params, values)
+    for i, best_trial in enumerate(study.best_trials):
+        print(f"The {i}-th Pareto solution was found at Trial#{best_trial.number}.")
+        print(f"  Params: {best_trial.params}")
+        f1, f2 = best_trial.values
+        print(f"  Values: {f1=}, {f2=}")

quadratic_simple_constraint.py

@@ -1,30 +1,41 @@
 """
 Optuna example that optimizes a simple quadratic function with constraints.
 
-In this example, we optimize a simple quadratic function with constraints.
-
-Please check https://optuna.readthedocs.io/en/stable/faq.html#id16 as well.
+This setup is called `constrained optimization`.
+Constrained optimization is useful when we would like to optimize some metrics under
+some constraints. For example, we would like to maximize the accuracy of a deep neural networks
+while guaranteeing that deep neural networks fit on your hardware, e.g. 8GB of memory consumption.
+In this case, constrained optimization aims to yield an optimal solution that satisfied
+such constraints.
+
+Note that Optuna cannot optimize an objective that will not return any results when some
+constraints violate. For example, when we run a memory-intensive algorithm and user sets
+the memory constraint very close to the limit, we may not get any results if the memory constraint
+is violated. However, Optuna cannot handle such situations.
+Please also check https://optuna.readthedocs.io/en/stable/faq.html#id16 as well.
 
 """
 
 import optuna
 
 
-# Define a simple 2-dimensional objective function with constraints.
+# Define a simple 2-dimensional objective function to minimize.
 def objective(trial):
     x = trial.suggest_float("x", -100, 100)
     y = trial.suggest_categorical("y", [-1, 0, 1])
     return x**2 + y
 
 
 # Define a function that returns constraints.
+# The constraints are to satisfy `c1 <= 0` and `c2 <= 0` simultaneously.
+# If we would like to make the constraint like `c1 <= a`,
+# we simply need to return `c1 - a` instead of `c1`.
 def constraints(trial):
     params = trial.params
     x, y = params["x"], params["y"]
-    # c1 <= 0 and c2 <= 0 must be satisfied.
     c1 = 3 * x * y + 10
     c2 = x * y + 30
-    return [c1, c2]
+    return c1, c2
 
 
 if __name__ == "__main__":
@@ -33,17 +44,20 @@ def constraints(trial):
     study = optuna.create_study(sampler=sampler)
     study.optimize(objective, n_trials=500, timeout=1)
 
-    best_trial_id, best_value, best_params = None, float("inf"), None
-    for t in study.trials:
-        infeasible = any(c > 0.0 for c in t.system_attrs["constraints"])
-        if infeasible:
-            continue
-        if best_value > t.value:
-            best_value = t.value
-            best_params = t.params.copy()
-            best_trial_id = t._trial_id
-
-    if best_trial_id is None:
-        print("All trials violated the constraints.")
+    print("Number of finished trials: ", len(study.trials))
+
+    feasible_trial_numbers = [
+        trial.number
+        for trial in study.trials
+        if all(c <= 0.0 for c in trial.system_attrs["constraints"])
+    ]
+    if len(feasible_trial_numbers) == 0:
+        print("No trials satisfied all the constraints.")
     else:
-        print(f"Best value is {best_value} with params {best_params}")
+        best_trial_number = sorted(feasible_trial_numbers, key=lambda i: study.trials[i].value)[0]
+        best_trial = study.trials[best_trial_number]
+        print(f"Best trial was found at Trial#{best_trial_number}")
+        print(f"  Params: {best_trial.params}")
+        print(f"  Value: {best_trial.value}")
+        c1, c2 = best_trial.system_attrs["constraints"]
+        print(f"  Constraints: {c1=}, {c2=}")