diff --git a/src/booster/erlpboost/qp_model.rs b/src/booster/erlpboost/qp_model.rs index ddcad08..49629d7 100644 --- a/src/booster/erlpboost/qp_model.rs +++ b/src/booster/erlpboost/qp_model.rs @@ -12,7 +12,7 @@ use crate::hypothesis::Classifier; use std::iter; -const QP_TOLERANCE: f64 = 1e-300; +const QP_TOLERANCE: f64 = 1e-9; /// A quadratic programming model for edge minimization. /// `QPModel` solves the entropy regularized edge minimization problem: @@ -45,33 +45,33 @@ const QP_TOLERANCE: f64 = 1e-300; /// ```txt /// # of /// rows γ d1 ... dm -/// ┏ ┃ ┓ ┏ ┓ -/// 1 ┃ 0 ┃ 1 ... 1 ┃ = ┃ 1 ┃ -/// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -/// ┃ 0 ┃ ┃ ≤ ┃ 0 ┃ -/// ┃ 0 ┃ ┃ ≤ ┃ 0 ┃ -/// ┃ . ┃ (-1) * Identity matrix ┃ . ┃ . ┃ -/// m ┃ . ┃ m x m ┃ . ┃ . ┃ -/// ┃ . ┃ ┃ . ┃ . ┃ -/// ┃ 0 ┃ ┃ ≤ ┃ 0 ┃ -/// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -/// ┃ 0 ┃ ┃ ≤ ┃ 0 ┃ -/// ┃ 0 ┃ ┃ ≤ ┃ 0 ┃ -/// ┃ . ┃ ┃ . ┃ . ┃ -/// m ┃ . ┃ Identity matrix ┃ . ┃ . ┃ -/// ┃ . ┃ m x m ┃ . ┃ . ┃ -/// ┃ 0 ┃ ┃ ≤ ┃ 0 ┃ -/// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -/// ┃ -1 ┃ y_1 h_1(x_1) ... y_m h_1(x_m) ┃ ≤ ┃ 0 ┃ -/// ┃ -1 ┃ y_1 h_2(x_1) ... y_m h_2(x_m) ┃ ≤ ┃ 0 ┃ -/// ┃ . ┃ . ... . ┃ . ┃ . ┃ -/// H ┃ . ┃ . ... . ┃ . ┃ . ┃ -/// ┃ . ┃ . ... . ┃ . ┃ . ┃ -/// ┃ -1 ┃ y_1 h_T(x_1) ... y_m h_T(x_m) ┃ ≤ ┃ 0 ┃ -/// ┗ ┃ ┛ ┗ ┛ +/// ┏ ┃ ┓ ┏ ┓ +/// 1 ┃ 0 ┃ 1 ... 1 ┃ = ┃ 1 ┃ +/// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +/// ┃ 0 ┃ ┃ ≤ ┃ 0 ┃ +/// ┃ 0 ┃ ┃ ≤ ┃ 0 ┃ +/// ┃ . ┃ (-1) * Identity matrix ┃ . ┃ . ┃ +/// m ┃ . ┃ m x m ┃ . ┃ . ┃ +/// ┃ . ┃ ┃ . ┃ . ┃ +/// ┃ 0 ┃ ┃ ≤ ┃ 0 ┃ +/// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +/// ┃ 0 ┃ ┃ ≤ ┃ 1/ν ┃ +/// ┃ 0 ┃ ┃ ≤ ┃ 1/ν ┃ +/// m ┃ . ┃ Identity matrix ┃ . ┃ . ┃ +/// ┃ . ┃ m x m ┃ . ┃ . ┃ +/// ┃ . ┃ ┃ . ┃ . ┃ +/// ┃ 0 ┃ ┃ ≤ ┃ 1/ν ┃ +/// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +/// ┃ -1 ┃ y_1 h_1(x_1) ... y_m h_1(x_m) ┃ ≤ ┃ 0 ┃ +/// ┃ -1 ┃ y_1 h_2(x_1) ... y_m h_2(x_m) ┃ ≤ ┃ 0 ┃ +/// ┃ . ┃ . ... . ┃ . ┃ . ┃ +/// H ┃ . ┃ . ... . ┃ . ┃ . ┃ +/// ┃ . ┃ . ... . ┃ . ┃ . ┃ +/// ┃ -1 ┃ y_1 h_T(x_1) ... y_m h_T(x_m) ┃ ≤ ┃ 0 ┃ +/// ┗ ┃ ┛ ┗ ┛ /// /// # of -/// cols 1 ┃ m +/// cols 1 ┃ m /// ``` pub(super) struct QPModel { pub(self) n_examples: usize, // number of columns @@ -124,7 +124,13 @@ impl QPModel { let rhs = self.build_rhs(); - let mut old_objval = 1e9; + let mut old_objval = 1e3; + + // Initialize `dist` as the uniform distribution. + dist.iter_mut() + .for_each(|di| { + *di = 1f64 / self.n_examples as f64; + }); loop { let settings = DefaultSettingsBuilder::default() .equilibrate_enable(true) @@ -172,7 +178,7 @@ impl QPModel { pub(self) fn build_linear_part_objective(&self, dist: &[f64]) -> Vec { let mut linear = Vec::with_capacity(1 + self.n_examples); - linear.push(0f64); + linear.push(1f64); let iter = dist.into_iter() .copied() .map(|di| (1f64 / self.eta) * di.ln()); @@ -194,6 +200,9 @@ impl QPModel { col_ptr.push(0usize); row_val.push(0usize); nonzero.push(1f64); + // NOTE: + // we do not need to multiply 0.5f64 + // since clarabel add it automatically. for (i, &di) in (1..).zip(dist) { col_ptr.push(i); row_val.push(i); @@ -238,7 +247,7 @@ impl QPModel { // capping constraint: `d_i ≤ 1/ν` row_val.push(self.n_examples + j); - nonzero.push(1f64); + nonzero.push(self.cap_inv); // margin constraints of `i`-th column for (i, &yh) in (0..).zip(margins) { diff --git a/src/booster/softboost/qp_model.rs b/src/booster/softboost/qp_model.rs index d30bdd7..7f9a8f6 100644 --- a/src/booster/softboost/qp_model.rs +++ b/src/booster/softboost/qp_model.rs @@ -47,30 +47,30 @@ const QP_TOLERANCE: f64 = 1e-9; /// ```txt /// # of /// rows d1 ... dm -/// ┏ ┓ ┏ ┓ -/// 1 ┃ 1 ... 1 ┃ = ┃ 1 ┃ -/// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -/// ┃ ┃ ≤ ┃ 0 ┃ -/// ┃ ┃ ≤ ┃ 0 ┃ -/// ┃ (-1) * Identity matrix ┃ . ┃ . ┃ -/// m ┃ m x m ┃ . ┃ . ┃ -/// ┃ ┃ . ┃ . ┃ -/// ┃ ┃ ≤ ┃ 0 ┃ -/// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -/// ┃ ┃ ≤ ┃ 0 ┃ -/// ┃ ┃ ≤ ┃ 0 ┃ -/// ┃ ┃ . ┃ . ┃ -/// m ┃ Identity matrix ┃ . ┃ . ┃ -/// ┃ m x m ┃ . ┃ . ┃ -/// ┃ ┃ ≤ ┃ 0 ┃ -/// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -/// ┃ y_1 h_1(x_1) ... y_m h_1(x_m) ┃ ≤ ┃ 0 ┃ -/// ┃ y_1 h_2(x_1) ... y_m h_2(x_m) ┃ ≤ ┃ 0 ┃ -/// ┃ . ... . ┃ . ┃ . ┃ -/// H ┃ . ... . ┃ . ┃ . ┃ -/// ┃ . ... . ┃ . ┃ . ┃ -/// ┃ y_1 h_T(x_1) ... y_m h_T(x_m) ┃ ≤ ┃ 0 ┃ -/// ┗ ┛ ┗ ┛ +/// ┏ ┓ ┏ ┓ +/// 1 ┃ 1 ... 1 ┃ = ┃ 1 ┃ +/// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +/// ┃ ┃ ≤ ┃ 0 ┃ +/// ┃ ┃ ≤ ┃ 0 ┃ +/// ┃ (-1) * Identity matrix ┃ . ┃ . ┃ +/// m ┃ m x m ┃ . ┃ . ┃ +/// ┃ ┃ . ┃ . ┃ +/// ┃ ┃ ≤ ┃ 0 ┃ +/// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +/// ┃ ┃ ≤ ┃ 1/ν ┃ +/// ┃ ┃ ≤ ┃ 1/ν ┃ +/// m ┃ Identity matrix ┃ . ┃ . ┃ +/// ┃ m x m ┃ . ┃ . ┃ +/// ┃ ┃ . ┃ . ┃ +/// ┃ ┃ ≤ ┃ 1/ν ┃ +/// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +/// ┃ y_1 h_1(x_1) ... y_m h_1(x_m) ┃ ≤ ┃ 0 ┃ +/// ┃ y_1 h_2(x_1) ... y_m h_2(x_m) ┃ ≤ ┃ 0 ┃ +/// ┃ . ... . ┃ . ┃ . ┃ +/// H ┃ . ... . ┃ . ┃ . ┃ +/// ┃ . ... . ┃ . ┃ . ┃ +/// ┃ y_1 h_T(x_1) ... y_m h_T(x_m) ┃ ≤ ┃ 0 ┃ +/// ┗ ┛ ┗ ┛ /// /// # of /// cols m @@ -236,7 +236,7 @@ impl QPModel { // capping constraint: `d_i ≤ 1/ν` row_val.push(self.n_examples + j + 1); - nonzero.push(1f64); + nonzero.push(self.cap_inv); // margin constraints of `i`-th column for (i, &yh) in (0..).zip(margins) {