From e351065ceda3295166cc2a1ea4c20578e3b75d25 Mon Sep 17 00:00:00 2001 From: Thierry Moudiki Date: Thu, 8 Aug 2024 20:15:40 +0200 Subject: [PATCH] add nonconformist to the mix --- .github/workflows/python-publish.yml | 16 +- unifiedbooster.egg-info/PKG-INFO | 29 + unifiedbooster.egg-info/SOURCES.txt | 23 + unifiedbooster.egg-info/dependency_links.txt | 1 + unifiedbooster.egg-info/entry_points.txt | 2 + unifiedbooster.egg-info/not-zip-safe | 1 + unifiedbooster.egg-info/requires.txt | 8 + unifiedbooster.egg-info/top_level.txt | 1 + unifiedbooster/nonconformist/LICENSE | 25 + unifiedbooster/nonconformist/__init__.py | 30 + unifiedbooster/nonconformist/acp.py | 381 ++++++++++++ unifiedbooster/nonconformist/base.py | 156 +++++ unifiedbooster/nonconformist/cp.py | 172 ++++++ unifiedbooster/nonconformist/evaluation.py | 486 +++++++++++++++ unifiedbooster/nonconformist/icp.py | 442 ++++++++++++++ unifiedbooster/nonconformist/nc.py | 610 +++++++++++++++++++ unifiedbooster/nonconformist/util.py | 9 + 17 files changed, 2384 insertions(+), 8 deletions(-) create mode 100644 unifiedbooster.egg-info/PKG-INFO create mode 100644 unifiedbooster.egg-info/SOURCES.txt create mode 100644 unifiedbooster.egg-info/dependency_links.txt create mode 100644 unifiedbooster.egg-info/entry_points.txt create mode 100644 unifiedbooster.egg-info/not-zip-safe create mode 100644 unifiedbooster.egg-info/requires.txt create mode 100644 unifiedbooster.egg-info/top_level.txt create mode 100644 unifiedbooster/nonconformist/LICENSE create mode 100644 unifiedbooster/nonconformist/__init__.py create mode 100644 unifiedbooster/nonconformist/acp.py create mode 100644 unifiedbooster/nonconformist/base.py create mode 100644 unifiedbooster/nonconformist/cp.py create mode 100644 unifiedbooster/nonconformist/evaluation.py create mode 100644 unifiedbooster/nonconformist/icp.py create mode 100644 unifiedbooster/nonconformist/nc.py create mode 100644 unifiedbooster/nonconformist/util.py diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 345ca77..41dad2e 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -27,11 +27,11 @@ jobs: - name: Build distribution run: python setup.py sdist bdist_wheel - - name: Run examples - run: pip install .&&find examples -maxdepth 2 -name "*.py" -exec python3 {} \; - - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.PYPI_GLOBAL_UB }} - repository-url: https://upload.pypi.org/legacy/ + #- name: Run examples + # run: pip install .&&find examples -maxdepth 2 -name "*.py" -exec python3 {} \; + + #- name: Publish to PyPI + # uses: pypa/gh-action-pypi-publish@release/v1 + # with: + # password: ${{ secrets.PYPI_GLOBAL_UB }} + # repository-url: https://upload.pypi.org/legacy/ diff --git a/unifiedbooster.egg-info/PKG-INFO b/unifiedbooster.egg-info/PKG-INFO new file mode 100644 index 0000000..fed0964 --- /dev/null +++ b/unifiedbooster.egg-info/PKG-INFO @@ -0,0 +1,29 @@ +Metadata-Version: 2.1 +Name: unifiedbooster +Version: 0.5.0 +Summary: Unified interface for Gradient Boosted Decision Trees +Home-page: https://github.com/thierrymoudiki/unifiedbooster +Author: T. Moudiki +Author-email: thierry.moudiki@gmail.com +License: BSD license +Keywords: unifiedbooster +Classifier: Development Status :: 2 - Pre-Alpha +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Natural Language :: English +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Requires-Python: >=3.6 +License-File: LICENSE +Requires-Dist: Cython +Requires-Dist: numpy +Requires-Dist: scikit-learn +Requires-Dist: xgboost +Requires-Dist: lightgbm +Requires-Dist: catboost +Requires-Dist: GPopt +Requires-Dist: nnetsauce + +Unified interface for Gradient Boosted Decision Trees diff --git a/unifiedbooster.egg-info/SOURCES.txt b/unifiedbooster.egg-info/SOURCES.txt new file mode 100644 index 0000000..f63f624 --- /dev/null +++ b/unifiedbooster.egg-info/SOURCES.txt @@ -0,0 +1,23 @@ +LICENSE +README.md +setup.py +unifiedbooster/__init__.py +unifiedbooster/gbdt.py +unifiedbooster/gbdt_classification.py +unifiedbooster/gbdt_regression.py +unifiedbooster/gpoptimization.py +unifiedbooster.egg-info/PKG-INFO +unifiedbooster.egg-info/SOURCES.txt +unifiedbooster.egg-info/dependency_links.txt +unifiedbooster.egg-info/entry_points.txt +unifiedbooster.egg-info/not-zip-safe +unifiedbooster.egg-info/requires.txt +unifiedbooster.egg-info/top_level.txt +unifiedbooster/nonconformist/__init__.py +unifiedbooster/nonconformist/acp.py +unifiedbooster/nonconformist/base.py +unifiedbooster/nonconformist/cp.py +unifiedbooster/nonconformist/evaluation.py +unifiedbooster/nonconformist/icp.py +unifiedbooster/nonconformist/nc.py +unifiedbooster/nonconformist/util.py \ No newline at end of file diff --git a/unifiedbooster.egg-info/dependency_links.txt b/unifiedbooster.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/unifiedbooster.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/unifiedbooster.egg-info/entry_points.txt b/unifiedbooster.egg-info/entry_points.txt new file mode 100644 index 0000000..3fc43f8 --- /dev/null +++ b/unifiedbooster.egg-info/entry_points.txt @@ -0,0 +1,2 @@ +[console_scripts] +unifiedbooster = unifiedbooster.cli:main diff --git a/unifiedbooster.egg-info/not-zip-safe b/unifiedbooster.egg-info/not-zip-safe new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/unifiedbooster.egg-info/not-zip-safe @@ -0,0 +1 @@ + diff --git a/unifiedbooster.egg-info/requires.txt b/unifiedbooster.egg-info/requires.txt new file mode 100644 index 0000000..0d8325e --- /dev/null +++ b/unifiedbooster.egg-info/requires.txt @@ -0,0 +1,8 @@ +Cython +numpy +scikit-learn +xgboost +lightgbm +catboost +GPopt +nnetsauce diff --git a/unifiedbooster.egg-info/top_level.txt b/unifiedbooster.egg-info/top_level.txt new file mode 100644 index 0000000..75a229a --- /dev/null +++ b/unifiedbooster.egg-info/top_level.txt @@ -0,0 +1 @@ +unifiedbooster diff --git a/unifiedbooster/nonconformist/LICENSE b/unifiedbooster/nonconformist/LICENSE new file mode 100644 index 0000000..cfacd8b --- /dev/null +++ b/unifiedbooster/nonconformist/LICENSE @@ -0,0 +1,25 @@ +The MIT License (MIT) + +nonconformist package: +Copyright (c) 2015 Henrik Linusson + +Other extensions: +Copyright (c) 2019 Yaniv Romano + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/unifiedbooster/nonconformist/__init__.py b/unifiedbooster/nonconformist/__init__.py new file mode 100644 index 0000000..ddddc18 --- /dev/null +++ b/unifiedbooster/nonconformist/__init__.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python + +""" +docstring +""" + +# Authors: Henrik Linusson +# Yaniv Romano modified np.py file to include CQR +# T. Moudiki modified __init__.py to import classes + +# __version__ = '2.1.0' + +from .nc import ( + AbsErrorErrFunc, + QuantileRegErrFunc, + RegressorNc, + RegressorNormalizer, +) +from .cp import IcpRegressor, TcpClassifier +from .icp import IcpClassifier +from .base import RegressorAdapter + +__all__ = [ + "AbsErrorErrFunc", + "QuantileRegErrFunc", + "RegressorAdapter", + "RegressorNc", + "RegressorNormalizer", + "IcpRegressor", +] diff --git a/unifiedbooster/nonconformist/acp.py b/unifiedbooster/nonconformist/acp.py new file mode 100644 index 0000000..6e51384 --- /dev/null +++ b/unifiedbooster/nonconformist/acp.py @@ -0,0 +1,381 @@ +#!/usr/bin/env python + +""" +Aggregated conformal predictors +""" + +# Authors: Henrik Linusson + +import numpy as np +from sklearn.model_selection import KFold, StratifiedKFold +from sklearn.model_selection import ShuffleSplit, StratifiedShuffleSplit +from sklearn.base import clone +from nonconformist.base import BaseEstimator +from nonconformist.util import calc_p + + +# ----------------------------------------------------------------------------- +# Sampling strategies +# ----------------------------------------------------------------------------- +class BootstrapSampler(object): + """Bootstrap sampler. + + See also + -------- + CrossSampler, RandomSubSampler + + Examples + -------- + """ + + def gen_samples(self, y, n_samples, problem_type): + for i in range(n_samples): + idx = np.array(range(y.size)) + train = np.random.choice(y.size, y.size, replace=True) + cal_mask = np.array(np.ones(idx.size), dtype=bool) + for j in train: + cal_mask[j] = False + cal = idx[cal_mask] + + yield train, cal + + +class CrossSampler(object): + """Cross-fold sampler. + + See also + -------- + BootstrapSampler, RandomSubSampler + + Examples + -------- + """ + + def gen_samples(self, y, n_samples, problem_type): + if problem_type == "classification": + folds = StratifiedKFold(y, n_folds=n_samples) + else: + folds = KFold(y.size, n_folds=n_samples) + for train, cal in folds: + yield train, cal + + +class RandomSubSampler(object): + """Random subsample sampler. + + Parameters + ---------- + calibration_portion : float + Ratio (0-1) of examples to use for calibration. + + See also + -------- + BootstrapSampler, CrossSampler + + Examples + -------- + """ + + def __init__(self, calibration_portion=0.3): + self.cal_portion = calibration_portion + + def gen_samples(self, y, n_samples, problem_type): + if problem_type == "classification": + splits = StratifiedShuffleSplit( + y, n_iter=n_samples, test_size=self.cal_portion + ) + else: + splits = ShuffleSplit( + y.size, n_iter=n_samples, test_size=self.cal_portion + ) + + for train, cal in splits: + yield train, cal + + +# ----------------------------------------------------------------------------- +# Conformal ensemble +# ----------------------------------------------------------------------------- +class AggregatedCp(BaseEstimator): + """Aggregated conformal predictor. + + Combines multiple IcpClassifier or IcpRegressor predictors into an + aggregated model. + + Parameters + ---------- + predictor : object + Prototype conformal predictor (e.g. IcpClassifier or IcpRegressor) + used for defining conformal predictors included in the aggregate model. + + sampler : object + Sampler object used to generate training and calibration examples + for the underlying conformal predictors. + + aggregation_func : callable + Function used to aggregate the predictions of the underlying + conformal predictors. Defaults to ``numpy.mean``. + + n_models : int + Number of models to aggregate. + + Attributes + ---------- + predictor : object + Prototype conformal predictor. + + predictors : list + List of underlying conformal predictors. + + sampler : object + Sampler object used to generate training and calibration examples. + + agg_func : callable + Function used to aggregate the predictions of the underlying + conformal predictors + + References + ---------- + .. [1] Vovk, V. (2013). Cross-conformal predictors. Annals of Mathematics + and Artificial Intelligence, 1-20. + + .. [2] Carlsson, L., Eklund, M., & Norinder, U. (2014). Aggregated + Conformal Prediction. In Artificial Intelligence Applications and + Innovations (pp. 231-240). Springer Berlin Heidelberg. + + Examples + -------- + """ + + def __init__( + self, + predictor, + sampler=BootstrapSampler(), + aggregation_func=None, + n_models=10, + ): + self.predictors = [] + self.n_models = n_models + self.predictor = predictor + self.sampler = sampler + + if aggregation_func is not None: + self.agg_func = aggregation_func + else: + self.agg_func = lambda x: np.mean(x, axis=2) + + def fit(self, x, y): + """Fit underlying conformal predictors. + + Parameters + ---------- + x : numpy array of shape [n_samples, n_features] + Inputs of examples for fitting the underlying conformal predictors. + + y : numpy array of shape [n_samples] + Outputs of examples for fitting the underlying conformal predictors. + + Returns + ------- + None + """ + self.n_train = y.size + self.predictors = [] + idx = np.random.permutation(y.size) + x, y = x[idx, :], y[idx] + problem_type = self.predictor.__class__.get_problem_type() + samples = self.sampler.gen_samples(y, self.n_models, problem_type) + for train, cal in samples: + predictor = clone(self.predictor) + predictor.fit(x[train, :], y[train]) + predictor.calibrate(x[cal, :], y[cal]) + self.predictors.append(predictor) + + if problem_type == "classification": + self.classes = self.predictors[0].classes + + def predict(self, x, significance=None): + """Predict the output values for a set of input patterns. + + Parameters + ---------- + x : numpy array of shape [n_samples, n_features] + Inputs of patters for which to predict output values. + + significance : float or None + Significance level (maximum allowed error rate) of predictions. + Should be a float between 0 and 1. If ``None``, then the p-values + are output rather than the predictions. Note: ``significance=None`` + is applicable to classification problems only. + + Returns + ------- + p : numpy array of shape [n_samples, n_classes] or [n_samples, 2] + For classification problems: If significance is ``None``, then p + contains the p-values for each sample-class pair; if significance + is a float between 0 and 1, then p is a boolean array denoting + which labels are included in the prediction sets. + + For regression problems: Prediction interval (minimum and maximum + boundaries) for the set of test patterns. + """ + is_regression = ( + self.predictor.__class__.get_problem_type() == "regression" + ) + + n_examples = x.shape[0] + + if is_regression and significance is None: + signs = np.arange(0.01, 1.0, 0.01) + pred = np.zeros((n_examples, 2, signs.size)) + for i, s in enumerate(signs): + predictions = np.dstack( + [p.predict(x, s) for p in self.predictors] + ) + predictions = self.agg_func(predictions) + pred[:, :, i] = predictions + return pred + else: + + def f(p, x): + return p.predict(x, significance if is_regression else None) + + predictions = np.dstack([f(p, x) for p in self.predictors]) + predictions = self.agg_func(predictions) + + if significance and not is_regression: + return predictions >= significance + else: + return predictions + + +class CrossConformalClassifier(AggregatedCp): + """Cross-conformal classifier. + + Combines multiple IcpClassifiers into a cross-conformal classifier. + + Parameters + ---------- + predictor : object + Prototype conformal predictor (e.g. IcpClassifier or IcpRegressor) + used for defining conformal predictors included in the aggregate model. + + aggregation_func : callable + Function used to aggregate the predictions of the underlying + conformal predictors. Defaults to ``numpy.mean``. + + n_models : int + Number of models to aggregate. + + Attributes + ---------- + predictor : object + Prototype conformal predictor. + + predictors : list + List of underlying conformal predictors. + + sampler : object + Sampler object used to generate training and calibration examples. + + agg_func : callable + Function used to aggregate the predictions of the underlying + conformal predictors + + References + ---------- + .. [1] Vovk, V. (2013). Cross-conformal predictors. Annals of Mathematics + and Artificial Intelligence, 1-20. + + Examples + -------- + """ + + def __init__(self, predictor, n_models=10): + super(CrossConformalClassifier, self).__init__( + predictor, CrossSampler(), n_models + ) + + def predict(self, x, significance=None): + ncal_ngt_neq = np.stack( + [p._get_stats(x) for p in self.predictors], axis=3 + ) + ncal_ngt_neq = ncal_ngt_neq.sum(axis=3) + + p = calc_p( + ncal_ngt_neq[:, :, 0], + ncal_ngt_neq[:, :, 1], + ncal_ngt_neq[:, :, 2], + smoothing=self.predictors[0].smoothing, + ) + + if significance: + return p > significance + else: + return p + + +class BootstrapConformalClassifier(AggregatedCp): + """Bootstrap conformal classifier. + + Combines multiple IcpClassifiers into a bootstrap conformal classifier. + + Parameters + ---------- + predictor : object + Prototype conformal predictor (e.g. IcpClassifier or IcpRegressor) + used for defining conformal predictors included in the aggregate model. + + aggregation_func : callable + Function used to aggregate the predictions of the underlying + conformal predictors. Defaults to ``numpy.mean``. + + n_models : int + Number of models to aggregate. + + Attributes + ---------- + predictor : object + Prototype conformal predictor. + + predictors : list + List of underlying conformal predictors. + + sampler : object + Sampler object used to generate training and calibration examples. + + agg_func : callable + Function used to aggregate the predictions of the underlying + conformal predictors + + References + ---------- + .. [1] Vovk, V. (2013). Cross-conformal predictors. Annals of Mathematics + and Artificial Intelligence, 1-20. + + Examples + -------- + """ + + def __init__(self, predictor, n_models=10): + super(BootstrapConformalClassifier, self).__init__( + predictor, BootstrapSampler(), n_models + ) + + def predict(self, x, significance=None): + ncal_ngt_neq = np.stack( + [p._get_stats(x) for p in self.predictors], axis=3 + ) + ncal_ngt_neq = ncal_ngt_neq.sum(axis=3) + + p = calc_p( + ncal_ngt_neq[:, :, 0] + ncal_ngt_neq[:, :, 0] / self.n_train, + ncal_ngt_neq[:, :, 1] + ncal_ngt_neq[:, :, 0] / self.n_train, + ncal_ngt_neq[:, :, 2], + smoothing=self.predictors[0].smoothing, + ) + + if significance: + return p > significance + else: + return p diff --git a/unifiedbooster/nonconformist/base.py b/unifiedbooster/nonconformist/base.py new file mode 100644 index 0000000..baea8e9 --- /dev/null +++ b/unifiedbooster/nonconformist/base.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python + +""" +docstring +""" + +# Authors: Henrik Linusson + +import abc +import numpy as np + +from sklearn.base import BaseEstimator + + +class RegressorMixin(object): + def __init__(self): + super(RegressorMixin, self).__init__() + + @classmethod + def get_problem_type(cls): + return "regression" + + +class ClassifierMixin(object): + def __init__(self): + super(ClassifierMixin, self).__init__() + + @classmethod + def get_problem_type(cls): + return "classification" + + +class BaseModelAdapter(BaseEstimator): + __metaclass__ = abc.ABCMeta + + def __init__(self, model, fit_params=None): + super(BaseModelAdapter, self).__init__() + + self.model = model + self.last_x, self.last_y = None, None + self.clean = False + self.fit_params = {} if fit_params is None else fit_params + + def fit(self, x, y): + """Fits the model. + + Parameters + ---------- + x : numpy array of shape [n_samples, n_features] + Inputs of examples for fitting the model. + + y : numpy array of shape [n_samples] + Outputs of examples for fitting the model. + + Returns + ------- + None + """ + + self.model.fit(x, y, **self.fit_params) + self.clean = False + + def predict(self, x): + """Returns the prediction made by the underlying model. + + Parameters + ---------- + x : numpy array of shape [n_samples, n_features] + Inputs of test examples. + + Returns + ------- + y : numpy array of shape [n_samples] + Predicted outputs of test examples. + """ + if ( + not self.clean + or self.last_x is None + or self.last_y is None + or not np.array_equal(self.last_x, x) + ): + self.last_x = x + self.last_y = self._underlying_predict(x) + self.clean = True + + return self.last_y.copy() + + @abc.abstractmethod + def _underlying_predict(self, x): + """Produces a prediction using the encapsulated model. + + Parameters + ---------- + x : numpy array of shape [n_samples, n_features] + Inputs of test examples. + + Returns + ------- + y : numpy array of shape [n_samples] + Predicted outputs of test examples. + """ + pass + + +class ClassifierAdapter(BaseModelAdapter): + def __init__(self, model, fit_params=None): + super(ClassifierAdapter, self).__init__(model, fit_params) + + def _underlying_predict(self, x): + return self.model.predict_proba(x) + + +class RegressorAdapter(BaseModelAdapter): + def __init__(self, model, fit_params=None): + super(RegressorAdapter, self).__init__(model, fit_params) + + def _underlying_predict(self, x): + return self.model.predict(x) + + +class OobMixin(object): + def __init__(self, model, fit_params=None): + super(OobMixin, self).__init__(model, fit_params) + self.train_x = None + + def fit(self, x, y): + super(OobMixin, self).fit(x, y) + self.train_x = x + + def _underlying_predict(self, x): + # TODO: sub-sampling of ensemble for test patterns + oob = x == self.train_x + + if hasattr(oob, "all"): + oob = oob.all() + + if oob: + return self._oob_prediction() + else: + return super(OobMixin, self)._underlying_predict(x) + + +class OobClassifierAdapter(OobMixin, ClassifierAdapter): + def __init__(self, model, fit_params=None): + super(OobClassifierAdapter, self).__init__(model, fit_params) + + def _oob_prediction(self): + return self.model.oob_decision_function_ + + +class OobRegressorAdapter(OobMixin, RegressorAdapter): + def __init__(self, model, fit_params=None): + super(OobRegressorAdapter, self).__init__(model, fit_params) + + def _oob_prediction(self): + return self.model.oob_prediction_ diff --git a/unifiedbooster/nonconformist/cp.py b/unifiedbooster/nonconformist/cp.py new file mode 100644 index 0000000..4cb4dcd --- /dev/null +++ b/unifiedbooster/nonconformist/cp.py @@ -0,0 +1,172 @@ +from .icp import * + +# TODO: move contents from nonconformist.icp here + + +# ----------------------------------------------------------------------------- +# TcpClassifier +# ----------------------------------------------------------------------------- +class TcpClassifier(BaseEstimator, ClassifierMixin): + """Transductive conformal classifier. + + Parameters + ---------- + nc_function : BaseScorer + Nonconformity scorer object used to calculate nonconformity of + calibration examples and test patterns. Should implement ``fit(x, y)`` + and ``calc_nc(x, y)``. + + smoothing : boolean + Decides whether to use stochastic smoothing of p-values. + + Attributes + ---------- + train_x : numpy array of shape [n_cal_examples, n_features] + Inputs of training set. + + train_y : numpy array of shape [n_cal_examples] + Outputs of calibration set. + + nc_function : BaseScorer + Nonconformity scorer object used to calculate nonconformity scores. + + classes : numpy array of shape [n_classes] + List of class labels, with indices corresponding to output columns + of TcpClassifier.predict() + + See also + -------- + IcpClassifier + + References + ---------- + .. [1] Vovk, V., Gammerman, A., & Shafer, G. (2005). Algorithmic learning + in a random world. Springer Science & Business Media. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.datasets import load_iris + >>> from sklearn.svm import SVC + >>> from nonconformist.base import ClassifierAdapter + >>> from nonconformist.cp import TcpClassifier + >>> from nonconformist.nc import ClassifierNc, MarginErrFunc + >>> iris = load_iris() + >>> idx = np.random.permutation(iris.target.size) + >>> train = idx[:int(idx.size / 2)] + >>> test = idx[int(idx.size / 2):] + >>> model = ClassifierAdapter(SVC(probability=True)) + >>> nc = ClassifierNc(model, MarginErrFunc()) + >>> tcp = TcpClassifier(nc) + >>> tcp.fit(iris.data[train, :], iris.target[train]) + >>> tcp.predict(iris.data[test, :], significance=0.10) + ... # doctest: +SKIP + array([[ True, False, False], + [False, True, False], + ..., + [False, True, False], + [False, True, False]], dtype=bool) + """ + + def __init__(self, nc_function, condition=None, smoothing=True): + self.train_x, self.train_y = None, None + self.nc_function = nc_function + super(TcpClassifier, self).__init__() + + # Check if condition-parameter is the default function (i.e., + # lambda x: 0). This is so we can safely clone the object without + # the clone accidentally having self.conditional = True. + def default_condition(x): + return 0 + + is_default = callable(condition) and ( + condition.__code__.co_code == default_condition.__code__.co_code + ) + + if is_default: + self.condition = condition + self.conditional = False + elif callable(condition): + self.condition = condition + self.conditional = True + else: + self.condition = lambda x: 0 + self.conditional = False + + self.smoothing = smoothing + + self.base_icp = IcpClassifier( + self.nc_function, self.condition, self.smoothing + ) + + self.classes = None + + def fit(self, x, y): + self.train_x, self.train_y = x, y + self.classes = np.unique(y) + + def predict(self, x, significance=None): + """Predict the output values for a set of input patterns. + + Parameters + ---------- + x : numpy array of shape [n_samples, n_features] + Inputs of patters for which to predict output values. + + significance : float or None + Significance level (maximum allowed error rate) of predictions. + Should be a float between 0 and 1. If ``None``, then the p-values + are output rather than the predictions. + + Returns + ------- + p : numpy array of shape [n_samples, n_classes] + If significance is ``None``, then p contains the p-values for each + sample-class pair; if significance is a float between 0 and 1, then + p is a boolean array denoting which labels are included in the + prediction sets. + """ + n_test = x.shape[0] + n_train = self.train_x.shape[0] + p = np.zeros((n_test, self.classes.size)) + for i in range(n_test): + for j, y in enumerate(self.classes): + train_x = np.vstack([self.train_x, x[i, :]]) + train_y = np.hstack([self.train_y, y]) + self.base_icp.fit(train_x, train_y) + scores = self.base_icp.nc_function.score(train_x, train_y) + ngt = (scores[:-1] > scores[-1]).sum() + neq = (scores[:-1] == scores[-1]).sum() + + p[i, j] = calc_p(n_train, ngt, neq, self.smoothing) + + if significance is not None: + return p > significance + else: + return p + + def predict_conf(self, x): + """Predict the output values for a set of input patterns, using + the confidence-and-credibility output scheme. + + Parameters + ---------- + x : numpy array of shape [n_samples, n_features] + Inputs of patters for which to predict output values. + + Returns + ------- + p : numpy array of shape [n_samples, 3] + p contains three columns: the first column contains the most + likely class for each test pattern; the second column contains + the confidence in the predicted class label, and the third column + contains the credibility of the prediction. + """ + p = self.predict(x, significance=None) + label = p.argmax(axis=1) + credibility = p.max(axis=1) + for i, idx in enumerate(label): + p[i, idx] = -np.inf + confidence = 1 - p.max(axis=1) + + return np.array([label, confidence, credibility]).T diff --git a/unifiedbooster/nonconformist/evaluation.py b/unifiedbooster/nonconformist/evaluation.py new file mode 100644 index 0000000..e85d5e8 --- /dev/null +++ b/unifiedbooster/nonconformist/evaluation.py @@ -0,0 +1,486 @@ +#!/usr/bin/env python + +""" +Evaluation of conformal predictors. +""" + +# Authors: Henrik Linusson + +# TODO: cross_val_score/run_experiment should possibly allow multiple to be evaluated on identical folding + +from __future__ import division + +from nonconformist.base import RegressorMixin, ClassifierMixin + +import sys +import numpy as np +import pandas as pd + +from sklearn.cross_validation import StratifiedShuffleSplit +from sklearn.cross_validation import KFold +from sklearn.cross_validation import train_test_split +from sklearn.base import clone, BaseEstimator + + +class BaseIcpCvHelper(BaseEstimator): + """Base class for cross validation helpers.""" + + def __init__(self, icp, calibration_portion): + super(BaseIcpCvHelper, self).__init__() + self.icp = icp + self.calibration_portion = calibration_portion + + def predict(self, x, significance=None): + return self.icp.predict(x, significance) + + +class ClassIcpCvHelper(BaseIcpCvHelper, ClassifierMixin): + """Helper class for running the ``cross_val_score`` evaluation + method on IcpClassifiers. + + See also + -------- + IcpRegCrossValHelper + + Examples + -------- + >>> from sklearn.datasets import load_iris + >>> from sklearn.ensemble import RandomForestClassifier + >>> from nonconformist.icp import IcpClassifier + >>> from nonconformist.nc import ClassifierNc, MarginErrFunc + >>> from nonconformist.evaluation import ClassIcpCvHelper + >>> from nonconformist.evaluation import class_mean_errors + >>> from nonconformist.evaluation import cross_val_score + >>> data = load_iris() + >>> nc = ProbEstClassifierNc(RandomForestClassifier(), MarginErrFunc()) + >>> icp = IcpClassifier(nc) + >>> icp_cv = ClassIcpCvHelper(icp) + >>> cross_val_score(icp_cv, + ... data.data, + ... data.target, + ... iterations=2, + ... folds=2, + ... scoring_funcs=[class_mean_errors], + ... significance_levels=[0.1]) + ... # doctest: +SKIP + class_mean_errors fold iter significance + 0 0.013333 0 0 0.1 + 1 0.080000 1 0 0.1 + 2 0.053333 0 1 0.1 + 3 0.080000 1 1 0.1 + """ + + def __init__(self, icp, calibration_portion=0.25): + super(ClassIcpCvHelper, self).__init__(icp, calibration_portion) + + def fit(self, x, y): + split = StratifiedShuffleSplit( + y, n_iter=1, test_size=self.calibration_portion + ) + for train, cal in split: + self.icp.fit(x[train, :], y[train]) + self.icp.calibrate(x[cal, :], y[cal]) + + +class RegIcpCvHelper(BaseIcpCvHelper, RegressorMixin): + """Helper class for running the ``cross_val_score`` evaluation + method on IcpRegressors. + + See also + -------- + IcpClassCrossValHelper + + Examples + -------- + >>> from sklearn.datasets import load_boston + >>> from sklearn.ensemble import RandomForestRegressor + >>> from nonconformist.icp import IcpRegressor + >>> from nonconformist.nc import RegressorNc, AbsErrorErrFunc + >>> from nonconformist.evaluation import RegIcpCvHelper + >>> from nonconformist.evaluation import reg_mean_errors + >>> from nonconformist.evaluation import cross_val_score + >>> data = load_boston() + >>> nc = RegressorNc(RandomForestRegressor(), AbsErrorErrFunc()) + >>> icp = IcpRegressor(nc) + >>> icp_cv = RegIcpCvHelper(icp) + >>> cross_val_score(icp_cv, + ... data.data, + ... data.target, + ... iterations=2, + ... folds=2, + ... scoring_funcs=[reg_mean_errors], + ... significance_levels=[0.1]) + ... # doctest: +SKIP + fold iter reg_mean_errors significance + 0 0 0 0.185771 0.1 + 1 1 0 0.138340 0.1 + 2 0 1 0.071146 0.1 + 3 1 1 0.043478 0.1 + """ + + def __init__(self, icp, calibration_portion=0.25): + super(RegIcpCvHelper, self).__init__(icp, calibration_portion) + + def fit(self, x, y): + split = train_test_split(x, y, test_size=self.calibration_portion) + x_tr, x_cal, y_tr, y_cal = split[0], split[1], split[2], split[3] + self.icp.fit(x_tr, y_tr) + self.icp.calibrate(x_cal, y_cal) + + +# ----------------------------------------------------------------------------- +# +# ----------------------------------------------------------------------------- +def cross_val_score( + model, + x, + y, + iterations=10, + folds=10, + fit_params=None, + scoring_funcs=None, + significance_levels=None, + verbose=False, +): + """Evaluates a conformal predictor using cross-validation. + + Parameters + ---------- + model : object + Conformal predictor to evaluate. + + x : numpy array of shape [n_samples, n_features] + Inputs of data to use for evaluation. + + y : numpy array of shape [n_samples] + Outputs of data to use for evaluation. + + iterations : int + Number of iterations to use for evaluation. The data set is randomly + shuffled before each iteration. + + folds : int + Number of folds to use for evaluation. + + fit_params : dictionary + Parameters to supply to the conformal prediction object on training. + + scoring_funcs : iterable + List of evaluation functions to apply to the conformal predictor in each + fold. Each evaluation function should have a signature + ``scorer(prediction, y, significance)``. + + significance_levels : iterable + List of significance levels at which to evaluate the conformal + predictor. + + verbose : boolean + Indicates whether to output progress information during evaluation. + + Returns + ------- + scores : pandas DataFrame + Tabulated results for each iteration, fold and evaluation function. + """ + + fit_params = fit_params if fit_params else {} + significance_levels = ( + significance_levels + if significance_levels is not None + else np.arange(0.01, 1.0, 0.01) + ) + + df = pd.DataFrame() + + columns = [ + "iter", + "fold", + "significance", + ] + [f.__name__ for f in scoring_funcs] + for i in range(iterations): + idx = np.random.permutation(y.size) + x, y = x[idx, :], y[idx] + cv = KFold(y.size, folds) + for j, (train, test) in enumerate(cv): + if verbose: + sys.stdout.write( + "\riter {}/{} fold {}/{}".format( + i + 1, iterations, j + 1, folds + ) + ) + m = clone(model) + m.fit(x[train, :], y[train], **fit_params) + prediction = m.predict(x[test, :], significance=None) + for k, s in enumerate(significance_levels): + scores = [ + scoring_func(prediction, y[test], s) + for scoring_func in scoring_funcs + ] + df_score = pd.DataFrame([[i, j, s] + scores], columns=columns) + df = df.append(df_score, ignore_index=True) + + return df + + +def run_experiment( + models, + csv_files, + iterations=10, + folds=10, + fit_params=None, + scoring_funcs=None, + significance_levels=None, + normalize=False, + verbose=False, + header=0, +): + """Performs a cross-validation evaluation of one or several conformal + predictors on a collection of data sets in csv format. + + Parameters + ---------- + models : object or iterable + Conformal predictor(s) to evaluate. + + csv_files : iterable + List of file names (with absolute paths) containing csv-data, used to + evaluate the conformal predictor. + + iterations : int + Number of iterations to use for evaluation. The data set is randomly + shuffled before each iteration. + + folds : int + Number of folds to use for evaluation. + + fit_params : dictionary + Parameters to supply to the conformal prediction object on training. + + scoring_funcs : iterable + List of evaluation functions to apply to the conformal predictor in each + fold. Each evaluation function should have a signature + ``scorer(prediction, y, significance)``. + + significance_levels : iterable + List of significance levels at which to evaluate the conformal + predictor. + + verbose : boolean + Indicates whether to output progress information during evaluation. + + Returns + ------- + scores : pandas DataFrame + Tabulated results for each data set, iteration, fold and + evaluation function. + """ + df = pd.DataFrame() + if not hasattr(models, "__iter__"): + models = [models] + + for model in models: + is_regression = model.get_problem_type() == "regression" + + n_data_sets = len(csv_files) + for i, csv_file in enumerate(csv_files): + if verbose: + print("\n{} ({} / {})".format(csv_file, i + 1, n_data_sets)) + data = pd.read_csv(csv_file, header=header) + x, y = data.values[:, :-1], data.values[:, -1] + x = np.array(x, dtype=np.float64) + if normalize: + if is_regression: + y = y - y.min() / (y.max() - y.min()) + else: + for j, y_ in enumerate(np.unique(y)): + y[y == y_] = j + + scores = cross_val_score( + model, + x, + y, + iterations, + folds, + fit_params, + scoring_funcs, + significance_levels, + verbose, + ) + + ds_df = pd.DataFrame(scores) + ds_df["model"] = model.__class__.__name__ + try: + ds_df["data_set"] = csv_file.split("/")[-1] + except: + ds_df["data_set"] = csv_file + + df = df.append(ds_df) + + return df + + +# ----------------------------------------------------------------------------- +# Validity measures +# ----------------------------------------------------------------------------- +def reg_n_correct(prediction, y, significance=None): + """Calculates the number of correct predictions made by a conformal + regression model. + """ + if significance is not None: + idx = int(significance * 100 - 1) + prediction = prediction[:, :, idx] + + low = y >= prediction[:, 0] + high = y <= prediction[:, 1] + correct = low * high + + return y[correct].size + + +def reg_mean_errors(prediction, y, significance): + """Calculates the average error rate of a conformal regression model.""" + return 1 - reg_n_correct(prediction, y, significance) / y.size + + +def class_n_correct(prediction, y, significance): + """Calculates the number of correct predictions made by a conformal + classification model. + """ + labels, y = np.unique(y, return_inverse=True) + prediction = prediction > significance + correct = np.zeros((y.size,), dtype=bool) + for i, y_ in enumerate(y): + correct[i] = prediction[i, int(y_)] + return np.sum(correct) + + +def class_mean_errors(prediction, y, significance=None): + """Calculates the average error rate of a conformal classification model.""" + return 1 - (class_n_correct(prediction, y, significance) / y.size) + + +def class_one_err(prediction, y, significance=None): + """Calculates the error rate of conformal classifier predictions containing + only a single output label. + """ + labels, y = np.unique(y, return_inverse=True) + prediction = prediction > significance + idx = np.arange(0, y.size, 1) + idx = filter(lambda x: np.sum(prediction[x, :]) == 1, idx) + errors = filter(lambda x: not prediction[x, int(y[x])], idx) + + if len(idx) > 0: + return np.size(errors) / np.size(idx) + else: + return 0 + + +def class_mean_errors_one_class(prediction, y, significance, c=0): + """Calculates the average error rate of a conformal classification model, + considering only test examples belonging to class ``c``. Use + ``functools.partial`` in order to test other classes. + """ + labels, y = np.unique(y, return_inverse=True) + prediction = prediction > significance + idx = np.arange(0, y.size, 1)[y == c] + errs = np.sum(1 for _ in filter(lambda x: not prediction[x, c], idx)) + + if idx.size > 0: + return errs / idx.size + else: + return 0 + + +def class_one_err_one_class(prediction, y, significance, c=0): + """Calculates the error rate of conformal classifier predictions containing + only a single output label. Considers only test examples belonging to + class ``c``. Use ``functools.partial`` in order to test other classes. + """ + labels, y = np.unique(y, return_inverse=True) + prediction = prediction > significance + idx = np.arange(0, y.size, 1) + idx = filter(lambda x: prediction[x, c], idx) + idx = filter(lambda x: np.sum(prediction[x, :]) == 1, idx) + errors = filter(lambda x: int(y[x]) != c, idx) + + if len(idx) > 0: + return np.size(errors) / np.size(idx) + else: + return 0 + + +# ----------------------------------------------------------------------------- +# Efficiency measures +# ----------------------------------------------------------------------------- +def _reg_interval_size(prediction, y, significance): + idx = int(significance * 100 - 1) + prediction = prediction[:, :, idx] + + return prediction[:, 1] - prediction[:, 0] + + +def reg_min_size(prediction, y, significance): + return np.min(_reg_interval_size(prediction, y, significance)) + + +def reg_q1_size(prediction, y, significance): + return np.percentile(_reg_interval_size(prediction, y, significance), 25) + + +def reg_median_size(prediction, y, significance): + return np.median(_reg_interval_size(prediction, y, significance)) + + +def reg_q3_size(prediction, y, significance): + return np.percentile(_reg_interval_size(prediction, y, significance), 75) + + +def reg_max_size(prediction, y, significance): + return np.max(_reg_interval_size(prediction, y, significance)) + + +def reg_mean_size(prediction, y, significance): + """Calculates the average prediction interval size of a conformal + regression model. + """ + return np.mean(_reg_interval_size(prediction, y, significance)) + + +def class_avg_c(prediction, y, significance): + """Calculates the average number of classes per prediction of a conformal + classification model. + """ + prediction = prediction > significance + return np.sum(prediction) / prediction.shape[0] + + +def class_mean_p_val(prediction, y, significance): + """Calculates the mean of the p-values output by a conformal classification + model. + """ + return np.mean(prediction) + + +def class_one_c(prediction, y, significance): + """Calculates the rate of singleton predictions (prediction sets containing + only a single class label) of a conformal classification model. + """ + prediction = prediction > significance + n_singletons = np.sum( + 1 for _ in filter(lambda x: np.sum(x) == 1, prediction) + ) + return n_singletons / y.size + + +def class_empty(prediction, y, significance): + """Calculates the rate of singleton predictions (prediction sets containing + only a single class label) of a conformal classification model. + """ + prediction = prediction > significance + n_empty = np.sum(1 for _ in filter(lambda x: np.sum(x) == 0, prediction)) + return n_empty / y.size + + +def n_test(prediction, y, significance): + """Provides the number of test patters used in the evaluation.""" + return y.size diff --git a/unifiedbooster/nonconformist/icp.py b/unifiedbooster/nonconformist/icp.py new file mode 100644 index 0000000..2c22bc9 --- /dev/null +++ b/unifiedbooster/nonconformist/icp.py @@ -0,0 +1,442 @@ +#!/usr/bin/env python + +""" +Inductive conformal predictors. +""" + +# Authors: Henrik Linusson + +from __future__ import division +from collections import defaultdict +from functools import partial + +import numpy as np +from sklearn.base import BaseEstimator + +from .base import RegressorMixin, ClassifierMixin +from .util import calc_p + + +# ----------------------------------------------------------------------------- +# Base inductive conformal predictor +# ----------------------------------------------------------------------------- +class BaseIcp(BaseEstimator): + """Base class for inductive conformal predictors.""" + + def __init__(self, nc_function, condition=None): + self.cal_x, self.cal_y = None, None + self.nc_function = nc_function + + # Check if condition-parameter is the default function (i.e., + # lambda x: 0). This is so we can safely clone the object without + # the clone accidentally having self.conditional = True. + def default_condition(x): + return 0 + + is_default = callable(condition) and ( + condition.__code__.co_code == default_condition.__code__.co_code + ) + + if is_default: + self.condition = condition + self.conditional = False + elif callable(condition): + self.condition = condition + self.conditional = True + else: + self.condition = lambda x: 0 + self.conditional = False + + def fit(self, x, y): + """Fit underlying nonconformity scorer. + + Parameters + ---------- + x : numpy array of shape [n_samples, n_features] + Inputs of examples for fitting the nonconformity scorer. + + y : numpy array of shape [n_samples] + Outputs of examples for fitting the nonconformity scorer. + + Returns + ------- + None + """ + # TODO: incremental? + self.nc_function.fit(x, y) + + def calibrate(self, x, y, increment=False): + """Calibrate conformal predictor based on underlying nonconformity + scorer. + + Parameters + ---------- + x : numpy array of shape [n_samples, n_features] + Inputs of examples for calibrating the conformal predictor. + + y : numpy array of shape [n_samples, n_features] + Outputs of examples for calibrating the conformal predictor. + + increment : boolean + If ``True``, performs an incremental recalibration of the conformal + predictor. The supplied ``x`` and ``y`` are added to the set of + previously existing calibration examples, and the conformal + predictor is then calibrated on both the old and new calibration + examples. + + Returns + ------- + None + """ + self._calibrate_hook(x, y, increment) + self._update_calibration_set(x, y, increment) + + if self.conditional: + category_map = np.array( + [self.condition((x[i, :], y[i])) for i in range(y.size)] + ) + self.categories = np.unique(category_map) + self.cal_scores = defaultdict(partial(np.ndarray, 0)) + + for cond in self.categories: + idx = category_map == cond + cal_scores = self.nc_function.score( + self.cal_x[idx, :], self.cal_y[idx] + ) + self.cal_scores[cond] = np.sort(cal_scores, 0)[::-1] + else: + self.categories = np.array([0]) + cal_scores = self.nc_function.score(self.cal_x, self.cal_y) + self.cal_scores = {0: np.sort(cal_scores, 0)[::-1]} + + def _calibrate_hook(self, x, y, increment): + pass + + def _update_calibration_set(self, x, y, increment): + if increment and self.cal_x is not None and self.cal_y is not None: + self.cal_x = np.vstack([self.cal_x, x]) + self.cal_y = np.hstack([self.cal_y, y]) + else: + self.cal_x, self.cal_y = x, y + + +# ----------------------------------------------------------------------------- +# Inductive conformal classifier +# ----------------------------------------------------------------------------- +class IcpClassifier(BaseIcp, ClassifierMixin): + """Inductive conformal classifier. + + Parameters + ---------- + nc_function : BaseScorer + Nonconformity scorer object used to calculate nonconformity of + calibration examples and test patterns. Should implement ``fit(x, y)`` + and ``calc_nc(x, y)``. + + smoothing : boolean + Decides whether to use stochastic smoothing of p-values. + + Attributes + ---------- + cal_x : numpy array of shape [n_cal_examples, n_features] + Inputs of calibration set. + + cal_y : numpy array of shape [n_cal_examples] + Outputs of calibration set. + + nc_function : BaseScorer + Nonconformity scorer object used to calculate nonconformity scores. + + classes : numpy array of shape [n_classes] + List of class labels, with indices corresponding to output columns + of IcpClassifier.predict() + + See also + -------- + IcpRegressor + + References + ---------- + .. [1] Papadopoulos, H., & Haralambous, H. (2011). Reliable prediction + intervals with regression neural networks. Neural Networks, 24(8), + 842-851. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.datasets import load_iris + >>> from sklearn.tree import DecisionTreeClassifier + >>> from nonconformist.base import ClassifierAdapter + >>> from nonconformist.icp import IcpClassifier + >>> from nonconformist.nc import ClassifierNc, MarginErrFunc + >>> iris = load_iris() + >>> idx = np.random.permutation(iris.target.size) + >>> train = idx[:int(idx.size / 3)] + >>> cal = idx[int(idx.size / 3):int(2 * idx.size / 3)] + >>> test = idx[int(2 * idx.size / 3):] + >>> model = ClassifierAdapter(DecisionTreeClassifier()) + >>> nc = ClassifierNc(model, MarginErrFunc()) + >>> icp = IcpClassifier(nc) + >>> icp.fit(iris.data[train, :], iris.target[train]) + >>> icp.calibrate(iris.data[cal, :], iris.target[cal]) + >>> icp.predict(iris.data[test, :], significance=0.10) + ... # doctest: +SKIP + array([[ True, False, False], + [False, True, False], + ..., + [False, True, False], + [False, True, False]], dtype=bool) + """ + + def __init__(self, nc_function, condition=None, smoothing=True): + super(IcpClassifier, self).__init__(nc_function, condition) + self.classes = None + self.smoothing = smoothing + + def _calibrate_hook(self, x, y, increment=False): + self._update_classes(y, increment) + + def _update_classes(self, y, increment): + if self.classes is None or not increment: + self.classes = np.unique(y) + else: + self.classes = np.unique(np.hstack([self.classes, y])) + + def predict(self, x, significance=None): + """Predict the output values for a set of input patterns. + + Parameters + ---------- + x : numpy array of shape [n_samples, n_features] + Inputs of patters for which to predict output values. + + significance : float or None + Significance level (maximum allowed error rate) of predictions. + Should be a float between 0 and 1. If ``None``, then the p-values + are output rather than the predictions. + + Returns + ------- + p : numpy array of shape [n_samples, n_classes] + If significance is ``None``, then p contains the p-values for each + sample-class pair; if significance is a float between 0 and 1, then + p is a boolean array denoting which labels are included in the + prediction sets. + """ + # TODO: if x == self.last_x ... + n_test_objects = x.shape[0] + p = np.zeros((n_test_objects, self.classes.size)) + + ncal_ngt_neq = self._get_stats(x) + + for i in range(len(self.classes)): + for j in range(n_test_objects): + p[j, i] = calc_p( + ncal_ngt_neq[j, i, 0], + ncal_ngt_neq[j, i, 1], + ncal_ngt_neq[j, i, 2], + self.smoothing, + ) + + if significance is not None: + return p > significance + else: + return p + + def _get_stats(self, x): + n_test_objects = x.shape[0] + ncal_ngt_neq = np.zeros((n_test_objects, self.classes.size, 3)) + for i, c in enumerate(self.classes): + test_class = np.zeros(x.shape[0], dtype=self.classes.dtype) + test_class.fill(c) + + # TODO: maybe calculate p-values using cython or similar + # TODO: interpolated p-values + + # TODO: nc_function.calc_nc should take X * {y1, y2, ... ,yn} + test_nc_scores = self.nc_function.score(x, test_class) + for j, nc in enumerate(test_nc_scores): + cal_scores = self.cal_scores[self.condition((x[j, :], c))][::-1] + n_cal = cal_scores.size + + idx_left = np.searchsorted(cal_scores, nc, "left") + idx_right = np.searchsorted(cal_scores, nc, "right") + + ncal_ngt_neq[j, i, 0] = n_cal + ncal_ngt_neq[j, i, 1] = n_cal - idx_right + ncal_ngt_neq[j, i, 2] = idx_right - idx_left + + return ncal_ngt_neq + + def predict_conf(self, x): + """Predict the output values for a set of input patterns, using + the confidence-and-credibility output scheme. + + Parameters + ---------- + x : numpy array of shape [n_samples, n_features] + Inputs of patters for which to predict output values. + + Returns + ------- + p : numpy array of shape [n_samples, 3] + p contains three columns: the first column contains the most + likely class for each test pattern; the second column contains + the confidence in the predicted class label, and the third column + contains the credibility of the prediction. + """ + p = self.predict(x, significance=None) + label = p.argmax(axis=1) + credibility = p.max(axis=1) + for i, idx in enumerate(label): + p[i, idx] = -np.inf + confidence = 1 - p.max(axis=1) + + return np.array([label, confidence, credibility]).T + + +# ----------------------------------------------------------------------------- +# Inductive conformal regressor +# ----------------------------------------------------------------------------- +class IcpRegressor(BaseIcp, RegressorMixin): + """Inductive conformal regressor. + + Parameters + ---------- + nc_function : BaseScorer + Nonconformity scorer object used to calculate nonconformity of + calibration examples and test patterns. Should implement ``fit(x, y)``, + ``calc_nc(x, y)`` and ``predict(x, nc_scores, significance)``. + + Attributes + ---------- + cal_x : numpy array of shape [n_cal_examples, n_features] + Inputs of calibration set. + + cal_y : numpy array of shape [n_cal_examples] + Outputs of calibration set. + + nc_function : BaseScorer + Nonconformity scorer object used to calculate nonconformity scores. + + See also + -------- + IcpClassifier + + References + ---------- + .. [1] Papadopoulos, H., Proedrou, K., Vovk, V., & Gammerman, A. (2002). + Inductive confidence machines for regression. In Machine Learning: ECML + 2002 (pp. 345-356). Springer Berlin Heidelberg. + + .. [2] Papadopoulos, H., & Haralambous, H. (2011). Reliable prediction + intervals with regression neural networks. Neural Networks, 24(8), + 842-851. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.datasets import load_boston + >>> from sklearn.tree import DecisionTreeRegressor + >>> from nonconformist.base import RegressorAdapter + >>> from nonconformist.icp import IcpRegressor + >>> from nonconformist.nc import RegressorNc, AbsErrorErrFunc + >>> boston = load_boston() + >>> idx = np.random.permutation(boston.target.size) + >>> train = idx[:int(idx.size / 3)] + >>> cal = idx[int(idx.size / 3):int(2 * idx.size / 3)] + >>> test = idx[int(2 * idx.size / 3):] + >>> model = RegressorAdapter(DecisionTreeRegressor()) + >>> nc = RegressorNc(model, AbsErrorErrFunc()) + >>> icp = IcpRegressor(nc) + >>> icp.fit(boston.data[train, :], boston.target[train]) + >>> icp.calibrate(boston.data[cal, :], boston.target[cal]) + >>> icp.predict(boston.data[test, :], significance=0.10) + ... # doctest: +SKIP + array([[ 5. , 20.6], + [ 15.5, 31.1], + ..., + [ 14.2, 29.8], + [ 11.6, 27.2]]) + """ + + def __init__(self, nc_function, condition=None): + super(IcpRegressor, self).__init__(nc_function, condition) + + def predict(self, x, significance=None): + """Predict the output values for a set of input patterns. + + Parameters + ---------- + x : numpy array of shape [n_samples, n_features] + Inputs of patters for which to predict output values. + + significance : float + Significance level (maximum allowed error rate) of predictions. + Should be a float between 0 and 1. If ``None``, then intervals for + all significance levels (0.01, 0.02, ..., 0.99) are output in a + 3d-matrix. + + Returns + ------- + p : numpy array of shape [n_samples, 2] or [n_samples, 2, 99} + If significance is ``None``, then p contains the interval (minimum + and maximum boundaries) for each test pattern, and each significance + level (0.01, 0.02, ..., 0.99). If significance is a float between + 0 and 1, then p contains the prediction intervals (minimum and + maximum boundaries) for the set of test patterns at the chosen + significance level. + """ + # TODO: interpolated p-values + + n_significance = ( + 99 if significance is None else np.array(significance).size + ) + + if n_significance > 1: + prediction = np.zeros((x.shape[0], 2, n_significance)) + else: + prediction = np.zeros((x.shape[0], 2)) + + condition_map = np.array( + [self.condition((x[i, :], None)) for i in range(x.shape[0])] + ) + + for condition in self.categories: + idx = condition_map == condition + if np.sum(idx) > 0: + p = self.nc_function.predict( + x[idx, :], self.cal_scores[condition], significance + ) + if n_significance > 1: + prediction[idx, :, :] = p + else: + prediction[idx, :] = p + + return prediction + + +class OobCpClassifier(IcpClassifier): + def __init__(self, nc_function, condition=None, smoothing=True): + super(OobCpClassifier, self).__init__(nc_function, condition, smoothing) + + def fit(self, x, y): + super(OobCpClassifier, self).fit(x, y) + super(OobCpClassifier, self).calibrate(x, y, False) + + def calibrate(self, x, y, increment=False): + # Should throw exception (or really not be implemented for oob) + pass + + +class OobCpRegressor(IcpRegressor): + def __init__(self, nc_function, condition=None): + super(OobCpRegressor, self).__init__(nc_function, condition) + + def fit(self, x, y): + super(OobCpRegressor, self).fit(x, y) + super(OobCpRegressor, self).calibrate(x, y, False) + + def calibrate(self, x, y, increment=False): + # Should throw exception (or really not be implemented for oob) + pass diff --git a/unifiedbooster/nonconformist/nc.py b/unifiedbooster/nonconformist/nc.py new file mode 100644 index 0000000..7574258 --- /dev/null +++ b/unifiedbooster/nonconformist/nc.py @@ -0,0 +1,610 @@ +#!/usr/bin/env python + +""" +Nonconformity functions. +""" + +# Authors: Henrik Linusson +# Yaniv Romano modified RegressorNc class to include CQR + +from __future__ import division + +import abc +import numpy as np +import sklearn.base +from .base import ClassifierAdapter, RegressorAdapter +from .base import OobClassifierAdapter, OobRegressorAdapter + +# ----------------------------------------------------------------------------- +# Error functions +# ----------------------------------------------------------------------------- + + +class ClassificationErrFunc(object): + """Base class for classification model error functions.""" + + __metaclass__ = abc.ABCMeta + + def __init__(self): + super(ClassificationErrFunc, self).__init__() + + @abc.abstractmethod + def apply(self, prediction, y): + """Apply the nonconformity function. + + Parameters + ---------- + prediction : numpy array of shape [n_samples, n_classes] + Class probability estimates for each sample. + + y : numpy array of shape [n_samples] + True output labels of each sample. + + Returns + ------- + nc : numpy array of shape [n_samples] + Nonconformity scores of the samples. + """ + pass + + +class RegressionErrFunc(object): + """Base class for regression model error functions.""" + + __metaclass__ = abc.ABCMeta + + def __init__(self): + super(RegressionErrFunc, self).__init__() + + @abc.abstractmethod + def apply(self, prediction, y): # , norm=None, beta=0): + """Apply the nonconformity function. + + Parameters + ---------- + prediction : numpy array of shape [n_samples, n_classes] + Class probability estimates for each sample. + + y : numpy array of shape [n_samples] + True output labels of each sample. + + Returns + ------- + nc : numpy array of shape [n_samples] + Nonconformity scores of the samples. + """ + pass + + @abc.abstractmethod + def apply_inverse(self, nc, significance): # , norm=None, beta=0): + """Apply the inverse of the nonconformity function (i.e., + calculate prediction interval). + + Parameters + ---------- + nc : numpy array of shape [n_calibration_samples] + Nonconformity scores obtained for conformal predictor. + + significance : float + Significance level (0, 1). + + Returns + ------- + interval : numpy array of shape [n_samples, 2] + Minimum and maximum interval boundaries for each prediction. + """ + pass + + +class InverseProbabilityErrFunc(ClassificationErrFunc): + """Calculates the probability of not predicting the correct class. + + For each correct output in ``y``, nonconformity is defined as + + .. math:: + 1 - \hat{P}(y_i | x) \, . + """ + + def __init__(self): + super(InverseProbabilityErrFunc, self).__init__() + + def apply(self, prediction, y): + prob = np.zeros(y.size, dtype=np.float32) + for i, y_ in enumerate(y): + if y_ >= prediction.shape[1]: + prob[i] = 0 + else: + prob[i] = prediction[i, int(y_)] + return 1 - prob + + +class MarginErrFunc(ClassificationErrFunc): + """ + Calculates the margin error. + + For each correct output in ``y``, nonconformity is defined as + + .. math:: + 0.5 - \dfrac{\hat{P}(y_i | x) - max_{y \, != \, y_i} \hat{P}(y | x)}{2} + """ + + def __init__(self): + super(MarginErrFunc, self).__init__() + + def apply(self, prediction, y): + prob = np.zeros(y.size, dtype=np.float32) + for i, y_ in enumerate(y): + if y_ >= prediction.shape[1]: + prob[i] = 0 + else: + prob[i] = prediction[i, int(y_)] + prediction[i, int(y_)] = -np.inf + return 0.5 - ((prob - prediction.max(axis=1)) / 2) + + +class AbsErrorErrFunc(RegressionErrFunc): + """Calculates absolute error nonconformity for regression problems. + + For each correct output in ``y``, nonconformity is defined as + + .. math:: + | y_i - \hat{y}_i | + """ + + def __init__(self): + super(AbsErrorErrFunc, self).__init__() + + def apply(self, prediction, y): + return np.abs(prediction - y) + + def apply_inverse(self, nc, significance): + nc = np.sort(nc)[::-1] + border = int(np.floor(significance * (nc.size + 1))) - 1 + # TODO: should probably warn against too few calibration examples + border = min(max(border, 0), nc.size - 1) + return np.vstack([nc[border], nc[border]]) + + +class SignErrorErrFunc(RegressionErrFunc): + """Calculates signed error nonconformity for regression problems. + + For each correct output in ``y``, nonconformity is defined as + + .. math:: + y_i - \hat{y}_i + + References + ---------- + .. [1] Linusson, Henrik, Ulf Johansson, and Tuve Lofstrom. + Signed-error conformal regression. Pacific-Asia Conference on Knowledge + Discovery and Data Mining. Springer International Publishing, 2014. + """ + + def __init__(self): + super(SignErrorErrFunc, self).__init__() + + def apply(self, prediction, y): + return prediction - y + + def apply_inverse(self, nc, significance): + + err_high = -nc + err_low = nc + + err_high = np.reshape(err_high, (nc.shape[0], 1)) + err_low = np.reshape(err_low, (nc.shape[0], 1)) + + nc = np.concatenate((err_low, err_high), 1) + + nc = np.sort(nc, 0) + index = int(np.ceil((1 - significance / 2) * (nc.shape[0] + 1))) - 1 + index = min(max(index, 0), nc.shape[0] - 1) + return np.vstack([nc[index, 0], nc[index, 1]]) + + +# CQR symmetric error function +class QuantileRegErrFunc(RegressionErrFunc): + """Calculates conformalized quantile regression error. + + For each correct output in ``y``, nonconformity is defined as + + .. math:: + max{\hat{q}_low - y, y - \hat{q}_high} + + """ + + def __init__(self): + super(QuantileRegErrFunc, self).__init__() + + def apply(self, prediction, y): + y_lower = prediction[:, 0] + y_upper = prediction[:, -1] + error_low = y_lower - y + error_high = y - y_upper + err = np.maximum(error_high, error_low) + return err + + def apply_inverse(self, nc, significance): + nc = np.sort(nc, 0) + index = int(np.ceil((1 - significance) * (nc.shape[0] + 1))) - 1 + index = min(max(index, 0), nc.shape[0] - 1) + return np.vstack([nc[index], nc[index]]) + + +# CQR asymmetric error function +class QuantileRegAsymmetricErrFunc(RegressionErrFunc): + """Calculates conformalized quantile regression asymmetric error function. + + For each correct output in ``y``, nonconformity is defined as + + .. math:: + E_low = \hat{q}_low - y + E_high = y - \hat{q}_high + + """ + + def __init__(self): + super(QuantileRegAsymmetricErrFunc, self).__init__() + + def apply(self, prediction, y): + y_lower = prediction[:, 0] + y_upper = prediction[:, -1] + + error_high = y - y_upper + error_low = y_lower - y + + err_high = np.reshape(error_high, (y_upper.shape[0], 1)) + err_low = np.reshape(error_low, (y_lower.shape[0], 1)) + + return np.concatenate((err_low, err_high), 1) + + def apply_inverse(self, nc, significance): + nc = np.sort(nc, 0) + index = int(np.ceil((1 - significance / 2) * (nc.shape[0] + 1))) - 1 + index = min(max(index, 0), nc.shape[0] - 1) + return np.vstack([nc[index, 0], nc[index, 1]]) + + +# ----------------------------------------------------------------------------- +# Base nonconformity scorer +# ----------------------------------------------------------------------------- +class BaseScorer(sklearn.base.BaseEstimator): + __metaclass__ = abc.ABCMeta + + def __init__(self): + super(BaseScorer, self).__init__() + + @abc.abstractmethod + def fit(self, x, y): + pass + + @abc.abstractmethod + def score(self, x, y=None): + pass + + +class RegressorNormalizer(BaseScorer): + def __init__(self, base_model, normalizer_model, err_func): + super(RegressorNormalizer, self).__init__() + self.base_model = base_model + self.normalizer_model = normalizer_model + self.err_func = err_func + + def fit(self, x, y): + residual_prediction = self.base_model.predict(x) + residual_error = np.abs(self.err_func.apply(residual_prediction, y)) + + ###################################################################### + # Optional: use logarithmic function as in the original implementation + # available in https://github.com/donlnz/nonconformist + # + # CODE: + # residual_error += 0.00001 # Add small term to avoid log(0) + # log_err = np.log(residual_error) + ###################################################################### + + log_err = residual_error + self.normalizer_model.fit(x, log_err) + + def score(self, x, y=None): + + ###################################################################### + # Optional: use logarithmic function as in the original implementation + # available in https://github.com/donlnz/nonconformist + # + # CODE: + # norm = np.exp(self.normalizer_model.predict(x)) + ###################################################################### + + norm = np.abs(self.normalizer_model.predict(x)) + return norm + + +class NcFactory(object): + @staticmethod + def create_nc(model, err_func=None, normalizer_model=None, oob=False): + if normalizer_model is not None: + normalizer_adapter = RegressorAdapter(normalizer_model) + else: + normalizer_adapter = None + + if isinstance(model, sklearn.base.ClassifierMixin): + err_func = MarginErrFunc() if err_func is None else err_func + if oob: + c = sklearn.base.clone(model) + c.fit([[0], [1]], [0, 1]) + if hasattr(c, "oob_decision_function_"): + adapter = OobClassifierAdapter(model) + else: + raise AttributeError( + "Cannot use out-of-bag " + "calibration with {}".format(model.__class__.__name__) + ) + else: + adapter = ClassifierAdapter(model) + + if normalizer_adapter is not None: + normalizer = RegressorNormalizer( + adapter, normalizer_adapter, err_func + ) + return ClassifierNc(adapter, err_func, normalizer) + else: + return ClassifierNc(adapter, err_func) + + elif isinstance(model, sklearn.base.RegressorMixin): + err_func = AbsErrorErrFunc() if err_func is None else err_func + if oob: + c = sklearn.base.clone(model) + c.fit([[0], [1]], [0, 1]) + if hasattr(c, "oob_prediction_"): + adapter = OobRegressorAdapter(model) + else: + raise AttributeError( + "Cannot use out-of-bag " + "calibration with {}".format(model.__class__.__name__) + ) + else: + adapter = RegressorAdapter(model) + + if normalizer_adapter is not None: + normalizer = RegressorNormalizer( + adapter, normalizer_adapter, err_func + ) + return RegressorNc(adapter, err_func, normalizer) + else: + return RegressorNc(adapter, err_func) + + +class BaseModelNc(BaseScorer): + """Base class for nonconformity scorers based on an underlying model. + + Parameters + ---------- + model : ClassifierAdapter or RegressorAdapter + Underlying classification model used for calculating nonconformity + scores. + + err_func : ClassificationErrFunc or RegressionErrFunc + Error function object. + + normalizer : BaseScorer + Normalization model. + + beta : float + Normalization smoothing parameter. As the beta-value increases, + the normalized nonconformity function approaches a non-normalized + equivalent. + """ + + def __init__(self, model, err_func, normalizer=None, beta=1e-6): + super(BaseModelNc, self).__init__() + self.err_func = err_func + self.model = model + self.normalizer = normalizer + self.beta = beta + + # If we use sklearn.base.clone (e.g., during cross-validation), + # object references get jumbled, so we need to make sure that the + # normalizer has a reference to the proper model adapter, if applicable. + if self.normalizer is not None and hasattr( + self.normalizer, "base_model" + ): + self.normalizer.base_model = self.model + + self.last_x, self.last_y = None, None + self.last_prediction = None + self.clean = False + + def fit(self, x, y): + """Fits the underlying model of the nonconformity scorer. + + Parameters + ---------- + x : numpy array of shape [n_samples, n_features] + Inputs of examples for fitting the underlying model. + + y : numpy array of shape [n_samples] + Outputs of examples for fitting the underlying model. + + Returns + ------- + None + """ + self.model.fit(x, y) + if self.normalizer is not None: + self.normalizer.fit(x, y) + self.clean = False + + def score(self, x, y=None): + """Calculates the nonconformity score of a set of samples. + + Parameters + ---------- + x : numpy array of shape [n_samples, n_features] + Inputs of examples for which to calculate a nonconformity score. + + y : numpy array of shape [n_samples] + Outputs of examples for which to calculate a nonconformity score. + + Returns + ------- + nc : numpy array of shape [n_samples] + Nonconformity scores of samples. + """ + prediction = self.model.predict(x) + n_test = x.shape[0] + if self.normalizer is not None: + norm = self.normalizer.score(x) + self.beta + else: + norm = np.ones(n_test) + if prediction.ndim > 1: + ret_val = self.err_func.apply(prediction, y) + else: + ret_val = self.err_func.apply(prediction, y) / norm + return ret_val + + +# ----------------------------------------------------------------------------- +# Classification nonconformity scorers +# ----------------------------------------------------------------------------- +class ClassifierNc(BaseModelNc): + """Nonconformity scorer using an underlying class probability estimating + model. + + Parameters + ---------- + model : ClassifierAdapter + Underlying classification model used for calculating nonconformity + scores. + + err_func : ClassificationErrFunc + Error function object. + + normalizer : BaseScorer + Normalization model. + + beta : float + Normalization smoothing parameter. As the beta-value increases, + the normalized nonconformity function approaches a non-normalized + equivalent. + + Attributes + ---------- + model : ClassifierAdapter + Underlying model object. + + err_func : ClassificationErrFunc + Scorer function used to calculate nonconformity scores. + + See also + -------- + RegressorNc, NormalizedRegressorNc + """ + + def __init__( + self, model, err_func=MarginErrFunc(), normalizer=None, beta=1e-6 + ): + super(ClassifierNc, self).__init__(model, err_func, normalizer, beta) + + +# ----------------------------------------------------------------------------- +# Regression nonconformity scorers +# ----------------------------------------------------------------------------- +class RegressorNc(BaseModelNc): + """Nonconformity scorer using an underlying regression model. + + Parameters + ---------- + model : RegressorAdapter + Underlying regression model used for calculating nonconformity scores. + + err_func : RegressionErrFunc + Error function object. + + normalizer : BaseScorer + Normalization model. + + beta : float + Normalization smoothing parameter. As the beta-value increases, + the normalized nonconformity function approaches a non-normalized + equivalent. + + Attributes + ---------- + model : RegressorAdapter + Underlying model object. + + err_func : RegressionErrFunc + Scorer function used to calculate nonconformity scores. + + See also + -------- + ProbEstClassifierNc, NormalizedRegressorNc + """ + + def __init__( + self, model, err_func=AbsErrorErrFunc(), normalizer=None, beta=1e-6 + ): + super(RegressorNc, self).__init__(model, err_func, normalizer, beta) + + def predict(self, x, nc, significance=None): + """Constructs prediction intervals for a set of test examples. + + Predicts the output of each test pattern using the underlying model, + and applies the (partial) inverse nonconformity function to each + prediction, resulting in a prediction interval for each test pattern. + + Parameters + ---------- + x : numpy array of shape [n_samples, n_features] + Inputs of patters for which to predict output values. + + significance : float + Significance level (maximum allowed error rate) of predictions. + Should be a float between 0 and 1. If ``None``, then intervals for + all significance levels (0.01, 0.02, ..., 0.99) are output in a + 3d-matrix. + + Returns + ------- + p : numpy array of shape [n_samples, 2] or [n_samples, 2, 99] + If significance is ``None``, then p contains the interval (minimum + and maximum boundaries) for each test pattern, and each significance + level (0.01, 0.02, ..., 0.99). If significance is a float between + 0 and 1, then p contains the prediction intervals (minimum and + maximum boundaries) for the set of test patterns at the chosen + significance level. + """ + n_test = x.shape[0] + prediction = self.model.predict(x) + if self.normalizer is not None: + norm = self.normalizer.score(x) + self.beta + else: + norm = np.ones(n_test) + + if significance: + intervals = np.zeros((x.shape[0], 2)) + err_dist = self.err_func.apply_inverse(nc, significance) + err_dist = np.hstack([err_dist] * n_test) + if prediction.ndim > 1: # CQR + intervals[:, 0] = prediction[:, 0] - err_dist[0, :] + intervals[:, 1] = prediction[:, -1] + err_dist[1, :] + else: # regular conformal prediction + err_dist *= norm + intervals[:, 0] = prediction - err_dist[0, :] + intervals[:, 1] = prediction + err_dist[1, :] + + return intervals + else: # Not tested for CQR + significance = np.arange(0.01, 1.0, 0.01) + intervals = np.zeros((x.shape[0], 2, significance.size)) + + for i, s in enumerate(significance): + err_dist = self.err_func.apply_inverse(nc, s) + err_dist = np.hstack([err_dist] * n_test) + err_dist *= norm + + intervals[:, 0, i] = prediction - err_dist[0, :] + intervals[:, 1, i] = prediction + err_dist[0, :] + + return intervals diff --git a/unifiedbooster/nonconformist/util.py b/unifiedbooster/nonconformist/util.py new file mode 100644 index 0000000..c02f83d --- /dev/null +++ b/unifiedbooster/nonconformist/util.py @@ -0,0 +1,9 @@ +from __future__ import division +import numpy as np + + +def calc_p(ncal, ngt, neq, smoothing=False): + if smoothing: + return (ngt + (neq + 1) * np.random.uniform(0, 1)) / (ncal + 1) + else: + return (ngt + neq + 1) / (ncal + 1)