From 09abcfc935c7a0efe180e8c5bc3e468e29228d56 Mon Sep 17 00:00:00 2001 From: Alejandro Moreo Date: Wed, 18 Jan 2023 19:46:19 +0100 Subject: [PATCH] adding calibration methods from the abstension package to quapy --- quapy/CHANGE_LOG.txt | 3 +- quapy/classification/calibration.py | 166 ++++++++++++++++++++++++++++ quapy/method/aggregative.py | 26 ++++- quapy/plot.py | 1 - 4 files changed, 191 insertions(+), 5 deletions(-) create mode 100644 quapy/classification/calibration.py diff --git a/quapy/CHANGE_LOG.txt b/quapy/CHANGE_LOG.txt index 20e0759..090afc8 100644 --- a/quapy/CHANGE_LOG.txt +++ b/quapy/CHANGE_LOG.txt @@ -34,7 +34,8 @@ - newer versions of numpy raise a warning when accessing types (e.g., np.float). I have replaced all such instances with the plain python type (e.g., float). -- new dependency "abstention" (to add to the project requirements and setup) +- new dependency "abstention" (to add to the project requirements and setup). Calibration methods from + https://github.com/kundajelab/abstention added. Things to fix: - calibration with recalibration methods has to be fixed for exact_train_prev in EMQ (conflicts with clone, deepcopy, etc.) diff --git a/quapy/classification/calibration.py b/quapy/classification/calibration.py new file mode 100644 index 0000000..9ea5576 --- /dev/null +++ b/quapy/classification/calibration.py @@ -0,0 +1,166 @@ +from copy import deepcopy + +from abstention.calibration import NoBiasVectorScaling, TempScaling, VectorScaling +from sklearn.base import BaseEstimator, clone +from sklearn.model_selection import cross_val_predict, train_test_split +import numpy as np + + +# Wrappers of calibration defined by Alexandari et al. in paper +# requires "pip install abstension" +# see https://github.com/kundajelab/abstention + + +class RecalibratedClassifier: + pass + + +class RecalibratedClassifierBase(BaseEstimator, RecalibratedClassifier): + """ + Applies a (re)calibration method from abstention.calibration, as defined in + `Alexandari et al. paper `_: + + :param estimator: a scikit-learn probabilistic classifier + :param calibrator: the calibration object (an instance of abstention.calibration.CalibratorFactory) + :param val_split: indicate an integer k for performing kFCV to obtain the posterior prevalences, or a float p + in (0,1) to indicate that the posteriors are obtained in a stratified validation split containing p% of the + training instances (the rest is used for training). In any case, the classifier is retrained in the whole + training set afterwards. + :param n_jobs: indicate the number of parallel workers (only when val_split is an integer) + :param verbose: whether or not to display information in the standard output + """ + + def __init__(self, estimator, calibrator, val_split=5, n_jobs=1, verbose=False): + self.estimator = estimator + self.calibrator = calibrator + self.val_split = val_split + self.n_jobs = n_jobs + self.verbose = verbose + + def fit(self, X, y): + k = self.val_split + if isinstance(k, int): + if k < 2: + raise ValueError('wrong value for val_split: the number of folds must be > 2') + return self.fit_cv(X, y) + elif isinstance(k, float): + if not (0 < k < 1): + raise ValueError('wrong value for val_split: the proportion of validation documents must be in (0,1)') + return self.fit_cv(X, y) + + def fit_cv(self, X, y): + posteriors = cross_val_predict( + self.estimator, X, y, cv=self.val_split, n_jobs=self.n_jobs, verbose=self.verbose, method="predict_proba" + ) + self.estimator.fit(X, y) + nclasses = len(np.unique(y)) + self.calibration_function = self.calibrator(posteriors, np.eye(nclasses)[y], posterior_supplied=True) + return self + + def fit_tr_val(self, X, y): + Xtr, Xva, ytr, yva = train_test_split(X, y, test_size=self.val_split, stratify=y) + self.estimator.fit(Xtr, ytr) + posteriors = self.estimator.predict_proba(Xva) + nclasses = len(np.unique(yva)) + self.calibrator = self.calibrator(posteriors, np.eye(nclasses)[yva], posterior_supplied=True) + return self + + def predict(self, X): + return self.estimator.predict(X) + + def predict_proba(self, X): + posteriors = self.estimator.predict_proba(X) + return self.calibration_function(posteriors) + + @property + def classes_(self): + return self.estimator.classes_ + + +class NBVSCalibration(RecalibratedClassifierBase): + """ + Applies the No-Bias Vector Scaling (NBVS) calibration method from abstention.calibration, as defined in + `Alexandari et al. paper `_: + + :param estimator: a scikit-learn probabilistic classifier + :param val_split: indicate an integer k for performing kFCV to obtain the posterior prevalences, or a float p + in (0,1) to indicate that the posteriors are obtained in a stratified validation split containing p% of the + training instances (the rest is used for training). In any case, the classifier is retrained in the whole + training set afterwards. + :param n_jobs: indicate the number of parallel workers (only when val_split is an integer) + :param verbose: whether or not to display information in the standard output + """ + + def __init__(self, estimator, val_split=5, n_jobs=1, verbose=False): + self.estimator = estimator + self.calibrator = NoBiasVectorScaling(verbose=verbose) + self.val_split = val_split + self.n_jobs = n_jobs + self.verbose = verbose + + +class BCTSCalibration(RecalibratedClassifierBase): + """ + Applies the Bias-Corrected Temperature Scaling (BCTS) calibration method from abstention.calibration, as defined in + `Alexandari et al. paper `_: + + :param estimator: a scikit-learn probabilistic classifier + :param val_split: indicate an integer k for performing kFCV to obtain the posterior prevalences, or a float p + in (0,1) to indicate that the posteriors are obtained in a stratified validation split containing p% of the + training instances (the rest is used for training). In any case, the classifier is retrained in the whole + training set afterwards. + :param n_jobs: indicate the number of parallel workers (only when val_split is an integer) + :param verbose: whether or not to display information in the standard output + """ + + def __init__(self, estimator, val_split=5, n_jobs=1, verbose=False): + self.estimator = estimator + self.calibrator = TempScaling(verbose=verbose, bias_positions='all') + self.val_split = val_split + self.n_jobs = n_jobs + self.verbose = verbose + + +class TSCalibration(RecalibratedClassifierBase): + """ + Applies the Temperature Scaling (TS) calibration method from abstention.calibration, as defined in + `Alexandari et al. paper `_: + + :param estimator: a scikit-learn probabilistic classifier + :param val_split: indicate an integer k for performing kFCV to obtain the posterior prevalences, or a float p + in (0,1) to indicate that the posteriors are obtained in a stratified validation split containing p% of the + training instances (the rest is used for training). In any case, the classifier is retrained in the whole + training set afterwards. + :param n_jobs: indicate the number of parallel workers (only when val_split is an integer) + :param verbose: whether or not to display information in the standard output + """ + + def __init__(self, estimator, val_split=5, n_jobs=1, verbose=False): + self.estimator = estimator + self.calibrator = TempScaling(verbose=verbose) + self.val_split = val_split + self.n_jobs = n_jobs + self.verbose = verbose + + +class VSCalibration(RecalibratedClassifierBase): + """ + Applies the Vector Scaling (VS) calibration method from abstention.calibration, as defined in + `Alexandari et al. paper `_: + + :param estimator: a scikit-learn probabilistic classifier + :param val_split: indicate an integer k for performing kFCV to obtain the posterior prevalences, or a float p + in (0,1) to indicate that the posteriors are obtained in a stratified validation split containing p% of the + training instances (the rest is used for training). In any case, the classifier is retrained in the whole + training set afterwards. + :param n_jobs: indicate the number of parallel workers (only when val_split is an integer) + :param verbose: whether or not to display information in the standard output + """ + + def __init__(self, estimator, val_split=5, n_jobs=1, verbose=False): + self.estimator = estimator + self.calibrator = VectorScaling(verbose=verbose) + self.val_split = val_split + self.n_jobs = n_jobs + self.verbose = verbose + diff --git a/quapy/method/aggregative.py b/quapy/method/aggregative.py index 9e5338d..d77f1ed 100644 --- a/quapy/method/aggregative.py +++ b/quapy/method/aggregative.py @@ -10,7 +10,8 @@ from sklearn.model_selection import StratifiedKFold, cross_val_predict from tqdm import tqdm import quapy as qp import quapy.functional as F -from classification.calibration import RecalibratedClassifier +from classification.calibration import RecalibratedClassifier, NBVSCalibration, BCTSCalibration, TSCalibration, \ + VSCalibration from quapy.classification.svmperf import SVMperf from quapy.data import LabelledCollection from quapy.method.base import BaseQuantifier, BinaryQuantifier @@ -138,8 +139,11 @@ class AggregativeProbabilisticQuantifier(AggregativeQuantifier): else: key_prefix = 'base_estimator__' parameters = {key_prefix + k: v for k, v in parameters.items()} + elif isinstance(self.learner, RecalibratedClassifier): + parameters = {'estimator__' + k: v for k, v in parameters.items()} self.learner.set_params(**parameters) + return self # Helper @@ -511,22 +515,38 @@ class EMQ(AggregativeProbabilisticQuantifier): or set to False for computing the training prevalence as an estimate, akin to PCC, i.e., as the expected value of the posterior probabilities of the training instances as suggested in `Alexandari et al. paper `_: + :param recalib: a string indicating the method of recalibration. Available choices include "nbvs" (No-Bias Vector + Scaling), "bcts" (Bias-Corrected Temperature Scaling), "ts" (Temperature Scaling), and "vs" (Vector Scaling). + The default value is None, indicating no recalibration. """ MAX_ITER = 1000 EPSILON = 1e-4 - def __init__(self, learner: BaseEstimator, exact_train_prev=True): + def __init__(self, learner: BaseEstimator, exact_train_prev=True, recalib=None): self.learner = learner self.exact_train_prev = exact_train_prev + self.recalib = recalib def fit(self, data: LabelledCollection, fit_learner=True): + if self.recalib is not None: + if self.recalib == 'nbvs': + self.learner = NBVSCalibration(self.learner) + elif self.recalib == 'bcts': + self.learner = BCTSCalibration(self.learner) + elif self.recalib == 'ts': + self.learner = TSCalibration(self.learner) + elif self.recalib == 'vs': + self.learner = VSCalibration(self.learner) + else: + raise ValueError('invalid param argument for recalibration method; available ones are ' + '"nbvs", "bcts", "ts", and "vs".') self.learner, _ = _training_helper(self.learner, data, fit_learner, ensure_probabilistic=True) if self.exact_train_prev: self.train_prevalence = F.prevalence_from_labels(data.labels, self.classes_) else: self.train_prevalence = qp.model_selection.cross_val_predict( - quantifier=PCC(clone(self.learner)), + quantifier=PCC(deepcopy(self.learner)), data=data, nfolds=3, random_state=0 diff --git a/quapy/plot.py b/quapy/plot.py index 7d94012..b63eba6 100644 --- a/quapy/plot.py +++ b/quapy/plot.py @@ -323,7 +323,6 @@ def error_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, for vline in vlines: ax.axvline(vline, 0, 1, linestyle='--', color='k') - ax.set_xlim(min_x, max_x) if show_legend: