CHANGE_LOG.txt

@@ -1,4 +1,4 @@
-Change Log 0.1.8g
+Change Log 0.1.8
 ----------------
 
 - Added Kernel Density Estimation methods (KDEyML, KDEyCS, KDEyHD) as proposed in the paper:

LeQua2024/baselines.py

@@ -3,12 +3,9 @@ import pickle
 import os
 import sys
 from os.path import join
-
-import numpy as np
 from sklearn.linear_model import LogisticRegression as LR
 
 from scripts.constants import SAMPLE_SIZE
-from scripts.evaluate import normalized_match_distance
 from LeQua2024._lequa2024 import LEQUA2024_TASKS, fetch_lequa2024, LEQUA2024_ZENODO
 from quapy.method.aggregative import *
 from quapy.method.non_aggregative import MaximumLikelihoodPrevalenceEstimation as MLPE
@@ -38,18 +35,11 @@ def wrap_params(cls_params:dict, prefix:str):
 def baselines():
 
     q_params = wrap_params(lr_params, 'classifier')
-    kde_params = {**q_params, 'bandwidth': np.linspace(0.01, 0.20, 20)}
-    dm_params = {**q_params, 'nbins': [2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 64]}
 
     yield CC(new_cls()), "CC", q_params
     yield ACC(new_cls()), "ACC", q_params
     yield PCC(new_cls()), "PCC", q_params
     yield PACC(new_cls()), "PACC", q_params
-    yield SLD(new_cls()), "SLD", q_params
-    #yield KDEyML(new_cls()), "KDEy-ML", kde_params
-    #yield KDEyHD(new_cls()), "KDEy-HD", kde_params
-    # yield KDEyCS(new_cls()), "KDEy-CS", kde_params
-    #yield DMy(new_cls()), "DMy", dm_params
 
 
 def main(args):
@@ -87,7 +77,7 @@ def main(args):
                 quantifier,
                 param_grid,
                 protocol=gen_val,
-                error=normalized_match_distance if args.task=='T3' else qp.error.mrae,
+                error=qp.error.mrae,
                 refit=False,
                 verbose=True,
                 n_jobs=-1

LeQua2024/predict.py

@@ -7,7 +7,6 @@ from tqdm import tqdm
 from scripts.data import gen_load_samples
 from glob import glob
 from scripts import constants
-from regressor import KDEyRegressor, RegressionToSimplex
 
 """
 LeQua2024 prediction script 

LeQua2024/regressor.py

@@ -1,133 +0,0 @@
-import pickle
-
-import numpy as np
-from sklearn.base import BaseEstimator, TransformerMixin
-from sklearn.model_selection import GridSearchCV
-from sklearn.multioutput import MultiOutputRegressor
-from sklearn.pipeline import Pipeline
-from sklearn.svm import SVR
-
-from LeQua2024._lequa2024 import fetch_lequa2024
-from quapy.data import LabelledCollection
-from quapy.protocol import AbstractProtocol
-from quapy.method.base import BaseQuantifier
-import quapy.functional as F
-from tqdm import tqdm
-from scripts.evaluate import normalized_match_distance, match_distance
-
-
-def projection_simplex_sort(unnormalized_arr) -> np.ndarray:
-    """Projects a point onto the probability simplex.
-    [This code is taken from the devel branch, that will correspond to the future QuaPy 0.1.9]
-
-    The code is adapted from Mathieu Blondel's BSD-licensed
-    `implementation <https://gist.github.com/mblondel/6f3b7aaad90606b98f71>`_
-    (see function `projection_simplex_sort` in their repo) which is accompanying the paper
-
-    Mathieu Blondel, Akinori Fujino, and Naonori Ueda.
-    Large-scale Multiclass Support Vector Machine Training via Euclidean Projection onto the Simplex,
-    ICPR 2014, `URL <http://www.mblondel.org/publications/mblondel-icpr2014.pdf>`_
-
-    :param `unnormalized_arr`: point in n-dimensional space, shape `(n,)`
-    :return: projection of `unnormalized_arr` onto the (n-1)-dimensional probability simplex, shape `(n,)`
-    """
-    unnormalized_arr = np.asarray(unnormalized_arr)
-    n = len(unnormalized_arr)
-    u = np.sort(unnormalized_arr)[::-1]
-    cssv = np.cumsum(u) - 1.0
-    ind = np.arange(1, n + 1)
-    cond = u - cssv / ind > 0
-    rho = ind[cond][-1]
-    theta = cssv[cond][-1] / float(rho)
-    return np.maximum(unnormalized_arr - theta, 0)
-
-
-class RegressionToSimplex(BaseEstimator):
-    """
-    A very simple regressor of probability distributions.
-    Internally, this class works by invoking an SVR regressor multioutput
-    followed by a mapping onto the probability simplex.
-
-    :param C: regularziation parameter for SVR
-    """
-
-    def __init__(self, C=1):
-        self.C = C
-
-    def fit(self, X, y):
-        """
-        Learns the correction
-
-        :param X: array-like of shape `(n_instances, n_classes)` with uncorrected prevalence vectors
-        :param y: array-like of shape `(n_instances, n_classes)` with true prevalence vectors
-        :return: self
-        """
-        self.reg = MultiOutputRegressor(SVR(C=self.C), n_jobs=-1)
-        self.reg.fit(X, y)
-        return self
-
-    def predict(self, X):
-        """
-        Corrects the a vector of prevalence values
-
-        :param X: array-like of shape `(n_classes,)` with one vector of uncorrected prevalence values
-        :return: array-like of shape `(n_classes,)` with one vector of corrected prevalence values
-        """
-        y_ = self.reg.predict(X)
-        y_ = np.asarray([projection_simplex_sort(y_i) for y_i in y_])
-        return y_
-
-
-class KDEyRegressor(BaseQuantifier):
-    """
-    This class implements a regressor-based correction on top of a quantifier.
-    The quantifier is taken to be KDEy-ML, which is considered to be already trained (this
-    method simply loads a pickled object).
-    The method then optimizes a regressor that corrects prevalence vectors using the
-    validation samples as training data.
-    The regressor is based on a multioutput SVR and relies on a post-processing to guarantee
-    that the output lies on the probability simplex (see also RegressionToSimplex)
-    """
-
-    def __init__(self, kde_path, Cs=np.logspace(-3,3,7)):
-        self.kde_path = kde_path
-        self.Cs = Cs
-
-    def fit(self, val_data: AbstractProtocol):
-        print(f'loading kde from {self.kde_path}')
-        self.kdey = pickle.load(open(self.kde_path, 'rb'))
-
-        print('representing val data with kde')
-        pbar = tqdm(val_data(), total=val_data.total())
-        Xs, Ys = [], []
-        for sample, prev in pbar:
-            prev_hat = self.kdey.quantify(sample)
-            Xs.append(prev_hat)
-            Ys.append(prev)
-
-        Xs = np.asarray(Xs)
-        Ys = np.asarray(Ys)
-
-        def scorer(estimator, X, y):
-            y_hat = estimator.predict(X)
-            md = normalized_match_distance(y, y_hat)
-            return (-md)
-
-        grid = {'C': self.Cs}
-        optim = GridSearchCV(
-            RegressionToSimplex(), param_grid=grid, scoring=scorer, verbose=0, cv=10, n_jobs=64
-        ).fit(Xs, Ys)
-        self.regressor = optim.best_estimator_
-        return self
-
-    def quantify(self, instances):
-        prev_hat = self.kdey.quantify(instances)
-        return self.regressor.predict([prev_hat])[0]
-
-
-if __name__ == '__main__':
-    train, gen_val, _ = fetch_lequa2024(task='T3', data_home='./data', merge_T3=True)
-    kdey_r = KDEyRegressor('./models/T3/KDEy-ML.pkl')
-    kdey_r.fit(gen_val)
-    pickle.dump(kdey_r, open('./models/T3/KDEyRegressor.pkl', 'wb'), protocol=pickle.HIGHEST_PROTOCOL)
-

LeQua2024/run_baselines.sh

@@ -1,6 +1,15 @@
 #!/bin/bash
 set -x
 
+# download the official scripts 
+if [ ! -d "scripts" ]; then
+   echo "Downloading the official scripts from the LeQua 2024 github repo"
+   wget https://github.com/HLT-ISTI/LeQua2024_scripts/archive/refs/heads/main.zip
+   unzip main.zip 
+   mv LeQua2024_scripts-main scripts
+   rm main.zip
+fi
+
 # T1: binary (n=2)
 # T2: multiclass (n=28)
 # T3: ordinal (n=5)

LeQua2024/util_scripts/covariate_shift_plot.py

@@ -1,120 +0,0 @@
-import os
-from os.path import join
-import pandas as pd
-import quapy as qp
-import numpy as np
-import matplotlib.pyplot as plt
-import seaborn as sns
-
-os.chdir('/home/moreo/QuaPy/LeQua2024/util_scripts')
-print(os.getcwd())
-
-qp.environ['SAMPLE_SIZE']=250
-
-true_prevs_path = '../TruePrevalences/T4.test_prevalences/T4/public/test_prevalences.txt'
-domain_prevs_path = '../T4_domain_prevalence/test_domain_prevalences.txt'
-folder = '../Results_CODALAB_2024/extracted/TASK_4'
-
-def load_result_file(path):
-    df = pd.read_csv(path, index_col=0)
-    id = df.index.to_numpy()
-    prevs = df.values
-    return id, prevs
-
-method_files = [
-    #'ACC.csv',
-    #'CC.csv',
-    #'DistMatching-y.csv',
-    #'KDEy.csv',
-    #'PACC.csv',
-    'PCC.csv',
-    #'SLD.csv',
-    #'TeamCUFE.csv',
-    #'TeamGMNet.csv',
-    'tobiaslotz.csv'
-]
-
-method_names_nice={
-    'DistMatching-y': 'DM',
-    'TeamGMNet': 'UniOviedo(Team1)',
-    'tobiaslotz': 'Lamarr'
-}
-
-desired_order=[
-    'Lamarr',
-    'SLD',
-    'DM',
-    'KDEy',
-    'UniOviedo(Team1)'
-]
-desired_order=[
-    'PCC', 'Lamarr'
-]
-
-# load the true values (sentiment prevalence, domain prevalence)
-true_id, true_prevs = load_result_file(true_prevs_path)
-dom_id, dom_prevs = load_result_file(domain_prevs_path)
-assert (true_id == dom_id).all(), 'unmatched files'
-
-# define the loss for evaluation
-error_name = 'RAE'
-error_log = False
-
-if error_name == 'RAE':
-    err_function_ = qp.error.rae
-elif error_name == 'AE':
-    err_function_ = qp.error.ae
-else:
-    raise ValueError()
-
-if error_log:
-    error_name = f'log({error_name})'
-    err_function = lambda x,y: np.log(err_function_(x,y))
-else:
-    err_function = err_function_
-
-# load the participant and baseline results
-errors = {}
-for method_file in method_files:
-    method_name = method_file.replace('.csv', '')
-    id, method_prevs = load_result_file(join(folder, method_file))
-    print(method_file)
-    assert (true_id == id).all(), f'unmatched files for {method_file}'
-    method_error = err_function(true_prevs, method_prevs)
-    method_name = method_names_nice.get(method_name, method_name)
-    errors[method_name] = method_error
-
-dom_A_prevs = dom_prevs[:,0]
-
-n_bins = 5
-bins = np.linspace(dom_A_prevs.min(), dom_A_prevs.max(), n_bins + 1)
-
-# Crear un DataFrame para los datos
-df = pd.DataFrame({'dom_A_prevs': dom_A_prevs})
-for method, err in errors.items():
-    df[method] = err
-
-# Asignar cada valor de dom_A_prevs a un bin
-df['bin'] = pd.cut(df['dom_A_prevs'], bins=bins, labels=False, include_lowest=True)
-
-# Convertir el DataFrame a formato largo
-df_long = df.melt(id_vars=['dom_A_prevs', 'bin'], value_vars=errors.keys(), var_name='Método', value_name='Error')
-
-# Crear etiquetas de los bins para el eje X
-bin_labels = [f"[{bins[i]:.3f}-{bins[i + 1]:.3f}" + (']' if i == n_bins-1 else ')') for i in range(n_bins)]
-df_long['bin_label'] = df_long['bin'].map(dict(enumerate(bin_labels)))
-
-# Crear el gráfico de boxplot en Seaborn
-plt.figure(figsize=(14, 8))
-sns.boxplot(x='bin', y='Error', hue='Método', data=df_long, palette='Set2', showfliers=False, hue_order=desired_order)
-
-# Configurar etiquetas del eje X con los rangos de los bins
-plt.xticks(ticks=range(n_bins), labels=bin_labels, rotation=0)
-plt.xlabel("Prevalence of Books")
-plt.ylabel(error_name)
-#plt.title("Boxplots de Errores por Método dentro de Bins de dom_A_prevs")
-plt.legend(loc='upper left', bbox_to_anchor=(1, 1))
-plt.tight_layout()
-plt.grid(True, which='both', linestyle='--', linewidth=0.5)
-#plt.show()
-plt.savefig(f'./t4_{error_name}_pcc.png')

LeQua2024/util_scripts/prior_shift_plot.py

@@ -1,168 +0,0 @@
-import os
-from os.path import join
-import pandas as pd
-
-from quapy.data.base import LabelledCollection
-import sys
-sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../')))
-sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../')))
-sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), './')))
-#from LeQua2024.scripts import constants
-#from LeQua2024._lequa2024 import fetch_lequa2024
-import quapy as qp
-import numpy as np
-import matplotlib.pyplot as plt
-import seaborn as sns
-from pathlib import Path
-import glob
-
-
-os.chdir('/home/moreo/QuaPy/LeQua2024')
-print(os.getcwd())
-
-
-qp.environ['SAMPLE_SIZE']=250
-
-TASK=1
-
-true_prevs_path = f'./TruePrevalences/T{TASK}.test_prevalences/T{TASK}/public/test_prevalences.txt'
-folder = F'./Results_CODALAB_2024/extracted/TASK_{TASK}'
-
-def load_result_file(path):
-    df = pd.read_csv(path, index_col=0)
-    id = df.index.to_numpy()
-    prevs = df.values
-    return id, prevs
-
-
-method_files = glob.glob(f"{folder}/*.csv")
-
-
-method_names_nice={
-    'DistMatching-y': 'DM',
-    'TeamGMNet': 'UniOviedo(Team1)',
-    'tobiaslotz': 'Lamarr'
-}
-
-exclude_methods=[
-    'TeamCUFE',
-    'hustav',
-    'PCC',
-    'CC'
-]
-
-
-# desired_order=[
-#     'Lamarr',
-#     'SLD',
-#     'DM',
-#     'KDEy',
-#     'UniOviedo(Team1)'
-# ]
-# desired_order=[
-#     'PCC', 'Lamarr'
-# ]
-
-# load the true values (sentiment prevalence, domain prevalence)
-true_id, true_prevs = load_result_file(true_prevs_path)
-
-
-# define the loss for evaluation
-error_name = 'RAE'
-error_log = False
-
-if error_name == 'RAE':
-    err_function_ = qp.error.rae
-elif error_name == 'AE':
-    err_function_ = qp.error.ae
-else:
-    raise ValueError()
-
-if error_log:
-    error_name = f'log({error_name})'
-    err_function = lambda x,y: np.log(err_function_(x,y))
-else:
-    err_function = err_function_
-
-
-def load_vector_documents(path):
-    """
-    Loads vectorized documents. In case the sample is unlabelled,
-    the labels returned are None
-
-    :param path: path to the data sample containing the raw documents
-    :return: a tuple with the documents (np.ndarray of shape `(n,256)`) and the labels (a np.ndarray of shape `(n,)` if
-        the sample is labelled, or None if the sample is unlabelled), with `n` the number of instances in the sample
-        (250 for T1 and T4, 1000 for T2, and 200 for T3)
-    """
-    D = pd.read_csv(path).to_numpy(dtype=float)
-    labelled = D.shape[1] == 257
-    if labelled:
-        X, y = D[:,1:], D[:,0].astype(int).flatten()
-    else:
-        X, y = D, None
-    return X, y
-
-#train_prevalence = fetch_lequa2024(task=f'T{TASK}', data_home='./data')
-train = LabelledCollection.load(f'/home/moreo/QuaPy/LeQua2024/data/lequa2024/T{TASK}/public/training_data.txt', loader_func=load_vector_documents)
-train_prev = train.prevalence()
-#train_prev = np.tile(train_prev, (len(true_id),1))
-
-from quapy.plot import error_by_drift
-
-# load the participant and baseline results
-method_names, estim_prevs = [], []
-for method_file in method_files:
-    method_name = Path(method_file).name.replace('.csv', '')
-    if method_name in exclude_methods:
-        continue
-    id, method_prevs = load_result_file(join(folder, method_name+'.csv'))
-    assert (true_id == id).all(), f'unmatched files for {method_file}'
-    method_name = method_names_nice.get(method_name, method_name)
-    method_names.append(method_name)
-    estim_prevs.append(method_prevs)
-
-true_prevs = [true_prevs]*len(method_names)
-tr_prevs =[train.prevalence()]*len(method_names)
-error_by_drift(method_names, 
-               true_prevs, 
-               estim_prevs, 
-               tr_prevs, 
-               error_name='mrae', show_std=True,
-               show_density=True, vlines=True, savepath=f'./util_scripts/t{TASK}_{error_name}_pcc.png')
-sys.exit()
-
-shift=qp.error.ae(train_prev, true_prevs)
-
-n_bins = 5
-bins = np.linspace(shift.min(), shift.max(), n_bins + 1)
-
-# Crear un DataFrame para los datos
-df = pd.DataFrame({'dom_A_prevs': shift})
-for method, err in errors.items():
-    df[method] = err
-
-# Asignar cada valor de dom_A_prevs a un bin
-df['bin'] = pd.cut(df['dom_A_prevs'], bins=bins, labels=False, include_lowest=True)
-
-# Convertir el DataFrame a formato largo
-df_long = df.melt(id_vars=['dom_A_prevs', 'bin'], value_vars=errors.keys(), var_name='Método', value_name='Error')
-
-# Crear etiquetas de los bins para el eje X
-bin_labels = [f"[{bins[i]:.3f}-{bins[i + 1]:.3f}" + (']' if i == n_bins-1 else ')') for i in range(n_bins)]
-df_long['bin_label'] = df_long['bin'].map(dict(enumerate(bin_labels)))
-
-# Crear el gráfico de boxplot en Seaborn
-plt.figure(figsize=(14, 8))
-sns.boxplot(x='bin', y='Error', hue='Método', data=df_long, palette='Set2', showfliers=False)
-
-# Configurar etiquetas del eje X con los rangos de los bins
-plt.xticks(ticks=range(n_bins), labels=bin_labels, rotation=0)
-plt.xlabel("Amount of PPS between the training prevalence and the test prevalences, in terms of AE ")
-plt.ylabel(error_name)
-#plt.title("Boxplots de Errores por Método dentro de Bins de dom_A_prevs")
-plt.legend(loc='upper left', bbox_to_anchor=(1, 1))
-plt.tight_layout()
-plt.grid(True, which='both', linestyle='--', linewidth=0.5)
-#plt.show()
-plt.savefig(f'./util_scripts/t{TASK}_{error_name}_pcc.png')

quapy/plot.py

@@ -1,6 +1,6 @@
 from collections import defaultdict
 import matplotlib.pyplot as plt
-from matplotlib.pyplot import get_cmap
+from matplotlib.cm import get_cmap
 import numpy as np
 from matplotlib import cm
 from scipy.stats import ttest_ind_from_stats