import os from os.path import join import pandas as pd from quapy.data.base import LabelledCollection import sys sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../'))) sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../'))) sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), './'))) #from LeQua2024.scripts import constants #from LeQua2024._lequa2024 import fetch_lequa2024 import quapy as qp import numpy as np import matplotlib.pyplot as plt import seaborn as sns from pathlib import Path import glob os.chdir('/home/moreo/QuaPy/LeQua2024') print(os.getcwd()) qp.environ['SAMPLE_SIZE']=250 TASK=1 true_prevs_path = f'./TruePrevalences/T{TASK}.test_prevalences/T{TASK}/public/test_prevalences.txt' folder = F'./Results_CODALAB_2024/extracted/TASK_{TASK}' def load_result_file(path): df = pd.read_csv(path, index_col=0) id = df.index.to_numpy() prevs = df.values return id, prevs method_files = glob.glob(f"{folder}/*.csv") method_names_nice={ 'DistMatching-y': 'DM', 'TeamGMNet': 'UniOviedo(Team1)', 'tobiaslotz': 'Lamarr' } exclude_methods=[ 'TeamCUFE', 'hustav', 'PCC', 'CC' ] # desired_order=[ # 'Lamarr', # 'SLD', # 'DM', # 'KDEy', # 'UniOviedo(Team1)' # ] # desired_order=[ # 'PCC', 'Lamarr' # ] # load the true values (sentiment prevalence, domain prevalence) true_id, true_prevs = load_result_file(true_prevs_path) # define the loss for evaluation error_name = 'RAE' error_log = False if error_name == 'RAE': err_function_ = qp.error.rae elif error_name == 'AE': err_function_ = qp.error.ae else: raise ValueError() if error_log: error_name = f'log({error_name})' err_function = lambda x,y: np.log(err_function_(x,y)) else: err_function = err_function_ def load_vector_documents(path): """ Loads vectorized documents. In case the sample is unlabelled, the labels returned are None :param path: path to the data sample containing the raw documents :return: a tuple with the documents (np.ndarray of shape `(n,256)`) and the labels (a np.ndarray of shape `(n,)` if the sample is labelled, or None if the sample is unlabelled), with `n` the number of instances in the sample (250 for T1 and T4, 1000 for T2, and 200 for T3) """ D = pd.read_csv(path).to_numpy(dtype=float) labelled = D.shape[1] == 257 if labelled: X, y = D[:,1:], D[:,0].astype(int).flatten() else: X, y = D, None return X, y #train_prevalence = fetch_lequa2024(task=f'T{TASK}', data_home='./data') train = LabelledCollection.load(f'/home/moreo/QuaPy/LeQua2024/data/lequa2024/T{TASK}/public/training_data.txt', loader_func=load_vector_documents) train_prev = train.prevalence() #train_prev = np.tile(train_prev, (len(true_id),1)) from quapy.plot import error_by_drift # load the participant and baseline results method_names, estim_prevs = [], [] for method_file in method_files: method_name = Path(method_file).name.replace('.csv', '') if method_name in exclude_methods: continue id, method_prevs = load_result_file(join(folder, method_name+'.csv')) assert (true_id == id).all(), f'unmatched files for {method_file}' method_name = method_names_nice.get(method_name, method_name) method_names.append(method_name) estim_prevs.append(method_prevs) true_prevs = [true_prevs]*len(method_names) tr_prevs =[train.prevalence()]*len(method_names) error_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, error_name='mrae', show_std=True, show_density=True, vlines=True, savepath=f'./util_scripts/t{TASK}_{error_name}_pcc.png') sys.exit() shift=qp.error.ae(train_prev, true_prevs) n_bins = 5 bins = np.linspace(shift.min(), shift.max(), n_bins + 1) # Crear un DataFrame para los datos df = pd.DataFrame({'dom_A_prevs': shift}) for method, err in errors.items(): df[method] = err # Asignar cada valor de dom_A_prevs a un bin df['bin'] = pd.cut(df['dom_A_prevs'], bins=bins, labels=False, include_lowest=True) # Convertir el DataFrame a formato largo df_long = df.melt(id_vars=['dom_A_prevs', 'bin'], value_vars=errors.keys(), var_name='Método', value_name='Error') # Crear etiquetas de los bins para el eje X bin_labels = [f"[{bins[i]:.3f}-{bins[i + 1]:.3f}" + (']' if i == n_bins-1 else ')') for i in range(n_bins)] df_long['bin_label'] = df_long['bin'].map(dict(enumerate(bin_labels))) # Crear el gráfico de boxplot en Seaborn plt.figure(figsize=(14, 8)) sns.boxplot(x='bin', y='Error', hue='Método', data=df_long, palette='Set2', showfliers=False) # Configurar etiquetas del eje X con los rangos de los bins plt.xticks(ticks=range(n_bins), labels=bin_labels, rotation=0) plt.xlabel("Amount of PPS between the training prevalence and the test prevalences, in terms of AE ") plt.ylabel(error_name) #plt.title("Boxplots de Errores por Método dentro de Bins de dom_A_prevs") plt.legend(loc='upper left', bbox_to_anchor=(1, 1)) plt.tight_layout() plt.grid(True, which='both', linestyle='--', linewidth=0.5) #plt.show() plt.savefig(f'./util_scripts/t{TASK}_{error_name}_pcc.png')