diff --git a/distribution_matching/commons.py b/distribution_matching/commons.py index 39bf5ec..bc9c833 100644 --- a/distribution_matching/commons.py +++ b/distribution_matching/commons.py @@ -8,7 +8,7 @@ from distribution_matching.method_dirichlety import DIRy from sklearn.linear_model import LogisticRegression from method_kdey_closed_efficient import KDEyclosed_efficient -METHODS = ['ACC', 'PACC', 'HDy-OvA', 'DIR', 'DM-T', 'DM-HD', 'KDEy-DMhd3', 'DM-CS', 'KDEy-closed++', 'EMQ', 'KDEy-ML'] #['ACC', 'PACC', 'HDy-OvA', 'DIR', 'DM', 'KDEy-DMhd3', 'KDEy-closed++', 'EMQ', 'KDEy-ML'] #, 'KDEy-DMhd2'] #, 'KDEy-DMhd2', 'DM-HD'] 'KDEy-DMjs', 'KDEy-DM', 'KDEy-ML+', 'KDEy-DMhd3+', 'EMQ-C', +METHODS = ['ACC', 'PACC', 'HDy-OvA', 'DM-T', 'DM-HD', 'KDEy-DMhd3', 'DM-CS', 'KDEy-closed++', 'DIR', 'EMQ', 'KDEy-ML'] #['ACC', 'PACC', 'HDy-OvA', 'DIR', 'DM', 'KDEy-DMhd3', 'KDEy-closed++', 'EMQ', 'KDEy-ML'] #, 'KDEy-DMhd2'] #, 'KDEy-DMhd2', 'DM-HD'] 'KDEy-DMjs', 'KDEy-DM', 'KDEy-ML+', 'KDEy-DMhd3+', 'EMQ-C', BIN_METHODS = [x.replace('-OvA', '') for x in METHODS] diff --git a/distribution_matching/figures/sensibility_plot.py b/distribution_matching/figures/sensibility_plot.py index 7de9cbd..ed4396a 100644 --- a/distribution_matching/figures/sensibility_plot.py +++ b/distribution_matching/figures/sensibility_plot.py @@ -9,18 +9,42 @@ Plots results for MAE, MRAE, and KLD The rest of hyperparameters were set to their default values """ -df_tweet = pd.read_csv('../results/tweet/sensibility/KDEy-ML.csv', sep='\t') -df_lequa = pd.read_csv('../results/lequa/sensibility/KDEy-ML.csv', sep='\t') -df = pd.concat([df_tweet, df_lequa]) -for err in ['MAE', 'MRAE', 'KLD']: - piv = df.pivot_table(index='Bandwidth', columns='Dataset', values=err) - g = sns.lineplot(data=piv, markers=True, dashes=False) - g.set(xlim=(0.01, 0.2)) - g.legend(loc="center left", bbox_to_anchor=(1, 0.5)) - g.set_ylabel(err) - g.set_xticks(np.linspace(0.01, 0.2, 20)) - plt.xticks(rotation=90) - plt.grid() - plt.savefig(f'./sensibility_{err}.pdf', bbox_inches='tight') - plt.clf() \ No newline at end of file + +log_mrae = True + +for method, param, xlim, xticks in [ + ('KDEy-ML', 'Bandwidth', (0.01, 0.2), np.linspace(0.01, 0.2, 20)), + ('DM-HD', 'nbins', (2,32), list(range(2,10)) + list(range(10,34,2))) +]: + + for dataset in ['tweet', 'lequa', 'uciml']: + + if dataset == 'tweet': + df = pd.read_csv(f'../results/tweet/sensibility/{method}.csv', sep='\t') + ylim = (0.03, 0.21) + elif dataset == 'lequa': + df = pd.read_csv(f'../results/lequa/T1B/sensibility/{method}.csv', sep='\t') + ylim = (0.0125, 0.03) + elif dataset == 'uciml': + ylim = (0, 0.23) + df = pd.read_csv(f'../results/ucimulti/sensibility/{method}.csv', sep='\t') + + for err in ['MAE']: #, 'MRAE']: + piv = df.pivot_table(index=param, columns='Dataset', values=err) + g = sns.lineplot(data=piv, markers=True, dashes=False) + g.set(xlim=xlim) + g.legend(loc="center left", bbox_to_anchor=(1, 0.5)) + + if log_mrae and err=='MRAE': + plt.yscale('log') + g.set_ylabel('log('+err+')') + else: + g.set_ylabel(err) + + g.set_ylim(ylim) + g.set_xticks(xticks) + plt.xticks(rotation=90) + plt.grid() + plt.savefig(f'./sensibility_{method}_{dataset}_{err}.pdf', bbox_inches='tight') + plt.clf() \ No newline at end of file diff --git a/distribution_matching/figures/simplex_density_plot.py b/distribution_matching/figures/simplex_density_plot.py index dc6f5ef..36c32bb 100644 --- a/distribution_matching/figures/simplex_density_plot.py +++ b/distribution_matching/figures/simplex_density_plot.py @@ -1,10 +1,8 @@ -import ternary import math import numpy as np from sklearn.linear_model import LogisticRegression from sklearn.model_selection import train_test_split from sklearn.neighbors import KernelDensity -import plotly.figure_factory as ff from data import LabelledCollection @@ -15,6 +13,7 @@ scale = 200 # con plotly salen los contornos bien, pero es un poco un jaleo porque utiliza el navegador... def plot_simplex_(ax, density, title='', fontsize=9, points=None): + import ternary tax = ternary.TernaryAxesSubplot(ax=ax, scale=scale) tax.heatmapf(density, boundary=True, style="triangular", colorbar=False, cmap='viridis') #cmap='magma') @@ -34,6 +33,7 @@ def plot_simplex_(ax, density, title='', fontsize=9, points=None): def plot_simplex(ax, coord, kde_scores, title='', fontsize=11, points=None, savepath=None): + import plotly.figure_factory as ff tax = ff.create_ternary_contour(coord.T, kde_scores, pole_labels=['y=1', 'y=2', 'y=3'], interp_mode='cartesian', @@ -49,6 +49,8 @@ def plot_simplex(ax, coord, kde_scores, title='', fontsize=11, points=None, save from mpl_toolkits.axes_grid1 import make_axes_locatable def plot_3class_problem(post_c1, post_c2, post_c3, post_test, alpha, bandwidth): + import ternary + post_c1 = np.flip(post_c1, axis=1) post_c2 = np.flip(post_c2, axis=1) post_c3 = np.flip(post_c3, axis=1) diff --git a/distribution_matching/lequa_bandwidth_sensibility.py b/distribution_matching/lequa_bandwidth_sensibility.py deleted file mode 100644 index d212ab3..0000000 --- a/distribution_matching/lequa_bandwidth_sensibility.py +++ /dev/null @@ -1,56 +0,0 @@ -import numpy as np -from sklearn.linear_model import LogisticRegression -import os -import pandas as pd -import quapy as qp -from method_kdey import KDEy - - -SEED=1 - -def task(bandwidth): - print('job-init', dataset, bandwidth) - train, val_gen, test_gen = qp.datasets.fetch_lequa2022(dataset) - - with qp.util.temp_seed(SEED): - quantifier = KDEy(LogisticRegression(), target='max_likelihood', val_split=10, bandwidth=bandwidth) - quantifier.fit(train) - report = qp.evaluation.evaluation_report( - quantifier, protocol=test_gen, error_metrics=['mae', 'mrae', 'kld'], verbose=True) - return report - - -if __name__ == '__main__': - - qp.environ['SAMPLE_SIZE'] = qp.datasets.LEQUA2022_SAMPLE_SIZE['T1B'] - qp.environ['N_JOBS'] = -1 - result_dir = f'results_lequa_sensibility' - - os.makedirs(result_dir, exist_ok=True) - - method = 'KDEy-MLE' - - global_result_path = f'{result_dir}/{method}' - - if not os.path.exists(global_result_path+'.csv'): - with open(global_result_path+'.csv', 'wt') as csv: - csv.write(f'Method\tDataset\tBandwidth\tMAE\tMRAE\tKLD\n') - - dataset = 'T1B' - bandwidths = np.linspace(0.01, 0.2, 20) - - reports = qp.util.parallel(task, bandwidths, n_jobs=-1) - with open(global_result_path + '.csv', 'at') as csv: - for bandwidth, report in zip(bandwidths, reports): - means = report.mean() - local_result_path = global_result_path + '_' + dataset + f'_{bandwidth:.3f}' - report.to_csv(f'{local_result_path}.dataframe') - csv.write(f'{method}\tLeQua-T1B\t{bandwidth}\t{means["mae"]:.5f}\t{means["mrae"]:.5f}\t{means["kld"]:.5f}\n') - csv.flush() - - df = pd.read_csv(global_result_path + '.csv', sep='\t') - - pd.set_option('display.max_columns', None) - pd.set_option('display.max_rows', None) - pv = df.pivot_table(index='Dataset', columns="Method", values=["MAE", "MRAE"]) - print(pv) diff --git a/distribution_matching/lequa_sensibility_analysis.py b/distribution_matching/lequa_sensibility_analysis.py new file mode 100644 index 0000000..e1de526 --- /dev/null +++ b/distribution_matching/lequa_sensibility_analysis.py @@ -0,0 +1,56 @@ +import numpy as np +from sklearn.linear_model import LogisticRegression +import os +import quapy as qp +from distribution_matching.commons import show_results +from method_kdey import KDEy +from quapy.method.aggregative import DistributionMatching + + +SEED=1 + +def task(val): + print('job-init', val) + train, val_gen, test_gen = qp.datasets.fetch_lequa2022('T1B') + + with qp.util.temp_seed(SEED): + if method=='KDEy-ML': + quantifier = KDEy(LogisticRegression(), target='max_likelihood', val_split=10, bandwidth=val) + elif method == 'DM-HD': + quantifier = DistributionMatching(LogisticRegression(), val_split=10, nbins=val, divergence='HD') + + quantifier.fit(train) + report = qp.evaluation.evaluation_report( + quantifier, protocol=test_gen, error_metrics=['mae', 'mrae', 'kld'], verbose=True) + return report + + +if __name__ == '__main__': + + qp.environ['SAMPLE_SIZE'] = qp.datasets.LEQUA2022_SAMPLE_SIZE['T1B'] + qp.environ['N_JOBS'] = -1 + result_dir = f'results/lequa/T1B/sensibility' + + os.makedirs(result_dir, exist_ok=True) + + for method, param, grid in [ + ('KDEy-ML', 'Bandwidth', np.linspace(0.01, 0.2, 20)), + ('DM-HD', 'nbins', list(range(2, 10)) + list(range(10, 34, 2))) + ]: + + global_result_path = f'{result_dir}/{method}' + + if not os.path.exists(global_result_path+'.csv'): + with open(global_result_path+'.csv', 'wt') as csv: + csv.write(f'Method\tDataset\t{param}\tMAE\tMRAE\tKLD\n') + + reports = qp.util.parallel(task, grid, n_jobs=-1) + with open(global_result_path + '.csv', 'at') as csv: + for val, report in zip(grid, reports): + means = report.mean() + local_result_path = global_result_path + '_T1B' + (f'_{val:.3f}' if isinstance(val, float) else f'{val}') + report.to_csv(f'{local_result_path}.dataframe') + csv.write(f'{method}\tLeQua-T1B\t{val}\t{means["mae"]:.5f}\t{means["mrae"]:.5f}\t{means["kld"]:.5f}\n') + csv.flush() + + show_results(global_result_path) diff --git a/distribution_matching/tweets_bandwidth_sensibility.py b/distribution_matching/tweets_bandwidth_sensibility.py deleted file mode 100644 index 245a1b9..0000000 --- a/distribution_matching/tweets_bandwidth_sensibility.py +++ /dev/null @@ -1,59 +0,0 @@ -import pickle -import numpy as np -from sklearn.linear_model import LogisticRegression -import os -import sys -import pandas as pd - -import quapy as qp -from quapy.method.aggregative import EMQ, DistributionMatching, PACC, ACC, CC, PCC, HDy, OneVsAllAggregative -from method_kdey import KDEy -from method_dirichlety import DIRy -from quapy.model_selection import GridSearchQ -from quapy.protocol import UPP - -SEED=1 - -if __name__ == '__main__': - - qp.environ['SAMPLE_SIZE'] = 100 - qp.environ['N_JOBS'] = -1 - n_bags_val = 250 - n_bags_test = 1000 - result_dir = f'results_tweet_sensibility' - - os.makedirs(result_dir, exist_ok=True) - - method = 'KDEy-MLE' - - global_result_path = f'{result_dir}/{method}' - - if not os.path.exists(global_result_path+'.csv'): - with open(global_result_path+'.csv', 'wt') as csv: - csv.write(f'Method\tDataset\tBandwidth\tMAE\tMRAE\tKLD\n') - - with open(global_result_path+'.csv', 'at') as csv: - for bandwidth in np.linspace(0.01, 0.2, 20): - for dataset in qp.datasets.TWITTER_SENTIMENT_DATASETS_TEST: - print('init', dataset) - - local_result_path = global_result_path + '_' + dataset + f'_{bandwidth:.3f}' - - with qp.util.temp_seed(SEED): - - data = qp.datasets.fetch_twitter(dataset, min_df=3, pickle=True, for_model_selection=False) - quantifier = KDEy(LogisticRegression(), target='max_likelihood', val_split=10, bandwidth=bandwidth) - quantifier.fit(data.training) - protocol = UPP(data.test, repeats=n_bags_test) - report = qp.evaluation.evaluation_report(quantifier, protocol, error_metrics=['mae', 'mrae', 'kld'], verbose=True) - report.to_csv(f'{local_result_path}.dataframe') - means = report.mean() - csv.write(f'{method}\t{data.name}\t{bandwidth}\t{means["mae"]:.5f}\t{means["mrae"]:.5f}\t{means["kld"]:.5f}\n') - csv.flush() - - df = pd.read_csv(global_result_path+'.csv', sep='\t') - - pd.set_option('display.max_columns', None) - pd.set_option('display.max_rows', None) - pv = df.pivot_table(index='Dataset', columns="Method", values=["MAE", "MRAE"]) - print(pv) diff --git a/distribution_matching/ucimulticlass_experiments.py b/distribution_matching/ucimulticlass_experiments.py index d2d0fc8..192c25f 100644 --- a/distribution_matching/ucimulticlass_experiments.py +++ b/distribution_matching/ucimulticlass_experiments.py @@ -25,9 +25,6 @@ if __name__ == '__main__': os.makedirs(result_dir, exist_ok=True) for method in METHODS: - #if method == 'HDy-OvA': continue - #if method == 'DIR': continue - # if method != 'EMQ-C': continue print('Init method', method)