diff --git a/Ordinal/experiments_lr_vs_ordlr.py b/Ordinal/experiments_lr_vs_ordlr.py index ff4f56a..a6712ca 100644 --- a/Ordinal/experiments_lr_vs_ordlr.py +++ b/Ordinal/experiments_lr_vs_ordlr.py @@ -3,11 +3,11 @@ import quapy as qp import os from sklearn.linear_model import LogisticRegression from sklearn.preprocessing import StandardScaler -from Ordinal.model import RegressionQuantification, LogisticAT, LogisticSE, LogisticIT, LAD, OrdinalRidge +from Ordinal.model import LogisticAT, LogisticSE, LogisticIT, LAD, OrdinalRidge #, RegressionQuantification from quapy.method.aggregative import PACC, CC, EMQ, PCC, ACC from os.path import join from utils import load_samples_folder, load_single_sample_pkl -from evaluation import nmd, mnmd +from Ordinal.evaluation import nmd, mnmd from tqdm import tqdm @@ -37,11 +37,11 @@ def quantifiers(): yield 'PACC(OLR-AT)', PACC(LogisticAT()), params_OLR yield 'SLD(OLR-AT)', EMQ(LogisticAT()), params_OLR - yield 'CC(OLR-SE)', CC(LogisticSE()), params_OLR - yield 'PCC(OLR-SE)', PCC(LogisticSE()), params_OLR - yield 'ACC(OLR-SE)', ACC(LogisticSE()), params_OLR - yield 'PACC(OLR-SE)', PACC(LogisticSE()), params_OLR - yield 'SLD(OLR-SE)', EMQ(LogisticSE()), params_OLR + # yield 'CC(OLR-SE)', CC(LogisticSE()), params_OLR + # yield 'PCC(OLR-SE)', PCC(LogisticSE()), params_OLR + # yield 'ACC(OLR-SE)', ACC(LogisticSE()), params_OLR + # yield 'PACC(OLR-SE)', PACC(LogisticSE()), params_OLR + # yield 'SLD(OLR-SE)', EMQ(LogisticSE()), params_OLR yield 'CC(OLR-IT)', CC(LogisticIT()), params_OLR yield 'PCC(OLR-IT)', PCC(LogisticIT()), params_OLR @@ -53,6 +53,7 @@ def quantifiers(): # regression-based ordinal regression (see https://pythonhosted.org/mord/) yield 'CC(LAD)', CC(LAD()), params_SVR yield 'ACC(LAD)', ACC(LAD()), params_SVR + yield 'CC(ORidge)', CC(OrdinalRidge()), params_Ridge yield 'ACC(ORidge)', ACC(OrdinalRidge()), params_Ridge @@ -60,7 +61,7 @@ def quantifiers(): def run_experiment(params): qname, q, param_grid = params qname += posfix - resultfile = join(resultpath, f'{qname}.all.csv') + resultfile = join(resultpath, f'{qname}.all.APP-OQ.csv') if os.path.exists(resultfile): print(f'result file {resultfile} already exists: continue') return None @@ -105,16 +106,16 @@ def run_experiment(params): print(f'{qname}: {mean_nmd:.4f} +-{std_nmd:.4f}') report.to_csv(resultfile, index=False) - print('[learning regressor-based adjustment]') - q = RegressionQuantification(q.best_model(), val_samples_generator=load_dev_samples) - q.fit(None) + # print('[learning regressor-based adjustment]') + # q = RegressionQuantification(q.best_model(), val_samples_generator=load_dev_samples) + # q.fit(None) - report = qp.evaluation.gen_prevalence_report(q, gen_fn=load_test_samples, error_metrics=[nmd]) - mean_nmd = report['nmd'].mean() - std_nmd = report['nmd'].std() - print(f'[{qname} regression-correction] {mean_nmd:.4f} +-{std_nmd:.4f}') - resultfile = join(resultpath, f'{qname}.all.reg.csv') - report.to_csv(resultfile, index=False) + # report = qp.evaluation.gen_prevalence_report(q, gen_fn=load_test_samples, error_metrics=[nmd]) + # mean_nmd = report['nmd'].mean() + # std_nmd = report['nmd'].std() + # print(f'[{qname} regression-correction] {mean_nmd:.4f} +-{std_nmd:.4f}') + # resultfile = join(resultpath, f'{qname}.all.reg.csv') + # report.to_csv(resultfile, index=False) return hyperparams diff --git a/Ordinal/utils.py b/Ordinal/utils.py index 182851d..fc74962 100644 --- a/Ordinal/utils.py +++ b/Ordinal/utils.py @@ -12,6 +12,9 @@ import quapy as qp from quapy.data import LabelledCollection +def jaggedness(p): + return (1/min(6, len(p)+1)) * sum((-p_prev + 2*p_i - p_next)**2 for p_prev, p_i, p_next in zip(p[:-2], p[1:-1], p[2:])) + def load_simple_sample_npytxt(parentdir, filename, classes=None): samplepath = join(parentdir, filename+'.txt')