diff --git a/KDEy/kdey_devel.py b/KDEy/kdey_devel.py
index 958c870..57cc487 100644
--- a/KDEy/kdey_devel.py
+++ b/KDEy/kdey_devel.py
@@ -17,6 +17,8 @@ import quapy.functional as F
epsilon = 1e-10
+BANDWIDTH_RANGE = (0.001, 0.2)
+
class KDEyMLauto(KDEyML):
def __init__(self, classifier: BaseEstimator = None, val_split=5, random_state=None, optim='two_steps'):
self.classifier = qp._get_classifier(classifier)
@@ -218,7 +220,7 @@ class KDEyMLauto(KDEyML):
def choose_bandwidth_maxlikelihood_search(self, tr_posteriors, tr_y, te_posteriors, classes):
n_classes = len(classes)
- init_prev = np.full(fill_value=1 / n_classes, shape=(n_classes,))
+ init_prev = F.uniform_prevalence(n_classes)
def neglikelihood_band(bandwidth):
mix_densities = self.get_mixture_components(tr_posteriors, tr_y, classes, bandwidth[0])
@@ -258,7 +260,7 @@ def optim_minimize(loss: Callable, init_prev: np.ndarray, return_loss=False):
constraints = ({'type': 'eq', 'fun': lambda x: 1 - sum(x)}) # values summing up to 1
r = optimize.minimize(loss, x0=init_prev, method='SLSQP', bounds=bounds, constraints=constraints)
# print(f'iterations-prevalence={r.nit}')
- assert r.success, 'Process did not converge!'
+ # assert r.success, 'Process did not converge!'
if return_loss:
return r.x, r.fun
else:
@@ -268,7 +270,7 @@ def optim_minimize(loss: Callable, init_prev: np.ndarray, return_loss=False):
class KDEyMLauto2(KDEyML):
- def __init__(self, classifier: BaseEstimator=None, val_split=5, bandwidth=0.1, random_state=None, reduction=100, max_reduced=500, target='likelihood'):
+ def __init__(self, classifier: BaseEstimator=None, val_split=5, bandwidth=0.1, random_state=None, reduction=100, max_reduced=500, target='likelihood', search='grid'):
"""
reduction: number of examples per class for automatically setting the bandwidth
"""
@@ -281,8 +283,10 @@ class KDEyMLauto2(KDEyML):
self.reduction = reduction
self.max_reduced = max_reduced
self.random_state = random_state
- assert target in ['likelihood', 'likelihood+'] or target in qp.error.QUANTIFICATION_ERROR_NAMES, 'unknown target for auto'
+ assert target in ['likelihood'] or target in qp.error.QUANTIFICATION_ERROR_NAMES, 'unknown target for auto'
+ assert search in ['grid', 'optim'], 'unknown value for search'
self.target = target
+ self.search = search
def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
if self.bandwidth == 'auto':
@@ -303,65 +307,42 @@ class KDEyMLauto2(KDEyML):
if len(train) > tr_length:
train = train.sampling(tr_length)
- init_prev = np.full(fill_value=1 / n_classes, shape=(n_classes,))
+ init_prev = F.uniform_prevalence(n_classes=n_classes)
repeats = 25
prot = UPP(val, sample_size=self.reduction, repeats=repeats, random_state=self.random_state)
- if self.target == 'likelihood+':
+ def eval_bandwidth(bandwidth):
+ mix_densities = self.get_mixture_components(*train.Xy, train.classes_, bandwidth)
+ loss_accum = 0
+ for (sample, prevtrue) in prot():
+ test_densities = [self.pdf(kde_i, sample) for kde_i in mix_densities]
- def neg_loglikelihood_bandwidth(bandwidth):
- mix_densities = self.get_mixture_components(*train.Xy, train.classes_, bandwidth)
+ def neg_loglikelihood_prev(prev):
+ test_mixture_likelihood = sum(prev_i * dens_i for prev_i, dens_i in zip(prev, test_densities))
+ test_loglikelihood = np.log(test_mixture_likelihood + epsilon)
+ nll = -np.sum(test_loglikelihood)
+ return nll
- loss_accum = 0
- for (sample, prevtrue) in prot():
- test_densities = [self.pdf(kde_i, sample) for kde_i in mix_densities]
+ if self.target == 'likelihood':
+ loss_fn = neg_loglikelihood_prev
+ else:
+ loss_fn = lambda prev_hat: qp.error.from_name(self.target)(prev, prev_hat)
- def neg_loglikelihood_prev(prev):
- test_mixture_likelihood = sum(prev_i * dens_i for prev_i, dens_i in zip(prev, test_densities))
- test_loglikelihood = np.log(test_mixture_likelihood + epsilon)
- nll = -np.sum(test_loglikelihood)
- return nll
+ pred_prev, neglikelihood = optim_minimize(loss_fn, init_prev, return_loss=True)
+ loss_accum += neglikelihood
+ return loss_accum
- pred_prev, neglikelihood = optim_minimize(neg_loglikelihood_prev, init_prev, return_loss=True)
- # print(f'\t\tprev={F.strprev(pred_prev)} (true={F.strprev(prev)}) got {neglikelihood=}')
- loss_accum += neglikelihood
- return loss_accum
-
- r = optimize.minimize_scalar(neg_loglikelihood_bandwidth, bounds=(0.00001, 0.2))
+ if self.search == 'optim':
+ r = optimize.minimize_scalar(eval_bandwidth, bounds=(0.001, 0.2), options={'xatol': 0.005})
best_band = r.x
best_loss_value = r.fun
nit = r.nit
# assert r.success, 'Process did not converge!'
- #found bandwidth=0.00994664 after nit=3 iterations loss_val=-212247.24305)
- else:
- best_band = None
- best_loss_value = None
- init_prev = np.full(fill_value=1 / n_classes, shape=(n_classes,))
- for bandwidth in np.logspace(-4, np.log10(0.2), 20):
- mix_densities = self.get_mixture_components(*train.Xy, train.classes_, bandwidth)
-
- loss_accum = 0
- for (sample, prev) in tqdm(prot(), total=repeats):
- test_densities = [self.pdf(kde_i, sample) for kde_i in mix_densities]
-
- def neg_loglikelihood_prev_(prev):
- test_mixture_likelihood = sum(prev_i * dens_i for prev_i, dens_i in zip(prev, test_densities))
- test_loglikelihood = np.log(test_mixture_likelihood + epsilon)
- return -np.sum(test_loglikelihood)
-
- if self.target == 'likelihood':
- loss_fn = neg_loglikelihood_prev_
- else:
- loss_fn = lambda prev_hat: qp.error.from_name(self.target)(prev, prev_hat)
-
- pred_prev, loss_val = optim_minimize(loss_fn, init_prev, return_loss=True)
- loss_accum += loss_val
-
- if best_loss_value is None or loss_accum < best_loss_value:
- best_loss_value = loss_accum
- best_band = bandwidth
+ elif self.search=='grid':
nit=20
+ band_evals = [(band, eval_bandwidth(band)) for band in np.logspace(-4, np.log10(0.2), num=nit)]
+ best_band, best_loss_value = sorted(band_evals, key=lambda x:x[1])[0]
print(f'found bandwidth={best_band:.8f} after {nit=} iterations loss_val={best_loss_value:.5f})')
self.bandwidth_ = best_band
diff --git a/KDEy/quantification_evaluation.py b/KDEy/quantification_evaluation.py
index 06d8b7f..b35dfe8 100644
--- a/KDEy/quantification_evaluation.py
+++ b/KDEy/quantification_evaluation.py
@@ -22,7 +22,7 @@ def newLR():
# typical hyperparameters explored for Logistic Regression
logreg_grid = {
- 'C': np.logspace(-3,3,7),
+ 'C': np.logspace(-4,4,9),
'class_weight': [None, 'balanced']
}
@@ -34,14 +34,16 @@ def wrap_hyper(classifier_hyper_grid: dict):
METHODS = [
('PACC', PACC(newLR()), wrap_hyper(logreg_grid)),
('EMQ', EMQ(newLR()), wrap_hyper(logreg_grid)),
- ('KDEy-ML', KDEyML(newLR()), {**wrap_hyper(logreg_grid), **{'bandwidth': np.logspace(-4, np.log10(0.2), 20)}}),
+ ('KDEy', KDEyML(newLR()), {**wrap_hyper(logreg_grid), **{'bandwidth': np.logspace(-4, np.log10(0.2), 20)}}),
# ('KDEy-MLred', KDEyMLred(newLR()), {**wrap_hyper(logreg_grid), **{'bandwidth': np.logspace(-4, np.log10(0.2), 20)}}),
- ('KDEy-ML-scott', KDEyML(newLR(), bandwidth='scott'), wrap_hyper(logreg_grid)),
- ('KDEy-ML-silver', KDEyML(newLR(), bandwidth='silverman'), wrap_hyper(logreg_grid)),
- ('KDEy-ML-autoLike', KDEyMLauto2(newLR(), bandwidth='auto', target='likelihood'), wrap_hyper(logreg_grid)),
- ('KDEy-ML-autoLike+', KDEyMLauto2(newLR(), bandwidth='auto', target='likelihood+'), wrap_hyper(logreg_grid)),
- ('KDEy-ML-autoAE', KDEyMLauto2(newLR(), bandwidth='auto', target='mae'), wrap_hyper(logreg_grid)),
- ('KDEy-ML-autoRAE', KDEyMLauto2(newLR(), bandwidth='auto', target='mrae'), wrap_hyper(logreg_grid)),
+ ('KDEy-scott', KDEyML(newLR(), bandwidth='scott'), wrap_hyper(logreg_grid)),
+ ('KDEy-silver', KDEyML(newLR(), bandwidth='silverman'), wrap_hyper(logreg_grid)),
+ ('KDEy-NLL', KDEyMLauto2(newLR(), bandwidth='auto', target='likelihood', search='grid'), wrap_hyper(logreg_grid)),
+ ('KDEy-NLL+', KDEyMLauto2(newLR(), bandwidth='auto', target='likelihood', search='optim'), wrap_hyper(logreg_grid)),
+ ('KDEy-AE', KDEyMLauto2(newLR(), bandwidth='auto', target='mae', search='grid'), wrap_hyper(logreg_grid)),
+ ('KDEy-AE+', KDEyMLauto2(newLR(), bandwidth='auto', target='mae', search='optim'), wrap_hyper(logreg_grid)),
+ ('KDEy-RAE', KDEyMLauto2(newLR(), bandwidth='auto', target='mrae', search='grid'), wrap_hyper(logreg_grid)),
+ ('KDEy-RAE', KDEyMLauto2(newLR(), bandwidth='auto', target='mrae', search='optim'), wrap_hyper(logreg_grid)),
]
@@ -80,12 +82,80 @@ def show_results(result_path):
print(pv)
+def run_experiment(method_name, quantifier, param_grid):
+
+ print('Init method', method_name)
+
+ with open(global_result_path + '.csv', 'at') as csv:
+ for dataset in qp.datasets.UCI_MULTICLASS_DATASETS:
+ print('init', dataset)
+
+ # run_experiment(global_result_path, method_name, quantifier, param_grid, dataset)
+ local_result_path = os.path.join(Path(global_result_path).parent, method_name + '_' + dataset + '.dataframe')
+
+ if os.path.exists(local_result_path):
+ print(f'result file {local_result_path} already exist; skipping')
+ report = qp.util.load_report(local_result_path)
+
+ else:
+ with qp.util.temp_seed(SEED):
+
+ data = qp.datasets.fetch_UCIMulticlassDataset(dataset, verbose=True)
+ train, test = data.train_test
+
+ transductive_names = [name for (name, *_) in TRANSDUCTIVE_METHODS]
+
+ if method_name not in transductive_names:
+ if len(param_grid) == 0:
+ t_init = time()
+ quantifier.fit(train)
+ train_time = time() - t_init
+ else:
+ # model selection (train)
+ train, val = train.split_stratified(random_state=SEED)
+ protocol = UPP(val, repeats=n_bags_val)
+ modsel = GridSearchQ(
+ quantifier, param_grid, protocol, refit=True, n_jobs=-1, verbose=1, error='mae'
+ )
+ t_init = time()
+ try:
+ modsel.fit(train)
+ print(f'best params {modsel.best_params_}')
+ print(f'best score {modsel.best_score_}')
+ quantifier = modsel.best_model()
+ except:
+ print('something went wrong... trying to fit the default model')
+ quantifier.fit(train)
+ train_time = time() - t_init
+ else:
+ # transductive
+ t_init = time()
+ quantifier.fit(train) # <-- nothing actually (proyects the X into posteriors only)
+ train_time = time() - t_init
+
+ # test
+ t_init = time()
+ protocol = UPP(test, repeats=n_bags_test)
+ report = qp.evaluation.evaluation_report(
+ quantifier, protocol, error_metrics=['mae', 'mrae', 'kld'], verbose=True
+ )
+ test_time = time() - t_init
+ report['tr_time'] = train_time
+ report['te_time'] = test_time
+ report.to_csv(local_result_path)
+
+ means = report.mean(numeric_only=True)
+ csv.write(f'{method_name}\t{dataset}\t{means["mae"]:.5f}\t{means["mrae"]:.5f}\t{means["kld"]:.5f}\t{means["tr_time"]:.3f}\t{means["te_time"]:.3f}\n')
+ csv.flush()
+
+
+
if __name__ == '__main__':
qp.environ['SAMPLE_SIZE'] = 500
qp.environ['N_JOBS'] = -1
- n_bags_val = 25
- n_bags_test = 100
+ n_bags_val = 100
+ n_bags_test = 500
result_dir = f'results_quantification/ucimulti'
os.makedirs(result_dir, exist_ok=True)
@@ -95,69 +165,6 @@ if __name__ == '__main__':
csv.write(f'Method\tDataset\tMAE\tMRAE\tKLD\tTR-TIME\tTE-TIME\n')
for method_name, quantifier, param_grid in METHODS + TRANSDUCTIVE_METHODS:
-
- print('Init method', method_name)
-
- with open(global_result_path + '.csv', 'at') as csv:
- for dataset in qp.datasets.UCI_MULTICLASS_DATASETS:
- print('init', dataset)
-
- # run_experiment(global_result_path, method_name, quantifier, param_grid, dataset)
- local_result_path = os.path.join(Path(global_result_path).parent, method_name + '_' + dataset + '.dataframe')
-
- if os.path.exists(local_result_path):
- print(f'result file {local_result_path} already exist; skipping')
- report = qp.util.load_report(local_result_path)
-
- else:
- with qp.util.temp_seed(SEED):
-
- data = qp.datasets.fetch_UCIMulticlassDataset(dataset, verbose=True)
- train, test = data.train_test
-
- transductive_names = [name for (name, *_) in TRANSDUCTIVE_METHODS]
-
- if method_name not in transductive_names:
- if len(param_grid) == 0:
- t_init = time()
- quantifier.fit(train)
- train_time = time() - t_init
- else:
- # model selection (train)
- train, val = train.split_stratified(random_state=SEED)
- protocol = UPP(val, repeats=n_bags_val)
- modsel = GridSearchQ(
- quantifier, param_grid, protocol, refit=True, n_jobs=-1, verbose=1, error='mae'
- )
- t_init = time()
- try:
- modsel.fit(train)
- print(f'best params {modsel.best_params_}')
- print(f'best score {modsel.best_score_}')
- quantifier = modsel.best_model()
- except:
- print('something went wrong... trying to fit the default model')
- quantifier.fit(train)
- train_time = time() - t_init
- else:
- # transductive
- t_init = time()
- quantifier.fit(train) # <-- nothing actually (proyects the X into posteriors only)
- train_time = time() - t_init
-
- # test
- t_init = time()
- protocol = UPP(test, repeats=n_bags_test)
- report = qp.evaluation.evaluation_report(
- quantifier, protocol, error_metrics=['mae', 'mrae', 'kld'], verbose=True
- )
- test_time = time() - t_init
- report['tr_time'] = train_time
- report['te_time'] = test_time
- report.to_csv(local_result_path)
-
- means = report.mean(numeric_only=True)
- csv.write(f'{method_name}\t{dataset}\t{means["mae"]:.5f}\t{means["mrae"]:.5f}\t{means["kld"]:.5f}\t{means["tr_time"]:.3f}\t{means["te_time"]:.3f}\n')
- csv.flush()
+ run_experiment(method_name, quantifier, param_grid)
show_results(global_result_path)
\ No newline at end of file
diff --git a/KDEy/quantification_evaluation_debug.py b/KDEy/quantification_evaluation_debug.py
index a2f5e69..073fe75 100644
--- a/KDEy/quantification_evaluation_debug.py
+++ b/KDEy/quantification_evaluation_debug.py
@@ -21,7 +21,127 @@ SEED = 1
def newLR():
- return LogisticRegression(max_iter=1000)#, C=1, class_weight='balanced')
+ return LogisticRegression(max_iter=1000)
+
+
+def plot(xaxis, metrics_measurements, metrics_names, suffix):
+ fig, ax1 = plt.subplots(figsize=(8, 6))
+
+ def add_plot(ax, mean_error, std_error, name, color, marker):
+ ax.plot(xaxis, mean_error, label=name, marker=marker, color=color)
+ if std_error is not None:
+ ax.fill_between(xaxis, mean_error - std_error, mean_error + std_error, color=color, alpha=0.2)
+
+ colors = ['b', 'g', 'r', 'c', 'purple']
+
+ def get_mean_std(measurement):
+ measurement = np.asarray(measurement)
+ measurement_mean = np.mean(measurement, axis=0)
+ if measurement.ndim == 2:
+ measurement_std = np.std(measurement, axis=0)
+ else:
+ measurement_std = None
+ return measurement_mean, measurement_std
+
+ for i, (measurement, name) in enumerate(zip(metrics_measurements, metrics_names)):
+ color = colors[i%len(colors)]
+ add_plot(ax1, *get_mean_std(measurement), name, color=color, marker='o')
+
+ ax1.set_xscale('log')
+
+ # Configurar etiquetas para el primer eje Y
+ ax1.set_xlabel('Bandwidth')
+ ax1.set_ylabel('Normalized value')
+ ax1.grid(True)
+ ax1.legend(loc='upper left')
+
+ # Crear un segundo eje Y que comparte el eje X
+ # ax2 = ax1.twinx()
+
+ # Pintar likelihood_val en el segundo eje Y
+ # add_plot(ax2, *get_mean_std(likelihood_measurements), name='NLL', color='purple', marker='x')
+
+ # Configurar etiquetas para el segundo eje Y
+ # ax1.set_ylabel('neg log likelihood')
+ # ax1.legend(loc='upper right')
+
+ # Mostrar el gráfico
+ plt.title(dataset)
+ # plt.show()
+ os.makedirs('./plots/likelihood/', exist_ok=True)
+
+ plt.savefig(f'./plots/likelihood/{dataset}-fig{suffix}.png')
+ plt.close()
+
+
+def generate_data(from_train=False):
+ data = qp.datasets.fetch_UCIMulticlassDataset(dataset)
+ n_classes = data.n_classes
+ print(f'{i=}')
+ print(f'{dataset=}')
+ print(f'{n_classes=}')
+ print(len(data.training))
+ print(len(data.test))
+
+ train, test = data.train_test
+ if from_train:
+ train, test = train.split_stratified(0.5)
+ train_prev = train.prevalence()
+ test_prev = test.prevalence()
+
+ print(f'train-prev = {F.strprev(train_prev)}')
+ print(f'test-prev = {F.strprev(test_prev)}')
+
+ repeats = 10
+ prot = UPP(test, sample_size=SAMPLE_SIZE, repeats=repeats)
+ kde = KDEyMLauto(newLR())
+ kde.fit(train)
+ AE_error, RAE_error, MSE_error, KLD_error, LIKE_value = [], [], [], [], []
+ tr_posteriors, tr_y = kde.classif_predictions.Xy
+ for sample_no, (sample, prev) in tqdm(enumerate(prot()), total=repeats):
+ te_posteriors = kde.classifier.predict_proba(sample)
+ classes = train.classes_
+
+ xaxis = []
+ ae_error = []
+ rae_error = []
+ mse_error = []
+ kld_error = []
+ likelihood_value = []
+
+ # for bandwidth in np.linspace(0.01, 0.2, 50):
+ for bandwidth in np.logspace(-5, np.log10(0.2), 50):
+ mix_densities = kde.get_mixture_components(tr_posteriors, tr_y, classes, bandwidth)
+ test_densities = [kde.pdf(kde_i, te_posteriors) for kde_i in mix_densities]
+
+ def neg_loglikelihood_prev(prev):
+ test_mixture_likelihood = sum(prev_i * dens_i for prev_i, dens_i in zip(prev, test_densities))
+ test_loglikelihood = np.log(test_mixture_likelihood + epsilon)
+ return -np.sum(test_loglikelihood)
+
+ init_prev = np.full(fill_value=1 / n_classes, shape=(n_classes,))
+ pred_prev, likelihood = optim_minimize(neg_loglikelihood_prev, init_prev, return_loss=True)
+
+ xaxis.append(bandwidth)
+ ae_error.append(qp.error.ae(prev, pred_prev))
+ rae_error.append(qp.error.rae(prev, pred_prev))
+ mse_error.append(qp.error.mse(prev, pred_prev))
+ kld_error.append(qp.error.kld(prev, pred_prev))
+ likelihood_value.append(likelihood)
+
+ AE_error.append(ae_error)
+ RAE_error.append(rae_error)
+ MSE_error.append(mse_error)
+ KLD_error.append(kld_error)
+ LIKE_value.append(likelihood_value)
+
+ return xaxis, AE_error, RAE_error, MSE_error, KLD_error, LIKE_value
+
+
+def normalize_metric(Error_matrix):
+ max_val, min_val = np.max(Error_matrix), np.min(Error_matrix)
+ return (np.asarray(Error_matrix) - min_val) / (max_val - min_val)
+
SAMPLE_SIZE=150
qp.environ['SAMPLE_SIZE'] = SAMPLE_SIZE
@@ -30,158 +150,57 @@ show_ae = True
show_rae = True
show_mse = False
show_kld = True
+normalize = True
epsilon = 1e-10
-# n_bags_test = 2
-# DATASETS = [qp.datasets.UCI_MULTICLASS_DATASETS[21]]
DATASETS = qp.datasets.UCI_MULTICLASS_DATASETS
-for i, dataset in enumerate(DATASETS):
+for i, dataset in enumerate(tqdm(DATASETS, desc='processing datasets', total=len(DATASETS))):
- def generate_data():
- data = qp.datasets.fetch_UCIMulticlassDataset(dataset)
- n_classes = data.n_classes
- print(f'{i=}')
- print(f'{dataset=}')
- print(f'{n_classes=}')
- print(len(data.training))
- print(len(data.test))
- train, test = data.train_test
- train_prev = train.prevalence()
- test_prev = test.prevalence()
+ xaxis, AE_error_te, RAE_error_te, MSE_error_te, KLD_error_te, LIKE_value_te = qp.util.pickled_resource(
+ f'./plots/likelihood/pickles/{dataset}.pkl', generate_data, False
+ )
- print(f'train-prev = {F.strprev(train_prev)}')
- print(f'test-prev = {F.strprev(test_prev)}')
+ xaxis, AE_error_tr, RAE_error_tr, MSE_error_tr, KLD_error_tr, LIKE_value_tr = qp.util.pickled_resource(
+ f'./plots/likelihood/pickles/{dataset}_tr.pkl', generate_data, True
+ )
- repeats = 10
- prot = UPP(test, sample_size=SAMPLE_SIZE, repeats=repeats)
- kde = KDEyMLauto(newLR())
- kde.fit(train)
- AE_error, RAE_error, MSE_error, KLD_error, LIKE_value = [], [], [], [], []
- tr_posteriors, tr_y = kde.classif_predictions.Xy
- for it, (sample, prev) in tqdm(enumerate(prot()), total=repeats):
- te_posteriors = kde.classifier.predict_proba(sample)
- classes = train.classes_
- xaxis = []
- ae_error = []
- rae_error = []
- mse_error = []
- kld_error = []
- likelihood_value = []
-
- # for bandwidth in np.linspace(0.01, 0.2, 50):
- for bandwidth in np.logspace(-5, 0.5, 50):
- mix_densities = kde.get_mixture_components(tr_posteriors, tr_y, classes, bandwidth)
- test_densities = [kde.pdf(kde_i, te_posteriors) for kde_i in mix_densities]
-
- def neg_loglikelihood_prev(prev):
- test_mixture_likelihood = sum(prev_i * dens_i for prev_i, dens_i in zip(prev, test_densities))
- test_loglikelihood = np.log(test_mixture_likelihood + epsilon)
- return -np.sum(test_loglikelihood)
-
- init_prev = np.full(fill_value=1 / n_classes, shape=(n_classes,))
- pred_prev, likelihood = optim_minimize(neg_loglikelihood_prev, init_prev, return_loss=True)
-
- xaxis.append(bandwidth)
- ae_error.append(qp.error.ae(prev, pred_prev))
- rae_error.append(qp.error.rae(prev, pred_prev))
- mse_error.append(qp.error.mse(prev, pred_prev))
- kld_error.append(qp.error.kld(prev, pred_prev))
- likelihood_value.append(likelihood)
-
- AE_error.append(ae_error)
- RAE_error.append(rae_error)
- MSE_error.append(mse_error)
- KLD_error.append(kld_error)
- LIKE_value.append(likelihood_value)
-
- return xaxis, AE_error, RAE_error, MSE_error, KLD_error, LIKE_value
-
- xaxis, AE_error, RAE_error, MSE_error, KLD_error, LIKE_value = qp.util.pickled_resource(
- f'./plots/likelihood/pickles/{dataset}.pkl', generate_data)
-
- for row in range(len(AE_error)):
-
- # Crear la figura
- # ----------------------------------------------------------------------------------------------------
- fig, ax1 = plt.subplots(figsize=(8, 6))
-
- # Pintar las series ae_error, rae_error, y kld_error en el primer eje Y
- if show_ae:
- ax1.plot(xaxis, AE_error[row], label='AE', marker='o', color='b')
- if show_rae:
- ax1.plot(xaxis, RAE_error[row], label='RAE', marker='s', color='g')
- if show_kld:
- ax1.plot(xaxis, KLD_error[row], label='KLD', marker='^', color='r')
- if show_mse:
- ax1.plot(xaxis, MSE_error[row], label='MSE', marker='^', color='c')
- ax1.set_xscale('log')
-
- # Configurar etiquetas para el primer eje Y
- ax1.set_xlabel('Bandwidth')
- ax1.set_ylabel('Error Value')
- ax1.grid(True)
- ax1.legend(loc='upper left')
-
- # Crear un segundo eje Y que comparte el eje X
- ax2 = ax1.twinx()
-
- # Pintar likelihood_val en el segundo eje Y
- ax2.plot(xaxis, LIKE_value[row], label='(neg)Likelihood', marker='x', color='purple')
-
- # Configurar etiquetas para el segundo eje Y
- ax2.set_ylabel('Likelihood Value')
- ax2.legend(loc='upper right')
-
- # Mostrar el gráfico
- plt.title('Error Metrics vs Bandwidth')
- # plt.show()
- os.makedirs('./plots/likelihood/', exist_ok=True)
- plt.savefig(f'./plots/likelihood/{dataset}-fig{row}.png')
- plt.close()
-
- # Crear la figura con las medias
+ # Test measurements
# ----------------------------------------------------------------------------------------------------
- fig, ax1 = plt.subplots(figsize=(8, 6))
+ measurements = []
+ measurement_names = []
+ if show_ae:
+ measurements.append(AE_error_te)
+ measurement_names.append('AE')
+ if show_rae:
+ measurements.append(RAE_error_te)
+ measurement_names.append('RAE')
+ if show_kld:
+ measurements.append(KLD_error_te)
+ measurement_names.append('KLD')
+ if show_mse:
+ measurements.append(MSE_error_te)
+ measurement_names.append('MSE')
+ measurements.append(normalize_metric(LIKE_value_te))
+ measurements.append(normalize_metric(LIKE_value_tr))
+ measurement_names.append('NLL(te)')
+ measurement_names.append('NLL(tr)')
- def add_plot(ax, vals_error, name, color, marker, show):
- if not show:
- return
- vals_error = np.asarray(vals_error)
- vals_ave = np.mean(vals_error, axis=0)
- vals_std = np.std(vals_error, axis=0)
- ax.plot(xaxis, vals_ave, label=name, marker=marker, color=color)
- ax.fill_between(xaxis, vals_ave - vals_std, vals_ave + vals_std, color=color, alpha=0.2)
+ if normalize:
+ measurements = [normalize_metric(m) for m in measurements]
- add_plot(ax1, AE_error, 'AE', color='b', marker='o', show=show_ae)
- add_plot(ax1, RAE_error, 'RAE', color='g', marker='s', show=show_rae)
- add_plot(ax1, KLD_error, 'KLD', color='r', marker='^', show=show_kld)
- add_plot(ax1, MSE_error, 'MSE', color='c', marker='^', show=show_mse)
- ax1.set_xscale('log')
+ # plot(xaxis, measurements, measurement_names, suffix='AVE')
- # Configurar etiquetas para el primer eje Y
- ax1.set_xlabel('Bandwidth')
- ax1.set_ylabel('Error Value')
- ax1.grid(True)
- ax1.legend(loc='upper left')
-
- # Crear un segundo eje Y que comparte el eje X
- ax2 = ax1.twinx()
-
- # Pintar likelihood_val en el segundo eje Y
- add_plot(ax2, LIKE_value, '(neg)Likelihood', color='purple', marker='x', show=True)
-
- # Configurar etiquetas para el segundo eje Y
- ax2.set_ylabel('Likelihood Value')
- ax2.legend(loc='upper right')
-
- # Mostrar el gráfico
- plt.title('Error Metrics vs Bandwidth')
- # plt.show()
- os.makedirs('./plots/likelihood/', exist_ok=True)
- plt.savefig(f'./plots/likelihood/{dataset}-figAve.png')
- plt.close()
+ # Train-Test measurements
+ # ----------------------------------------------------------------------------------------------------
+ # measurements = []
+ # measurement_names = []
+ # measurements.append(normalize_metric(LIKE_value_te))
+ # measurements.append(normalize_metric(LIKE_value_tr))
+ # measurement_names.append('NLL(te)')
+ # measurement_names.append('NLL(tr)')
+ plot(xaxis, measurements, measurement_names, suffix='AVEtr')