From 2d6ac4af0d2fa52bb7caf07b2acd6fe69edc84da Mon Sep 17 00:00:00 2001
From: Alejandro Moreo <alejandro.moreo@isti.cnr.it>
Date: Mon, 4 Sep 2023 12:05:25 +0200
Subject: [PATCH] testing the sensibility of KDEy to the bandwidth

---
 distribution_matching/show_results.py         |  2 +-
 .../tweets_bandwidth_sensibility.py           | 63 +++++++++++++++++++
 distribution_matching/tweets_experiments.py   |  6 +-
 3 files changed, 67 insertions(+), 4 deletions(-)
 create mode 100644 distribution_matching/tweets_bandwidth_sensibility.py

diff --git a/distribution_matching/show_results.py b/distribution_matching/show_results.py
index c2b8eeb..918d12e 100644
--- a/distribution_matching/show_results.py
+++ b/distribution_matching/show_results.py
@@ -2,7 +2,7 @@ import sys
 from pathlib import Path
 import pandas as pd
 
-result_dir = 'results_tweet_1000_mrae'
+result_dir = 'results_tweet_mae_redohyper'
 #result_dir = 'results_lequa_mrae'
 
 dfs = []
diff --git a/distribution_matching/tweets_bandwidth_sensibility.py b/distribution_matching/tweets_bandwidth_sensibility.py
new file mode 100644
index 0000000..f3564fc
--- /dev/null
+++ b/distribution_matching/tweets_bandwidth_sensibility.py
@@ -0,0 +1,63 @@
+import pickle
+import numpy as np
+from sklearn.linear_model import LogisticRegression
+import os
+import sys
+import pandas as pd
+
+import quapy as qp
+from quapy.method.aggregative import EMQ, DistributionMatching, PACC, ACC, CC, PCC, HDy, OneVsAllAggregative
+from method_kdey import KDEy
+from method_dirichlety import DIRy
+from quapy.model_selection import GridSearchQ
+from quapy.protocol import UPP
+
+SEED=1
+
+if __name__ == '__main__':
+
+    qp.environ['SAMPLE_SIZE'] = 100
+    qp.environ['N_JOBS'] = -1
+    n_bags_val = 250
+    n_bags_test = 1000
+    result_dir = f'results_tweet_sensibility'
+
+    os.makedirs(result_dir, exist_ok=True)
+
+    method = 'KDEy-MLE'
+        
+    global_result_path = f'{result_dir}/{method}'
+    
+    if not os.path.exists(global_result_path+'.csv'):
+        with open(global_result_path+'.csv', 'wt') as csv:
+            csv.write(f'Method\tDataset\tBandwidth\tMAE\tMRAE\tKLD\n')    
+
+    with open(global_result_path+'.csv', 'at') as csv:
+        # four semeval dataset share the training, so it is useless to optimize hyperparameters four times;
+        # this variable controls that the mod sel has already been done, and skip this otherwise
+        semeval_trained = False
+
+        for bandwidth in np.linspace(0.01, 0.2, 20):                        
+            for dataset in qp.datasets.TWITTER_SENTIMENT_DATASETS_TEST:
+                print('init', dataset)
+
+                local_result_path = global_result_path + '_' + dataset + f'_{bandwidth:.3f}'
+                
+                with qp.util.temp_seed(SEED):
+
+                    data = qp.datasets.fetch_twitter(dataset, min_df=3, pickle=True, for_model_selection=False)
+                    quantifier = KDEy(LogisticRegression(), target='max_likelihood', val_split=10, bandwidth=bandwidth)
+                    quantifier.fit(data.training)
+                    protocol = UPP(data.test, repeats=n_bags_test)
+                    report = qp.evaluation.evaluation_report(quantifier, protocol, error_metrics=['mae', 'mrae', 'kld'], verbose=True)
+                    report.to_csv(f'{local_result_path}.dataframe')
+                    means = report.mean()
+                    csv.write(f'{method}\t{data.name}\t{bandwidth}\t{means["mae"]:.5f}\t{means["mrae"]:.5f}\t{means["kld"]:.5f}\n')
+                    csv.flush()
+
+    df = pd.read_csv(global_result_path+'.csv', sep='\t')
+
+    pd.set_option('display.max_columns', None)
+    pd.set_option('display.max_rows', None)
+    pv = df.pivot_table(index='Dataset', columns="Method", values=["MAE", "MRAE"])
+    print(pv)
diff --git a/distribution_matching/tweets_experiments.py b/distribution_matching/tweets_experiments.py
index ed3847a..8d2e542 100644
--- a/distribution_matching/tweets_experiments.py
+++ b/distribution_matching/tweets_experiments.py
@@ -20,13 +20,13 @@ if __name__ == '__main__':
     qp.environ['N_JOBS'] = -1
     n_bags_val = 250
     n_bags_test = 1000
-    optim = 'mrae'
-    result_dir = f'results_tweet_{optim}'
+    optim = 'mae'
+    result_dir = f'results_tweet_{optim}_redohyper'
 
     os.makedirs(result_dir, exist_ok=True)
 
     hyper_LR = {
-        'classifier__C': np.logspace(-4,4,9),
+        'classifier__C': np.logspace(-3,3,7),
         'classifier__class_weight': ['balanced', None]
     }