From 98560f673ca478b6b48337d6b020f39df0102a36 Mon Sep 17 00:00:00 2001
From: Lorenzo Volpi <lorenzo.volpi@outlook.com>
Date: Fri, 10 Nov 2023 01:24:42 +0100
Subject: [PATCH] confidence methods added

---
 quacc/evaluation/method.py | 71 +++++++++++++++++++++++---------------
 quacc/main_test.py         | 14 ++++----
 2 files changed, 50 insertions(+), 35 deletions(-)

diff --git a/quacc/evaluation/method.py b/quacc/evaluation/method.py
index 8419c52..47854d2 100644
--- a/quacc/evaluation/method.py
+++ b/quacc/evaluation/method.py
@@ -2,7 +2,7 @@ import inspect
 from functools import wraps
 
 import numpy as np
-from quapy.method.aggregative import PACC, SLD, CC
+from quapy.method.aggregative import CC, PACC, SLD
 from quapy.protocol import UPP, AbstractProtocol
 from sklearn.linear_model import LogisticRegression
 
@@ -17,14 +17,15 @@ _sld_param_grid = {
     "q__classifier__C": np.logspace(-3, 3, 7),
     "q__classifier__class_weight": [None, "balanced"],
     "q__recalib": [None, "bcts"],
-    "q__exact_train_prev": [True],
-    "confidence": [None, "max_conf", "entropy"],
+    "confidence": [["max_conf", "entropy"]],
 }
 _pacc_param_grid = {
     "q__classifier__C": np.logspace(-3, 3, 7),
     "q__classifier__class_weight": [None, "balanced"],
-    "confidence": [None, "max_conf", "entropy"],
+    "confidence": [["max_conf", "entropy"]],
 }
+
+
 def method(func):
     @wraps(func)
     def wrapper(c_model, validation, protocol):
@@ -43,7 +44,7 @@ def evaluation_report(
     report = EvaluationReport(name=method_name)
     for sample in protocol():
         e_sample = estimator.extend(sample)
-        estim_prev = estimator.estimate(e_sample.X, ext=True)
+        estim_prev = estimator.estimate(e_sample.eX)
         acc_score = qc.error.acc(estim_prev)
         f1_score = qc.error.f1(estim_prev)
         report.append_row(
@@ -75,6 +76,32 @@ def mul_sld(c_model, validation, protocol) -> EvaluationReport:
     )
 
 
+@method
+def binc_sld(c_model, validation, protocol) -> EvaluationReport:
+    est = BQAE(
+        c_model,
+        SLD(LogisticRegression()),
+        confidence=["max_conf", "entropy"],
+    ).fit(validation)
+    return evaluation_report(
+        estimator=est,
+        protocol=protocol,
+    )
+
+
+@method
+def mulc_sld(c_model, validation, protocol) -> EvaluationReport:
+    est = MCAE(
+        c_model,
+        SLD(LogisticRegression()),
+        confidence=["max_conf", "entropy"],
+    ).fit(validation)
+    return evaluation_report(
+        estimator=est,
+        protocol=protocol,
+    )
+
+
 @method
 def binmc_sld(c_model, validation, protocol) -> EvaluationReport:
     est = BQAE(
@@ -218,8 +245,12 @@ def mul_pacc(c_model, validation, protocol) -> EvaluationReport:
 
 
 @method
-def binmc_pacc(c_model, validation, protocol) -> EvaluationReport:
-    est = BQAE(c_model, PACC(LogisticRegression()), confidence="max_conf").fit(validation)
+def binc_pacc(c_model, validation, protocol) -> EvaluationReport:
+    est = BQAE(
+        c_model,
+        PACC(LogisticRegression()),
+        confidence=["max_conf", "entropy"],
+    ).fit(validation)
     return evaluation_report(
         estimator=est,
         protocol=protocol,
@@ -227,26 +258,12 @@ def binmc_pacc(c_model, validation, protocol) -> EvaluationReport:
 
 
 @method
-def mulmc_pacc(c_model, validation, protocol) -> EvaluationReport:
-    est = MCAE(c_model, PACC(LogisticRegression()), confidence="max_conf").fit(validation)
-    return evaluation_report(
-        estimator=est,
-        protocol=protocol,
-    )
- 
-
-@method
-def binne_pacc(c_model, validation, protocol) -> EvaluationReport:
-    est = BQAE(c_model, PACC(LogisticRegression()), confidence="entropy").fit(validation)
-    return evaluation_report(
-        estimator=est,
-        protocol=protocol,
-    )
-
-
-@method
-def mulne_pacc(c_model, validation, protocol) -> EvaluationReport:
-    est = MCAE(c_model, PACC(LogisticRegression()), confidence="entropy").fit(validation)
+def mulc_pacc(c_model, validation, protocol) -> EvaluationReport:
+    est = MCAE(
+        c_model,
+        PACC(LogisticRegression()),
+        confidence=["max_conf", "entropy"],
+    ).fit(validation)
     return evaluation_report(
         estimator=est,
         protocol=protocol,
diff --git a/quacc/main_test.py b/quacc/main_test.py
index f5b2009..56aa3a0 100644
--- a/quacc/main_test.py
+++ b/quacc/main_test.py
@@ -2,7 +2,6 @@ from copy import deepcopy
 from time import time
 
 import numpy as np
-import win11toast
 from quapy.method.aggregative import SLD
 from quapy.protocol import APP, UPP
 from sklearn.linear_model import LogisticRegression
@@ -54,8 +53,8 @@ def test_gs():
     )
     for sample in protocol():
         e_sample = gs_estimator.extend(sample)
-        estim_prev_b = estimator.estimate(e_sample.X, ext=True)
-        estim_prev_gs = gs_estimator.estimate(e_sample.X, ext=True)
+        estim_prev_b = estimator.estimate(e_sample.eX)
+        estim_prev_gs = gs_estimator.estimate(e_sample.eX)
         erb.append_row(
             sample.prevalence(),
             acc=abs(acc(e_sample.prevalence()) - acc(estim_prev_b)),
@@ -74,7 +73,6 @@ def test_gs():
 
     print(cr.table())
     print(f"[took {time() - tstart:.3f}s]")
-    win11toast.notify("Test", "completed")
 
 
 def test_mc():
@@ -108,12 +106,12 @@ def test_et():
     estimator = MCAE(
         classifier,
         SLD(LogisticRegression(), exact_train_prev=False),
-        confidence="max_conf",
+        confidence="entropy",
     ).fit(d.validation)
     e_test = estimator.extend(d.test)
-    ep = estimator.estimate(e_test.X, ext=True)
-    print(f"{qc.error.acc(ep) = }")
-    print(f"{qc.error.acc(e_test.prevalence()) = }")
+    ep = estimator.estimate(e_test.eX)
+    print(f"estim prev = {qc.error.acc(ep)}")
+    print(f"true prev {qc.error.acc(e_test.prevalence())}")
 
 
 if __name__ == "__main__":