lorenzo.volpi
/
QuAcc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

confidence methods added

This commit is contained in:
Lorenzo Volpi 2023-11-10 01:24:42 +01:00
parent e5f631d4bc
commit 98560f673c
2 changed files with 50 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
quacc/evaluation/method.py
View File

@ -2,7 +2,7 @@ import inspect
from functools import wraps
import numpy as np
from quapy.method.aggregative import PACC, SLD, CC
from quapy.method.aggregative import CC, PACC, SLD
from quapy.protocol import UPP, AbstractProtocol
from sklearn.linear_model import LogisticRegression
@ -17,14 +17,15 @@ _sld_param_grid = {
"q__classifier__C": np.logspace(-3, 3, 7),
"q__classifier__class_weight": [None, "balanced"],
"q__recalib": [None, "bcts"],
"q__exact_train_prev": [True],
"confidence": [None, "max_conf", "entropy"],
"confidence": [["max_conf", "entropy"]],
}
_pacc_param_grid = {
"q__classifier__C": np.logspace(-3, 3, 7),
"q__classifier__class_weight": [None, "balanced"],
"confidence": [None, "max_conf", "entropy"],
"confidence": [["max_conf", "entropy"]],
}
def method(func):
@wraps(func)
def wrapper(c_model, validation, protocol):
@ -43,7 +44,7 @@ def evaluation_report(
report = EvaluationReport(name=method_name)
for sample in protocol():
e_sample = estimator.extend(sample)
estim_prev = estimator.estimate(e_sample.X, ext=True)
estim_prev = estimator.estimate(e_sample.eX)
acc_score = qc.error.acc(estim_prev)
f1_score = qc.error.f1(estim_prev)
report.append_row(
@ -75,6 +76,32 @@ def mul_sld(c_model, validation, protocol) -> EvaluationReport:
)
@method
def binc_sld(c_model, validation, protocol) -> EvaluationReport:
est = BQAE(
c_model,
SLD(LogisticRegression()),
confidence=["max_conf", "entropy"],
).fit(validation)
return evaluation_report(
estimator=est,
protocol=protocol,
)
@method
def mulc_sld(c_model, validation, protocol) -> EvaluationReport:
est = MCAE(
c_model,
SLD(LogisticRegression()),
confidence=["max_conf", "entropy"],
).fit(validation)
return evaluation_report(
estimator=est,
protocol=protocol,
)
@method
def binmc_sld(c_model, validation, protocol) -> EvaluationReport:
est = BQAE(
@ -218,8 +245,12 @@ def mul_pacc(c_model, validation, protocol) -> EvaluationReport:
@method
def binmc_pacc(c_model, validation, protocol) -> EvaluationReport:
est = BQAE(c_model, PACC(LogisticRegression()), confidence="max_conf").fit(validation)
def binc_pacc(c_model, validation, protocol) -> EvaluationReport:
est = BQAE(
c_model,
PACC(LogisticRegression()),
confidence=["max_conf", "entropy"],
).fit(validation)
return evaluation_report(
estimator=est,
protocol=protocol,
@ -227,26 +258,12 @@ def binmc_pacc(c_model, validation, protocol) -> EvaluationReport:
@method
def mulmc_pacc(c_model, validation, protocol) -> EvaluationReport:
est = MCAE(c_model, PACC(LogisticRegression()), confidence="max_conf").fit(validation)
return evaluation_report(
estimator=est,
protocol=protocol,
)
@method
def binne_pacc(c_model, validation, protocol) -> EvaluationReport:
est = BQAE(c_model, PACC(LogisticRegression()), confidence="entropy").fit(validation)
return evaluation_report(
estimator=est,
protocol=protocol,
)
@method
def mulne_pacc(c_model, validation, protocol) -> EvaluationReport:
est = MCAE(c_model, PACC(LogisticRegression()), confidence="entropy").fit(validation)
def mulc_pacc(c_model, validation, protocol) -> EvaluationReport:
est = MCAE(
c_model,
PACC(LogisticRegression()),
confidence=["max_conf", "entropy"],
).fit(validation)
return evaluation_report(
estimator=est,
protocol=protocol,

14
quacc/main_test.py
View File

@ -2,7 +2,6 @@ from copy import deepcopy
from time import time
import numpy as np
import win11toast
from quapy.method.aggregative import SLD
from quapy.protocol import APP, UPP
from sklearn.linear_model import LogisticRegression
@ -54,8 +53,8 @@ def test_gs():
)
for sample in protocol():
e_sample = gs_estimator.extend(sample)
estim_prev_b = estimator.estimate(e_sample.X, ext=True)
estim_prev_gs = gs_estimator.estimate(e_sample.X, ext=True)
estim_prev_b = estimator.estimate(e_sample.eX)
estim_prev_gs = gs_estimator.estimate(e_sample.eX)
erb.append_row(
sample.prevalence(),
acc=abs(acc(e_sample.prevalence()) - acc(estim_prev_b)),
@ -74,7 +73,6 @@ def test_gs():
print(cr.table())
print(f"[took {time() - tstart:.3f}s]")
win11toast.notify("Test", "completed")
def test_mc():
@ -108,12 +106,12 @@ def test_et():
estimator = MCAE(
classifier,
SLD(LogisticRegression(), exact_train_prev=False),
confidence="max_conf",
confidence="entropy",
).fit(d.validation)
e_test = estimator.extend(d.test)
ep = estimator.estimate(e_test.X, ext=True)
print(f"{qc.error.acc(ep) = }")
print(f"{qc.error.acc(e_test.prevalence()) = }")
ep = estimator.estimate(e_test.eX)
print(f"estim prev = {qc.error.acc(ep)}")
print(f"true prev {qc.error.acc(e_test.prevalence())}")
if __name__ == "__main__":