lorenzo.volpi
/
QuAcc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mul3w implemented

This commit is contained in:
Lorenzo Volpi 2023-11-11 18:46:27 +01:00
parent 64e546b8fd
commit 9f6333efb5
17 changed files with 1973 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -5,6 +5,7 @@ quavenv/*
__pycache__/*
baselines/__pycache__/*
baselines/densratio/__pycache__/*
qcpanel/__pycache__/*
quacc/__pycache__/*
quacc/evaluation/__pycache__/*
quacc/method/__pycache__/*

2
.vscode/launch.json vendored
View File

@ -9,7 +9,7 @@
"name": "main",
"type": "python",
"request": "launch",
"program": "C:\\Users\\Lorenzo Volpi\\source\\tesi\\quacc\\main.py",
"program": "~/source/tesi/quacc/main.py",
"console": "integratedTerminal",
"justMyCode": true
},

6
TODO.md
View File

@ -42,7 +42,7 @@
- [x] testare anche su imbd
- [x] aggiungere esecuzione remota via ssh
- [ ] testare confidence con sia max_conf che exntropy
- [x] testare confidence con sia max_conf che exntropy
- [x] implementare mul3
- [ ] rivedere nuove baselines
- [ ] importare nuovi dataset
- [ ] implementare mul3
- [ ] importare nuovi dataset

13
conf.yaml
View File

@ -5,16 +5,13 @@ debug_conf: &debug_conf
DATASET_N_PREVS: 9
confs:
- DATASET_NAME: rcv1
DATASET_TARGET: CCAT
- DATASET_NAME: imdb
plot_confs:
debug:
debug_gs:
PLOT_ESTIMATORS:
- binc_sld
- binmc_sld
- binne_sld
- bin_sld
- mul_sld
- mul3w_sld
- atc_mc
PLOT_STDEV: true
@ -224,4 +221,4 @@ main_conf: &main_conf
- atc_ne
- doc_feat
exec: *mc_conf
exec: *debug_conf

1084
poetry.lock generated
View File

File diff suppressed because it is too large Load Diff

6
pyproject.toml
View File

@ -16,6 +16,8 @@ logging = "^0.4.9.6"
[tool.poetry.scripts]
main = "quacc.main:main"
run = "run:run"
panel = "qcpanel.run:run"
sync_up = "remote:sync_code"
[tool.poetry.group.dev.dependencies]
@ -25,6 +27,10 @@ pytest-mock = "^3.11.1"
pytest-cov = "^4.1.0"
tabulate = "^0.9.0"
paramiko = "^3.3.1"
panel = "^1.3.1"
ipympl = "^0.9.3"
ipykernel = "^6.26.0"
ipywidgets-bokeh = "^1.5.0"
[tool.pytest.ini_options]
addopts = "--cov=quacc --capture=tee-sys"

BIN
qcpanel/__pycache__/__init__.cpython-310.pyc
View File

Binary file not shown.

BIN
qcpanel/__pycache__/run.cpython-310.pyc
View File

Binary file not shown.

598
quacc.log
View File

@ -5035,3 +5035,601 @@
09/11/23 21:08:08| DEBUG [BinaryQuantifierAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 10.0, 'quantifier__classifier__class_weight': None, 'quantifier__recalib': None, 'confidence': ['max_conf', 'entropy']} (score=0.00688) [took 1027.9305s]
09/11/23 21:15:36| INFO bin_sld_gs finished [took 1476.4334s]
09/11/23 21:15:36| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs finished [took 1479.7993s]
----------------------------------------------------------------------------------------------------
10/11/23 13:06:41| INFO dataset rcv1_CCAT_9prevs
10/11/23 13:06:47| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs started
10/11/23 13:06:52| WARNING Method mul_sld failed. Exception: ExtendedCollection.from_lc() got an unexpected keyword argument 'pred_proba'
10/11/23 13:06:56| WARNING Method mul3w_sld failed. Exception: ExtendedCollection.from_lc() got an unexpected keyword argument 'pred_proba'
----------------------------------------------------------------------------------------------------
10/11/23 13:07:50| INFO dataset rcv1_CCAT_9prevs
10/11/23 13:07:55| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs started
10/11/23 13:09:28| WARNING Method mul_sld failed. Exception: labels ({1, 2}) contain values not included in classes_ ({0})
10/11/23 13:09:29| WARNING Method mul3w_sld failed. Exception: labels ({1, 2}) contain values not included in classes_ ({0})
10/11/23 13:09:46| INFO ref finished [took 88.3729s]
10/11/23 13:09:54| INFO atc_mc finished [took 90.4167s]
10/11/23 13:09:54| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs finished [took 118.7000s]
10/11/23 13:09:54| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs started
10/11/23 13:11:25| WARNING Method mul3w_sld failed. Exception: labels ({1, 2}) contain values not included in classes_ ({0})
10/11/23 13:11:26| WARNING Method mul_sld failed. Exception: labels ({1, 2}) contain values not included in classes_ ({0})
----------------------------------------------------------------------------------------------------
10/11/23 13:19:00| INFO dataset rcv1_CCAT_9prevs
10/11/23 13:19:05| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs started
10/11/23 13:19:10| WARNING Method mul_sld failed. Exception: 'int' object is not iterable
10/11/23 13:19:13| WARNING Method mul3w_sld failed. Exception: 'int' object is not iterable
----------------------------------------------------------------------------------------------------
10/11/23 13:20:12| INFO dataset rcv1_CCAT_9prevs
10/11/23 13:20:17| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs started
10/11/23 13:21:36| WARNING Method mul_sld failed. Exception: labels ({1, 2}) contain values not included in classes_ ({0})
10/11/23 13:21:39| WARNING Method mul3w_sld failed. Exception: labels ({1, 2}) contain values not included in classes_ ({0, 1})
10/11/23 13:21:54| INFO ref finished [took 77.8358s]
10/11/23 13:22:02| INFO atc_mc finished [took 81.6210s]
10/11/23 13:22:02| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs finished [took 104.5107s]
10/11/23 13:22:02| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs started
10/11/23 13:23:31| WARNING Method mul3w_sld failed. Exception: labels ({1, 2}) contain values not included in classes_ ({0, 1})
10/11/23 13:23:31| WARNING Method mul_sld failed. Exception: labels ({1, 2}) contain values not included in classes_ ({0})
10/11/23 13:23:47| INFO ref finished [took 83.8010s]
10/11/23 13:23:55| INFO atc_mc finished [took 82.0180s]
10/11/23 13:23:55| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs finished [took 113.0759s]
10/11/23 13:23:55| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs started
10/11/23 13:25:35| WARNING Method mul3w_sld failed. Exception: labels ({1, 2}) contain values not included in classes_ ({0, 1})
10/11/23 13:25:35| WARNING Method mul_sld failed. Exception: labels ({1, 2}) contain values not included in classes_ ({0})
10/11/23 13:25:52| INFO ref finished [took 91.9242s]
10/11/23 13:26:01| INFO atc_mc finished [took 91.0173s]
10/11/23 13:26:01| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs finished [took 125.8152s]
10/11/23 13:26:01| INFO Dataset sample 0.40 of dataset rcv1_CCAT_9prevs started
----------------------------------------------------------------------------------------------------
10/11/23 14:35:06| INFO dataset rcv1_CCAT_1prevs
10/11/23 14:35:16| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
10/11/23 14:36:54| WARNING Method mul_sld failed. Exception: labels ({1, 2}) contain values not included in classes_ ({0})
10/11/23 14:37:32| INFO ref finished [took 114.8732s]
----------------------------------------------------------------------------------------------------
10/11/23 14:45:14| INFO dataset rcv1_CCAT_1prevs
10/11/23 14:45:20| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
10/11/23 14:46:56| INFO ref finished [took 77.5985s]
10/11/23 14:47:09| INFO atc_mc finished [took 83.9760s]
10/11/23 14:56:11| WARNING Method mul_sld failed. Exception: labels ({1, 2}) contain values not included in classes_ ({0})
10/11/23 14:56:13| WARNING Method mul3w_sld failed. Exception: labels ({1, 2}) contain values not included in classes_ ({0, 1})
10/11/23 14:56:13| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs finished [took 652.8237s]
----------------------------------------------------------------------------------------------------
10/11/23 14:56:25| INFO dataset rcv1_CCAT_1prevs
----------------------------------------------------------------------------------------------------
10/11/23 14:56:31| INFO dataset rcv1_CCAT_1prevs
10/11/23 14:56:37| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
10/11/23 14:58:44| INFO ref finished [took 106.4738s]
10/11/23 14:59:09| INFO atc_mc finished [took 117.9679s]
10/11/23 15:16:17| INFO mul3w_sld finished [took 1166.3690s]
----------------------------------------------------------------------------------------------------
10/11/23 15:49:18| INFO dataset rcv1_CCAT_1prevs
10/11/23 15:49:23| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
10/11/23 15:50:15| INFO ref finished [took 44.7880s]
----------------------------------------------------------------------------------------------------
10/11/23 16:24:50| INFO dataset rcv1_CCAT_1prevs
10/11/23 16:24:57| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
----------------------------------------------------------------------------------------------------
10/11/23 16:26:54| INFO dataset rcv1_CCAT_1prevs
10/11/23 16:27:01| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
----------------------------------------------------------------------------------------------------
10/11/23 16:28:15| INFO dataset rcv1_CCAT_1prevs
10/11/23 16:28:22| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
----------------------------------------------------------------------------------------------------
10/11/23 16:29:17| INFO dataset rcv1_CCAT_1prevs
10/11/23 16:29:25| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
10/11/23 16:30:30| INFO ref finished [took 52.9566s]
----------------------------------------------------------------------------------------------------
10/11/23 16:33:04| INFO dataset rcv1_CCAT_1prevs
10/11/23 16:33:11| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
10/11/23 16:34:10| INFO ref finished [took 50.4274s]
10/11/23 16:34:22| INFO mul_sld finished [took 66.6494s]
10/11/23 16:34:22| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs finished [took 70.8885s]
----------------------------------------------------------------------------------------------------
10/11/23 16:36:29| INFO dataset rcv1_CCAT_9prevs
10/11/23 16:36:34| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs started
10/11/23 16:37:26| WARNING Method mul3w_sld failed. Exception: index 3 is out of bounds for axis 0 with size 3
10/11/23 16:37:40| INFO ref finished [took 50.7235s]
10/11/23 16:37:47| INFO atc_mc finished [took 54.6993s]
----------------------------------------------------------------------------------------------------
10/11/23 16:45:15| INFO dataset rcv1_CCAT_9prevs
10/11/23 16:45:20| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs started
10/11/23 16:46:11| WARNING Method mul_sld failed. Exception: unsupported operand type(s) for /: 'NoneType' and 'int'
10/11/23 16:46:12| WARNING Method mul3w_sld failed. Exception: unsupported operand type(s) for /: 'NoneType' and 'int'
10/11/23 16:46:25| INFO ref finished [took 50.7336s]
10/11/23 16:46:33| INFO atc_mc finished [took 54.1424s]
10/11/23 16:46:33| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs finished [took 72.4632s]
10/11/23 16:46:33| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs started
10/11/23 16:47:28| WARNING Method mul_sld failed. Exception: unsupported operand type(s) for /: 'NoneType' and 'int'
10/11/23 16:47:29| WARNING Method mul3w_sld failed. Exception: unsupported operand type(s) for /: 'NoneType' and 'int'
----------------------------------------------------------------------------------------------------
10/11/23 16:51:39| INFO dataset rcv1_CCAT_9prevs
10/11/23 16:51:44| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs started
10/11/23 16:52:50| INFO ref finished [took 51.5277s]
10/11/23 16:52:55| INFO mul3w_sld finished [took 62.7316s]
10/11/23 16:52:58| INFO atc_mc finished [took 55.6057s]
10/11/23 16:53:25| INFO mul_sld finished [took 97.7123s]
10/11/23 16:53:25| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs finished [took 100.9808s]
10/11/23 16:53:25| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs started
10/11/23 16:54:36| INFO ref finished [took 55.7445s]
10/11/23 16:54:43| INFO atc_mc finished [took 56.7203s]
10/11/23 16:54:45| INFO mul3w_sld finished [took 71.0450s]
10/11/23 16:54:53| INFO mul_sld finished [took 84.6525s]
10/11/23 16:54:53| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs finished [took 88.0027s]
10/11/23 16:54:53| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs started
10/11/23 16:56:06| INFO ref finished [took 54.1395s]
10/11/23 16:56:11| INFO mul3w_sld finished [took 70.1307s]
10/11/23 16:56:13| INFO atc_mc finished [took 57.6850s]
10/11/23 16:56:16| INFO mul_sld finished [took 79.3229s]
10/11/23 16:56:16| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs finished [took 83.1293s]
10/11/23 16:56:16| INFO Dataset sample 0.40 of dataset rcv1_CCAT_9prevs started
10/11/23 16:57:25| INFO ref finished [took 53.7706s]
10/11/23 16:57:31| INFO mul3w_sld finished [took 67.0318s]
10/11/23 16:57:32| INFO atc_mc finished [took 57.7642s]
10/11/23 16:57:35| INFO mul_sld finished [took 76.4066s]
10/11/23 16:57:35| INFO Dataset sample 0.40 of dataset rcv1_CCAT_9prevs finished [took 79.0392s]
10/11/23 16:57:35| INFO Dataset sample 0.50 of dataset rcv1_CCAT_9prevs started
10/11/23 16:58:46| INFO ref finished [took 53.7141s]
10/11/23 16:58:51| INFO mul3w_sld finished [took 68.1234s]
10/11/23 16:58:53| INFO atc_mc finished [took 57.5702s]
10/11/23 16:58:59| INFO mul_sld finished [took 80.1266s]
10/11/23 16:58:59| INFO Dataset sample 0.50 of dataset rcv1_CCAT_9prevs finished [took 83.5102s]
10/11/23 16:58:59| INFO Dataset sample 0.60 of dataset rcv1_CCAT_9prevs started
10/11/23 17:00:08| INFO ref finished [took 54.7955s]
10/11/23 17:00:12| INFO mul3w_sld finished [took 65.3780s]
10/11/23 17:00:16| INFO atc_mc finished [took 58.0304s]
10/11/23 17:00:35| INFO mul_sld finished [took 92.9140s]
10/11/23 17:00:35| INFO Dataset sample 0.60 of dataset rcv1_CCAT_9prevs finished [took 95.8577s]
10/11/23 17:00:35| INFO Dataset sample 0.70 of dataset rcv1_CCAT_9prevs started
10/11/23 17:01:48| INFO ref finished [took 56.4922s]
10/11/23 17:01:53| INFO atc_mc finished [took 57.1146s]
10/11/23 17:01:53| INFO mul3w_sld finished [took 70.4300s]
10/11/23 17:02:03| INFO mul_sld finished [took 84.4971s]
10/11/23 17:02:03| INFO Dataset sample 0.70 of dataset rcv1_CCAT_9prevs finished [took 88.5686s]
10/11/23 17:02:03| INFO Dataset sample 0.80 of dataset rcv1_CCAT_9prevs started
10/11/23 17:03:17| INFO ref finished [took 55.9443s]
10/11/23 17:03:19| INFO mul3w_sld finished [took 68.1298s]
10/11/23 17:03:21| INFO atc_mc finished [took 55.8708s]
10/11/23 17:03:29| INFO mul_sld finished [took 81.8552s]
10/11/23 17:03:29| INFO Dataset sample 0.80 of dataset rcv1_CCAT_9prevs finished [took 85.5029s]
10/11/23 17:03:29| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs started
10/11/23 17:04:36| INFO ref finished [took 52.4327s]
10/11/23 17:04:42| INFO mul3w_sld finished [took 65.1205s]
10/11/23 17:04:44| INFO atc_mc finished [took 56.4258s]
10/11/23 17:05:07| INFO mul_sld finished [took 95.0175s]
10/11/23 17:05:07| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs finished [took 98.2742s]
----------------------------------------------------------------------------------------------------
10/11/23 17:11:56| INFO dataset rcv1_CCAT_9prevs
10/11/23 17:12:01| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs started
10/11/23 17:13:39| INFO ref finished [took 70.2028s]
10/11/23 17:13:41| INFO mul3w_sld finished [took 92.9234s]
10/11/23 17:13:43| INFO mul3wc_sld finished [took 81.4985s]
10/11/23 17:13:47| INFO atc_mc finished [took 69.8737s]
10/11/23 17:13:49| INFO mulc_sld finished [took 91.9609s]
10/11/23 17:14:09| INFO mul_sld finished [took 125.3021s]
10/11/23 17:14:09| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs finished [took 128.4443s]
10/11/23 17:14:09| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs started
10/11/23 17:15:49| INFO ref finished [took 69.9186s]
10/11/23 17:15:52| INFO mul3w_sld finished [took 94.1565s]
10/11/23 17:15:53| INFO mul3wc_sld finished [took 81.7286s]
10/11/23 17:15:55| INFO atc_mc finished [took 68.0996s]
10/11/23 17:15:59| INFO mulc_sld finished [took 93.4749s]
10/11/23 17:16:02| INFO mul_sld finished [took 109.9866s]
10/11/23 17:16:02| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs finished [took 113.2243s]
10/11/23 17:16:02| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs started
10/11/23 17:17:46| INFO ref finished [took 69.7954s]
10/11/23 17:17:49| INFO mul3w_sld finished [took 97.4538s]
10/11/23 17:17:52| INFO atc_mc finished [took 68.3256s]
10/11/23 17:17:53| INFO mul3wc_sld finished [took 84.1359s]
10/11/23 17:17:54| INFO mul_sld finished [took 107.4652s]
10/11/23 17:17:55| INFO mulc_sld finished [took 95.2790s]
10/11/23 17:17:55| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs finished [took 112.5479s]
10/11/23 17:17:55| INFO Dataset sample 0.40 of dataset rcv1_CCAT_9prevs started
10/11/23 17:19:33| INFO ref finished [took 69.0242s]
10/11/23 17:19:37| INFO mul3w_sld finished [took 94.6726s]
10/11/23 17:19:39| INFO mul3wc_sld finished [took 83.5757s]
10/11/23 17:19:40| INFO atc_mc finished [took 70.0895s]
10/11/23 17:19:42| INFO mul_sld finished [took 104.3169s]
10/11/23 17:19:45| INFO mulc_sld finished [took 96.4947s]
10/11/23 17:19:45| INFO Dataset sample 0.40 of dataset rcv1_CCAT_9prevs finished [took 109.8588s]
10/11/23 17:19:45| INFO Dataset sample 0.50 of dataset rcv1_CCAT_9prevs started
10/11/23 17:21:25| INFO ref finished [took 68.6827s]
10/11/23 17:21:28| INFO mul3w_sld finished [took 95.7660s]
10/11/23 17:21:33| INFO mul3wc_sld finished [took 84.6669s]
10/11/23 17:21:34| INFO atc_mc finished [took 68.2059s]
10/11/23 17:21:38| INFO mul_sld finished [took 109.7998s]
10/11/23 17:21:41| INFO mulc_sld finished [took 100.2322s]
10/11/23 17:21:41| INFO Dataset sample 0.50 of dataset rcv1_CCAT_9prevs finished [took 116.3329s]
10/11/23 17:21:41| INFO Dataset sample 0.60 of dataset rcv1_CCAT_9prevs started
10/11/23 17:23:20| INFO ref finished [took 68.4213s]
10/11/23 17:23:21| INFO mul3w_sld finished [took 92.4175s]
10/11/23 17:23:25| INFO mul3wc_sld finished [took 84.2143s]
10/11/23 17:23:28| INFO atc_mc finished [took 70.9748s]
10/11/23 17:23:33| INFO mul_sld finished [took 108.9404s]
10/11/23 17:23:35| INFO mulc_sld finished [took 98.6542s]
10/11/23 17:23:35| INFO Dataset sample 0.60 of dataset rcv1_CCAT_9prevs finished [took 114.0658s]
10/11/23 17:23:35| INFO Dataset sample 0.70 of dataset rcv1_CCAT_9prevs started
10/11/23 17:25:15| INFO ref finished [took 67.6212s]
10/11/23 17:25:19| INFO mul3w_sld finished [took 94.4720s]
10/11/23 17:25:23| INFO mul3wc_sld finished [took 84.2353s]
10/11/23 17:25:24| INFO atc_mc finished [took 68.8926s]
10/11/23 17:25:29| INFO mul_sld finished [took 110.0608s]
10/11/23 17:25:31| INFO mulc_sld finished [took 98.8512s]
10/11/23 17:25:31| INFO Dataset sample 0.70 of dataset rcv1_CCAT_9prevs finished [took 116.1487s]
10/11/23 17:25:31| INFO Dataset sample 0.80 of dataset rcv1_CCAT_9prevs started
10/11/23 17:27:13| INFO ref finished [took 67.6916s]
10/11/23 17:27:14| INFO mul3wc_sld finished [took 77.9027s]
10/11/23 17:27:16| INFO mul3w_sld finished [took 95.6551s]
10/11/23 17:27:21| INFO atc_mc finished [took 69.1522s]
10/11/23 17:27:25| INFO mul_sld finished [took 110.5946s]
10/11/23 17:27:26| INFO mulc_sld finished [took 97.1366s]
10/11/23 17:27:26| INFO Dataset sample 0.80 of dataset rcv1_CCAT_9prevs finished [took 115.0205s]
10/11/23 17:27:26| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs started
10/11/23 17:28:57| INFO ref finished [took 60.3214s]
10/11/23 17:29:05| INFO mul3w_sld finished [took 89.8969s]
10/11/23 17:29:08| INFO mul3wc_sld finished [took 79.4281s]
10/11/23 17:29:09| INFO mulc_sld finished [took 89.2493s]
10/11/23 17:29:10| INFO atc_mc finished [took 69.0056s]
10/11/23 17:29:29| INFO mul_sld finished [took 119.2894s]
10/11/23 17:29:29| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs finished [took 122.1553s]
----------------------------------------------------------------------------------------------------
10/11/23 17:53:55| INFO dataset rcv1_CCAT_9prevs
10/11/23 17:54:01| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs started
10/11/23 17:54:04| WARNING Method mul_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 17:54:06| WARNING Method mul3w_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 17:54:53| INFO ref finished [took 44.4485s]
10/11/23 17:55:01| INFO atc_mc finished [took 49.8926s]
10/11/23 17:55:01| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs finished [took 60.0268s]
10/11/23 17:55:01| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs started
10/11/23 17:55:04| WARNING Method mul_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 17:55:07| WARNING Method mul3w_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 17:55:54| INFO ref finished [took 44.7491s]
10/11/23 17:56:02| INFO atc_mc finished [took 49.9022s]
10/11/23 17:56:02| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs finished [took 60.8386s]
10/11/23 17:56:02| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs started
10/11/23 17:56:05| WARNING Method mul_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 17:56:08| WARNING Method mul3w_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 17:56:56| INFO ref finished [took 45.2058s]
10/11/23 17:57:03| INFO atc_mc finished [took 49.0981s]
10/11/23 17:57:03| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs finished [took 61.1000s]
10/11/23 17:57:03| INFO Dataset sample 0.40 of dataset rcv1_CCAT_9prevs started
10/11/23 17:57:05| WARNING Method mul_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 17:57:08| WARNING Method mul3w_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 17:57:54| INFO ref finished [took 44.3211s]
10/11/23 17:58:01| INFO atc_mc finished [took 48.9814s]
10/11/23 17:58:01| INFO Dataset sample 0.40 of dataset rcv1_CCAT_9prevs finished [took 58.0909s]
10/11/23 17:58:01| INFO Dataset sample 0.50 of dataset rcv1_CCAT_9prevs started
10/11/23 17:58:04| WARNING Method mul_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 17:58:06| WARNING Method mul3w_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 17:58:53| INFO ref finished [took 43.7895s]
10/11/23 17:59:01| INFO atc_mc finished [took 49.8161s]
10/11/23 17:59:01| INFO Dataset sample 0.50 of dataset rcv1_CCAT_9prevs finished [took 60.3531s]
10/11/23 17:59:01| INFO Dataset sample 0.60 of dataset rcv1_CCAT_9prevs started
10/11/23 17:59:04| WARNING Method mul_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 17:59:07| WARNING Method mul3w_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 17:59:53| INFO ref finished [took 43.8473s]
10/11/23 18:00:02| INFO atc_mc finished [took 50.1003s]
10/11/23 18:00:02| INFO Dataset sample 0.60 of dataset rcv1_CCAT_9prevs finished [took 60.5473s]
10/11/23 18:00:02| INFO Dataset sample 0.70 of dataset rcv1_CCAT_9prevs started
10/11/23 18:00:05| WARNING Method mul_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 18:00:08| WARNING Method mul3w_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 18:00:55| INFO ref finished [took 44.2168s]
10/11/23 18:01:02| INFO atc_mc finished [took 49.0013s]
10/11/23 18:01:02| INFO Dataset sample 0.70 of dataset rcv1_CCAT_9prevs finished [took 60.5261s]
10/11/23 18:01:02| INFO Dataset sample 0.80 of dataset rcv1_CCAT_9prevs started
10/11/23 18:01:06| WARNING Method mul_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 18:01:08| WARNING Method mul3w_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 18:01:55| INFO ref finished [took 44.0374s]
10/11/23 18:02:02| INFO atc_mc finished [took 49.0107s]
10/11/23 18:02:02| INFO Dataset sample 0.80 of dataset rcv1_CCAT_9prevs finished [took 60.0171s]
10/11/23 18:02:02| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs started
10/11/23 18:02:06| WARNING Method mul_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 18:02:08| WARNING Method mul3w_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 18:02:55| INFO ref finished [took 44.8341s]
10/11/23 18:03:02| INFO atc_mc finished [took 49.3663s]
10/11/23 18:03:02| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs finished [took 59.9146s]
----------------------------------------------------------------------------------------------------
10/11/23 18:25:20| INFO dataset rcv1_CCAT_1prevs
10/11/23 18:25:26| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
10/11/23 18:25:33| WARNING Method mul_sld_gs failed. Exception: '>=' not supported between instances of 'AttributeError' and 'int'
----------------------------------------------------------------------------------------------------
10/11/23 18:27:03| INFO dataset rcv1_CCAT_1prevs
10/11/23 18:27:08| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
----------------------------------------------------------------------------------------------------
10/11/23 18:32:33| INFO dataset rcv1_CCAT_1prevs
10/11/23 18:32:38| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
10/11/23 18:32:56| WARNING Method mul_sld_gs failed. Exception: '>=' not supported between instances of 'AssertionError' and 'int'
----------------------------------------------------------------------------------------------------
10/11/23 18:34:35| INFO dataset rcv1_CCAT_1prevs
10/11/23 18:34:40| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
10/11/23 18:36:25| INFO ref finished [took 96.5839s]
10/11/23 18:44:44| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 10.0, 'quantifier__classifier__class_weight': None, 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.00784) [took 599.7376s]
10/11/23 18:45:34| INFO mul_sld_gs finished [took 649.9561s]
10/11/23 18:45:34| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs finished [took 654.0138s]
----------------------------------------------------------------------------------------------------
10/11/23 19:09:37| INFO dataset rcv1_CCAT_9prevs
10/11/23 19:09:42| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs started
10/11/23 19:12:34| INFO ref finished [took 161.2859s]
10/11/23 19:13:10| INFO atc_mc finished [took 195.4184s]
10/11/23 19:23:09| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 0.1, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.00980) [took 799.2257s]
10/11/23 19:25:06| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 0.1, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.01058) [took 920.3112s]
10/11/23 19:25:20| INFO mul3w_sld_gs finished [took 930.7628s]
10/11/23 19:25:54| INFO mul_sld_gs finished [took 968.7880s]
10/11/23 19:25:54| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs finished [took 972.0481s]
10/11/23 19:25:54| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs started
10/11/23 19:29:20| INFO ref finished [took 194.4623s]
10/11/23 19:29:41| INFO atc_mc finished [took 212.3133s]
10/11/23 19:39:18| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 0.01, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.01783) [took 796.9196s]
10/11/23 19:41:39| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 0.01, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.01419) [took 940.9938s]
10/11/23 19:41:44| INFO mul3w_sld_gs finished [took 942.8448s]
10/11/23 19:42:29| INFO mul_sld_gs finished [took 991.5975s]
10/11/23 19:42:29| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs finished [took 995.0756s]
10/11/23 19:42:29| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs started
10/11/23 19:45:53| INFO ref finished [took 192.2438s]
10/11/23 19:46:27| INFO atc_mc finished [took 222.4684s]
10/11/23 19:56:09| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 1000.0, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.00855) [took 811.9404s]
10/11/23 19:58:17| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 10.0, 'quantifier__classifier__class_weight': None, 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.00987) [took 944.2852s]
10/11/23 19:58:26| INFO mul3w_sld_gs finished [took 949.1296s]
10/11/23 19:59:09| INFO mul_sld_gs finished [took 995.5734s]
10/11/23 19:59:09| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs finished [took 999.2987s]
10/11/23 19:59:09| INFO Dataset sample 0.40 of dataset rcv1_CCAT_9prevs started
10/11/23 20:02:30| INFO ref finished [took 192.8661s]
10/11/23 20:02:59| INFO atc_mc finished [took 217.1095s]
10/11/23 20:12:26| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 100.0, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': None, 'confidence': ['max_conf', 'entropy']} (score=0.01004) [took 792.0336s]
10/11/23 20:14:45| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 100.0, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': None, 'confidence': ['max_conf', 'entropy']} (score=0.01023) [took 933.3629s]
10/11/23 20:14:49| INFO mul3w_sld_gs finished [took 935.2472s]
10/11/23 20:15:33| INFO mul_sld_gs finished [took 981.2718s]
10/11/23 20:15:33| INFO Dataset sample 0.40 of dataset rcv1_CCAT_9prevs finished [took 983.8857s]
10/11/23 20:15:33| INFO Dataset sample 0.50 of dataset rcv1_CCAT_9prevs started
10/11/23 20:18:54| INFO ref finished [took 190.6586s]
10/11/23 20:19:31| INFO atc_mc finished [took 222.0446s]
10/11/23 20:28:53| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 0.1, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.00628) [took 794.2257s]
10/11/23 20:31:06| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 1.0, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.00601) [took 930.8288s]
10/11/23 20:31:12| INFO mul3w_sld_gs finished [took 932.7508s]
10/11/23 20:31:58| INFO mul_sld_gs finished [took 982.4600s]
10/11/23 20:31:58| INFO Dataset sample 0.50 of dataset rcv1_CCAT_9prevs finished [took 985.3991s]
10/11/23 20:31:58| INFO Dataset sample 0.60 of dataset rcv1_CCAT_9prevs started
10/11/23 20:35:16| INFO ref finished [took 187.4145s]
10/11/23 20:35:52| INFO atc_mc finished [took 219.1775s]
10/11/23 20:45:24| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 0.1, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.00683) [took 799.7247s]
10/11/23 20:47:38| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 100.0, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.00635) [took 936.5460s]
10/11/23 20:47:46| INFO mul3w_sld_gs finished [took 941.6249s]
10/11/23 20:48:28| INFO mul_sld_gs finished [took 987.3566s]
10/11/23 20:48:28| INFO Dataset sample 0.60 of dataset rcv1_CCAT_9prevs finished [took 990.3392s]
10/11/23 20:48:28| INFO Dataset sample 0.70 of dataset rcv1_CCAT_9prevs started
10/11/23 20:52:00| INFO ref finished [took 199.2882s]
10/11/23 20:52:27| INFO atc_mc finished [took 221.3995s]
10/11/23 21:01:50| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 0.01, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.01112) [took 794.5357s]
10/11/23 21:03:48| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 0.01, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.01010) [took 916.6784s]
10/11/23 21:04:04| INFO mul3w_sld_gs finished [took 928.2783s]
10/11/23 21:04:45| INFO mul_sld_gs finished [took 972.9080s]
10/11/23 21:04:45| INFO Dataset sample 0.70 of dataset rcv1_CCAT_9prevs finished [took 976.3012s]
10/11/23 21:04:45| INFO Dataset sample 0.80 of dataset rcv1_CCAT_9prevs started
10/11/23 21:07:55| INFO ref finished [took 179.6820s]
10/11/23 21:08:25| INFO atc_mc finished [took 205.4880s]
10/11/23 21:17:51| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 100.0, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.01448) [took 778.9722s]
10/11/23 21:20:04| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 100.0, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.01845) [took 915.8827s]
10/11/23 21:20:10| INFO mul3w_sld_gs finished [took 917.5008s]
10/11/23 21:20:53| INFO mul_sld_gs finished [took 965.1015s]
10/11/23 21:20:53| INFO Dataset sample 0.80 of dataset rcv1_CCAT_9prevs finished [took 968.7572s]
10/11/23 21:20:53| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs started
10/11/23 21:23:59| INFO ref finished [took 175.7336s]
10/11/23 21:24:36| INFO atc_mc finished [took 208.4952s]
10/11/23 21:34:01| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 100.0, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': None, 'confidence': ['max_conf', 'entropy']} (score=0.00971) [took 780.9362s]
10/11/23 21:35:59| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 10.0, 'quantifier__classifier__class_weight': None, 'quantifier__recalib': None, 'confidence': ['max_conf', 'entropy']} (score=0.00915) [took 902.5701s]
10/11/23 21:36:12| INFO mul3w_sld_gs finished [took 911.8556s]
10/11/23 21:36:47| INFO mul_sld_gs finished [took 950.1102s]
10/11/23 21:36:47| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs finished [took 953.3643s]
----------------------------------------------------------------------------------------------------
10/11/23 23:35:14| INFO dataset rcv1_CCAT_1prevs
10/11/23 23:35:19| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
10/11/23 23:36:36| INFO ref finished [took 57.4602s]
10/11/23 23:36:41| INFO mul3w_sld finished [took 72.3696s]
10/11/23 23:36:43| INFO atc_mc finished [took 57.1442s]
10/11/23 23:36:48| INFO mul_sld finished [took 84.9653s]
10/11/23 23:36:48| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs finished [took 88.9861s]
10/11/23 23:36:48| ERROR Evaluation over rcv1_CCAT_1prevs failed. Exception: file must have a 'write' attribute
----------------------------------------------------------------------------------------------------
10/11/23 23:40:17| INFO dataset rcv1_CCAT_1prevs
10/11/23 23:40:23| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
10/11/23 23:41:49| INFO ref finished [took 68.3161s]
10/11/23 23:41:53| INFO mul3w_sld finished [took 80.2113s]
10/11/23 23:41:56| INFO atc_mc finished [took 67.4302s]
10/11/23 23:42:01| INFO mul_sld finished [took 94.1596s]
10/11/23 23:42:01| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs finished [took 98.1999s]
10/11/23 23:42:01| ERROR Evaluation over rcv1_CCAT_1prevs failed. Exception: write() argument must be str, not bytes
----------------------------------------------------------------------------------------------------
10/11/23 23:46:43| INFO dataset rcv1_CCAT_1prevs
10/11/23 23:46:48| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
10/11/23 23:48:04| INFO ref finished [took 56.6494s]
10/11/23 23:48:10| INFO mul3w_sld finished [took 71.5272s]
10/11/23 23:48:12| INFO atc_mc finished [took 56.8870s]
10/11/23 23:48:15| INFO mul_sld finished [took 83.1487s]
10/11/23 23:48:15| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs finished [took 86.6542s]
----------------------------------------------------------------------------------------------------
11/11/23 02:18:21| ERROR Evaluation over rcv1_CCAT_3prevs failed. Exception: 'Invalid estimator: estimator mul3w_slds does not exist'
11/11/23 02:18:21| ERROR Failed while saving configuration rcv1_CCAT_debug_gs of rcv1_CCAT_3prevs. Exception: local variable 'dr' referenced before assignment
----------------------------------------------------------------------------------------------------
11/11/23 02:20:09| INFO dataset rcv1_CCAT_3prevs
11/11/23 02:20:15| INFO Dataset sample 0.20 of dataset rcv1_CCAT_3prevs started
11/11/23 02:21:26| INFO ref finished [took 54.1901s]
11/11/23 02:21:33| INFO mul3w_sld finished [took 69.3537s]
11/11/23 02:21:34| INFO atc_mc finished [took 56.0851s]
11/11/23 02:21:37| INFO mul_sld finished [took 78.6476s]
11/11/23 02:21:37| INFO Dataset sample 0.20 of dataset rcv1_CCAT_3prevs finished [took 82.3064s]
11/11/23 02:21:37| INFO Dataset sample 0.50 of dataset rcv1_CCAT_3prevs started
11/11/23 02:22:46| INFO ref finished [took 53.7198s]
11/11/23 02:22:52| INFO mul3w_sld finished [took 67.7595s]
11/11/23 02:22:54| INFO atc_mc finished [took 57.2473s]
11/11/23 02:23:00| INFO mul_sld finished [took 80.1637s]
11/11/23 02:23:00| INFO Dataset sample 0.50 of dataset rcv1_CCAT_3prevs finished [took 83.2895s]
11/11/23 02:23:00| INFO Dataset sample 0.80 of dataset rcv1_CCAT_3prevs started
11/11/23 02:24:11| INFO ref finished [took 53.3185s]
11/11/23 02:24:16| INFO mul3w_sld finished [took 67.0687s]
11/11/23 02:24:19| INFO atc_mc finished [took 56.5244s]
11/11/23 02:24:26| INFO mul_sld finished [took 82.6075s]
11/11/23 02:24:26| INFO Dataset sample 0.80 of dataset rcv1_CCAT_3prevs finished [took 85.5541s]
----------------------------------------------------------------------------------------------------
11/11/23 03:03:37| INFO dataset rcv1_CCAT_9prevs
11/11/23 03:03:42| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs started
11/11/23 03:04:49| INFO ref finished [took 51.5735s]
11/11/23 03:04:53| INFO mul3w_sld finished [took 62.2816s]
11/11/23 03:04:57| INFO atc_mc finished [took 56.1819s]
11/11/23 03:05:22| INFO mul_sld finished [took 96.8027s]
11/11/23 03:05:22| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs finished [took 100.0654s]
11/11/23 03:05:22| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs started
11/11/23 03:06:32| INFO ref finished [took 53.7892s]
11/11/23 03:06:37| INFO mul3w_sld finished [took 66.3243s]
11/11/23 03:06:40| INFO atc_mc finished [took 57.0767s]
11/11/23 03:06:48| INFO mul_sld finished [took 82.1666s]
11/11/23 03:06:48| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs finished [took 85.6326s]
11/11/23 03:06:48| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs started
11/11/23 03:07:59| INFO ref finished [took 53.4098s]
11/11/23 03:08:05| INFO mul3w_sld finished [took 67.3941s]
11/11/23 03:08:06| INFO atc_mc finished [took 56.8497s]
11/11/23 03:08:09| INFO mul_sld finished [took 77.4972s]
11/11/23 03:08:09| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs finished [took 81.1647s]
11/11/23 03:08:09| INFO Dataset sample 0.40 of dataset rcv1_CCAT_9prevs started
11/11/23 03:09:18| INFO ref finished [took 54.4416s]
11/11/23 03:09:23| INFO mul3w_sld finished [took 67.3565s]
11/11/23 03:09:25| INFO atc_mc finished [took 57.5687s]
11/11/23 03:09:29| INFO mul_sld finished [took 77.1110s]
11/11/23 03:09:29| INFO Dataset sample 0.40 of dataset rcv1_CCAT_9prevs finished [took 79.8726s]
11/11/23 03:09:29| INFO Dataset sample 0.50 of dataset rcv1_CCAT_9prevs started
11/11/23 03:10:38| INFO ref finished [took 54.2066s]
11/11/23 03:10:44| INFO mul3w_sld finished [took 66.9062s]
11/11/23 03:10:46| INFO atc_mc finished [took 57.1371s]
11/11/23 03:10:52| INFO mul_sld finished [took 79.7498s]
11/11/23 03:10:52| INFO Dataset sample 0.50 of dataset rcv1_CCAT_9prevs finished [took 83.1506s]
11/11/23 03:10:52| INFO Dataset sample 0.60 of dataset rcv1_CCAT_9prevs started
11/11/23 03:11:59| INFO ref finished [took 51.1698s]
11/11/23 03:12:03| INFO mul3w_sld finished [took 63.7260s]
11/11/23 03:12:07| INFO atc_mc finished [took 56.1393s]
11/11/23 03:12:15| INFO mul_sld finished [took 79.3176s]
11/11/23 03:12:15| INFO Dataset sample 0.60 of dataset rcv1_CCAT_9prevs finished [took 82.4614s]
11/11/23 03:12:15| INFO Dataset sample 0.70 of dataset rcv1_CCAT_9prevs started
11/11/23 03:13:23| INFO ref finished [took 51.5505s]
11/11/23 03:13:28| INFO mul3w_sld finished [took 64.8809s]
11/11/23 03:13:32| INFO atc_mc finished [took 56.2766s]
11/11/23 03:13:39| INFO mul_sld finished [took 80.9284s]
11/11/23 03:13:39| INFO Dataset sample 0.70 of dataset rcv1_CCAT_9prevs finished [took 84.2727s]
11/11/23 03:13:39| INFO Dataset sample 0.80 of dataset rcv1_CCAT_9prevs started
11/11/23 03:14:50| INFO ref finished [took 54.6314s]
11/11/23 03:14:55| INFO mul3w_sld finished [took 67.8281s]
11/11/23 03:14:58| INFO atc_mc finished [took 57.4433s]
11/11/23 03:15:04| INFO mul_sld finished [took 82.1945s]
11/11/23 03:15:04| INFO Dataset sample 0.80 of dataset rcv1_CCAT_9prevs finished [took 85.5484s]
11/11/23 03:15:04| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs started
11/11/23 03:16:12| INFO ref finished [took 54.0583s]
11/11/23 03:16:19| INFO mul3w_sld finished [took 65.9654s]
11/11/23 03:16:20| INFO atc_mc finished [took 56.7869s]
11/11/23 03:16:42| INFO mul_sld finished [took 94.5979s]
11/11/23 03:16:42| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs finished [took 97.8705s]
----------------------------------------------------------------------------------------------------
11/11/23 14:55:30| INFO dataset rcv1_CCAT_9prevs
11/11/23 14:55:35| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs started
11/11/23 14:56:42| INFO ref finished [took 51.5446s]
11/11/23 14:56:45| INFO mul3w_sld finished [took 61.1724s]
11/11/23 14:56:49| INFO atc_mc finished [took 55.8947s]
11/11/23 14:57:16| INFO mul_sld finished [took 97.5932s]
11/11/23 14:57:16| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs finished [took 100.9322s]
11/11/23 14:57:16| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs started
11/11/23 14:58:25| INFO ref finished [took 52.3894s]
11/11/23 14:58:32| INFO mul3w_sld finished [took 66.8082s]
11/11/23 14:58:34| INFO atc_mc finished [took 57.1608s]
11/11/23 14:58:43| INFO mul_sld finished [took 82.8613s]
11/11/23 14:58:43| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs finished [took 86.2458s]
11/11/23 14:58:43| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs started
11/11/23 14:59:55| INFO ref finished [took 53.4860s]
11/11/23 15:00:00| INFO mul3w_sld finished [took 68.6692s]
11/11/23 15:00:02| INFO atc_mc finished [took 56.8460s]
11/11/23 15:00:04| INFO mul_sld finished [took 77.8829s]
11/11/23 15:00:04| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs finished [took 81.6584s]
11/11/23 15:00:04| INFO Dataset sample 0.40 of dataset rcv1_CCAT_9prevs started
11/11/23 15:01:12| INFO ref finished [took 52.5890s]
11/11/23 15:01:18| INFO mul3w_sld finished [took 66.3191s]
11/11/23 15:01:18| INFO atc_mc finished [took 55.8094s]
11/11/23 15:01:23| INFO mul_sld finished [took 76.2650s]
11/11/23 15:01:23| INFO Dataset sample 0.40 of dataset rcv1_CCAT_9prevs finished [took 78.8604s]
11/11/23 15:01:23| INFO Dataset sample 0.50 of dataset rcv1_CCAT_9prevs started
11/11/23 15:02:30| INFO ref finished [took 52.1273s]
11/11/23 15:02:39| INFO mul3w_sld finished [took 67.7736s]
11/11/23 15:02:41| INFO atc_mc finished [took 58.1392s]
11/11/23 15:02:47| INFO mul_sld finished [took 80.6840s]
11/11/23 15:02:47| INFO Dataset sample 0.50 of dataset rcv1_CCAT_9prevs finished [took 84.0856s]
11/11/23 15:02:47| INFO Dataset sample 0.60 of dataset rcv1_CCAT_9prevs started
11/11/23 15:03:54| INFO ref finished [took 51.3613s]
11/11/23 15:03:59| INFO mul3w_sld finished [took 64.3562s]
11/11/23 15:04:03| INFO atc_mc finished [took 57.0352s]
11/11/23 15:04:10| INFO mul_sld finished [took 79.8549s]
11/11/23 15:04:10| INFO Dataset sample 0.60 of dataset rcv1_CCAT_9prevs finished [took 82.9290s]
11/11/23 15:04:10| INFO Dataset sample 0.70 of dataset rcv1_CCAT_9prevs started
11/11/23 15:05:20| INFO ref finished [took 52.6105s]
11/11/23 15:05:25| INFO mul3w_sld finished [took 66.7591s]
11/11/23 15:05:27| INFO atc_mc finished [took 55.8870s]
11/11/23 15:05:34| INFO mul_sld finished [took 80.9824s]
11/11/23 15:05:34| INFO Dataset sample 0.70 of dataset rcv1_CCAT_9prevs finished [took 84.2673s]
11/11/23 15:05:34| INFO Dataset sample 0.80 of dataset rcv1_CCAT_9prevs started
11/11/23 15:06:45| INFO ref finished [took 53.6830s]
11/11/23 15:06:50| INFO mul3w_sld finished [took 67.4974s]
11/11/23 15:06:53| INFO atc_mc finished [took 57.2897s]
11/11/23 15:06:59| INFO mul_sld finished [took 81.3697s]
11/11/23 15:06:59| INFO Dataset sample 0.80 of dataset rcv1_CCAT_9prevs finished [took 84.7980s]
11/11/23 15:06:59| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs started
11/11/23 15:08:08| INFO ref finished [took 54.0902s]
11/11/23 15:08:14| INFO mul3w_sld finished [took 66.9136s]
11/11/23 15:08:16| INFO atc_mc finished [took 57.8269s]
11/11/23 15:08:39| INFO mul_sld finished [took 96.4137s]
11/11/23 15:08:39| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs finished [took 99.5541s]
----------------------------------------------------------------------------------------------------
11/11/23 18:04:49| INFO dataset imdb_9prevs
11/11/23 18:05:00| INFO Dataset sample 0.10 of dataset imdb_9prevs started
11/11/23 18:05:30| INFO ref finished [took 26.1862s]
11/11/23 18:05:37| INFO atc_mc finished [took 32.4255s]
11/11/23 18:05:42| INFO mul3w_sld finished [took 38.9162s]
11/11/23 18:06:19| INFO mul_sld finished [took 76.8053s]
11/11/23 18:06:19| INFO Dataset sample 0.10 of dataset imdb_9prevs finished [took 79.1732s]
11/11/23 18:06:19| INFO Dataset sample 0.20 of dataset imdb_9prevs started
11/11/23 18:06:51| INFO ref finished [took 27.6356s]
11/11/23 18:06:58| INFO atc_mc finished [took 33.5521s]
11/11/23 18:07:01| INFO mul3w_sld finished [took 38.2698s]
11/11/23 18:07:26| INFO mul_sld finished [took 63.9553s]
11/11/23 18:07:26| INFO Dataset sample 0.20 of dataset imdb_9prevs finished [took 66.8514s]
11/11/23 18:07:26| INFO Dataset sample 0.30 of dataset imdb_9prevs started
11/11/23 18:08:02| INFO ref finished [took 27.3880s]
11/11/23 18:08:09| INFO atc_mc finished [took 34.5060s]
11/11/23 18:08:12| INFO mul3w_sld finished [took 38.3408s]
11/11/23 18:08:23| INFO mul_sld finished [took 50.5943s]
11/11/23 18:08:23| INFO Dataset sample 0.30 of dataset imdb_9prevs finished [took 57.8801s]
11/11/23 18:08:23| INFO Dataset sample 0.40 of dataset imdb_9prevs started
11/11/23 18:08:57| INFO ref finished [took 27.9013s]
11/11/23 18:09:03| INFO atc_mc finished [took 33.7677s]
11/11/23 18:09:06| INFO mul3w_sld finished [took 37.6649s]
11/11/23 18:09:18| INFO mul_sld finished [took 51.5975s]
11/11/23 18:09:18| INFO Dataset sample 0.40 of dataset imdb_9prevs finished [took 54.8989s]
11/11/23 18:09:18| INFO Dataset sample 0.50 of dataset imdb_9prevs started
11/11/23 18:09:53| INFO ref finished [took 28.6798s]
11/11/23 18:10:00| INFO atc_mc finished [took 35.3206s]
11/11/23 18:10:03| INFO mul3w_sld finished [took 38.9082s]
11/11/23 18:10:15| INFO mul_sld finished [took 52.0709s]
11/11/23 18:10:15| INFO Dataset sample 0.50 of dataset imdb_9prevs finished [took 56.2785s]
11/11/23 18:10:15| INFO Dataset sample 0.60 of dataset imdb_9prevs started
11/11/23 18:10:48| INFO ref finished [took 28.5387s]
11/11/23 18:10:55| INFO atc_mc finished [took 34.9152s]
11/11/23 18:10:58| INFO mul3w_sld finished [took 39.0343s]
11/11/23 18:11:07| INFO mul_sld finished [took 49.9013s]
11/11/23 18:11:07| INFO Dataset sample 0.60 of dataset imdb_9prevs finished [took 52.8028s]
11/11/23 18:11:07| INFO Dataset sample 0.70 of dataset imdb_9prevs started
11/11/23 18:11:43| INFO ref finished [took 29.2208s]
11/11/23 18:11:50| INFO atc_mc finished [took 36.2494s]
11/11/23 18:11:54| INFO mul3w_sld finished [took 39.4380s]
11/11/23 18:12:02| INFO mul_sld finished [took 50.3782s]
11/11/23 18:12:02| INFO Dataset sample 0.70 of dataset imdb_9prevs finished [took 54.9711s]
11/11/23 18:12:02| INFO Dataset sample 0.80 of dataset imdb_9prevs started
11/11/23 18:12:37| INFO ref finished [took 28.2535s]
11/11/23 18:12:45| INFO atc_mc finished [took 34.9431s]
11/11/23 18:12:47| INFO mul3w_sld finished [took 38.4049s]
11/11/23 18:12:56| INFO mul_sld finished [took 48.3663s]
11/11/23 18:12:56| INFO Dataset sample 0.80 of dataset imdb_9prevs finished [took 53.7749s]
11/11/23 18:12:56| INFO Dataset sample 0.90 of dataset imdb_9prevs started
11/11/23 18:13:28| INFO ref finished [took 27.5002s]
11/11/23 18:13:35| INFO atc_mc finished [took 34.2726s]
11/11/23 18:13:41| INFO mul3w_sld finished [took 41.9968s]
11/11/23 18:13:54| INFO mul_sld finished [took 56.7537s]
11/11/23 18:13:54| INFO Dataset sample 0.90 of dataset imdb_9prevs finished [took 58.1750s]

59
quacc/data.py
View File

@ -20,13 +20,20 @@ from quapy.data import LabelledCollection
#
class ExtensionPolicy:
def __init__(self, collapse_false=False):
self.collapse_false = collapse_false
class ExtendedData:
def __init__(
self,
instances: np.ndarray | sp.csr_matrix,
pred_proba: np.ndarray,
ext: np.ndarray = None,
extpol=None,
):
self.extpol = ExtensionPolicy() if extpol is None else extpol
self.b_instances_ = instances
self.pred_proba_ = pred_proba
self.ext_ = ext
@ -89,14 +96,31 @@ class ExtendedData:
class ExtendedLabels:
def __init__(self, true: np.ndarray, pred: np.ndarray, ncl: np.ndarray):
def __init__(
self,
true: np.ndarray,
pred: np.ndarray,
ncl: np.ndarray,
extpol: ExtensionPolicy = None,
):
self.extpol = ExtensionPolicy() if extpol is None else extpol
self.true = true
self.pred = pred
self.ncl = ncl
@property
def y(self):
return self.true * self.ncl + self.pred
if self.extpol.collapse_false:
return self.true + self.pred
else:
return self.true * self.ncl + self.pred
@property
def classes(self):
if self.extpol.collapse_false:
return np.arange(self.ncl + 1)
else:
return np.arange(self.ncl**2)
def __getitem__(self, idx):
return ExtendedLabels(self.true[idx], self.pred[idx], self.ncl)
@ -109,8 +133,10 @@ class ExtendedCollection(LabelledCollection):
labels: np.ndarray,
pred_proba: np.ndarray = None,
ext: np.ndarray = None,
extpol=None,
):
e_data, e_labels, _classes = self.__extend_collection(
self.extpol = ExtensionPolicy() if extpol is None else extpol
e_data, e_labels = self.__extend_collection(
instances=instances,
labels=labels,
pred_proba=pred_proba,
@ -118,16 +144,19 @@ class ExtendedCollection(LabelledCollection):
)
self.e_data_ = e_data
self.e_labels_ = e_labels
super().__init__(e_data.X, e_labels.y, classes=_classes)
super().__init__(e_data.X, e_labels.y, classes=e_labels.classes)
@classmethod
def from_lc(
cls,
lc: LabelledCollection,
predict_proba: np.ndarray,
pred_proba: np.ndarray,
ext: np.ndarray = None,
extpol=None,
):
return ExtendedCollection(lc.X, lc.y, pred_proba=predict_proba, ext=ext)
return ExtendedCollection(
lc.X, lc.y, pred_proba=pred_proba, ext=ext, extpol=extpol
)
@property
def pred_proba(self):
@ -145,6 +174,13 @@ class ExtendedCollection(LabelledCollection):
def ey(self):
return self.e_labels_
def counts(self):
_counts = super().counts()
if self.extpol.collapse_false:
_counts = np.insert(_counts, 2, 0)
return _counts
def split_by_pred(self):
_ncl = len(self.pred_proba)
_instances, _indexes = self.e_data_.split_by_pred(return_indexes=True)
@ -160,13 +196,14 @@ class ExtendedCollection(LabelledCollection):
labels: np.ndarray,
pred_proba: np.ndarray,
ext: np.ndarray = None,
) -> Tuple[ExtendedData, ExtendedLabels, np.ndarray]:
n_classes = np.unique(labels).shape[0]
extpol=None,
) -> Tuple[ExtendedData, ExtendedLabels]:
n_classes = pred_proba.shape[1]
# n_X = [ X | predicted probs. ]
e_instances = ExtendedData(instances, pred_proba, ext=ext)
e_instances = ExtendedData(instances, pred_proba, ext=ext, extpol=self.extpol)
# n_y = (exptected y, predicted y)
preds = np.argmax(pred_proba, axis=-1)
e_labels = ExtendedLabels(labels, preds, n_classes)
e_labels = ExtendedLabels(labels, preds, n_classes, extpol=self.extpol)
return e_instances, e_labels, np.arange(n_classes**2)
return e_instances, e_labels

40
quacc/evaluation/method.py
View File

@ -76,6 +76,15 @@ def mul_sld(c_model, validation, protocol) -> EvaluationReport:
)
@method
def mul3w_sld(c_model, validation, protocol) -> EvaluationReport:
est = MCAE(c_model, SLD(LogisticRegression()), collapse_false=True).fit(validation)
return evaluation_report(
estimator=est,
protocol=protocol,
)
@method
def binc_sld(c_model, validation, protocol) -> EvaluationReport:
est = BQAE(
@ -102,6 +111,20 @@ def mulc_sld(c_model, validation, protocol) -> EvaluationReport:
)
@method
def mul3wc_sld(c_model, validation, protocol) -> EvaluationReport:
est = MCAE(
c_model,
SLD(LogisticRegression()),
confidence=["max_conf", "entropy"],
collapse_false=True,
).fit(validation)
return evaluation_report(
estimator=est,
protocol=protocol,
)
@method
def binmc_sld(c_model, validation, protocol) -> EvaluationReport:
est = BQAE(
@ -188,6 +211,23 @@ def mul_sld_gs(c_model, validation, protocol) -> EvaluationReport:
)
@method
def mul3w_sld_gs(c_model, validation, protocol) -> EvaluationReport:
v_train, v_val = validation.split_stratified(0.6, random_state=0)
model = MCAE(c_model, SLD(LogisticRegression()), collapse_false=True)
est = GridSearchAE(
model=model,
param_grid=_sld_param_grid,
refit=False,
protocol=UPP(v_val, repeats=100),
verbose=True,
).fit(v_train)
return evaluation_report(
estimator=est,
protocol=protocol,
)
@method
def bin_sld_gsq(c_model, validation, protocol) -> EvaluationReport:
est = BQAEgsq(

187
quacc/evaluation/report.py
View File

@ -1,3 +1,4 @@
import pickle
from pathlib import Path
from typing import List, Tuple
@ -145,7 +146,12 @@ class CompReport:
return avg_p
def get_plots(
self, mode="delta", metric="acc", estimators=None, conf="default", stdev=False
self,
mode="delta",
metric="acc",
estimators=None,
conf="default",
return_fig=False,
) -> List[Tuple[str, Path]]:
if mode == "delta":
avg_data = self.avg_by_prevs(metric=metric, estimators=estimators)
@ -156,6 +162,7 @@ class CompReport:
metric=metric,
name=conf,
train_prev=self.train_prev,
return_fig=return_fig,
)
elif mode == "delta_stdev":
avg_data = self.avg_by_prevs(metric=metric, estimators=estimators)
@ -168,6 +175,7 @@ class CompReport:
name=conf,
train_prev=self.train_prev,
stdevs=st_data.T.to_numpy(),
return_fig=return_fig,
)
elif mode == "diagonal":
f_data = self.data(metric=metric + "_score", estimators=estimators)
@ -180,6 +188,7 @@ class CompReport:
metric=metric,
name=conf,
train_prev=self.train_prev,
return_fig=return_fig,
)
elif mode == "shift":
_shift_data = self.shift_data(metric=metric, estimators=estimators)
@ -197,6 +206,7 @@ class CompReport:
name=conf,
train_prev=self.train_prev,
counts=shift_counts.T.to_numpy(),
return_fig=return_fig,
)
def to_md(self, conf="default", metric="acc", estimators=None, stdev=False) -> str:
@ -219,7 +229,6 @@ class CompReport:
metric=metric,
estimators=estimators,
conf=conf,
stdev=stdev,
)
res += f"![plot_{mode}]({op.relative_to(env.OUT_DIR).as_posix()})\n"
@ -287,6 +296,95 @@ class DatasetReport:
self.add(cr)
return self
def get_plots(
self,
data=None,
mode="delta_train",
metric="acc",
estimators=None,
conf="default",
return_fig=False,
):
if mode == "delta_train":
_data = self.data(metric, estimators) if data is None else data
avg_on_train = _data.groupby(level=1).mean()
prevs_on_train = np.sort(avg_on_train.index.unique(0))
return plot.plot_delta(
base_prevs=np.around(
[(1.0 - p, p) for p in prevs_on_train], decimals=2
),
columns=avg_on_train.columns.to_numpy(),
data=avg_on_train.T.to_numpy(),
metric=metric,
name=conf,
train_prev=None,
avg="train",
return_fig=return_fig,
)
elif mode == "stdev_train":
_data = self.data(metric, estimators) if data is None else data
avg_on_train = _data.groupby(level=1).mean()
prevs_on_train = np.sort(avg_on_train.index.unique(0))
stdev_on_train = _data.groupby(level=1).std()
return plot.plot_delta(
base_prevs=np.around(
[(1.0 - p, p) for p in prevs_on_train], decimals=2
),
columns=avg_on_train.columns.to_numpy(),
data=avg_on_train.T.to_numpy(),
metric=metric,
name=conf,
train_prev=None,
stdevs=stdev_on_train.T.to_numpy(),
avg="train",
return_fig=return_fig,
)
elif mode == "delta_test":
_data = self.data(metric, estimators) if data is None else data
avg_on_test = _data.groupby(level=0).mean()
prevs_on_test = np.sort(avg_on_test.index.unique(0))
return plot.plot_delta(
base_prevs=np.around([(1.0 - p, p) for p in prevs_on_test], decimals=2),
columns=avg_on_test.columns.to_numpy(),
data=avg_on_test.T.to_numpy(),
metric=metric,
name=conf,
train_prev=None,
avg="test",
return_fig=return_fig,
)
elif mode == "stdev_test":
_data = self.data(metric, estimators) if data is None else data
avg_on_test = _data.groupby(level=0).mean()
prevs_on_test = np.sort(avg_on_test.index.unique(0))
stdev_on_test = _data.groupby(level=0).std()
return plot.plot_delta(
base_prevs=np.around([(1.0 - p, p) for p in prevs_on_test], decimals=2),
columns=avg_on_test.columns.to_numpy(),
data=avg_on_test.T.to_numpy(),
metric=metric,
name=conf,
train_prev=None,
stdevs=stdev_on_test.T.to_numpy(),
avg="test",
return_fig=return_fig,
)
elif mode == "shift":
_shift_data = self.shift_data(metric, estimators) if data is None else data
avg_shift = _shift_data.groupby(level=0).mean()
count_shift = _shift_data.groupby(level=0).count()
prevs_shift = np.sort(avg_shift.index.unique(0))
return plot.plot_shift(
shift_prevs=np.around([(1.0 - p, p) for p in prevs_shift], decimals=2),
columns=avg_shift.columns.to_numpy(),
data=avg_shift.T.to_numpy(),
metric=metric,
name=conf,
train_prev=None,
counts=count_shift.T.to_numpy(),
return_fig=return_fig,
)
def to_md(self, conf="default", metric="acc", estimators=[], stdev=False):
res = f"# {self.name}\n\n"
for cr in self.crs:
@ -300,95 +398,82 @@ class DatasetReport:
######################## avg on train ########################
res += "### avg on train\n"
avg_on_train = _data.groupby(level=1).mean()
prevs_on_train = np.sort(avg_on_train.index.unique(0))
stdev_on_train = _data.groupby(level=1).std() if stdev else None
avg_on_train_tbl = _data.groupby(level=1).mean()
avg_on_train_tbl.loc["avg", :] = _data.mean()
res += avg_on_train_tbl.to_html() + "\n\n"
delta_op = plot.plot_delta(
base_prevs=np.around([(1.0 - p, p) for p in prevs_on_train], decimals=2),
columns=avg_on_train.columns.to_numpy(),
data=avg_on_train.T.to_numpy(),
delta_op = self.get_plots(
data=_data,
mode="delta_train",
metric=metric,
name=conf,
train_prev=None,
avg="train",
estimators=estimators,
conf=conf,
)
res += f"![plot_delta]({delta_op.relative_to(env.OUT_DIR).as_posix()})\n"
if stdev:
delta_stdev_op = plot.plot_delta(
base_prevs=np.around(
[(1.0 - p, p) for p in prevs_on_train], decimals=2
),
columns=avg_on_train.columns.to_numpy(),
data=avg_on_train.T.to_numpy(),
delta_stdev_op = self.get_plots(
data=_data,
mode="stdev_train",
metric=metric,
name=conf,
train_prev=None,
stdevs=stdev_on_train.T.to_numpy(),
avg="train",
estimators=estimators,
conf=conf,
)
res += f"![plot_delta_stdev]({delta_stdev_op.relative_to(env.OUT_DIR).as_posix()})\n"
######################## avg on test ########################
res += "### avg on test\n"
avg_on_test = _data.groupby(level=0).mean()
prevs_on_test = np.sort(avg_on_test.index.unique(0))
stdev_on_test = _data.groupby(level=0).std() if stdev else None
avg_on_test_tbl = _data.groupby(level=0).mean()
avg_on_test_tbl.loc["avg", :] = _data.mean()
res += avg_on_test_tbl.to_html() + "\n\n"
delta_op = plot.plot_delta(
base_prevs=np.around([(1.0 - p, p) for p in prevs_on_test], decimals=2),
columns=avg_on_test.columns.to_numpy(),
data=avg_on_test.T.to_numpy(),
delta_op = self.get_plots(
data=_data,
mode="delta_test",
metric=metric,
name=conf,
train_prev=None,
avg="test",
estimators=estimators,
conf=conf,
)
res += f"![plot_delta]({delta_op.relative_to(env.OUT_DIR).as_posix()})\n"
if stdev:
delta_stdev_op = plot.plot_delta(
base_prevs=np.around([(1.0 - p, p) for p in prevs_on_test], decimals=2),
columns=avg_on_test.columns.to_numpy(),
data=avg_on_test.T.to_numpy(),
delta_stdev_op = self.get_plots(
data=_data,
mode="stdev_test",
metric=metric,
name=conf,
train_prev=None,
stdevs=stdev_on_test.T.to_numpy(),
avg="test",
estimators=estimators,
conf=conf,
)
res += f"![plot_delta_stdev]({delta_stdev_op.relative_to(env.OUT_DIR).as_posix()})\n"
######################## avg shift ########################
res += "### avg dataset shift\n"
avg_shift = _shift_data.groupby(level=0).mean()
count_shift = _shift_data.groupby(level=0).count()
prevs_shift = np.sort(avg_shift.index.unique(0))
shift_op = plot.plot_shift(
shift_prevs=np.around([(1.0 - p, p) for p in prevs_shift], decimals=2),
columns=avg_shift.columns.to_numpy(),
data=avg_shift.T.to_numpy(),
shift_op = self.get_plots(
data=_shift_data,
mode="shift",
metric=metric,
name=conf,
train_prev=None,
counts=count_shift.T.to_numpy(),
estimators=estimators,
conf=conf,
)
res += f"![plot_shift]({shift_op.relative_to(env.OUT_DIR).as_posix()})\n"
return res
def pickle(self, pickle_path: Path):
with open(pickle_path, "wb") as f:
pickle.dump(self, f)
@classmethod
def unpickle(cls, pickle_path: Path):
with open(pickle_path, "rb") as f:
dr = pickle.load(f)
return dr
def __iter__(self):
return (cr for cr in self.crs)

1
quacc/main.py
View File

@ -25,6 +25,7 @@ def estimate_comparison():
dataset,
estimators=CE.name[env.COMP_ESTIMATORS],
)
dr.pickle(env.OUT_DIR / f"{dataset.name}.pickle")
except Exception as e:
log.error(f"Evaluation over {dataset.name} failed. Exception: {e}")
traceback(e)

31
quacc/method/base.py
View File

@ -8,7 +8,7 @@ from quapy.data import LabelledCollection
from quapy.method.aggregative import BaseQuantifier
from sklearn.base import BaseEstimator
from quacc.data import ExtendedCollection, ExtendedData
from quacc.data import ExtendedCollection, ExtendedData, ExtensionPolicy
class BaseAccuracyEstimator(BaseQuantifier):
@ -19,6 +19,7 @@ class BaseAccuracyEstimator(BaseQuantifier):
):
self.__check_classifier(classifier)
self.quantifier = quantifier
self.extpol = ExtensionPolicy()
def __check_classifier(self, classifier):
if not hasattr(classifier, "predict_proba"):
@ -31,13 +32,15 @@ class BaseAccuracyEstimator(BaseQuantifier):
if pred_proba is None:
pred_proba = self.classifier.predict_proba(coll.X)
return ExtendedCollection.from_lc(coll, pred_proba=pred_proba)
return ExtendedCollection.from_lc(
coll, pred_proba=pred_proba, extpol=self.extpol
)
def _extend_instances(self, instances: np.ndarray | sp.csr_matrix, pred_proba=None):
if pred_proba is None:
pred_proba = self.classifier.predict_proba(instances)
return ExtendedData(instances, pred_proba=pred_proba)
return ExtendedData(instances, pred_proba=pred_proba, extpol=self.extpol)
@abstractmethod
def fit(self, train: LabelledCollection | ExtendedCollection):
@ -106,7 +109,9 @@ class ConfidenceBasedAccuracyEstimator(BaseAccuracyEstimator):
pred_proba = self.classifier.predict_proba(coll.X)
_ext = self.__get_ext(pred_proba)
return ExtendedCollection.from_lc(coll, pred_proba=pred_proba, ext=_ext)
return ExtendedCollection.from_lc(
coll, pred_proba=pred_proba, ext=_ext, extpol=self.extpol
)
def _extend_instances(
self,
@ -117,7 +122,9 @@ class ConfidenceBasedAccuracyEstimator(BaseAccuracyEstimator):
pred_proba = self.classifier.predict_proba(instances)
_ext = self.__get_ext(pred_proba)
return ExtendedData(instances, pred_proba=pred_proba, ext=_ext)
return ExtendedData(
instances, pred_proba=pred_proba, ext=_ext, extpol=self.extpol
)
class MultiClassAccuracyEstimator(ConfidenceBasedAccuracyEstimator):
@ -126,6 +133,7 @@ class MultiClassAccuracyEstimator(ConfidenceBasedAccuracyEstimator):
classifier: BaseEstimator,
quantifier: BaseQuantifier,
confidence: str = None,
collapse_false=False,
):
super().__init__(
classifier=classifier,
@ -133,6 +141,7 @@ class MultiClassAccuracyEstimator(ConfidenceBasedAccuracyEstimator):
confidence=confidence,
)
self.e_train = None
self.extpol = ExtensionPolicy(collapse_false=collapse_false)
def fit(self, train: LabelledCollection):
self.e_train = self.extend(train)
@ -149,7 +158,13 @@ class MultiClassAccuracyEstimator(ConfidenceBasedAccuracyEstimator):
e_inst = self._extend_instances(instances)
estim_prev = self.quantifier.quantify(e_inst.X)
return self._check_prevalence_classes(estim_prev, self.quantifier.classes_)
estim_prev = self._check_prevalence_classes(
estim_prev, self.quantifier.classes_
)
if self.extpol.collapse_false:
estim_prev = np.insert(estim_prev, 2, 0.0)
return estim_prev
def _check_prevalence_classes(self, estim_prev, estim_classes) -> np.ndarray:
true_classes = self.e_train.classes_
@ -158,6 +173,10 @@ class MultiClassAccuracyEstimator(ConfidenceBasedAccuracyEstimator):
estim_prev = np.insert(estim_prev, _cls, [0.0], axis=0)
return estim_prev
@property
def collapse_false(self):
return self.extpol.collapse_false
class BinaryQuantifierAccuracyEstimator(ConfidenceBasedAccuracyEstimator):
def __init__(

1
quacc/method/model_selection.py
View File

@ -174,6 +174,7 @@ class GridSearchAE(BaseAccuracyEstimator):
except Exception as e:
self._sout(f"something went wrong for config {params}; skipping:")
self._sout(f"\tException: {e}")
score = None
return params, score, model

16
quacc/plot.py
View File

@ -29,6 +29,7 @@ def plot_delta(
train_prev=None,
legend=True,
avg=None,
return_fig=False,
) -> Path:
_base_title = "delta_stdev" if stdevs is not None else "delta"
if train_prev is not None:
@ -84,9 +85,12 @@ def plot_delta(
if legend:
ax.legend(loc="center left", bbox_to_anchor=(1, 0.5))
if return_fig:
return fig
output_path = env.PLOT_OUT_DIR / f"{title}.png"
fig.savefig(output_path, bbox_inches="tight")
return output_path
@ -100,6 +104,7 @@ def plot_diagonal(
name="default",
train_prev=None,
legend=True,
return_fig=False,
):
if train_prev is not None:
t_prev_pos = int(round(train_prev[pos_class] * 100))
@ -169,6 +174,10 @@ def plot_diagonal(
if legend:
ax.legend(loc="center left", bbox_to_anchor=(1, 0.5))
if return_fig:
return fig
output_path = env.PLOT_OUT_DIR / f"{title}.png"
fig.savefig(output_path, bbox_inches="tight")
return output_path
@ -185,6 +194,7 @@ def plot_shift(
name="default",
train_prev=None,
legend=True,
return_fig=False,
) -> Path:
if train_prev is not None:
t_prev_pos = int(round(train_prev[pos_class] * 100))
@ -233,6 +243,10 @@ def plot_shift(
if legend:
ax.legend(loc="center left", bbox_to_anchor=(1, 0.5))
if return_fig:
return fig
output_path = env.PLOT_OUT_DIR / f"{title}.png"
fig.savefig(output_path, bbox_inches="tight")

17
remote.py
View File

@ -21,10 +21,12 @@ __to_sync_up = {
"dir": [
"quacc",
"baselines",
"qcpanel",
],
"file": [
"conf.yaml",
"run.py",
"pyproject.toml",
],
}
__to_sync_down = {
@ -60,10 +62,13 @@ def put_dir(sftp: paramiko.SFTPClient, from_: Path, to_: Path):
elif (from_ / f).is_dir():
_ex_list += put_dir(sftp, from_ / f, to_ / f)
to_list = sftp.listdir(str(to_))
for f in to_list:
if f not in from_list:
_ex_list += prune_remote(sftp, to_ / f)
try:
to_list = sftp.listdir(str(to_))
for f in to_list:
if f not in from_list:
_ex_list += prune_remote(sftp, to_ / f)
except FileNotFoundError:
pass
return _ex_list
@ -78,10 +83,10 @@ def get_dir(sftp: paramiko.SFTPClient, from_: Path, to_: Path):
mode = sftp.stat(str(from_ / f)).st_mode
if stat.S_ISDIR(mode):
_ex_list += get_dir(sftp, from_ / f, to_ / f)
_ex_list.append([sftp.rmdir, str(from_ / f)])
# _ex_list.append([sftp.rmdir, str(from_ / f)])
elif stat.S_ISREG(mode):
_ex_list.append([sftp.get, str(from_ / f), str(to_ / f)])
_ex_list.append([sftp.remove, str(from_ / f)])
# _ex_list.append([sftp.remove, str(from_ / f)])
return _ex_list