From 526cf80b66503c4df94f58771c032638de6a9b3d Mon Sep 17 00:00:00 2001 From: andrea Date: Thu, 22 Oct 2020 15:33:54 +0200 Subject: [PATCH] moved .sh files --- src/experiment_scripts/10run_jrc_combinations.sh | 14 +++++++------- src/experiment_scripts/10run_rcv_combinations.sh | 14 +++++++------- src/{ => experiment_scripts}/extract_features.sh | 0 .../main_deep_learning.py | 2 +- .../main_embeddings_cls.py | 1 - .../main_majorityvoting_cls.py | 5 +---- src/{ => experiment_scripts}/main_mbert.py | 0 .../main_mbert_extractor.py | 2 +- .../main_qualitative_analysis.py | 0 src/experiment_scripts/run_combinations_jrc.sh | 16 ++++++++-------- src/experiment_scripts/run_combinations_rcv.sh | 16 ++++++++-------- src/{ => experiment_scripts}/run_fun_bert_jrc.sh | 0 src/{ => experiment_scripts}/run_fun_bert_rcv.sh | 0 src/{ => experiment_scripts}/run_mbert_jrc.sh | 0 src/{ => experiment_scripts}/run_mbert_rcv.sh | 0 src/{main_multimodal_cls.py => main_gFun.py} | 0 src/util/common.py | 2 +- 17 files changed, 34 insertions(+), 38 deletions(-) rename src/{ => experiment_scripts}/extract_features.sh (100%) rename src/{ => experiment_scripts}/main_deep_learning.py (99%) rename src/{ => experiment_scripts}/main_embeddings_cls.py (99%) rename src/{ => experiment_scripts}/main_majorityvoting_cls.py (95%) rename src/{ => experiment_scripts}/main_mbert.py (100%) rename src/{ => experiment_scripts}/main_mbert_extractor.py (99%) rename src/{ => experiment_scripts}/main_qualitative_analysis.py (100%) rename src/{ => experiment_scripts}/run_fun_bert_jrc.sh (100%) rename src/{ => experiment_scripts}/run_fun_bert_rcv.sh (100%) rename src/{ => experiment_scripts}/run_mbert_jrc.sh (100%) rename src/{ => experiment_scripts}/run_mbert_rcv.sh (100%) rename src/{main_multimodal_cls.py => main_gFun.py} (100%) diff --git a/src/experiment_scripts/10run_jrc_combinations.sh b/src/experiment_scripts/10run_jrc_combinations.sh index 904e7e9..156a0a5 100644 --- a/src/experiment_scripts/10run_jrc_combinations.sh +++ b/src/experiment_scripts/10run_jrc_combinations.sh @@ -5,12 +5,12 @@ runs='6 7 8 9' #0 1 2 3 4 5 for run in $runs do dataset=$dataset_path$run.pickle - #python main_multimodal_cls.py $dataset -o $logfile -P -U -S -c -r -z --l2 --allprob # last combination for CIKM 3 Pr(views) concatenated (done up to run5) + #python main_gFun.py $dataset -o $logfile -P -U -S -c -r -z --l2 --allprob # last combination for CIKM 3 Pr(views) concatenated (done up to run5) python main_multimodal_cls.py $dataset -o $logfile -P -U -S -c -r -z --l2 --allprob # last combination for CIKM 3 views concatenated - #python main_multimodal_cls.py $dataset -o $logfile -P -U -S -c -r -a -z --l2 --allprob - #python main_multimodal_cls.py $dataset -o $logfile -P -U -c -r -a -z --l2 --allprob - #python main_multimodal_cls.py $dataset -o $logfile -P -S -c -r -z --l2 --allprob - #python main_multimodal_cls.py $dataset -o $logfile -P -U -c -r -z --l2 --allprob - #python main_multimodal_cls.py $dataset -o $logfile -c -P -U -r -z --l2 - #python main_multimodal_cls.py $dataset -o $logfile -c -P -U -S -r -z --l2 + #python main_gFun.py $dataset -o $logfile -P -U -S -c -r -a -z --l2 --allprob + #python main_gFun.py $dataset -o $logfile -P -U -c -r -a -z --l2 --allprob + #python main_gFun.py $dataset -o $logfile -P -S -c -r -z --l2 --allprob + #python main_gFun.py $dataset -o $logfile -P -U -c -r -z --l2 --allprob + #python main_gFun.py $dataset -o $logfile -c -P -U -r -z --l2 + #python main_gFun.py $dataset -o $logfile -c -P -U -S -r -z --l2 done \ No newline at end of file diff --git a/src/experiment_scripts/10run_rcv_combinations.sh b/src/experiment_scripts/10run_rcv_combinations.sh index e993327..b5d8a3b 100644 --- a/src/experiment_scripts/10run_rcv_combinations.sh +++ b/src/experiment_scripts/10run_rcv_combinations.sh @@ -5,12 +5,12 @@ runs='0 1 2 3 4 5 6 7 8 9' for run in $runs do dataset=$dataset_path$run.pickle - #python main_multimodal_cls.py $dataset -o $logfile -P -U -S -c -r -z --l2 --allprob # last combination for CIKM 3 Pr(views) concatenated + #python main_gFun.py $dataset -o $logfile -P -U -S -c -r -z --l2 --allprob # last combination for CIKM 3 Pr(views) concatenated python main_multimodal_cls.py $dataset -o $logfile -P -U -S -c -r -z --l2 --allprob # last combination for CIKM 3 views concatenated - #python main_multimodal_cls.py $dataset -o $logfile -P -U -c -r -a -z --l2 --allprob - #python main_multimodal_cls.py $dataset -o $logfile -P -U -S -c -r -a -z --l2 --allprob - #python main_multimodal_cls.py $dataset -o $logfile -P -S -c -r -z --l2 --allprob - #python main_multimodal_cls.py $dataset -o $logfile -P -U -c -r -z --l2 --allprob - #python main_multimodal_cls.py $dataset -o $logfile -c -P -U -r -z --l2 - #python main_multimodal_cls.py $dataset -o $logfile -c -P -U -S -r -z --l2 + #python main_gFun.py $dataset -o $logfile -P -U -c -r -a -z --l2 --allprob + #python main_gFun.py $dataset -o $logfile -P -U -S -c -r -a -z --l2 --allprob + #python main_gFun.py $dataset -o $logfile -P -S -c -r -z --l2 --allprob + #python main_gFun.py $dataset -o $logfile -P -U -c -r -z --l2 --allprob + #python main_gFun.py $dataset -o $logfile -c -P -U -r -z --l2 + #python main_gFun.py $dataset -o $logfile -c -P -U -S -r -z --l2 done \ No newline at end of file diff --git a/src/extract_features.sh b/src/experiment_scripts/extract_features.sh similarity index 100% rename from src/extract_features.sh rename to src/experiment_scripts/extract_features.sh diff --git a/src/main_deep_learning.py b/src/experiment_scripts/main_deep_learning.py similarity index 99% rename from src/main_deep_learning.py rename to src/experiment_scripts/main_deep_learning.py index a911b14..ee56054 100755 --- a/src/main_deep_learning.py +++ b/src/experiment_scripts/main_deep_learning.py @@ -1,6 +1,6 @@ import argparse import torch.nn as nn -from torch.optim.lr_scheduler import StepLR, MultiStepLR +from torch.optim.lr_scheduler import StepLR from dataset_builder import MultilingualDataset from learning.transformers import load_muse_embeddings from models.lstm_class import RNNMultilingualClassifier diff --git a/src/main_embeddings_cls.py b/src/experiment_scripts/main_embeddings_cls.py similarity index 99% rename from src/main_embeddings_cls.py rename to src/experiment_scripts/main_embeddings_cls.py index 51cafc8..08552d3 100644 --- a/src/main_embeddings_cls.py +++ b/src/experiment_scripts/main_embeddings_cls.py @@ -1,6 +1,5 @@ import os from dataset_builder import MultilingualDataset -from learning.learners import * from util.evaluation import * from optparse import OptionParser from util.file import exists diff --git a/src/main_majorityvoting_cls.py b/src/experiment_scripts/main_majorityvoting_cls.py similarity index 95% rename from src/main_majorityvoting_cls.py rename to src/experiment_scripts/main_majorityvoting_cls.py index 0ae4f1e..ee5efe5 100644 --- a/src/main_majorityvoting_cls.py +++ b/src/experiment_scripts/main_majorityvoting_cls.py @@ -2,15 +2,12 @@ import os from dataset_builder import MultilingualDataset # from learning.learners import * # from learning.learners import FunnellingMultimodal -from learning.transformers import Funnelling, PosteriorProbabilitiesEmbedder, MetaClassifier, \ - TfidfVectorizerMultilingual, DocEmbedderList, WordClassEmbedder, MuseEmbedder, FeatureSet2Posteriors, Voting +from learning.transformers import PosteriorProbabilitiesEmbedder, TfidfVectorizerMultilingual, WordClassEmbedder, MuseEmbedder, FeatureSet2Posteriors, Voting from util.evaluation import * from optparse import OptionParser from util.file import exists from util.results import PolylingualClassificationResults from sklearn.svm import SVC -from util.util import get_learner, get_params -from sklearn.linear_model import LogisticRegression, LogisticRegressionCV parser = OptionParser() diff --git a/src/main_mbert.py b/src/experiment_scripts/main_mbert.py similarity index 100% rename from src/main_mbert.py rename to src/experiment_scripts/main_mbert.py diff --git a/src/main_mbert_extractor.py b/src/experiment_scripts/main_mbert_extractor.py similarity index 99% rename from src/main_mbert_extractor.py rename to src/experiment_scripts/main_mbert_extractor.py index f294fc5..16f09d3 100644 --- a/src/main_mbert_extractor.py +++ b/src/experiment_scripts/main_mbert_extractor.py @@ -1,4 +1,4 @@ -from main_mbert import * +from experiment_scripts.main_mbert import * import pickle diff --git a/src/main_qualitative_analysis.py b/src/experiment_scripts/main_qualitative_analysis.py similarity index 100% rename from src/main_qualitative_analysis.py rename to src/experiment_scripts/main_qualitative_analysis.py diff --git a/src/experiment_scripts/run_combinations_jrc.sh b/src/experiment_scripts/run_combinations_jrc.sh index fa7f0d1..a4aabde 100644 --- a/src/experiment_scripts/run_combinations_jrc.sh +++ b/src/experiment_scripts/run_combinations_jrc.sh @@ -7,10 +7,10 @@ logfile=./results/final_combinations_jrc.csv # (no one seems to improve over standard funnelling [the improved version after A.1] with posteriors probabilities...) # aggregation=concatenation -#python main_multimodal_cls.py $dataset -o $logfile -P -U -r -z --l2 -#python main_multimodal_cls.py $dataset -o $logfile -P -S -r -z --l2 -#python main_multimodal_cls.py $dataset -o $logfile -U -S -r -z --l2 -#python main_multimodal_cls.py $dataset -o $logfile -P -U -S -r -z --l2 +#python main_gFun.py $dataset -o $logfile -P -U -r -z --l2 +#python main_gFun.py $dataset -o $logfile -P -S -r -z --l2 +#python main_gFun.py $dataset -o $logfile -U -S -r -z --l2 +#python main_gFun.py $dataset -o $logfile -P -U -S -r -z --l2 # ##FeatureSetToPosteriors (aggregation mean) @@ -20,10 +20,10 @@ python main_multimodal_cls.py $dataset -o $logfile -U -S -r -a -z --l2 --allprob python main_multimodal_cls.py $dataset -o $logfile -P -U -S -r -a -z --l2 --allprob ##FeatureSetToPosteriors -#python main_multimodal_cls.py $dataset -o $logfile -P -U -r -z --l2 --allprob -#python main_multimodal_cls.py $dataset -o $logfile -P -S -r -z --l2 --allprob -#python main_multimodal_cls.py $dataset -o $logfile -U -S -r -z --l2 --allprob -#python main_multimodal_cls.py $dataset -o $logfile -P -U -S -r -z --l2 --allprob +#python main_gFun.py $dataset -o $logfile -P -U -r -z --l2 --allprob +#python main_gFun.py $dataset -o $logfile -P -S -r -z --l2 --allprob +#python main_gFun.py $dataset -o $logfile -U -S -r -z --l2 --allprob +#python main_gFun.py $dataset -o $logfile -P -U -S -r -z --l2 --allprob #MajorityVoting #python main_majorityvoting_cls.py $dataset -o $logfile -P -U -r diff --git a/src/experiment_scripts/run_combinations_rcv.sh b/src/experiment_scripts/run_combinations_rcv.sh index 1d48f9c..4e1acfb 100644 --- a/src/experiment_scripts/run_combinations_rcv.sh +++ b/src/experiment_scripts/run_combinations_rcv.sh @@ -7,10 +7,10 @@ logfile=./results/final_combinations_rcv.csv # (no one seems to improve over standard funnelling [the improved version after A.1] with posteriors probabilities...) # aggregation=concatenation -#python main_multimodal_cls.py $dataset -o $logfile -P -U -r -z --l2 -#python main_multimodal_cls.py $dataset -o $logfile -P -S -r -z --l2 -#python main_multimodal_cls.py $dataset -o $logfile -U -S -r -z --l2 -#python main_multimodal_cls.py $dataset -o $logfile -P -U -S -r -z --l2 +#python main_gFun.py $dataset -o $logfile -P -U -r -z --l2 +#python main_gFun.py $dataset -o $logfile -P -S -r -z --l2 +#python main_gFun.py $dataset -o $logfile -U -S -r -z --l2 +#python main_gFun.py $dataset -o $logfile -P -U -S -r -z --l2 # ##FeatureSetToPosteriors (aggregation mean) python main_multimodal_cls.py $dataset -o $logfile -P -U -r -a -z --l2 --allprob @@ -19,10 +19,10 @@ python main_multimodal_cls.py $dataset -o $logfile -U -S -r -a -z --l2 --allprob python main_multimodal_cls.py $dataset -o $logfile -P -U -S -r -a -z --l2 --allprob ##FeatureSetToPosteriors -#python main_multimodal_cls.py $dataset -o $logfile -P -U -r -z --l2 --allprob -#python main_multimodal_cls.py $dataset -o $logfile -P -S -r -z --l2 --allprob -#python main_multimodal_cls.py $dataset -o $logfile -U -S -r -z --l2 --allprob -#python main_multimodal_cls.py $dataset -o $logfile -P -U -S -r -z --l2 --allprob +#python main_gFun.py $dataset -o $logfile -P -U -r -z --l2 --allprob +#python main_gFun.py $dataset -o $logfile -P -S -r -z --l2 --allprob +#python main_gFun.py $dataset -o $logfile -U -S -r -z --l2 --allprob +#python main_gFun.py $dataset -o $logfile -P -U -S -r -z --l2 --allprob #MajorityVoting #python main_majorityvoting_cls.py $dataset -o $logfile -P -U -r diff --git a/src/run_fun_bert_jrc.sh b/src/experiment_scripts/run_fun_bert_jrc.sh similarity index 100% rename from src/run_fun_bert_jrc.sh rename to src/experiment_scripts/run_fun_bert_jrc.sh diff --git a/src/run_fun_bert_rcv.sh b/src/experiment_scripts/run_fun_bert_rcv.sh similarity index 100% rename from src/run_fun_bert_rcv.sh rename to src/experiment_scripts/run_fun_bert_rcv.sh diff --git a/src/run_mbert_jrc.sh b/src/experiment_scripts/run_mbert_jrc.sh similarity index 100% rename from src/run_mbert_jrc.sh rename to src/experiment_scripts/run_mbert_jrc.sh diff --git a/src/run_mbert_rcv.sh b/src/experiment_scripts/run_mbert_rcv.sh similarity index 100% rename from src/run_mbert_rcv.sh rename to src/experiment_scripts/run_mbert_rcv.sh diff --git a/src/main_multimodal_cls.py b/src/main_gFun.py similarity index 100% rename from src/main_multimodal_cls.py rename to src/main_gFun.py diff --git a/src/util/common.py b/src/util/common.py index b6b93dc..88134d3 100755 --- a/src/util/common.py +++ b/src/util/common.py @@ -235,7 +235,7 @@ class MultilingualIndex: def bert_embeddings(self, bert_path, max_len=512, batch_size=64, stored_embeddings=False): show_gpu('GPU memory before initializing mBert model:') # TODO: load dumped embeddings? - from main_mbert_extractor import do_tokenization, ExtractorDataset, DataLoader + from experiment_scripts.main_mbert_extractor import do_tokenization, ExtractorDataset, DataLoader from transformers import BertConfig, BertForSequenceClassification print('[mBERT] generating mBERT doc embeddings')