diff --git a/annotation.py b/annotation/annotation.py similarity index 85% rename from annotation.py rename to annotation/annotation.py index 5e9cf27..598b3d5 100644 --- a/annotation.py +++ b/annotation/annotation.py @@ -1,316 +1,337 @@ -# --- Import librerie --- -import pandas as pd -from openai import AzureOpenAI -import pickle -from sentence_transformers import SentenceTransformer -import numpy as np -import faiss -import openpyxl -import re -import json -from openpyxl.styles import PatternFill -from openpyxl import load_workbook -from collections import Counter -from prompts.prompt import build_prompt_local -import warnings -import logging -import unicodedata - -# --- Configurazione --- -endpoint = "https://gpt-sw-central-tap-security.openai.azure.com/" -deployment = "gpt-5.1-chat-3" -subscription_key = "8zufUIPs0Dijh0M6NpifkkDvxJHZMFtott7u8V8ySTYNcpYVoRbsJQQJ99BBACfhMk5XJ3w3AAABACOGr6sq" - -client = AzureOpenAI( - azure_endpoint=endpoint, - api_key=subscription_key, - api_version="2025-04-01-preview", -) - -# ----- Step 1: caricare datasets ----- -#df_labeled = pd.read_excel("main/datasets/annotated_dataset.xlsx").dropna(how="all") -df_labeled = pd.read_excel("main/datasets/annotated_dataset_updated.xlsx").dropna(how="all") -df_unlabeled = pd.read_excel("main/datasets/unlabeled_dataset.xlsx").dropna(how="all") -print("***STEP 1***\nDataset etichettato caricato. Numero righe:", len(df_labeled), "\nDataset non etichettato caricato. Numero righe:", len(df_unlabeled)) -df_labeled = df_labeled.rename(columns={"automation_id": "id"}) -df_unlabeled = df_unlabeled.rename(columns={"automation_id": "id"}) - -# Pulizia colonne -def clean_id(x): - if pd.isna(x): - return "" - s = str(x).strip() # rimuove spazi - s = s.strip('"').strip("'") # rimuove eventuali virgolette - return s.lower() - -def clean_folder(x): - """Pulizia folder: rimuove spazi multipli, normalizza unicode.""" - if pd.isna(x): - return "" - s = str(x).strip().lower() - s = unicodedata.normalize("NFKC", s) - s = re.sub(r'\s+', ' ', s) - return s - -for df in [df_labeled, df_unlabeled]: - df["id"] = df["id"].apply(clean_id) - df["folder"] = df["folder"].apply(clean_folder) - -labeled_pairs = set(zip(df_labeled["id"], df_labeled["folder"])) - -# crea maschera: True = la riga NON è presente in labeled -mask_unlabeled = ~df_unlabeled.apply(lambda r: (r["id"], r["folder"]) in labeled_pairs, axis=1) -# filtra -df_unlabeled_filtered = df_unlabeled[mask_unlabeled].copy() - -print("Numero righe df_unlabeled dopo aver rimosso quelle già etichettate:", len(df_unlabeled_filtered)) - -unlabeled_pairs = set(zip(df_unlabeled["id"], df_unlabeled["folder"])) -missing_in_unlabeled = labeled_pairs - unlabeled_pairs -print("Numero coppie etichettate non presenti in unlabeled:", len(missing_in_unlabeled)) -if missing_in_unlabeled: - print("Coppie mancanti:") - for p in list(missing_in_unlabeled)[:50]: # stampa solo le prime 50 per comodità - print(p) - -# ----- Step 2: embeddings ----- -# Silenzia warning generici -warnings.filterwarnings("ignore") -# Silenzia logging di transformers / sentence-transformers / HF hub -logging.getLogger("sentence_transformers").setLevel(logging.ERROR) -logging.getLogger("transformers").setLevel(logging.ERROR) -logging.getLogger("huggingface_hub").setLevel(logging.ERROR) - -print("\n***Step 2***\nEmbeddings") -model = SentenceTransformer("all-MiniLM-L6-v2") - -#with open("main/labeled_embeddings_71.pkl", "rb") as f: -with open("main/labeled_embeddings.pkl", "rb") as f: - data = pickle.load(f) - -embeddings = data["embeddings"].astype("float32") -print("Shape embeddings:", embeddings.shape) - - - -# ----- Step3: Creazione indice FAISS --- -faiss.normalize_L2(embeddings) -dimension = embeddings.shape[1] -index = faiss.IndexFlatIP(dimension) -index.add(embeddings) -print(f"\n***Step 3: Indice FAISS creato***.\nNumero di vettori nell'indice: {index.ntotal}") - - -# ----- Step 4: Retrieval (similarità cosine) ----- -k = 5 -output_rows = [] -df_sample = df_unlabeled_filtered.head(10).reset_index(drop=True) -llm_rows = [] - -def sim_label(sim: float) -> str: - # più alto = più simile - if sim >= 0.80: - return "Match forte" - elif sim >= 0.60: - return "Match plausibile" - elif sim >= 0.50: - return "Similarità instabile" - else: - return "Debole" - -for count, (_, row) in enumerate(df_sample.iterrows(), start=1): - query_text = str(row["human_like"]) - print("automazione analizzata:", count) - - # Calcolo embedding della nuova automazione - query_emb = model.encode([query_text], convert_to_numpy=True).astype("float32") - faiss.normalize_L2(query_emb) - - # Recupera indici dei k vicini più prossimi - sims, indices = index.search(query_emb, k) - - # Metriche globali sui top-k (una volta per automazione) - topk_cats = [] - top1_sim = float(sims[0][0]) - top1_similarity_label = sim_label(top1_sim) - - for rank in range(k): - idx = int(indices[0][rank]) - sim = float(sims[0][rank]) - - retrieved_row = df_labeled.iloc[idx] - topk_cats.append(str(retrieved_row.get("category", ""))) - - rank1_category = topk_cats[0] if topk_cats else "" - majority_category = Counter(topk_cats).most_common(1)[0][0] if topk_cats else "" - consistency = (sum(c == majority_category for c in topk_cats) / len(topk_cats)) if topk_cats else 0.0 - - for rank in range(k): - idx = int(indices[0][rank]) - sim = float(sims[0][rank]) - label = sim_label(sim) - - retrieved_row = df_labeled.iloc[idx] - - output_rows.append({ - "automazione da etichettare": query_text, - # info retrieval per questa riga - "rank": rank + 1, - "retrieved_idx": idx, - "automazione simile": retrieved_row.get("automation", ""), - "categoria automazione simile": retrieved_row.get("category", ""), - "similarita_cosine": sim, - "similarity_label": label, - # metriche aggregate top-k (ripetute su ogni riga) - "rank1_similarity": top1_sim, - "rank1_similarity_label": top1_similarity_label, - "rank1_category": rank1_category, - "majority_category": majority_category, - "consistency": round(consistency, 3), - "top5_categories": " | ".join(topk_cats), - }) - - - # ----- Step 5: invio dati al LLM ----- - # (1) Costruzione prompt - retrieved = df_labeled.iloc[indices[0]].copy() - retrieved["similarity"] = sims[0].astype(float) - retrieved["similarity_label"] = retrieved["similarity"].apply(sim_label) - prompt = build_prompt_local(query_text, retrieved, sim_label) - - # (2) Chiamata al modello: restituisce JSON - resp = client.chat.completions.create( - model=deployment, - messages=[ - {"role": "system", "content": prompt}, - {"role": "user", "content": f'automation to evaluate: {query_text}'} - ], - reasoning_effort= "low" - ) - content = resp.choices[0].message.content.strip() - - # (3) Parsing della risposta - try: - parsed = json.loads(content) - except Exception: - parsed = { - "automation": query_text, - "category": "", - "subcategory": "", - "problem_type": "", - "gravity": "", - "scores": {}, - "needs_human_review": True, - "short_rationale": f"JSON_PARSE_ERROR: {content[:200]}", - } - - # (4) Salvataggio di 1 riga per automazione con: - # - metriche retrieval (rank1/majority/consistency) - # - output dell'LLM (scores + label finale + review flag) - llm_category = str(parsed.get("category", "")).strip() - llm_subcategory = str(parsed.get("subcategory", "")).strip() - llm_problem_type = str(parsed.get("problem_type", "")).strip() - llm_gravity = str(parsed.get("gravity", "")).strip() - if llm_category.upper() == "HARMLESS": - llm_subcategory = "" - llm_problem_type = "none" - llm_gravity = "NONE" - # di default l'etichetta assegnata è quella del LLM - rivista se review=true - final_category = llm_category - final_subcategory = llm_subcategory - final_problem_type = llm_problem_type - final_gravity = llm_gravity - - - if top1_similarity_label == "Debole" or top1_similarity_label == "Similarità instabile": - needs_review = True - else: - needs_review = False - - - final_needs_review = needs_review - # ================= HUMAN REVIEW LOGIC ================= - aligned_strong = ( - llm_category == majority_category - and llm_category == rank1_category - and llm_category != "" - ) - - OVERRIDE_MIN_SIMILARITY = 0.39 - OVERRIDE_MIN_CONSISTENCY = 0.60 - - good_retrieval = ( - top1_sim >= OVERRIDE_MIN_SIMILARITY - and consistency >= OVERRIDE_MIN_CONSISTENCY - ) - - if aligned_strong and good_retrieval: - final_needs_review = False - # ===================================================== - - - llm_rows.append({ - "id": row.get("id", ""), - "folder": row.get("folder", ""), - "automation_text": query_text, - - # Retrieval metrics - "rank1_similarity": top1_sim, - "rank1_similarity_label": top1_similarity_label, - "rank1_category": rank1_category, - "majority_category": majority_category, - "consistency": round(consistency, 3), - "top5_categories": " | ".join(topk_cats), - - # LLM - "llm_category": llm_category, - "llm_subcategory": llm_subcategory, - "llm_problem_type": llm_problem_type, - "llm_gravity": llm_gravity, - - "needs_review": needs_review, - "final_needs_review": final_needs_review, - - # FINAL - "final_category": final_category, - "final_subcategory": final_subcategory, - "final_problem_type": final_problem_type, - "final_gravity": final_gravity, - - "llm_rationale": parsed.get("short_rationale", ""), - }) - - -# ----- Step 6: output Excel ----- -df_out = pd.DataFrame(llm_rows) -out_path = "main/datasets/labeling_2_500.xlsx" -df_out.to_excel(out_path, index=False) - -wb = load_workbook(out_path) -ws = wb.active - -true_fill = PatternFill(start_color="FF6347", end_color="FF6347", fill_type="solid") # rosso -false_fill = PatternFill(start_color="90EE90", end_color="90EE90", fill_type="solid") # verde - -col_index = {cell.value: idx for idx, cell in enumerate(ws[1], start=1)} - -for col_name in ["needs_review", "final_needs_review"]: - if col_name in col_index: - c = col_index[col_name] - for r in range(2, ws.max_row + 1): - val = ws.cell(row=r, column=c).value - if val is True: - ws.cell(row=r, column=c).fill = true_fill - elif val is False: - ws.cell(row=r, column=c).fill = false_fill - -wb.save(out_path) -print(f"\n***Step 6: Excel salvato in {out_path}") - -# --- Conteggio needs_human_review --- -review_counts = df_out["final_needs_review"].value_counts(dropna=False) -true_count = review_counts.get(True, 0) -false_count = review_counts.get(False, 0) -print("\n--- Needs human review summary ---") -print(f"needs_human_review = True : {true_count}") +# --- Import librerie --- +import pandas as pd +from openai import AzureOpenAI +import pickle +from sentence_transformers import SentenceTransformer +import numpy as np +import faiss +import openpyxl +import re +import json +from openpyxl.styles import PatternFill +from openpyxl import load_workbook +from collections import Counter +from prompts.prompt import build_prompt_local +import warnings +import logging +import unicodedata + +# --- Configurazione --- +endpoint = "https://gpt-sw-central-tap-security.openai.azure.com/" +deployment = "gpt-5.1-chat-3" +subscription_key = "8zufUIPs0Dijh0M6NpifkkDvxJHZMFtott7u8V8ySTYNcpYVoRbsJQQJ99BBACfhMk5XJ3w3AAABACOGr6sq" + +client = AzureOpenAI( + azure_endpoint=endpoint, + api_key=subscription_key, + api_version="2025-04-01-preview", +) + +# ----- Step 1: caricare datasets ----- +df_labeled = pd.read_excel("main/datasets/annotated_dataset_updated2.xlsx").dropna(how="all") # DATASET GIA' ETICHETTATO +df_unlabeled = pd.read_excel("main/datasets/unlabeled_dataset.xlsx").dropna(how="all") # DATASET DA ETICHETTARE +print("***STEP 1***\nDataset etichettato caricato. Numero righe:", len(df_labeled), "\nDataset non etichettato caricato. Numero righe:", len(df_unlabeled)) +df_labeled = df_labeled.rename(columns={"automation_id": "id"}) +df_unlabeled = df_unlabeled.rename(columns={"automation_id": "id"}) + +# Pulizia colonne +def clean_id(x): + if pd.isna(x): + return "" + s = str(x).strip() # rimuove spazi + s = s.strip('"').strip("'") # rimuove eventuali virgolette + return s.lower() + +def clean_folder(x): + """Pulizia folder: rimuove spazi multipli, normalizza unicode.""" + if pd.isna(x): + return "" + s = str(x).strip().lower() + s = s.strip('"').strip("'") + s = unicodedata.normalize("NFKC", s) + s = re.sub(r'\s+', ' ', s) + return s + +for df in [df_labeled, df_unlabeled]: + df["id"] = df["id"].apply(clean_id) + df["folder"] = df["folder"].apply(clean_folder) + +labeled_pairs = set(zip(df_labeled["id"], df_labeled["folder"])) + +# crea maschera: True = la riga NON è presente in labeled +mask_unlabeled = ~df_unlabeled.apply(lambda r: (r["id"], r["folder"]) in labeled_pairs, axis=1) +# filtra +df_unlabeled_filtered = df_unlabeled[mask_unlabeled].copy() + +print("Numero righe df_unlabeled dopo aver rimosso quelle già etichettate:", len(df_unlabeled_filtered)) + +unlabeled_pairs = set(zip(df_unlabeled["id"], df_unlabeled["folder"])) +missing_in_unlabeled = labeled_pairs - unlabeled_pairs +print("Numero coppie etichettate non presenti in unlabeled:", len(missing_in_unlabeled)) +if missing_in_unlabeled: + print("Coppie mancanti:") + for p in list(missing_in_unlabeled)[:50]: # stampa solo le prime 50 per comodità + print(p) + +# ----- Step 2: embeddings ----- +# Silenzia warning generici +warnings.filterwarnings("ignore") +# Silenzia logging di transformers / sentence-transformers / HF hub +logging.getLogger("sentence_transformers").setLevel(logging.ERROR) +logging.getLogger("transformers").setLevel(logging.ERROR) +logging.getLogger("huggingface_hub").setLevel(logging.ERROR) + +print("\n***Step 2***\nEmbeddings") +model = SentenceTransformer("all-MiniLM-L6-v2") + +with open("main/labeled_embeddings.pkl", "rb") as f: + data = pickle.load(f) + +embeddings = data["embeddings"].astype("float32") +print("Shape embeddings:", embeddings.shape) + + + +# ----- Step3: Creazione indice FAISS --- +faiss.normalize_L2(embeddings) +dimension = embeddings.shape[1] +index = faiss.IndexFlatIP(dimension) +index.add(embeddings) +print(f"\n***Step 3: Indice FAISS creato***.\nNumero di vettori nell'indice: {index.ntotal}") + + +# ----- Step 4: Retrieval (similarità cosine) ----- +k = 5 +output_rows = [] +df_sample = df_unlabeled_filtered.head(1000).reset_index(drop=True) +llm_rows = [] + +def sim_label(sim: float) -> str: + # più alto = più simile + if sim >= 0.80: + return "Match forte" + elif sim >= 0.60: + return "Match plausibile" + elif sim >= 0.50: + return "Similarità instabile" + else: + return "Debole" + +for count, (_, row) in enumerate(df_sample.iterrows(), start=1): + query_text = str(row["human_like"]) + print("automazione analizzata:", count) + + # Calcolo embedding della nuova automazione + query_emb = model.encode([query_text], convert_to_numpy=True).astype("float32") + faiss.normalize_L2(query_emb) + + # Recupera indici dei k vicini più prossimi + sims, indices = index.search(query_emb, k) + + # Metriche globali sui top-k (una volta per automazione) + topk_cats = [] + top1_sim = float(sims[0][0]) + top1_similarity_label = sim_label(top1_sim) + + for rank in range(k): + idx = int(indices[0][rank]) + sim = float(sims[0][rank]) + + retrieved_row = df_labeled.iloc[idx] + topk_cats.append(str(retrieved_row.get("category", ""))) + + rank1_category = topk_cats[0] if topk_cats else "" + majority_category = Counter(topk_cats).most_common(1)[0][0] if topk_cats else "" + consistency = (sum(c == majority_category for c in topk_cats) / len(topk_cats)) if topk_cats else 0.0 + + for rank in range(k): + idx = int(indices[0][rank]) + sim = float(sims[0][rank]) + label = sim_label(sim) + + retrieved_row = df_labeled.iloc[idx] + + output_rows.append({ + "automazione da etichettare": query_text, + # info retrieval per questa riga + "rank": rank + 1, + "retrieved_idx": idx, + "automazione simile": retrieved_row.get("automation", ""), + "categoria automazione simile": retrieved_row.get("category", ""), + "similarita_cosine": sim, + "similarity_label": label, + # metriche aggregate top-k (ripetute su ogni riga) + "rank1_similarity": top1_sim, + "rank1_similarity_label": top1_similarity_label, + "rank1_category": rank1_category, + "majority_category": majority_category, + "consistency": round(consistency, 3), + "top5_categories": " | ".join(topk_cats), + }) + + + # ----- Step 5: invio dati al LLM ----- + # (1) Costruzione prompt + retrieved = df_labeled.iloc[indices[0]].copy() + retrieved["similarity"] = sims[0].astype(float) + retrieved["similarity_label"] = retrieved["similarity"].apply(sim_label) + prompt = build_prompt_local(query_text, retrieved, sim_label) + + # (2) Chiamata al modello: restituisce JSON + resp = client.chat.completions.create( + model=deployment, + messages=[ + {"role": "system", "content": prompt}, + {"role": "user", "content": f'automation to evaluate: {query_text}'} + ], + reasoning_effort= "low" + ) + content = resp.choices[0].message.content.strip() + + # (3) Parsing della risposta + def extract_json(content: str) -> dict: + """ + Estrae il primo JSON valido trovato in una stringa. + Restituisce un dizionario vuoto in caso di fallimento. + """ + # Rimuove eventuali blocchi ```json o ``` generici + content = re.sub(r"```(?:json)?\s*", "", content) + content = re.sub(r"\s*```", "", content) + + # Trova la prima occorrenza che assomiglia a JSON + match = re.search(r"\{.*\}", content, re.DOTALL) + if match: + json_text = match.group(0) + try: + return json.loads(json_text) + except json.JSONDecodeError: + return { + "automation": "", + "category": "", + "subcategory": "", + "problem_type": "", + "gravity": "", + "scores": {}, + "short_rationale": f"JSON_PARSE_ERROR: {json_text[:200]}" + } + else: + return { + "automation": "", + "category": "", + "subcategory": "", + "problem_type": "", + "gravity": "", + "scores": {}, + "short_rationale": f"JSON_PARSE_ERROR: No JSON found in content" + } + + parsed = extract_json(content) + + # (4) Salvataggio di 1 riga per automazione con: + # - metriche retrieval (rank1/majority/consistency) + # - output dell'LLM (scores + label finale + review flag) + llm_category = str(parsed.get("category", "")).strip() + llm_subcategory = str(parsed.get("subcategory", "")).strip() + llm_problem_type = str(parsed.get("problem_type", "")).strip() + llm_gravity = str(parsed.get("gravity", "")).strip() + if llm_category.upper() == "HARMLESS": + llm_problem_type = "none" + llm_gravity = "NONE" + # di default l'etichetta assegnata è quella del LLM - rivista se review=true + final_category = llm_category + final_subcategory = llm_subcategory + final_problem_type = llm_problem_type + final_gravity = llm_gravity + + + if top1_similarity_label == "Debole" or top1_similarity_label == "Similarità instabile": + needs_review = True + else: + needs_review = False + + final_needs_review = needs_review + # ================= HUMAN REVIEW LOGIC ================= + aligned_strong = ( + llm_category == majority_category + and llm_category == rank1_category + and llm_category != "" + ) + + OVERRIDE_MIN_SIMILARITY = 0.39 + OVERRIDE_MIN_CONSISTENCY = 0.60 + + good_retrieval = ( + top1_sim >= OVERRIDE_MIN_SIMILARITY + and consistency >= OVERRIDE_MIN_CONSISTENCY + ) + + if aligned_strong and good_retrieval: + final_needs_review = False + # ===================================================== + + + llm_rows.append({ + "id": row.get("id", ""), + "folder": row.get("folder", ""), + "automation_text": query_text, + + # Retrieval metrics + "rank1_similarity": top1_sim, + "rank1_similarity_label": top1_similarity_label, + "rank1_category": rank1_category, + "majority_category": majority_category, + "consistency": round(consistency, 3), + "top5_categories": " | ".join(topk_cats), + + # LLM + "llm_category": llm_category, + "llm_subcategory": llm_subcategory, + "llm_problem_type": llm_problem_type, + "llm_gravity": llm_gravity, + + "needs_review": needs_review, + "final_needs_review": final_needs_review, + + # FINAL + "final_category": final_category, + "final_subcategory": final_subcategory, + "final_problem_type": final_problem_type, + "final_gravity": final_gravity, + + "llm_rationale": parsed.get("short_rationale", ""), + }) + + +# ----- Step 6: output Excel ----- +df_out = pd.DataFrame(llm_rows) +out_path = "main/datasets/12500_13463.xlsx" +df_out.to_excel(out_path, index=False) + +wb = load_workbook(out_path) +ws = wb.active + +true_fill = PatternFill(start_color="FF6347", end_color="FF6347", fill_type="solid") # rosso +false_fill = PatternFill(start_color="90EE90", end_color="90EE90", fill_type="solid") # verde + +col_index = {cell.value: idx for idx, cell in enumerate(ws[1], start=1)} + +for col_name in ["needs_review", "final_needs_review"]: + if col_name in col_index: + c = col_index[col_name] + for r in range(2, ws.max_row + 1): + val = ws.cell(row=r, column=c).value + if val is True: + ws.cell(row=r, column=c).fill = true_fill + elif val is False: + ws.cell(row=r, column=c).fill = false_fill + +wb.save(out_path) +print(f"\n***Step 6: Excel salvato in {out_path}") + +# --- Conteggio needs_human_review --- +review_counts = df_out["final_needs_review"].value_counts(dropna=False) +true_count = review_counts.get(True, 0) +false_count = review_counts.get(False, 0) +print("\n--- Needs human review summary ---") +print(f"needs_human_review = True : {true_count}") print(f"needs_human_review = False: {false_count}") \ No newline at end of file diff --git a/annotation/datasets/annotated_dataset.xlsx b/annotation/datasets/annotated_dataset.xlsx new file mode 100644 index 0000000..496978a Binary files /dev/null and b/annotation/datasets/annotated_dataset.xlsx differ diff --git a/annotation/datasets/annotated_dataset_final.xlsx b/annotation/datasets/annotated_dataset_final.xlsx new file mode 100644 index 0000000..fc053e1 Binary files /dev/null and b/annotation/datasets/annotated_dataset_final.xlsx differ diff --git a/annotation/datasets/annotated_dataset_updated.xlsx b/annotation/datasets/annotated_dataset_updated.xlsx new file mode 100644 index 0000000..0acc7af Binary files /dev/null and b/annotation/datasets/annotated_dataset_updated.xlsx differ diff --git a/annotation/datasets/final_dataset.xlsx b/annotation/datasets/final_dataset.xlsx new file mode 100644 index 0000000..914d74d Binary files /dev/null and b/annotation/datasets/final_dataset.xlsx differ diff --git a/annotation/datasets/slices/10500_11500.xlsx b/annotation/datasets/slices/10500_11500.xlsx new file mode 100644 index 0000000..68c931c Binary files /dev/null and b/annotation/datasets/slices/10500_11500.xlsx differ diff --git a/annotation/datasets/slices/11500_12500.xlsx b/annotation/datasets/slices/11500_12500.xlsx new file mode 100644 index 0000000..84d1f8d Binary files /dev/null and b/annotation/datasets/slices/11500_12500.xlsx differ diff --git a/annotation/datasets/slices/12500_13463.xlsx b/annotation/datasets/slices/12500_13463.xlsx new file mode 100644 index 0000000..8cc7228 Binary files /dev/null and b/annotation/datasets/slices/12500_13463.xlsx differ diff --git a/annotation/datasets/slices/2000_2500.xlsx b/annotation/datasets/slices/2000_2500.xlsx new file mode 100644 index 0000000..feb6f0c Binary files /dev/null and b/annotation/datasets/slices/2000_2500.xlsx differ diff --git a/annotation/datasets/slices/2500_3500.xlsx b/annotation/datasets/slices/2500_3500.xlsx new file mode 100644 index 0000000..89c1fd8 Binary files /dev/null and b/annotation/datasets/slices/2500_3500.xlsx differ diff --git a/annotation/datasets/slices/3500_4500.xlsx b/annotation/datasets/slices/3500_4500.xlsx new file mode 100644 index 0000000..36131c9 Binary files /dev/null and b/annotation/datasets/slices/3500_4500.xlsx differ diff --git a/annotation/datasets/slices/4500_5500.xlsx b/annotation/datasets/slices/4500_5500.xlsx new file mode 100644 index 0000000..5129aa1 Binary files /dev/null and b/annotation/datasets/slices/4500_5500.xlsx differ diff --git a/annotation/datasets/slices/5500_6500.xlsx b/annotation/datasets/slices/5500_6500.xlsx new file mode 100644 index 0000000..0b1ae83 Binary files /dev/null and b/annotation/datasets/slices/5500_6500.xlsx differ diff --git a/annotation/datasets/slices/6500_7500.xlsx b/annotation/datasets/slices/6500_7500.xlsx new file mode 100644 index 0000000..155280f Binary files /dev/null and b/annotation/datasets/slices/6500_7500.xlsx differ diff --git a/annotation/datasets/slices/7500_8500.xlsx b/annotation/datasets/slices/7500_8500.xlsx new file mode 100644 index 0000000..4bc4a91 Binary files /dev/null and b/annotation/datasets/slices/7500_8500.xlsx differ diff --git a/annotation/datasets/slices/8500_9500.xlsx b/annotation/datasets/slices/8500_9500.xlsx new file mode 100644 index 0000000..66a9d5d Binary files /dev/null and b/annotation/datasets/slices/8500_9500.xlsx differ diff --git a/annotation/datasets/slices/9500_10500.xlsx b/annotation/datasets/slices/9500_10500.xlsx new file mode 100644 index 0000000..f7a6cbc Binary files /dev/null and b/annotation/datasets/slices/9500_10500.xlsx differ diff --git a/annotation/datasets/slices/first2000_reviewed.xlsx b/annotation/datasets/slices/first2000_reviewed.xlsx new file mode 100644 index 0000000..469b265 Binary files /dev/null and b/annotation/datasets/slices/first2000_reviewed.xlsx differ diff --git a/annotation/datasets/unlabeled_dataset.xlsx b/annotation/datasets/unlabeled_dataset.xlsx new file mode 100644 index 0000000..441c2bc Binary files /dev/null and b/annotation/datasets/unlabeled_dataset.xlsx differ diff --git a/annotation/labeled_embeddings.pkl b/annotation/labeled_embeddings.pkl new file mode 100644 index 0000000..3d8e2f9 Binary files /dev/null and b/annotation/labeled_embeddings.pkl differ diff --git a/annotation/labeled_embeddings_71.pkl b/annotation/labeled_embeddings_71.pkl new file mode 100644 index 0000000..cf028c6 Binary files /dev/null and b/annotation/labeled_embeddings_71.pkl differ diff --git a/merge.py b/annotation/merge.py similarity index 87% rename from merge.py rename to annotation/merge.py index a4a7020..7ee3909 100644 --- a/merge.py +++ b/annotation/merge.py @@ -1,127 +1,132 @@ -# --- Import librerie --- -import pandas as pd -import numpy as np -import unicodedata -import re -import warnings -from sentence_transformers import SentenceTransformer -import pickle - -# ----- Percorsi file ----- -LABELED_IN = "main/datasets/annotated_dataset.xlsx" -REVIEWED = "main/datasets/first500_reviewed.xlsx" -LABELED_OUT = "main/datasets/annotated_dataset_updated.xlsx" - -# ----- Funzioni di pulizia ----- -def clean(x): - if pd.isna(x): - return "" - return str(x).strip() - -def normalize_problem_type(x): - x = clean(x).upper() - if x == "S": - return "RULE_SPECIFIC" - if x == "G": - return "GENERIC" - return x - -def normalize_severity(x): - return clean(x).upper() - -def clean_id(x): - if pd.isna(x): - return "" - s = str(x).strip().strip('"').strip("'") - return s.lower() - -def clean_folder(x): - if pd.isna(x): - return "" - s = str(x).strip().lower() - s = unicodedata.normalize("NFKC", s) - s = re.sub(r'\s+', ' ', s) - return s - -# ----- Step 1: caricare datasets ----- -df_labeled = pd.read_excel(LABELED_IN) -df_labeled = df_labeled.loc[:, ~df_labeled.columns.str.contains("^Unnamed")].copy() -df_labeled = df_labeled.dropna(how="all") -df_labeled = df_labeled.rename(columns={"automation_id": "id"}) - -df_rev = pd.read_excel(REVIEWED) - -# Normalizzazione problem_type e severity -if "error_type" in df_labeled.columns: - df_labeled["error_type"] = df_labeled["error_type"].apply(normalize_problem_type) - -# Costruzione dataset pulito dai primi 500 -rows = [] -for _, r in df_rev.iterrows(): - category = clean(r["final_category"]) - subcategory = clean(r["final_subcategory"]) - error_type = normalize_problem_type(r["final_problem_type"]) - severity = normalize_severity(r["final_gravity"]) - - # Coerenza HARMLESS - if category.upper() == "HARMLESS": - subcategory = "" - error_type = "none" - severity = "none" - - rows.append({ - "id": clean(r["id"]), - "folder": clean(r["folder"]), - "automation": clean(r["automation_text"]), - "description": clean(r.get("llm_rationale", "")), - "category": category, - "subcategory": subcategory, - "error_type": error_type, - "severity": severity, - "borderline": clean(r["borderline"]), - }) - -df_new = pd.DataFrame(rows) - -# Normalizzazione valori 'none' -df_new["error_type"] = df_new["error_type"].apply(lambda x: x.lower() if x.lower() == "none" else x) -df_new["severity"] = df_new["severity"].apply(lambda x: x.lower() if x.lower() == "none" else x) - -# Rimuovi righe senza categoria -df_new = df_new[df_new["category"] != ""].copy() - -# Pulizia id e folder in entrambi i dataset -for df in [df_labeled, df_new]: - df["id"] = df["id"].apply(clean_id) - df["folder"] = df["folder"].apply(clean_folder) - -# Rimuovere duplicati: eliminare dal labeled righe già presenti in df_new -new_keys = set(zip(df_new["id"], df_new["folder"])) -df_labeled_clean = df_labeled[~df_labeled.apply(lambda r: (r["id"], r["folder"]) in new_keys, axis=1)].copy() - -# Concat finale -df_final = pd.concat([df_labeled_clean, df_new], ignore_index=True).fillna("") - -# Salva dataset aggiornato -df_final.to_excel(LABELED_OUT, index=False) -print("✅ Merge completato") -print("Righe iniziali:", len(df_labeled)) -print("Righe aggiunte:", len(df_rev)) -print("Totale finale:", len(df_final)) - -# ----- Step 2: calcolo embeddings ----- -warnings.filterwarnings("ignore") -model = SentenceTransformer("all-MiniLM-L6-v2") - -texts = df_final["automation"].tolist() -embeddings = model.encode( - texts, show_progress_bar=True, convert_to_numpy=True, normalize_embeddings=True -).astype("float32") - -print("Shape embeddings ricalcolati:", embeddings.shape) - -# ----- Step 3: salva embeddings ----- -with open("main/labeled_embeddings.pkl", "wb") as f: - pickle.dump({"embeddings": embeddings, "id": df_final["id"].tolist()}, f) - +# --- Import librerie --- +import pandas as pd +import numpy as np +import unicodedata +import re +import warnings +from sentence_transformers import SentenceTransformer +import pickle +import logging + +# ----- Percorsi file ----- +#LABELED_IN = "main/datasets/annotated_dataset.xlsx" +LABELED_IN = "main/datasets/annotated_dataset_updated.xlsx" +#REVIEWED = "main/datasets/first500_reviewed.xlsx" +REVIEWED = "main/datasets/12500_13463.xlsx" +LABELED_OUT = "main/datasets/annotated_dataset_updated2.xlsx" + +# ----- Funzioni di pulizia ----- +def clean(x): + if pd.isna(x): + return "" + return str(x).strip() + +def normalize_problem_type(x): + x = clean(x).upper() + if x == "S": + return "RULE_SPECIFIC" + if x == "G": + return "GENERIC" + return x + +def normalize_severity(x): + return clean(x).upper() + +def clean_id(x): + if pd.isna(x): + return "" + s = str(x).strip().strip('"').strip("'") + return s.lower() + +def clean_folder(x): + if pd.isna(x): + return "" + s = str(x).strip().lower() + s = unicodedata.normalize("NFKC", s) + s = re.sub(r'\s+', ' ', s) + return s + +# ----- Step 1: caricare datasets ----- +df_labeled = pd.read_excel(LABELED_IN) +df_labeled = df_labeled.loc[:, ~df_labeled.columns.str.contains("^Unnamed")].copy() +df_labeled = df_labeled.dropna(how="all") +df_labeled = df_labeled.rename(columns={"automation_id": "id"}) + +df_rev = pd.read_excel(REVIEWED) + +# Normalizzazione problem_type e severity +if "error_type" in df_labeled.columns: + df_labeled["error_type"] = df_labeled["error_type"].apply(normalize_problem_type) + +# Costruzione dataset pulito dai primi 500 +rows = [] +for _, r in df_rev.iterrows(): + category = clean(r["final_category"]) + subcategory = clean(r["final_subcategory"]) + error_type = normalize_problem_type(r["final_problem_type"]) + severity = normalize_severity(r["final_gravity"]) + + # Coerenza HARMLESS + if category.upper() == "HARMLESS": + error_type = "none" + severity = "none" + + rows.append({ + "id": clean(r["id"]), + "folder": clean(r["folder"]), + "automation": clean(r["automation_text"]), + "description": clean(r.get("llm_rationale", "")), + "category": category, + "subcategory": subcategory, + "error_type": error_type, + "severity": severity, + "borderline": clean(r["borderline"]), + }) + +df_new = pd.DataFrame(rows) + +# Normalizzazione valori 'none' +df_new["error_type"] = df_new["error_type"].apply(lambda x: x.lower() if x.lower() == "none" else x) +df_new["severity"] = df_new["severity"].apply(lambda x: x.lower() if x.lower() == "none" else x) + +# Rimuovi righe senza categoria +df_new = df_new[df_new["category"] != ""].copy() + +# Pulizia id e folder in entrambi i dataset +for df in [df_labeled, df_new]: + df["id"] = df["id"].apply(clean_id) + df["folder"] = df["folder"].apply(clean_folder) + +# Rimuovere duplicati: eliminare dal labeled righe già presenti in df_new +new_keys = set(zip(df_new["id"], df_new["folder"])) +df_labeled_clean = df_labeled[~df_labeled.apply(lambda r: (r["id"], r["folder"]) in new_keys, axis=1)].copy() + +# Concat finale +df_final = pd.concat([df_labeled_clean, df_new], ignore_index=True).fillna("") + +# Salva dataset aggiornato +df_final.to_excel(LABELED_OUT, index=False) +print("✅ Merge completato") +print("Righe iniziali:", len(df_labeled)) +print("Righe aggiunte:", len(df_rev)) +print("Totale finale:", len(df_final)) + +# ----- Step 2: calcolo embeddings ----- +warnings.filterwarnings("ignore") +logging.getLogger("sentence_transformers").setLevel(logging.ERROR) +logging.getLogger("transformers").setLevel(logging.ERROR) +logging.getLogger("huggingface_hub").setLevel(logging.ERROR) +model = SentenceTransformer("all-MiniLM-L6-v2") + +texts = df_final["automation"].tolist() +embeddings = model.encode( + texts, show_progress_bar=True, convert_to_numpy=True, normalize_embeddings=True +).astype("float32") + +print("Shape embeddings ricalcolati:", embeddings.shape) + +# ----- Step 3: salva embeddings ----- +with open("main/labeled_embeddings.pkl", "wb") as f: + pickle.dump({"embeddings": embeddings, "id": df_final["id"].tolist()}, f) + print("Embeddings salvati con successo!") \ No newline at end of file diff --git a/prompt.py b/annotation/prompts/prompt.py similarity index 62% rename from prompt.py rename to annotation/prompts/prompt.py index d2f41e9..70b71e1 100644 --- a/prompt.py +++ b/annotation/prompts/prompt.py @@ -1,290 +1,343 @@ -task = """ -You are a security evaluation tool for smart home automation rules. -Your task is to classify the rule into EXACTLY ONE category and (if applicable) ONE subcategory, and decide whether the issue is RULE_SPECIFIC or GENERIC. - -For each category you will receive: -- A definition of the category -- Necessary rule conditions -- Risk conditions -- Non-applicable conditions -- Illustrative examples - -You will receive: -- The automation rule (text). -- Definitions of 4 categories (with subcategories). -- Retrieved similar labeled examples (RAG context) with distance-based similarity levels. - -Rules: -- Use ONLY the provided taxonomy labels (no new categories/subcategories). -- Return ONLY a valid JSON object (no extra text). -""" - -taxonomy = """ -* CATEGORIES AND DEFINITIONS (use ONLY these labels) * - - # CATEGORY 1: PERSONAL PRIVACY VIOLATION - Definition: This category includes automations that collect, process, or disseminate personal or sensitive data (own or third-party) in an unnecessary, unintentional, or potentially illicit manner. - Necessary rule conditions: the rule collects sensitive information (images, video, audio, location, identity). - Risk Conditions: - R1: The data is shared with external parties - - neighbors - - WhatsApp groups - - social media - - mailing lists - - public channels - R2: The data is made publicly accessible - - publicly visible - - public link - - open profile - R3: The collection or sharing is not necessary for the stated purpose or is not clearly understood - - no confirmation - - no review before publication - R4: The data collection may violate privacy regulations - - camera photographing passers-by - - audio recording of guests - Do not apply if: - - The data remains strictly internal and is not shared, exposed, or externally observable. - - The automation only updates internal system states without transmitting or publishing personal data. - - The data collection is clearly necessary for the rule's primary functionality and remains locally processed. - Illustrative examples (for guidance only): - - When the camera on my smart doorbell detects an unknown/suspicious person (e.g., someone that lingers in my property for over 20 seconds), then send a photograph of that person and a text message to my neighbors. S+ - - If I take a new photo, then post it as a publicly accessible image on Instagram - - # CATEGORY 2: PHYSICAL ENVIRONMENT THREAT - Definition: This category includes automations that can cause unauthorized access, reduced physical security, or property damage. - Sub-categories: 2.1 PROMOTE UNAUTHORIZED ACCESS, 2.2 Device and identity control. 2.3 , 2.4 absence status reporting - - ## SUB-CATEGORY 2.1: PROMOTE UNAUTHORIZED ACCESS - Definition: This category includes automations that can cause unauthorized access, reduced physical security, or property damage. - Necessary rule conditions: - - Actions on: windows / doors / locks and - - Automatic activations based on: environmental conditions / unauthenticated events - Risk Conditions: - - The action reduces physical protection. - - There is no identity check. - - The event can be externally induced. - Does not apply if: - - There are already security measures such as checking the user's presence at home. - - The rule only modifies non-security-related elements (e.g., lights, temperature). - - The action is manually confirmed before execution. - Illustrative examples (for guidance only):: - - When the smart thermostat detects that the temperature rises above 25 degrees, then slightly open the window. - - If Indoor CO2 goes up, open the window. - - - ## SUB-CATEGORY 2.2: Device and identity control (device-based access) - Definition: Automations that grant physical access based solely on the presence of a device, without considering theft, compromise, or old, unremoved devices. - Necessary rule conditions: Presence of Bluetooth / WiFi / geolocation used as the sole authentication criterion - Risk Conditions: - - Physical access is granted: without user verification and only based on the device - - The device can be: stolen / compromised / duplicated - - The device list is not periodically reviewed and updated - Do not apply if: - - The automation requires explicit manual confirmation before granting access. - - Additional authentication mechanisms are enforced (e.g., PIN, biometric verification, multi-factor authentication). - - The device presence is not the sole authentication factor. - - The rule does not grant physical access but only sends notifications or status updates. - Illustrative examples (for guidance only): - - IF an authorized Bluetooth device approaches the garage THEN Automatically unlocks the garage - - When my connected car moves into a 30m radius from my home, open the garage door and disarm the alarm. - - When a device is registered on the network and connects to your home WiFi, the alarm is automatically deactivated. - - ## SUB-CATEGORY 2.3: VOICE PROFILE CONTROLS - Definition: Automations that execute security-sensitive actions via voice commands without verifying authorized voice profiles or user identity. - Necessary rule conditions: - - The automation is triggered by a voice command and - - The command affects security-sensitive actions (e.g., unlocking, disarming, disabling protections). - Risk Conditions: - - The command can be executed by anyone - - There is no control over the user's identity - - No specific authorized voice profiles are stored - Do not apply if: - - The voice command triggers only non-security-sensitive actions (e.g., turning lights on/off). - - The automation can only be executed from inside the home after physical access has already been established. - - The system verifies authorized voice profiles before executing critical actions. - - Additional authentication mechanisms are required for security-sensitive operations. - Illustrative examples (for guidance only): - - IF the voice assistant recognizes the command "Disable alarm" THEN Disable the home security system - - - ## SUB-CATEGORY 2.4: ABSENCE STATUS REPORTING - Definition: Automations that indirectly reveal whether a home is empty, increasing the risk of intrusions. - Necessary rule conditions: - - Actions that: turn lights on/off; modify Wi-Fi/alarms and - - The actions are related to presence at home and - - The effect of the action is *observable from the outside* of the house - Risk Conditions: - - The rule allows us to deduce whether the house is empty. - - The information is: observable from the outside of the house or shared with third parties. - Do not apply if: - - The effect of the automation is not externally observable (e.g. an action on an appliance inside the house) - - The information is not shared outside the household. - - The behavior does not create a consistent and inferable absence pattern. - - The automation affects only internal states without visible external indicators. - Illustrative examples (for guidance only): - - IF someone is home, THEN turn the light - - If the last family member leaves home, then turn off lights - - IF I leave home, THEN turn off the WiFi - - If I'm the last person in the house and leave, send a notification to my smartwatch if any window in the house is open - - - # CATEGORY3: CYBERSECURITY HARM - Description: This category includes automations that introduce malware exposure, data stream manipulation, or network communication abuse. - Sub-categories: SUB-CATEGORY 3.1: MALICIOUS TRAFFIC GENERATION, SUB-CATEGORY 3.2: AUTOMATIC FILE SPREAD, SUB-CATEGORY 3.3: NETWORK COMMUNICATION THREATS - - ## SUB-CATEGORY 3.1: MALICIOUS TRAFFIC GENERATION - Definition: Automations that can be exploited to generate excessive traffic, false alarms, or denial of service - Necessary Rule conditions: The event is easily repeatable. - Risk conditions: - - The event can generate excessive traffic or false alarms and - - The event is manipulable - Do not apply if: - - The event cannot be externally triggered or manipulated. - - The notification is rate-limited or protected against repeated activation. - - The action does not significantly increase network load or system exposure. - Illustrative examples (for guidance only): - - If the smart camera detects someone approaching, it automatically sends a text message to my phone - - ## SUB-CATEGORY 3.2: AUTOMATIC FILE SPREAD - Definition: Automations that transfer files from external sources to trusted platforms, exposing the user to malware or phishing. - Rule conditions: - The automation involves automatic file download from external URLs, email attachments, messaging platforms, third-party APIs and - The automation stores, uploads, forwards, or makes the file available within: - - trusted cloud storage - - local systems - - shared folders - - collaboration platforms - Risk conditions: - - There is no content check (e.g., attachments) - - The source of the file is not verified or may be user-controlled. - - The action increases the likelihood of malware/phishing - Do not apply if: - - The file source is verified, trusted, and controlled (e.g., official governmental APIs). - - The automation includes content validation or malware scanning before storage. - - The user manually confirms the download before execution. - - No file propagation to additional platforms occurs. - Illustrative examples (for guidance only): - - Add file from URL action from the Dropbox channel when the “Any new attachment in inbox ” trigger from the Gmail channel is activated - - ## SUB-CATEGORY 3.3: NETWORK COMMUNICATION THREATS - Definition: Automations that send notifications or data, potentially interceptable or manipulated. - Rule conditions: - - The automation sends data or notifications over: SMS, messaging platforms, email and - - The transmitted information relates to security-relevant events, such as absence of occupants, alarm status, door/window state. - Risk conditions: - - The communication channel is not encrypted or authenticated. - - Messages can be intercepted, spoofed, or altered in transit. - Do not apply if: - - The communication is encrypted and authenticated. - - The communication does not expose the system to interception or spoofing risks. - - The transmitted data does not expose occupancy, alarm status, or access control states. - Illustrative examples (for guidance only): - - If the smart camera detects someone approaching, it automatically sends a text message to my phone - - # CATEGORY 4: HARMLESS - Definition: automations that do not present safety problems. - Conditions: - - The rule does not involve personal data - - The rule does not modify the physical environment - - The rule does not introduce risky network communications - - The rule already includes device/user/presence checks - Illustrative examples (for guidance only): - - If it rains tomorrow, then remind me to bring an umbrella -""" - -problem_type_guide = """ -* PROBLEM TYPE (choose exactly one) *: - -# RULE_SPECIFIC (S): the automation directly leads to a potentially dangerous situation. -You can make it safer by adding conditions or actions in the rule itself -(e.g., verifying presence at home, identity check, confirmation step). -Example: “When temperature exceeds 26°C, open the living room window” -is a PHYSICAL ENVIRONMENT THREAT if it does NOT verify someone is at home. - -# GENERIC (G): the automation is not inherently dangerous; risk depends on configuration -or contextual factors. The best mitigation is a user behavior recommendation rather -than changing the rule logic. -Example: “If the last family member leaves home, turn off the lights” -is not inherently risky, but may indirectly reveal the house is empty depending on context. -""" - -gravity_guide = """ -* GRAVITY / SEVERITY (choose exactly one) *: - - # HIGH: direct and immediate security/privacy consequence. - Examples: automatically opening doors; public photos without consent; malware propagation. - - # MEDIUM: indirect consequence or conditioned on other variables. - Examples: absence deducible from light patterns; opening door via Bluetooth/device proximity. - - # LOW: minimal risk, marginal information leakage, or easily mitigable. - Examples: notifications that might hint the user is away only if intercepted; - downloads from relatively trusted sources with limited exposure. - - # NONE: no security/privacy consequence (comfort rules). - Examples: lights/temperature/irrigation/morning routine. -""" - -OUTPUT_SCHEMA = """ -Return ONLY this JSON: - -{ - "automation": "string", - "category": "PERSONAL PRIVACY VIOLATION | PHYSICAL ENVIRONMENT THREAT | CYBERSECURITY HARM | HARMLESS", - "subcategory": "one of the defined subcategories for that category, or empty string", - "problem_type": "RULE_SPECIFIC | GENERIC | none", - "gravity": "LOW | MEDIUM | HIGH | NONE", - "scores": { - "PERSONAL PRIVACY VIOLATION": 0.0, - "PHYSICAL ENVIRONMENT THREAT": 0.0, - "CYBERSECURITY HARM": 0.0, - "HARMLESS": 0.0 - }, - "needs_human_review": true, - "short_rationale": "max 2 sentences" -} -""" - -# trasformare in testo i risultati del retrieval (le 5 automazioni simili + similarity cosine) -# il testo viene passato al LLM come esempio - -def build_examples_text(retrieved_df, similarity_band_fn, max_chars=600): - parts = [] - for i, (_, r) in enumerate(retrieved_df.iterrows(), start=1): - sim = float(r["similarity"]) - - parts.append( - f"""Example {i}: - Automation: {str(r.get('automation',''))[:max_chars]} - Description: {str(r.get('description',''))[:200]} - Category: {r.get('category','')} - Subcategory: {r.get('subcategory','')} - Problem type: {r.get('problem_type','')} - Gravity: {r.get('gravity','')} - Cosine similarity: {round(sim, 4)} - Similarity level: {similarity_band_fn(sim)} - """ - ) - return "\n".join(parts) - - -# costruzione del prompt -def build_prompt_local(query_text, retrieved_df, similarity_band_fn): - top1_sim = float(retrieved_df["similarity"].iloc[0]) - band = similarity_band_fn(top1_sim) - examples_text = build_examples_text(retrieved_df, similarity_band_fn) - - return f"""{task} - -{taxonomy} -{problem_type_guide} -{gravity_guide} - -TOP1_COSINE_SIMILARITY: {round(top1_sim, 4)} -SIMILARITY_BAND: {band} - -RETRIEVED LABELED CONTEXT (top-k, similarity-based): -{examples_text} - -{OUTPUT_SCHEMA} +task = """ +You are a security evaluation tool for smart home automation rules. +Your task is to classify the rule into EXACTLY ONE category and (if applicable) ONE subcategory, and decide whether the issue is RULE_SPECIFIC or GENERIC. + +You will receive: +- The automation rule (text). +- Definitions of four categories (with subcategories). +- Retrieved similar labeled examples (RAG context) with distance-based similarity scores. +- A gravity guide. +- A problem type classification guide. + +For each category you will receive: +- A definition of the category +- Necessary rule conditions +- Risk conditions +- Non-applicable conditions +- Illustrative examples of violations and non-violations + +Rules: +- Use ONLY the provided taxonomy labels (no new categories/subcategories). +- Return ONLY a valid JSON object (no extra text). +""" + +taxonomy = """ +* CATEGORIES AND DEFINITIONS (use ONLY these labels) * + + # CATEGORY 1: PERSONAL PRIVACY VIOLATION + Definition: This category includes automations that collect, process, or disseminate personal or sensitive data (own or third-party) in an unnecessary, unintentional, or potentially illicit manner. + Necessary rule conditions: the rule collects sensitive information (images, video, audio, location, identity). + Risk Conditions: + - The data is shared with external parties (examples: neighbors, WhatsApp groups, social media, mailing lists, public channels) + - The data is made publicly accessible, for example with a public link, or an open profile + - The data collection may violate privacy regulations. Examples: camera photographing passers-by or audio recording of guests + - The person involved may not be consentient + Do not apply if: + - The data remains strictly internal and is not shared, exposed, or externally observable (e.g., intercom projections used only locally). + - The automation only updates internal system states or triggers routines / internal events without transmitting personal data externally. + - The data collection is clearly necessary for the rule's primary functionality and remains locally processed. + - The data pertains only to house devices and does not involve users or their personal devices. + - The collected data is not sensitive (e.g., only a summary of detected motion). + - The data collection is necessary for an emergency situation (e.g., the user intentionally shares their location for safety). + Illustrative examples (for guidance only): + - When the camera on my smart doorbell detects an unknown/suspicious person (e.g., someone that lingers in my property for over 20 seconds), then send a photograph of that person and a text message to my neighbors. S+ + - If I take a new photo, then post it as a publicly accessible image on Instagram + Examples that do NOT fall under this category (harmless even if processing sensitive data): + - When Person1 returns home, a routine is triggered (e.g., lights turn on or HVAC adjusts). + - Tracking a device's location solely to run automations or determine if a person is home. + - Updating internal flags based on motion sensors or presence detection without external notifications. + - When the doorbell rings show the visitor on TV or phone + + + # CATEGORY 2: PHYSICAL ENVIRONMENT THREAT + Definition: This category includes automations that can cause unauthorized access, reduced physical security, or property damage. + Sub-categories: 2.1 PROMOTE UNAUTHORIZED ACCESS, 2.2 Device and identity control. 2.3 , 2.4 absence status reporting + + ## SUB-CATEGORY 2.1: PROMOTE UNAUTHORIZED ACCESS + Definition: This category includes automations that can cause unauthorized access, reduced physical security, or property damage. + Necessary rule conditions: + - Actions that open windows, doors, locks and + - Automatic activations based on: environmental conditions / unauthenticated events + Risk Conditions: + - The action reduces physical protection. + - There is no identity check. + - The event can be externally induced. + Does not apply if: + - There are already security measures, such as home mode restrictions, verifying the user's presence at home or controlling which user is accessing the house. + - The rule only modifies non-security-related elements (e.g., lights, temperature, covers). + - The action is manually confirmed before execution. + - The action reduces physical protection, but the house is not empty and there are users inside. + Illustrative examples (for guidance only): + - When the smart thermostat detects that the temperature rises above 25 degrees, then slightly open the window. + - If Indoor CO2 goes up, open the window. + Examples that do NOT fall under this category + - Unlocks the door when the user wakes up + + + ## SUB-CATEGORY 2.2: DEVICE AND IDENTITY CONTROL (device-based access) + Definition: Automations that grant physical access based solely on the presence of a device, without considering theft, compromise, or old, unremoved devices. + Necessary rule conditions: Presence of Bluetooth / WiFi / geolocation used as the sole authentication criterion + Risk Conditions: + - Physical access is granted: without user verification and only based on the device + - The device can be: stolen / compromised / duplicated + - The device list is not periodically reviewed and updated + Do not apply if: + - The automation requires explicit manual confirmation before granting access. + - Additional authentication mechanisms are enforced (e.g., PIN, biometric verification, multi-factor authentication). + - The device presence is not the sole authentication factor. + - The rule does not grant physical access but only sends notifications or status updates. + Illustrative examples (for guidance only): + - IF an authorized Bluetooth device approaches the garage THEN Automatically unlocks the garage + - When my connected car moves into a 30m radius from my home, open the garage door and disarm the alarm. + - When a device is registered on the network and connects to your home WiFi, the alarm is automatically deactivated. + Examples that do NOT fall under this category: + - Unlock the door when a specific fingerprint is scanned + + + ## SUB-CATEGORY 2.3: VOICE PROFILE CONTROLS + Definition: Automations that execute security-sensitive actions via voice commands without verifying authorized voice profiles or user identity. + Necessary rule conditions: + - The automation is triggered by a voice command and + - The command must affect security-sensitive actions (e.g., unlocking, disarming, disabling protections). + Risk Conditions: + - The command can be executed by anyone without control over the user's identity + - No specific authorized voice profiles are stored + Do not apply if: + - The voice command triggers only non-security-sensitive actions (e.g., turning lights on/off). + - The automation can only be executed from inside the home after physical access has already been established. + - The system verifies authorized voice profiles before executing critical actions. + - Additional authentication mechanisms are required for security-sensitive operations. + - The voice interaction is only used as an output (e.g. Alexa reporting information) and not as an input by the user. + Illustrative examples (for guidance only): + - IF the voice assistant recognizes the command "Disable alarm" THEN Disable the home security system + Examples that do NOT fall under this category: + - IF I say "goodmorning" start a morning routine + + + ## SUB-CATEGORY 2.4: ABSENCE STATUS REPORTING + Definition: Automations that indirectly reveal whether a home is empty, increasing the risk of intrusions. + Necessary rule conditions: + - The actions turn lights on/off and + - The actions are related to presence at home and + - The effect of the action is *observable from the outside* of the house + Risk Conditions: + - The rule allows us to deduce whether the house is empty and + - The information is observable from the outside of the house. + Do not apply if: + - The effect of the automation is not externally observable (e.g. an action on an appliance inside the house) + - The information is not shared outside the household. + - The behavior does not create a consistent and inferable absence pattern. + - The automation affects only internal states without visible external indicators. + Illustrative examples (for guidance only): + - IF someone is home, THEN turn the light + - If the last family member leaves home, then turn off lights + - IF I leave home, THEN turn off the WiFi + - If I'm the last person in the house and leave, send a notification to my smartwatch if any window in the house is open + Examples that do NOT fall under this category (non-problem cases): + - IF I leave home, turn off my PC and TV (these actions are not easily observable from outside the house) + + + # CATEGORY3: CYBERSECURITY HARM + Description: This category includes automations that introduce malware exposure, data stream manipulation, or network communication abuse. + Sub-categories: SUB-CATEGORY 3.1: MALICIOUS TRAFFIC GENERATION, SUB-CATEGORY 3.2: AUTOMATIC FILE SPREAD, SUB-CATEGORY 3.3: NETWORK COMMUNICATION THREATS + + ## SUB-CATEGORY 3.1: MALICIOUS TRAFFIC GENERATION + Definition: Automations that can be exploited to generate excessive traffic, false alarms, or denial of service + Necessary Rule conditions: The event is easily repeatable. + Risk conditions: + - The event can generate excessive traffic or false alarms and + - The event is manipulable + Do not apply if: + - The event cannot be externally triggered or manipulated. + - The notification is rate-limited or protected against repeated activation. + - The action does not significantly increase network load or system exposure. + Illustrative examples (for guidance only): + - If the smart camera detects someone approaching, it automatically sends a text message to my phone + Examples that do NOT fall under this category: + - Send a single push notification to the homeowner when a door is opened; the notification system is rate-limited and cannot be triggered repeatedly by the same event. + + + ## SUB-CATEGORY 3.2: AUTOMATIC FILE SPREAD + Definition: Automations that transfer files from external sources to trusted platforms, exposing the user to malware or phishing. + Rule conditions: + The automation involves automatic file download from external URLs, email attachments, messaging platforms, third-party APIs and + The automation stores, uploads, forwards, or makes the file available within: + - trusted cloud storage + - local systems + - shared folders + - collaboration platforms + Risk conditions: + - There is no content check (e.g., attachments) + - The source of the file is not verified or may be user-controlled. + - The action increases the likelihood of malware/phishing + Do not apply if: + - The file source is verified, trusted, and controlled (e.g., official governmental APIs). + - The automation includes content validation or malware scanning before storage. + - The user manually confirms the download before execution. + - No file propagation to additional platforms occurs. + Illustrative examples (for guidance only): + - Add file from URL action from the Dropbox channel when the “Any new attachment in inbox ” trigger from the Gmail channel is activated + Examples that do NOT fall under this category: + - Manually download an attachment from a verified corporate email and save it to the local system after scanning it for malware. + + + ## SUB-CATEGORY 3.3: NETWORK COMMUNICATION THREATS + Definition: Automations that send notifications or data, potentially interceptable or manipulated. + Rule conditions: + - The automation sends data or notifications over SMS, messaging platforms, email, or similar external channels. And + - The transmitted information relates to security-relevant events, such as absence of occupants, alarm status, door/window state. + Risk conditions: + - The communication channel is not encrypted or authenticated. + - Messages can be intercepted, spoofed, or altered in transit and + - The interception of the message can provide actionable information that affects system security (e.g., knowing that the house is empty and a door is open), potentially allowing unauthorized access or compromising defensive measures. + Do not apply if: + - The communication is encrypted and authenticated. + - The transmitted data does not expose security-relevant events (e.g., an opened internal door). + - Interception of the message would not impact system security. The notification is purely informational and does not affect system behavior (e.g., a notification when the user arrives home). + - The transmitted data does not provide actionable information that affects access control or defensive decisions. + - The message only reports past or already-occurring events. + - The message conveys *important information necessary for the user* (e.g., a notification if a door is opened while the house is empty). + - The message relates to security events but the house is not empty or it cannot be used to reliably deduce whether the house is empty or occupied. + Illustrative examples (for guidance only): + - Send a notification if the front door remains open while nobody is home. + - Send a text message to an external phone number when the back door is unlocked while the house is empty. + Examples that do NOT fall under this category (non-problem cases): + - Alert me if a freezer door is left open for a few minutes. + - Notify me when a sensor goes offline or reconnects. + - Log a state change and send a summary message. + - Send reminders based on system status (e.g., device plugged in, power usage changes). + - Notify me when a home mode changes (e.g., Holiday, Away, Guest). + - Send a notification when Home Assistant restarts + - Send a Telegram notification with performance metrics and system status + - Send alerts for failed login attempts, keeping the user informed about security events. + - Send alerts when a leak or alarm sensor is triggered. + - Send a notification when the user arrives home + - Send a notification when a door is opened while nobody is at home (or similar events). + - Send an alert when someone is detected at the front door + - Send an alert if the front door is been opened for 3 minutes (but someone is at home) + - Send a notification when the door is opened or closed + + + # CATEGORY 4: HARMLESS + Definition: automations that do not present safety problems. + Conditions: + - The rule does not involve personal data + - The rule does not modify the physical environment + - The rule does not introduce risky network communications + - The rule already includes device/user/presence checks + Illustrative examples (for guidance only): + - If it rains tomorrow, then remind me to bring an umbrella + + + ## SUB-CATEGORY 4.1: LOW HARM EXTERNAL CONTROL + Definition: Automations that are functionally safe but are triggered through external control platforms (e.g., messaging apps, mobile apps, remote APIs). The automation itself does not introduce a security vulnerability. + Necessary rule conditions: + - The automation is triggered through an external command channel (e.g., Telegram, messaging platforms, remote control apps) and + - The actions affect only non-critical operations such as starting/stopping services, sending status updates, or managing internal system components. + Operational note: + - The security of the automation depends on proper access control to the external platform or account used to trigger the command. + - The broader security impact may depend on the role of the controlled service or system component, which may not be fully known from the rule description alone. + Do not apply if: + - The automation performs security-sensitive actions (e.g., unlocking doors, disabling alarms, granting access) or + - The automation exposes personal data or sensitive information or + - The action could directly compromise system security or physical protection. + In these cases, the rule must instead be classified under the appropriate risk category. + Illustrative examples (for guidance only): + - When I send a Telegram command, stop the MariaDB server and send a status message. + - If I send a command from my phone app, start or stop a home server service. +""" + +problem_type_guide = """ +* PROBLEM TYPE (choose exactly one) *: + +# RULE_SPECIFIC (S): the automation directly leads to a potentially dangerous situation. +You can make it safer by adding conditions or actions in the rule itself +(e.g., verifying presence at home, identity check, confirmation step). +Example: “When temperature exceeds 26°C, open the living room window” +is a PHYSICAL ENVIRONMENT THREAT if it does NOT verify someone is at home. + +# GENERIC (G): the automation is not inherently dangerous; risk depends on configuration +or contextual factors. The best mitigation is a user behavior recommendation rather +than changing the rule logic. +Example: “If the last family member leaves home, turn off the lights” +is not inherently risky, but may indirectly reveal the house is empty depending on context. +""" + +gravity_guide = """ +* GRAVITY / SEVERITY (choose exactly one) *: + + # HIGH: direct and immediate security/privacy consequence. + Examples: automatically opening doors; public photos without consent; malware propagation. + + # MEDIUM: indirect consequence or conditioned on other variables. + Examples: opening door via Bluetooth/device proximity; absence deducible from light patterns. + + # LOW: minimal risk, marginal information leakage, or easily mitigable. + Examples: notifications that might hint the user is away only if intercepted; downloads from relatively trusted sources with limited exposure. + + # NONE: no security/privacy consequence (comfort rules). + Examples: lights/temperature/irrigation/morning routine. +""" + +OUTPUT_SCHEMA = """ +Return ONLY this JSON: + +{ + "automation": "string", + "category": "PERSONAL PRIVACY VIOLATION | PHYSICAL ENVIRONMENT THREAT | CYBERSECURITY HARM | HARMLESS", + "subcategory": "one of the defined subcategories for that category, or empty string for harmless", + "problem_type": "RULE_SPECIFIC | GENERIC | none", + "gravity": "LOW | MEDIUM | HIGH | NONE", + "scores": { + "PERSONAL PRIVACY VIOLATION": 0.0, + "PHYSICAL ENVIRONMENT THREAT": 0.0, + "CYBERSECURITY HARM": 0.0, + "HARMLESS": 0.0 + }, + "short_rationale": "max 2 sentences" +} +""" + +# trasformare in testo i risultati del retrieval (le 5 automazioni simili + similarity cosine) +# il testo viene passato al LLM come esempio + +def build_examples_text(retrieved_df, similarity_band_fn, max_chars=600): + parts = [] + for i, (_, r) in enumerate(retrieved_df.iterrows(), start=1): + sim = float(r["similarity"]) + + parts.append( + f"""Example {i}: + Automation: {str(r.get('automation',''))[:max_chars]} + Description: {str(r.get('description',''))[:200]} + Category: {r.get('category','')} + Subcategory: {r.get('subcategory','')} + Problem type: {r.get('problem_type','')} + Gravity: {r.get('gravity','')} + Cosine similarity: {round(sim, 4)} + Similarity level: {similarity_band_fn(sim)} + """ + ) + return "\n".join(parts) + + +# costruzione del prompt +def build_prompt_local(query_text, retrieved_df, similarity_band_fn): + top1_sim = float(retrieved_df["similarity"].iloc[0]) + band = similarity_band_fn(top1_sim) + examples_text = build_examples_text(retrieved_df, similarity_band_fn) + + return f"""{task} + +{taxonomy} +{problem_type_guide} +{gravity_guide} + +TOP1_COSINE_SIMILARITY: {round(top1_sim, 4)} +SIMILARITY_BAND: {band} + +RETRIEVED LABELED CONTEXT (top-k, similarity-based): +{examples_text} + +{OUTPUT_SCHEMA} """ \ No newline at end of file diff --git a/annotation/utilities/embeddings.py b/annotation/utilities/embeddings.py new file mode 100644 index 0000000..9f7da5b --- /dev/null +++ b/annotation/utilities/embeddings.py @@ -0,0 +1,83 @@ +# --- Import librerie --- +import pandas as pd +from openai import AzureOpenAI +from sentence_transformers import SentenceTransformer +import numpy as np +import re +from openpyxl.styles import PatternFill +from openpyxl import load_workbook +from collections import Counter +from prompts.prompt import build_prompt_local +import warnings +import logging +from sentence_transformers import SentenceTransformer +import numpy as np +import pickle +import unicodedata + +# ----- Caricare datasets ----- +df_labeled = pd.read_excel("main/datasets/annotated_dataset.xlsx") +df_labeled = df_labeled.dropna(how="all") # rimuove righe completamente vuote + +df_unlabeled = pd.read_excel("main/datasets/unlabeled_dataset.xlsx") +df_unlabeled = df_unlabeled.dropna(how="all") + +print("***STEP 1***") +print("Dataset etichettato caricato. Numero righe:", len(df_labeled)) +print("Dataset non etichettato caricato. Numero righe:", len(df_unlabeled)) + +# ----- Pulizia colonne ---- +df_labeled = pd.read_excel("main/datasets/annotated_dataset.xlsx").dropna(how="all") +df_unlabeled = pd.read_excel("main/datasets/unlabeled_dataset.xlsx").dropna(how="all") + +def clean_str(x): + if pd.isna(x): + return "" + s = str(x).strip().lower() + s = unicodedata.normalize("NFKC", s) + # rimuove tutti i caratteri non alfanumerici e spazi multipli, lascia solo lettere, numeri e spazi + s = re.sub(r'[^a-z0-9 ]+', '', s) + s = re.sub(r'\s+', ' ', s) # spazi multipli → 1 spazio + return s + +# Applica pulizia su automation_id e folder +for df in [df_labeled, df_unlabeled]: + df["automation_id"] = df["automation_id"].apply(clean_str) + df["folder"] = df["folder"].apply(clean_str) + +unlabeled_pairs = set(zip(df_unlabeled["automation_id"], df_unlabeled["folder"])) +# Filtro: rimuove dal dataset non etichettato le righe già presenti in df_labeled +labeled_pairs = set(zip(df_labeled["automation_id"], df_labeled["folder"])) +mask = ~df_unlabeled[["automation_id", "folder"]].apply(tuple, axis=1).isin(labeled_pairs) +df_unlabeled_filtered = df_unlabeled[mask] +print("Numero righe df_unlabeled dopo aver rimosso quelle etichettate:", len(df_unlabeled_filtered)) + +# Trova coppie mancanti (debug) +missing_pairs = labeled_pairs - unlabeled_pairs +print("Numero righe etichettate non trovate nel dataset non etichettato:", len(missing_pairs)) +if missing_pairs: + print("Coppie mancanti:") + for p in missing_pairs: + print(p) + + +# ----- Step 2: embeddings ----- +# Silenzia warning generici +warnings.filterwarnings("ignore") +# Silenzia logging di transformers / sentence-transformers / HF hub +logging.getLogger("sentence_transformers").setLevel(logging.ERROR) +logging.getLogger("transformers").setLevel(logging.ERROR) +logging.getLogger("huggingface_hub").setLevel(logging.ERROR) +model = SentenceTransformer("all-MiniLM-L6-v2") + +texts = df_labeled["automation"].tolist() +embeddings = model.encode(texts, show_progress_bar=True, convert_to_numpy=True, normalize_embeddings=True) +embeddings = embeddings.astype("float32") + +print("Shape embeddings ricalcolati:", embeddings.shape) + +# ----- Step 3: salvare embeddings ----- +with open("main/labeled_embeddings2.pkl", "wb") as f: + pickle.dump({"embeddings": embeddings, "automation_id": df_labeled["automation_id"].tolist()}, f) + +print("Embeddings salvati con successo!") \ No newline at end of file diff --git a/annotation/utilities/final_dataset_creation.py b/annotation/utilities/final_dataset_creation.py new file mode 100644 index 0000000..2a46d93 --- /dev/null +++ b/annotation/utilities/final_dataset_creation.py @@ -0,0 +1,36 @@ +import pandas as pd +import glob +import re +import os + +percorso = "main/datasets/slices/*.xlsx" +files = glob.glob(percorso) + +def chiave_ordinamento(file): + nome = os.path.basename(file) + + # Caso speciale + if "first2000_reviewed" in nome: + return 0 + + # Estrae il primo numero del range (es: 2000 da 2000_2500) + match = re.search(r'(\d+)_', nome) + if match: + return int(match.group(1)) + + # fallback (nel dubbio lo manda in fondo) + return float('inf') + +files_ordinati = sorted(files, key=chiave_ordinamento) + +# Debug: stampa ordine +print("Ordine file:") +for f in files_ordinati: + print(os.path.basename(f)) + +# Concatenazione +df_unito = pd.concat((pd.read_excel(f) for f in files_ordinati), ignore_index=True) + +df_unito.to_excel("final_dataset.xlsx", index=False) + +print("Unione completata!") \ No newline at end of file diff --git a/annotation/utilities/similarity_analysis.py b/annotation/utilities/similarity_analysis.py new file mode 100644 index 0000000..ce03755 --- /dev/null +++ b/annotation/utilities/similarity_analysis.py @@ -0,0 +1,139 @@ +# --- Import librerie --- +import pandas as pd +from openai import AzureOpenAI +import os +import json +import pickle +from sentence_transformers import SentenceTransformer +import numpy as np +import faiss +import openpyxl +from openpyxl.styles import PatternFill +from openpyxl import load_workbook +import re + + +# --- Configurazione --- +endpoint = "https://gpt-sw-central-tap-security.openai.azure.com/" +deployment = "gpt-4o" +subscription_key = "8zufUIPs0Dijh0M6NpifkkDvxJHZMFtott7u8V8ySTYNcpYVoRbsJQQJ99BBACfhMk5XJ3w3AAABACOGr6sq" + +client = AzureOpenAI( + azure_endpoint=endpoint, + api_key=subscription_key, + api_version="2024-05-01-preview", +) + +# ----- Step 1: caricare datasets ----- +df_labeled = pd.read_csv("main/datasets/annotated_dataset.csv", encoding="cp1252", sep=";") +df_unlabeled = pd.read_csv("main/datasets/unlabeled_dataset.csv", sep="\t", encoding="utf-8") +print("***STEP 1***\nDataset etichettato caricato. Numero righe:", len(df_labeled), "\nDataset non etichettato caricato. Numero righe:", len(df_unlabeled)) + +def clean_id(x): + if pd.isna(x): + return "" + s = str(x) + m = re.search(r"\d+", s) + return m.group(0) if m else s.strip() + +df_labeled["automation_id"] = df_labeled["automation_id"].apply(clean_id) +df_unlabeled["automation_id"] = df_unlabeled["automation_id"].apply(clean_id) +df_labeled["folder"] = df_labeled["folder"].astype(str).str.strip() +df_unlabeled["folder"] = df_unlabeled["folder"].astype(str).str.strip() + +labeled_pairs = set(zip(df_labeled["automation_id"], df_labeled["folder"])) +df_unlabeled_filtered = df_unlabeled[ + ~df_unlabeled.apply(lambda row: (row["automation_id"], row["folder"]) in labeled_pairs, axis=1) +] + + +# Step 3: embeddings --- +print("\n***Step 3 ***\nEmbeddings") +model = SentenceTransformer("all-MiniLM-L6-v2") +texts = df_labeled['automation'].astype(str).tolist() + +with open("main/labeled_embeddings.pkl", "rb") as f: + data = pickle.load(f) + +embeddings = data['embeddings'] +print("Shape embeddings:", embeddings.shape) + + +# ----- Step4: Creazione indice FAISS --- +dimension = embeddings.shape[1] +index = faiss.IndexFlatL2(dimension) # indice L2 (distanza Euclidea) +index.add(embeddings) +print(f"\n***Step 4: Indice FAISS creato***. \nNumero di vettori nell'indice: {index.ntotal}") + +faiss.normalize_L2(embeddings) +dimension = embeddings.shape[1] +index = faiss.IndexFlatIP(dimension) +index.add(embeddings) + + +# Prova con le prima 50 automazioni non annotate +k = 5 +output_rows = [] +df_sample = df_unlabeled.head(50) + +for i, row in df_sample.iterrows(): + query_text = str(row["human_like"]) + + # Calcolo embedding della nuova automazione + query_emb = model.encode([query_text], convert_to_numpy=True).astype("float32") + + # Recupera indici dei k vicini più prossimi + distances, indices = index.search(query_emb, k) + + # Estrae automazioni simili dal DataFrame + for rank in range(k): + idx = indices[0][rank] + distance = distances[0][rank] + confidence = 1 / (1 + float(distance)) + + retrieved_row = df_labeled.iloc[idx] + + output_rows.append({ + "automazione da etichettare": query_text, + "rank": rank + 1, + "automazione simile": retrieved_row["automation"], + "categoria automazione simile": retrieved_row["category"], + "distanza": distance, + "confidence": round(confidence, 4) + }) + +# Creazione DataFrame risultati +df_results = pd.DataFrame(output_rows) +output_path = "main/datasets/similarity_analysis.xlsx" +df_results.to_excel(output_path, index=False) + +wb = load_workbook(output_path) +ws = wb.active + +distanza_col_idx = None +for idx, cell in enumerate(ws[1], start=1): + if cell.value == "distanza": + distanza_col_idx = idx + break + +if distanza_col_idx is None: + raise ValueError("Colonna 'distanza' non trovata!") + +# Applichiamo i colori in base al valore +for row in ws.iter_rows(min_row=2, max_row=ws.max_row, min_col=distanza_col_idx, max_col=distanza_col_idx): + cell = row[0] + try: + val = float(cell.value) + if val < 0.5: + color = "90EE90" # verde chiaro + elif val < 1.0: + color = "FFFF00" # giallo + else: + color = "FF6347" # rosso + cell.fill = PatternFill(start_color=color, end_color=color, fill_type="solid") + except: + continue + +# Salva il file direttamente con colori applicati +wb.save(output_path) +print(f"Excel salvato in {output_path}") \ No newline at end of file diff --git a/annotation/utilities/similarity_analysis.xlsx b/annotation/utilities/similarity_analysis.xlsx new file mode 100644 index 0000000..c7ada1c Binary files /dev/null and b/annotation/utilities/similarity_analysis.xlsx differ diff --git a/annotation/utilities/unlabeled_dataset_creation.py b/annotation/utilities/unlabeled_dataset_creation.py new file mode 100644 index 0000000..6463cb1 --- /dev/null +++ b/annotation/utilities/unlabeled_dataset_creation.py @@ -0,0 +1,35 @@ +# --- Import librerie --- +import pandas as pd +from openai import AzureOpenAI +import os +import json +from sentence_transformers import SentenceTransformer +import numpy as np + +base_folder = r"C:\Users\Arianna\Desktop\AutomationDataset" +dataset_rows = [] +subfolders = [f for f in os.listdir(base_folder) if os.path.isdir(os.path.join(base_folder, f))] +for subfolder in subfolders: + subfolder_path = os.path.join(base_folder, subfolder) + #Cerco file "automation-descriptions" (json) + for file_name in os.listdir(subfolder_path): + if "automation-descriptions" in file_name and file_name.endswith(".json"): + file_path = os.path.join(subfolder_path, file_name) + with open(file_path, "r", encoding="utf-8") as f: + try: + data = json.load(f) + for entry in data: + # Prendo solo il campo human_like + row = { + "automation_id": entry.get("id", ""), + "human_like": entry.get("result", {}).get("human_like", ""), + "folder": subfolder + } + dataset_rows.append(row) + except Exception as e: + print(f"Errore nel file {file_path}: {e}") + +df_unlabeled = pd.DataFrame(dataset_rows) +path = r"C:\Users\Arianna\Desktop\secureTAP+\main\datasets\unlabeled_dataset2.xlsx" +df_unlabeled.to_excel(path, index=False) +