upload annotation dataset
This commit is contained in:
parent
24f940e9af
commit
2b8f32c9c7
|
|
@ -1,316 +1,337 @@
|
|||
# --- Import librerie ---
|
||||
import pandas as pd
|
||||
from openai import AzureOpenAI
|
||||
import pickle
|
||||
from sentence_transformers import SentenceTransformer
|
||||
import numpy as np
|
||||
import faiss
|
||||
import openpyxl
|
||||
import re
|
||||
import json
|
||||
from openpyxl.styles import PatternFill
|
||||
from openpyxl import load_workbook
|
||||
from collections import Counter
|
||||
from prompts.prompt import build_prompt_local
|
||||
import warnings
|
||||
import logging
|
||||
import unicodedata
|
||||
|
||||
# --- Configurazione ---
|
||||
endpoint = "https://gpt-sw-central-tap-security.openai.azure.com/"
|
||||
deployment = "gpt-5.1-chat-3"
|
||||
subscription_key = "8zufUIPs0Dijh0M6NpifkkDvxJHZMFtott7u8V8ySTYNcpYVoRbsJQQJ99BBACfhMk5XJ3w3AAABACOGr6sq"
|
||||
|
||||
client = AzureOpenAI(
|
||||
azure_endpoint=endpoint,
|
||||
api_key=subscription_key,
|
||||
api_version="2025-04-01-preview",
|
||||
)
|
||||
|
||||
# ----- Step 1: caricare datasets -----
|
||||
#df_labeled = pd.read_excel("main/datasets/annotated_dataset.xlsx").dropna(how="all")
|
||||
df_labeled = pd.read_excel("main/datasets/annotated_dataset_updated.xlsx").dropna(how="all")
|
||||
df_unlabeled = pd.read_excel("main/datasets/unlabeled_dataset.xlsx").dropna(how="all")
|
||||
print("***STEP 1***\nDataset etichettato caricato. Numero righe:", len(df_labeled), "\nDataset non etichettato caricato. Numero righe:", len(df_unlabeled))
|
||||
df_labeled = df_labeled.rename(columns={"automation_id": "id"})
|
||||
df_unlabeled = df_unlabeled.rename(columns={"automation_id": "id"})
|
||||
|
||||
# Pulizia colonne
|
||||
def clean_id(x):
|
||||
if pd.isna(x):
|
||||
return ""
|
||||
s = str(x).strip() # rimuove spazi
|
||||
s = s.strip('"').strip("'") # rimuove eventuali virgolette
|
||||
return s.lower()
|
||||
|
||||
def clean_folder(x):
|
||||
"""Pulizia folder: rimuove spazi multipli, normalizza unicode."""
|
||||
if pd.isna(x):
|
||||
return ""
|
||||
s = str(x).strip().lower()
|
||||
s = unicodedata.normalize("NFKC", s)
|
||||
s = re.sub(r'\s+', ' ', s)
|
||||
return s
|
||||
|
||||
for df in [df_labeled, df_unlabeled]:
|
||||
df["id"] = df["id"].apply(clean_id)
|
||||
df["folder"] = df["folder"].apply(clean_folder)
|
||||
|
||||
labeled_pairs = set(zip(df_labeled["id"], df_labeled["folder"]))
|
||||
|
||||
# crea maschera: True = la riga NON è presente in labeled
|
||||
mask_unlabeled = ~df_unlabeled.apply(lambda r: (r["id"], r["folder"]) in labeled_pairs, axis=1)
|
||||
# filtra
|
||||
df_unlabeled_filtered = df_unlabeled[mask_unlabeled].copy()
|
||||
|
||||
print("Numero righe df_unlabeled dopo aver rimosso quelle già etichettate:", len(df_unlabeled_filtered))
|
||||
|
||||
unlabeled_pairs = set(zip(df_unlabeled["id"], df_unlabeled["folder"]))
|
||||
missing_in_unlabeled = labeled_pairs - unlabeled_pairs
|
||||
print("Numero coppie etichettate non presenti in unlabeled:", len(missing_in_unlabeled))
|
||||
if missing_in_unlabeled:
|
||||
print("Coppie mancanti:")
|
||||
for p in list(missing_in_unlabeled)[:50]: # stampa solo le prime 50 per comodità
|
||||
print(p)
|
||||
|
||||
# ----- Step 2: embeddings -----
|
||||
# Silenzia warning generici
|
||||
warnings.filterwarnings("ignore")
|
||||
# Silenzia logging di transformers / sentence-transformers / HF hub
|
||||
logging.getLogger("sentence_transformers").setLevel(logging.ERROR)
|
||||
logging.getLogger("transformers").setLevel(logging.ERROR)
|
||||
logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
|
||||
|
||||
print("\n***Step 2***\nEmbeddings")
|
||||
model = SentenceTransformer("all-MiniLM-L6-v2")
|
||||
|
||||
#with open("main/labeled_embeddings_71.pkl", "rb") as f:
|
||||
with open("main/labeled_embeddings.pkl", "rb") as f:
|
||||
data = pickle.load(f)
|
||||
|
||||
embeddings = data["embeddings"].astype("float32")
|
||||
print("Shape embeddings:", embeddings.shape)
|
||||
|
||||
|
||||
|
||||
# ----- Step3: Creazione indice FAISS ---
|
||||
faiss.normalize_L2(embeddings)
|
||||
dimension = embeddings.shape[1]
|
||||
index = faiss.IndexFlatIP(dimension)
|
||||
index.add(embeddings)
|
||||
print(f"\n***Step 3: Indice FAISS creato***.\nNumero di vettori nell'indice: {index.ntotal}")
|
||||
|
||||
|
||||
# ----- Step 4: Retrieval (similarità cosine) -----
|
||||
k = 5
|
||||
output_rows = []
|
||||
df_sample = df_unlabeled_filtered.head(10).reset_index(drop=True)
|
||||
llm_rows = []
|
||||
|
||||
def sim_label(sim: float) -> str:
|
||||
# più alto = più simile
|
||||
if sim >= 0.80:
|
||||
return "Match forte"
|
||||
elif sim >= 0.60:
|
||||
return "Match plausibile"
|
||||
elif sim >= 0.50:
|
||||
return "Similarità instabile"
|
||||
else:
|
||||
return "Debole"
|
||||
|
||||
for count, (_, row) in enumerate(df_sample.iterrows(), start=1):
|
||||
query_text = str(row["human_like"])
|
||||
print("automazione analizzata:", count)
|
||||
|
||||
# Calcolo embedding della nuova automazione
|
||||
query_emb = model.encode([query_text], convert_to_numpy=True).astype("float32")
|
||||
faiss.normalize_L2(query_emb)
|
||||
|
||||
# Recupera indici dei k vicini più prossimi
|
||||
sims, indices = index.search(query_emb, k)
|
||||
|
||||
# Metriche globali sui top-k (una volta per automazione)
|
||||
topk_cats = []
|
||||
top1_sim = float(sims[0][0])
|
||||
top1_similarity_label = sim_label(top1_sim)
|
||||
|
||||
for rank in range(k):
|
||||
idx = int(indices[0][rank])
|
||||
sim = float(sims[0][rank])
|
||||
|
||||
retrieved_row = df_labeled.iloc[idx]
|
||||
topk_cats.append(str(retrieved_row.get("category", "")))
|
||||
|
||||
rank1_category = topk_cats[0] if topk_cats else ""
|
||||
majority_category = Counter(topk_cats).most_common(1)[0][0] if topk_cats else ""
|
||||
consistency = (sum(c == majority_category for c in topk_cats) / len(topk_cats)) if topk_cats else 0.0
|
||||
|
||||
for rank in range(k):
|
||||
idx = int(indices[0][rank])
|
||||
sim = float(sims[0][rank])
|
||||
label = sim_label(sim)
|
||||
|
||||
retrieved_row = df_labeled.iloc[idx]
|
||||
|
||||
output_rows.append({
|
||||
"automazione da etichettare": query_text,
|
||||
# info retrieval per questa riga
|
||||
"rank": rank + 1,
|
||||
"retrieved_idx": idx,
|
||||
"automazione simile": retrieved_row.get("automation", ""),
|
||||
"categoria automazione simile": retrieved_row.get("category", ""),
|
||||
"similarita_cosine": sim,
|
||||
"similarity_label": label,
|
||||
# metriche aggregate top-k (ripetute su ogni riga)
|
||||
"rank1_similarity": top1_sim,
|
||||
"rank1_similarity_label": top1_similarity_label,
|
||||
"rank1_category": rank1_category,
|
||||
"majority_category": majority_category,
|
||||
"consistency": round(consistency, 3),
|
||||
"top5_categories": " | ".join(topk_cats),
|
||||
})
|
||||
|
||||
|
||||
# ----- Step 5: invio dati al LLM -----
|
||||
# (1) Costruzione prompt
|
||||
retrieved = df_labeled.iloc[indices[0]].copy()
|
||||
retrieved["similarity"] = sims[0].astype(float)
|
||||
retrieved["similarity_label"] = retrieved["similarity"].apply(sim_label)
|
||||
prompt = build_prompt_local(query_text, retrieved, sim_label)
|
||||
|
||||
# (2) Chiamata al modello: restituisce JSON
|
||||
resp = client.chat.completions.create(
|
||||
model=deployment,
|
||||
messages=[
|
||||
{"role": "system", "content": prompt},
|
||||
{"role": "user", "content": f'automation to evaluate: {query_text}'}
|
||||
],
|
||||
reasoning_effort= "low"
|
||||
)
|
||||
content = resp.choices[0].message.content.strip()
|
||||
|
||||
# (3) Parsing della risposta
|
||||
try:
|
||||
parsed = json.loads(content)
|
||||
except Exception:
|
||||
parsed = {
|
||||
"automation": query_text,
|
||||
"category": "",
|
||||
"subcategory": "",
|
||||
"problem_type": "",
|
||||
"gravity": "",
|
||||
"scores": {},
|
||||
"needs_human_review": True,
|
||||
"short_rationale": f"JSON_PARSE_ERROR: {content[:200]}",
|
||||
}
|
||||
|
||||
# (4) Salvataggio di 1 riga per automazione con:
|
||||
# - metriche retrieval (rank1/majority/consistency)
|
||||
# - output dell'LLM (scores + label finale + review flag)
|
||||
llm_category = str(parsed.get("category", "")).strip()
|
||||
llm_subcategory = str(parsed.get("subcategory", "")).strip()
|
||||
llm_problem_type = str(parsed.get("problem_type", "")).strip()
|
||||
llm_gravity = str(parsed.get("gravity", "")).strip()
|
||||
if llm_category.upper() == "HARMLESS":
|
||||
llm_subcategory = ""
|
||||
llm_problem_type = "none"
|
||||
llm_gravity = "NONE"
|
||||
# di default l'etichetta assegnata è quella del LLM - rivista se review=true
|
||||
final_category = llm_category
|
||||
final_subcategory = llm_subcategory
|
||||
final_problem_type = llm_problem_type
|
||||
final_gravity = llm_gravity
|
||||
|
||||
|
||||
if top1_similarity_label == "Debole" or top1_similarity_label == "Similarità instabile":
|
||||
needs_review = True
|
||||
else:
|
||||
needs_review = False
|
||||
|
||||
|
||||
final_needs_review = needs_review
|
||||
# ================= HUMAN REVIEW LOGIC =================
|
||||
aligned_strong = (
|
||||
llm_category == majority_category
|
||||
and llm_category == rank1_category
|
||||
and llm_category != ""
|
||||
)
|
||||
|
||||
OVERRIDE_MIN_SIMILARITY = 0.39
|
||||
OVERRIDE_MIN_CONSISTENCY = 0.60
|
||||
|
||||
good_retrieval = (
|
||||
top1_sim >= OVERRIDE_MIN_SIMILARITY
|
||||
and consistency >= OVERRIDE_MIN_CONSISTENCY
|
||||
)
|
||||
|
||||
if aligned_strong and good_retrieval:
|
||||
final_needs_review = False
|
||||
# =====================================================
|
||||
|
||||
|
||||
llm_rows.append({
|
||||
"id": row.get("id", ""),
|
||||
"folder": row.get("folder", ""),
|
||||
"automation_text": query_text,
|
||||
|
||||
# Retrieval metrics
|
||||
"rank1_similarity": top1_sim,
|
||||
"rank1_similarity_label": top1_similarity_label,
|
||||
"rank1_category": rank1_category,
|
||||
"majority_category": majority_category,
|
||||
"consistency": round(consistency, 3),
|
||||
"top5_categories": " | ".join(topk_cats),
|
||||
|
||||
# LLM
|
||||
"llm_category": llm_category,
|
||||
"llm_subcategory": llm_subcategory,
|
||||
"llm_problem_type": llm_problem_type,
|
||||
"llm_gravity": llm_gravity,
|
||||
|
||||
"needs_review": needs_review,
|
||||
"final_needs_review": final_needs_review,
|
||||
|
||||
# FINAL
|
||||
"final_category": final_category,
|
||||
"final_subcategory": final_subcategory,
|
||||
"final_problem_type": final_problem_type,
|
||||
"final_gravity": final_gravity,
|
||||
|
||||
"llm_rationale": parsed.get("short_rationale", ""),
|
||||
})
|
||||
|
||||
|
||||
# ----- Step 6: output Excel -----
|
||||
df_out = pd.DataFrame(llm_rows)
|
||||
out_path = "main/datasets/labeling_2_500.xlsx"
|
||||
df_out.to_excel(out_path, index=False)
|
||||
|
||||
wb = load_workbook(out_path)
|
||||
ws = wb.active
|
||||
|
||||
true_fill = PatternFill(start_color="FF6347", end_color="FF6347", fill_type="solid") # rosso
|
||||
false_fill = PatternFill(start_color="90EE90", end_color="90EE90", fill_type="solid") # verde
|
||||
|
||||
col_index = {cell.value: idx for idx, cell in enumerate(ws[1], start=1)}
|
||||
|
||||
for col_name in ["needs_review", "final_needs_review"]:
|
||||
if col_name in col_index:
|
||||
c = col_index[col_name]
|
||||
for r in range(2, ws.max_row + 1):
|
||||
val = ws.cell(row=r, column=c).value
|
||||
if val is True:
|
||||
ws.cell(row=r, column=c).fill = true_fill
|
||||
elif val is False:
|
||||
ws.cell(row=r, column=c).fill = false_fill
|
||||
|
||||
wb.save(out_path)
|
||||
print(f"\n***Step 6: Excel salvato in {out_path}")
|
||||
|
||||
# --- Conteggio needs_human_review ---
|
||||
review_counts = df_out["final_needs_review"].value_counts(dropna=False)
|
||||
true_count = review_counts.get(True, 0)
|
||||
false_count = review_counts.get(False, 0)
|
||||
print("\n--- Needs human review summary ---")
|
||||
print(f"needs_human_review = True : {true_count}")
|
||||
# --- Import librerie ---
|
||||
import pandas as pd
|
||||
from openai import AzureOpenAI
|
||||
import pickle
|
||||
from sentence_transformers import SentenceTransformer
|
||||
import numpy as np
|
||||
import faiss
|
||||
import openpyxl
|
||||
import re
|
||||
import json
|
||||
from openpyxl.styles import PatternFill
|
||||
from openpyxl import load_workbook
|
||||
from collections import Counter
|
||||
from prompts.prompt import build_prompt_local
|
||||
import warnings
|
||||
import logging
|
||||
import unicodedata
|
||||
|
||||
# --- Configurazione ---
|
||||
endpoint = "https://gpt-sw-central-tap-security.openai.azure.com/"
|
||||
deployment = "gpt-5.1-chat-3"
|
||||
subscription_key = "8zufUIPs0Dijh0M6NpifkkDvxJHZMFtott7u8V8ySTYNcpYVoRbsJQQJ99BBACfhMk5XJ3w3AAABACOGr6sq"
|
||||
|
||||
client = AzureOpenAI(
|
||||
azure_endpoint=endpoint,
|
||||
api_key=subscription_key,
|
||||
api_version="2025-04-01-preview",
|
||||
)
|
||||
|
||||
# ----- Step 1: caricare datasets -----
|
||||
df_labeled = pd.read_excel("main/datasets/annotated_dataset_updated2.xlsx").dropna(how="all") # DATASET GIA' ETICHETTATO
|
||||
df_unlabeled = pd.read_excel("main/datasets/unlabeled_dataset.xlsx").dropna(how="all") # DATASET DA ETICHETTARE
|
||||
print("***STEP 1***\nDataset etichettato caricato. Numero righe:", len(df_labeled), "\nDataset non etichettato caricato. Numero righe:", len(df_unlabeled))
|
||||
df_labeled = df_labeled.rename(columns={"automation_id": "id"})
|
||||
df_unlabeled = df_unlabeled.rename(columns={"automation_id": "id"})
|
||||
|
||||
# Pulizia colonne
|
||||
def clean_id(x):
|
||||
if pd.isna(x):
|
||||
return ""
|
||||
s = str(x).strip() # rimuove spazi
|
||||
s = s.strip('"').strip("'") # rimuove eventuali virgolette
|
||||
return s.lower()
|
||||
|
||||
def clean_folder(x):
|
||||
"""Pulizia folder: rimuove spazi multipli, normalizza unicode."""
|
||||
if pd.isna(x):
|
||||
return ""
|
||||
s = str(x).strip().lower()
|
||||
s = s.strip('"').strip("'")
|
||||
s = unicodedata.normalize("NFKC", s)
|
||||
s = re.sub(r'\s+', ' ', s)
|
||||
return s
|
||||
|
||||
for df in [df_labeled, df_unlabeled]:
|
||||
df["id"] = df["id"].apply(clean_id)
|
||||
df["folder"] = df["folder"].apply(clean_folder)
|
||||
|
||||
labeled_pairs = set(zip(df_labeled["id"], df_labeled["folder"]))
|
||||
|
||||
# crea maschera: True = la riga NON è presente in labeled
|
||||
mask_unlabeled = ~df_unlabeled.apply(lambda r: (r["id"], r["folder"]) in labeled_pairs, axis=1)
|
||||
# filtra
|
||||
df_unlabeled_filtered = df_unlabeled[mask_unlabeled].copy()
|
||||
|
||||
print("Numero righe df_unlabeled dopo aver rimosso quelle già etichettate:", len(df_unlabeled_filtered))
|
||||
|
||||
unlabeled_pairs = set(zip(df_unlabeled["id"], df_unlabeled["folder"]))
|
||||
missing_in_unlabeled = labeled_pairs - unlabeled_pairs
|
||||
print("Numero coppie etichettate non presenti in unlabeled:", len(missing_in_unlabeled))
|
||||
if missing_in_unlabeled:
|
||||
print("Coppie mancanti:")
|
||||
for p in list(missing_in_unlabeled)[:50]: # stampa solo le prime 50 per comodità
|
||||
print(p)
|
||||
|
||||
# ----- Step 2: embeddings -----
|
||||
# Silenzia warning generici
|
||||
warnings.filterwarnings("ignore")
|
||||
# Silenzia logging di transformers / sentence-transformers / HF hub
|
||||
logging.getLogger("sentence_transformers").setLevel(logging.ERROR)
|
||||
logging.getLogger("transformers").setLevel(logging.ERROR)
|
||||
logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
|
||||
|
||||
print("\n***Step 2***\nEmbeddings")
|
||||
model = SentenceTransformer("all-MiniLM-L6-v2")
|
||||
|
||||
with open("main/labeled_embeddings.pkl", "rb") as f:
|
||||
data = pickle.load(f)
|
||||
|
||||
embeddings = data["embeddings"].astype("float32")
|
||||
print("Shape embeddings:", embeddings.shape)
|
||||
|
||||
|
||||
|
||||
# ----- Step3: Creazione indice FAISS ---
|
||||
faiss.normalize_L2(embeddings)
|
||||
dimension = embeddings.shape[1]
|
||||
index = faiss.IndexFlatIP(dimension)
|
||||
index.add(embeddings)
|
||||
print(f"\n***Step 3: Indice FAISS creato***.\nNumero di vettori nell'indice: {index.ntotal}")
|
||||
|
||||
|
||||
# ----- Step 4: Retrieval (similarità cosine) -----
|
||||
k = 5
|
||||
output_rows = []
|
||||
df_sample = df_unlabeled_filtered.head(1000).reset_index(drop=True)
|
||||
llm_rows = []
|
||||
|
||||
def sim_label(sim: float) -> str:
|
||||
# più alto = più simile
|
||||
if sim >= 0.80:
|
||||
return "Match forte"
|
||||
elif sim >= 0.60:
|
||||
return "Match plausibile"
|
||||
elif sim >= 0.50:
|
||||
return "Similarità instabile"
|
||||
else:
|
||||
return "Debole"
|
||||
|
||||
for count, (_, row) in enumerate(df_sample.iterrows(), start=1):
|
||||
query_text = str(row["human_like"])
|
||||
print("automazione analizzata:", count)
|
||||
|
||||
# Calcolo embedding della nuova automazione
|
||||
query_emb = model.encode([query_text], convert_to_numpy=True).astype("float32")
|
||||
faiss.normalize_L2(query_emb)
|
||||
|
||||
# Recupera indici dei k vicini più prossimi
|
||||
sims, indices = index.search(query_emb, k)
|
||||
|
||||
# Metriche globali sui top-k (una volta per automazione)
|
||||
topk_cats = []
|
||||
top1_sim = float(sims[0][0])
|
||||
top1_similarity_label = sim_label(top1_sim)
|
||||
|
||||
for rank in range(k):
|
||||
idx = int(indices[0][rank])
|
||||
sim = float(sims[0][rank])
|
||||
|
||||
retrieved_row = df_labeled.iloc[idx]
|
||||
topk_cats.append(str(retrieved_row.get("category", "")))
|
||||
|
||||
rank1_category = topk_cats[0] if topk_cats else ""
|
||||
majority_category = Counter(topk_cats).most_common(1)[0][0] if topk_cats else ""
|
||||
consistency = (sum(c == majority_category for c in topk_cats) / len(topk_cats)) if topk_cats else 0.0
|
||||
|
||||
for rank in range(k):
|
||||
idx = int(indices[0][rank])
|
||||
sim = float(sims[0][rank])
|
||||
label = sim_label(sim)
|
||||
|
||||
retrieved_row = df_labeled.iloc[idx]
|
||||
|
||||
output_rows.append({
|
||||
"automazione da etichettare": query_text,
|
||||
# info retrieval per questa riga
|
||||
"rank": rank + 1,
|
||||
"retrieved_idx": idx,
|
||||
"automazione simile": retrieved_row.get("automation", ""),
|
||||
"categoria automazione simile": retrieved_row.get("category", ""),
|
||||
"similarita_cosine": sim,
|
||||
"similarity_label": label,
|
||||
# metriche aggregate top-k (ripetute su ogni riga)
|
||||
"rank1_similarity": top1_sim,
|
||||
"rank1_similarity_label": top1_similarity_label,
|
||||
"rank1_category": rank1_category,
|
||||
"majority_category": majority_category,
|
||||
"consistency": round(consistency, 3),
|
||||
"top5_categories": " | ".join(topk_cats),
|
||||
})
|
||||
|
||||
|
||||
# ----- Step 5: invio dati al LLM -----
|
||||
# (1) Costruzione prompt
|
||||
retrieved = df_labeled.iloc[indices[0]].copy()
|
||||
retrieved["similarity"] = sims[0].astype(float)
|
||||
retrieved["similarity_label"] = retrieved["similarity"].apply(sim_label)
|
||||
prompt = build_prompt_local(query_text, retrieved, sim_label)
|
||||
|
||||
# (2) Chiamata al modello: restituisce JSON
|
||||
resp = client.chat.completions.create(
|
||||
model=deployment,
|
||||
messages=[
|
||||
{"role": "system", "content": prompt},
|
||||
{"role": "user", "content": f'automation to evaluate: {query_text}'}
|
||||
],
|
||||
reasoning_effort= "low"
|
||||
)
|
||||
content = resp.choices[0].message.content.strip()
|
||||
|
||||
# (3) Parsing della risposta
|
||||
def extract_json(content: str) -> dict:
|
||||
"""
|
||||
Estrae il primo JSON valido trovato in una stringa.
|
||||
Restituisce un dizionario vuoto in caso di fallimento.
|
||||
"""
|
||||
# Rimuove eventuali blocchi ```json o ``` generici
|
||||
content = re.sub(r"```(?:json)?\s*", "", content)
|
||||
content = re.sub(r"\s*```", "", content)
|
||||
|
||||
# Trova la prima occorrenza che assomiglia a JSON
|
||||
match = re.search(r"\{.*\}", content, re.DOTALL)
|
||||
if match:
|
||||
json_text = match.group(0)
|
||||
try:
|
||||
return json.loads(json_text)
|
||||
except json.JSONDecodeError:
|
||||
return {
|
||||
"automation": "",
|
||||
"category": "",
|
||||
"subcategory": "",
|
||||
"problem_type": "",
|
||||
"gravity": "",
|
||||
"scores": {},
|
||||
"short_rationale": f"JSON_PARSE_ERROR: {json_text[:200]}"
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"automation": "",
|
||||
"category": "",
|
||||
"subcategory": "",
|
||||
"problem_type": "",
|
||||
"gravity": "",
|
||||
"scores": {},
|
||||
"short_rationale": f"JSON_PARSE_ERROR: No JSON found in content"
|
||||
}
|
||||
|
||||
parsed = extract_json(content)
|
||||
|
||||
# (4) Salvataggio di 1 riga per automazione con:
|
||||
# - metriche retrieval (rank1/majority/consistency)
|
||||
# - output dell'LLM (scores + label finale + review flag)
|
||||
llm_category = str(parsed.get("category", "")).strip()
|
||||
llm_subcategory = str(parsed.get("subcategory", "")).strip()
|
||||
llm_problem_type = str(parsed.get("problem_type", "")).strip()
|
||||
llm_gravity = str(parsed.get("gravity", "")).strip()
|
||||
if llm_category.upper() == "HARMLESS":
|
||||
llm_problem_type = "none"
|
||||
llm_gravity = "NONE"
|
||||
# di default l'etichetta assegnata è quella del LLM - rivista se review=true
|
||||
final_category = llm_category
|
||||
final_subcategory = llm_subcategory
|
||||
final_problem_type = llm_problem_type
|
||||
final_gravity = llm_gravity
|
||||
|
||||
|
||||
if top1_similarity_label == "Debole" or top1_similarity_label == "Similarità instabile":
|
||||
needs_review = True
|
||||
else:
|
||||
needs_review = False
|
||||
|
||||
final_needs_review = needs_review
|
||||
# ================= HUMAN REVIEW LOGIC =================
|
||||
aligned_strong = (
|
||||
llm_category == majority_category
|
||||
and llm_category == rank1_category
|
||||
and llm_category != ""
|
||||
)
|
||||
|
||||
OVERRIDE_MIN_SIMILARITY = 0.39
|
||||
OVERRIDE_MIN_CONSISTENCY = 0.60
|
||||
|
||||
good_retrieval = (
|
||||
top1_sim >= OVERRIDE_MIN_SIMILARITY
|
||||
and consistency >= OVERRIDE_MIN_CONSISTENCY
|
||||
)
|
||||
|
||||
if aligned_strong and good_retrieval:
|
||||
final_needs_review = False
|
||||
# =====================================================
|
||||
|
||||
|
||||
llm_rows.append({
|
||||
"id": row.get("id", ""),
|
||||
"folder": row.get("folder", ""),
|
||||
"automation_text": query_text,
|
||||
|
||||
# Retrieval metrics
|
||||
"rank1_similarity": top1_sim,
|
||||
"rank1_similarity_label": top1_similarity_label,
|
||||
"rank1_category": rank1_category,
|
||||
"majority_category": majority_category,
|
||||
"consistency": round(consistency, 3),
|
||||
"top5_categories": " | ".join(topk_cats),
|
||||
|
||||
# LLM
|
||||
"llm_category": llm_category,
|
||||
"llm_subcategory": llm_subcategory,
|
||||
"llm_problem_type": llm_problem_type,
|
||||
"llm_gravity": llm_gravity,
|
||||
|
||||
"needs_review": needs_review,
|
||||
"final_needs_review": final_needs_review,
|
||||
|
||||
# FINAL
|
||||
"final_category": final_category,
|
||||
"final_subcategory": final_subcategory,
|
||||
"final_problem_type": final_problem_type,
|
||||
"final_gravity": final_gravity,
|
||||
|
||||
"llm_rationale": parsed.get("short_rationale", ""),
|
||||
})
|
||||
|
||||
|
||||
# ----- Step 6: output Excel -----
|
||||
df_out = pd.DataFrame(llm_rows)
|
||||
out_path = "main/datasets/12500_13463.xlsx"
|
||||
df_out.to_excel(out_path, index=False)
|
||||
|
||||
wb = load_workbook(out_path)
|
||||
ws = wb.active
|
||||
|
||||
true_fill = PatternFill(start_color="FF6347", end_color="FF6347", fill_type="solid") # rosso
|
||||
false_fill = PatternFill(start_color="90EE90", end_color="90EE90", fill_type="solid") # verde
|
||||
|
||||
col_index = {cell.value: idx for idx, cell in enumerate(ws[1], start=1)}
|
||||
|
||||
for col_name in ["needs_review", "final_needs_review"]:
|
||||
if col_name in col_index:
|
||||
c = col_index[col_name]
|
||||
for r in range(2, ws.max_row + 1):
|
||||
val = ws.cell(row=r, column=c).value
|
||||
if val is True:
|
||||
ws.cell(row=r, column=c).fill = true_fill
|
||||
elif val is False:
|
||||
ws.cell(row=r, column=c).fill = false_fill
|
||||
|
||||
wb.save(out_path)
|
||||
print(f"\n***Step 6: Excel salvato in {out_path}")
|
||||
|
||||
# --- Conteggio needs_human_review ---
|
||||
review_counts = df_out["final_needs_review"].value_counts(dropna=False)
|
||||
true_count = review_counts.get(True, 0)
|
||||
false_count = review_counts.get(False, 0)
|
||||
print("\n--- Needs human review summary ---")
|
||||
print(f"needs_human_review = True : {true_count}")
|
||||
print(f"needs_human_review = False: {false_count}")
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -1,127 +1,132 @@
|
|||
# --- Import librerie ---
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import unicodedata
|
||||
import re
|
||||
import warnings
|
||||
from sentence_transformers import SentenceTransformer
|
||||
import pickle
|
||||
|
||||
# ----- Percorsi file -----
|
||||
LABELED_IN = "main/datasets/annotated_dataset.xlsx"
|
||||
REVIEWED = "main/datasets/first500_reviewed.xlsx"
|
||||
LABELED_OUT = "main/datasets/annotated_dataset_updated.xlsx"
|
||||
|
||||
# ----- Funzioni di pulizia -----
|
||||
def clean(x):
|
||||
if pd.isna(x):
|
||||
return ""
|
||||
return str(x).strip()
|
||||
|
||||
def normalize_problem_type(x):
|
||||
x = clean(x).upper()
|
||||
if x == "S":
|
||||
return "RULE_SPECIFIC"
|
||||
if x == "G":
|
||||
return "GENERIC"
|
||||
return x
|
||||
|
||||
def normalize_severity(x):
|
||||
return clean(x).upper()
|
||||
|
||||
def clean_id(x):
|
||||
if pd.isna(x):
|
||||
return ""
|
||||
s = str(x).strip().strip('"').strip("'")
|
||||
return s.lower()
|
||||
|
||||
def clean_folder(x):
|
||||
if pd.isna(x):
|
||||
return ""
|
||||
s = str(x).strip().lower()
|
||||
s = unicodedata.normalize("NFKC", s)
|
||||
s = re.sub(r'\s+', ' ', s)
|
||||
return s
|
||||
|
||||
# ----- Step 1: caricare datasets -----
|
||||
df_labeled = pd.read_excel(LABELED_IN)
|
||||
df_labeled = df_labeled.loc[:, ~df_labeled.columns.str.contains("^Unnamed")].copy()
|
||||
df_labeled = df_labeled.dropna(how="all")
|
||||
df_labeled = df_labeled.rename(columns={"automation_id": "id"})
|
||||
|
||||
df_rev = pd.read_excel(REVIEWED)
|
||||
|
||||
# Normalizzazione problem_type e severity
|
||||
if "error_type" in df_labeled.columns:
|
||||
df_labeled["error_type"] = df_labeled["error_type"].apply(normalize_problem_type)
|
||||
|
||||
# Costruzione dataset pulito dai primi 500
|
||||
rows = []
|
||||
for _, r in df_rev.iterrows():
|
||||
category = clean(r["final_category"])
|
||||
subcategory = clean(r["final_subcategory"])
|
||||
error_type = normalize_problem_type(r["final_problem_type"])
|
||||
severity = normalize_severity(r["final_gravity"])
|
||||
|
||||
# Coerenza HARMLESS
|
||||
if category.upper() == "HARMLESS":
|
||||
subcategory = ""
|
||||
error_type = "none"
|
||||
severity = "none"
|
||||
|
||||
rows.append({
|
||||
"id": clean(r["id"]),
|
||||
"folder": clean(r["folder"]),
|
||||
"automation": clean(r["automation_text"]),
|
||||
"description": clean(r.get("llm_rationale", "")),
|
||||
"category": category,
|
||||
"subcategory": subcategory,
|
||||
"error_type": error_type,
|
||||
"severity": severity,
|
||||
"borderline": clean(r["borderline"]),
|
||||
})
|
||||
|
||||
df_new = pd.DataFrame(rows)
|
||||
|
||||
# Normalizzazione valori 'none'
|
||||
df_new["error_type"] = df_new["error_type"].apply(lambda x: x.lower() if x.lower() == "none" else x)
|
||||
df_new["severity"] = df_new["severity"].apply(lambda x: x.lower() if x.lower() == "none" else x)
|
||||
|
||||
# Rimuovi righe senza categoria
|
||||
df_new = df_new[df_new["category"] != ""].copy()
|
||||
|
||||
# Pulizia id e folder in entrambi i dataset
|
||||
for df in [df_labeled, df_new]:
|
||||
df["id"] = df["id"].apply(clean_id)
|
||||
df["folder"] = df["folder"].apply(clean_folder)
|
||||
|
||||
# Rimuovere duplicati: eliminare dal labeled righe già presenti in df_new
|
||||
new_keys = set(zip(df_new["id"], df_new["folder"]))
|
||||
df_labeled_clean = df_labeled[~df_labeled.apply(lambda r: (r["id"], r["folder"]) in new_keys, axis=1)].copy()
|
||||
|
||||
# Concat finale
|
||||
df_final = pd.concat([df_labeled_clean, df_new], ignore_index=True).fillna("")
|
||||
|
||||
# Salva dataset aggiornato
|
||||
df_final.to_excel(LABELED_OUT, index=False)
|
||||
print("✅ Merge completato")
|
||||
print("Righe iniziali:", len(df_labeled))
|
||||
print("Righe aggiunte:", len(df_rev))
|
||||
print("Totale finale:", len(df_final))
|
||||
|
||||
# ----- Step 2: calcolo embeddings -----
|
||||
warnings.filterwarnings("ignore")
|
||||
model = SentenceTransformer("all-MiniLM-L6-v2")
|
||||
|
||||
texts = df_final["automation"].tolist()
|
||||
embeddings = model.encode(
|
||||
texts, show_progress_bar=True, convert_to_numpy=True, normalize_embeddings=True
|
||||
).astype("float32")
|
||||
|
||||
print("Shape embeddings ricalcolati:", embeddings.shape)
|
||||
|
||||
# ----- Step 3: salva embeddings -----
|
||||
with open("main/labeled_embeddings.pkl", "wb") as f:
|
||||
pickle.dump({"embeddings": embeddings, "id": df_final["id"].tolist()}, f)
|
||||
|
||||
# --- Import librerie ---
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import unicodedata
|
||||
import re
|
||||
import warnings
|
||||
from sentence_transformers import SentenceTransformer
|
||||
import pickle
|
||||
import logging
|
||||
|
||||
# ----- Percorsi file -----
|
||||
#LABELED_IN = "main/datasets/annotated_dataset.xlsx"
|
||||
LABELED_IN = "main/datasets/annotated_dataset_updated.xlsx"
|
||||
#REVIEWED = "main/datasets/first500_reviewed.xlsx"
|
||||
REVIEWED = "main/datasets/12500_13463.xlsx"
|
||||
LABELED_OUT = "main/datasets/annotated_dataset_updated2.xlsx"
|
||||
|
||||
# ----- Funzioni di pulizia -----
|
||||
def clean(x):
|
||||
if pd.isna(x):
|
||||
return ""
|
||||
return str(x).strip()
|
||||
|
||||
def normalize_problem_type(x):
|
||||
x = clean(x).upper()
|
||||
if x == "S":
|
||||
return "RULE_SPECIFIC"
|
||||
if x == "G":
|
||||
return "GENERIC"
|
||||
return x
|
||||
|
||||
def normalize_severity(x):
|
||||
return clean(x).upper()
|
||||
|
||||
def clean_id(x):
|
||||
if pd.isna(x):
|
||||
return ""
|
||||
s = str(x).strip().strip('"').strip("'")
|
||||
return s.lower()
|
||||
|
||||
def clean_folder(x):
|
||||
if pd.isna(x):
|
||||
return ""
|
||||
s = str(x).strip().lower()
|
||||
s = unicodedata.normalize("NFKC", s)
|
||||
s = re.sub(r'\s+', ' ', s)
|
||||
return s
|
||||
|
||||
# ----- Step 1: caricare datasets -----
|
||||
df_labeled = pd.read_excel(LABELED_IN)
|
||||
df_labeled = df_labeled.loc[:, ~df_labeled.columns.str.contains("^Unnamed")].copy()
|
||||
df_labeled = df_labeled.dropna(how="all")
|
||||
df_labeled = df_labeled.rename(columns={"automation_id": "id"})
|
||||
|
||||
df_rev = pd.read_excel(REVIEWED)
|
||||
|
||||
# Normalizzazione problem_type e severity
|
||||
if "error_type" in df_labeled.columns:
|
||||
df_labeled["error_type"] = df_labeled["error_type"].apply(normalize_problem_type)
|
||||
|
||||
# Costruzione dataset pulito dai primi 500
|
||||
rows = []
|
||||
for _, r in df_rev.iterrows():
|
||||
category = clean(r["final_category"])
|
||||
subcategory = clean(r["final_subcategory"])
|
||||
error_type = normalize_problem_type(r["final_problem_type"])
|
||||
severity = normalize_severity(r["final_gravity"])
|
||||
|
||||
# Coerenza HARMLESS
|
||||
if category.upper() == "HARMLESS":
|
||||
error_type = "none"
|
||||
severity = "none"
|
||||
|
||||
rows.append({
|
||||
"id": clean(r["id"]),
|
||||
"folder": clean(r["folder"]),
|
||||
"automation": clean(r["automation_text"]),
|
||||
"description": clean(r.get("llm_rationale", "")),
|
||||
"category": category,
|
||||
"subcategory": subcategory,
|
||||
"error_type": error_type,
|
||||
"severity": severity,
|
||||
"borderline": clean(r["borderline"]),
|
||||
})
|
||||
|
||||
df_new = pd.DataFrame(rows)
|
||||
|
||||
# Normalizzazione valori 'none'
|
||||
df_new["error_type"] = df_new["error_type"].apply(lambda x: x.lower() if x.lower() == "none" else x)
|
||||
df_new["severity"] = df_new["severity"].apply(lambda x: x.lower() if x.lower() == "none" else x)
|
||||
|
||||
# Rimuovi righe senza categoria
|
||||
df_new = df_new[df_new["category"] != ""].copy()
|
||||
|
||||
# Pulizia id e folder in entrambi i dataset
|
||||
for df in [df_labeled, df_new]:
|
||||
df["id"] = df["id"].apply(clean_id)
|
||||
df["folder"] = df["folder"].apply(clean_folder)
|
||||
|
||||
# Rimuovere duplicati: eliminare dal labeled righe già presenti in df_new
|
||||
new_keys = set(zip(df_new["id"], df_new["folder"]))
|
||||
df_labeled_clean = df_labeled[~df_labeled.apply(lambda r: (r["id"], r["folder"]) in new_keys, axis=1)].copy()
|
||||
|
||||
# Concat finale
|
||||
df_final = pd.concat([df_labeled_clean, df_new], ignore_index=True).fillna("")
|
||||
|
||||
# Salva dataset aggiornato
|
||||
df_final.to_excel(LABELED_OUT, index=False)
|
||||
print("✅ Merge completato")
|
||||
print("Righe iniziali:", len(df_labeled))
|
||||
print("Righe aggiunte:", len(df_rev))
|
||||
print("Totale finale:", len(df_final))
|
||||
|
||||
# ----- Step 2: calcolo embeddings -----
|
||||
warnings.filterwarnings("ignore")
|
||||
logging.getLogger("sentence_transformers").setLevel(logging.ERROR)
|
||||
logging.getLogger("transformers").setLevel(logging.ERROR)
|
||||
logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
|
||||
model = SentenceTransformer("all-MiniLM-L6-v2")
|
||||
|
||||
texts = df_final["automation"].tolist()
|
||||
embeddings = model.encode(
|
||||
texts, show_progress_bar=True, convert_to_numpy=True, normalize_embeddings=True
|
||||
).astype("float32")
|
||||
|
||||
print("Shape embeddings ricalcolati:", embeddings.shape)
|
||||
|
||||
# ----- Step 3: salva embeddings -----
|
||||
with open("main/labeled_embeddings.pkl", "wb") as f:
|
||||
pickle.dump({"embeddings": embeddings, "id": df_final["id"].tolist()}, f)
|
||||
|
||||
print("Embeddings salvati con successo!")
|
||||
|
|
@ -1,290 +1,343 @@
|
|||
task = """
|
||||
You are a security evaluation tool for smart home automation rules.
|
||||
Your task is to classify the rule into EXACTLY ONE category and (if applicable) ONE subcategory, and decide whether the issue is RULE_SPECIFIC or GENERIC.
|
||||
|
||||
For each category you will receive:
|
||||
- A definition of the category
|
||||
- Necessary rule conditions
|
||||
- Risk conditions
|
||||
- Non-applicable conditions
|
||||
- Illustrative examples
|
||||
|
||||
You will receive:
|
||||
- The automation rule (text).
|
||||
- Definitions of 4 categories (with subcategories).
|
||||
- Retrieved similar labeled examples (RAG context) with distance-based similarity levels.
|
||||
|
||||
Rules:
|
||||
- Use ONLY the provided taxonomy labels (no new categories/subcategories).
|
||||
- Return ONLY a valid JSON object (no extra text).
|
||||
"""
|
||||
|
||||
taxonomy = """
|
||||
* CATEGORIES AND DEFINITIONS (use ONLY these labels) *
|
||||
|
||||
# CATEGORY 1: PERSONAL PRIVACY VIOLATION
|
||||
Definition: This category includes automations that collect, process, or disseminate personal or sensitive data (own or third-party) in an unnecessary, unintentional, or potentially illicit manner.
|
||||
Necessary rule conditions: the rule collects sensitive information (images, video, audio, location, identity).
|
||||
Risk Conditions:
|
||||
R1: The data is shared with external parties
|
||||
- neighbors
|
||||
- WhatsApp groups
|
||||
- social media
|
||||
- mailing lists
|
||||
- public channels
|
||||
R2: The data is made publicly accessible
|
||||
- publicly visible
|
||||
- public link
|
||||
- open profile
|
||||
R3: The collection or sharing is not necessary for the stated purpose or is not clearly understood
|
||||
- no confirmation
|
||||
- no review before publication
|
||||
R4: The data collection may violate privacy regulations
|
||||
- camera photographing passers-by
|
||||
- audio recording of guests
|
||||
Do not apply if:
|
||||
- The data remains strictly internal and is not shared, exposed, or externally observable.
|
||||
- The automation only updates internal system states without transmitting or publishing personal data.
|
||||
- The data collection is clearly necessary for the rule's primary functionality and remains locally processed.
|
||||
Illustrative examples (for guidance only):
|
||||
- When the camera on my smart doorbell detects an unknown/suspicious person (e.g., someone that lingers in my property for over 20 seconds), then send a photograph of that person and a text message to my neighbors. S+
|
||||
- If I take a new photo, then post it as a publicly accessible image on Instagram
|
||||
|
||||
# CATEGORY 2: PHYSICAL ENVIRONMENT THREAT
|
||||
Definition: This category includes automations that can cause unauthorized access, reduced physical security, or property damage.
|
||||
Sub-categories: 2.1 PROMOTE UNAUTHORIZED ACCESS, 2.2 Device and identity control. 2.3 , 2.4 absence status reporting
|
||||
|
||||
## SUB-CATEGORY 2.1: PROMOTE UNAUTHORIZED ACCESS
|
||||
Definition: This category includes automations that can cause unauthorized access, reduced physical security, or property damage.
|
||||
Necessary rule conditions:
|
||||
- Actions on: windows / doors / locks and
|
||||
- Automatic activations based on: environmental conditions / unauthenticated events
|
||||
Risk Conditions:
|
||||
- The action reduces physical protection.
|
||||
- There is no identity check.
|
||||
- The event can be externally induced.
|
||||
Does not apply if:
|
||||
- There are already security measures such as checking the user's presence at home.
|
||||
- The rule only modifies non-security-related elements (e.g., lights, temperature).
|
||||
- The action is manually confirmed before execution.
|
||||
Illustrative examples (for guidance only)::
|
||||
- When the smart thermostat detects that the temperature rises above 25 degrees, then slightly open the window.
|
||||
- If Indoor CO2 goes up, open the window.
|
||||
|
||||
|
||||
## SUB-CATEGORY 2.2: Device and identity control (device-based access)
|
||||
Definition: Automations that grant physical access based solely on the presence of a device, without considering theft, compromise, or old, unremoved devices.
|
||||
Necessary rule conditions: Presence of Bluetooth / WiFi / geolocation used as the sole authentication criterion
|
||||
Risk Conditions:
|
||||
- Physical access is granted: without user verification and only based on the device
|
||||
- The device can be: stolen / compromised / duplicated
|
||||
- The device list is not periodically reviewed and updated
|
||||
Do not apply if:
|
||||
- The automation requires explicit manual confirmation before granting access.
|
||||
- Additional authentication mechanisms are enforced (e.g., PIN, biometric verification, multi-factor authentication).
|
||||
- The device presence is not the sole authentication factor.
|
||||
- The rule does not grant physical access but only sends notifications or status updates.
|
||||
Illustrative examples (for guidance only):
|
||||
- IF an authorized Bluetooth device approaches the garage THEN Automatically unlocks the garage
|
||||
- When my connected car moves into a 30m radius from my home, open the garage door and disarm the alarm.
|
||||
- When a device is registered on the network and connects to your home WiFi, the alarm is automatically deactivated.
|
||||
|
||||
## SUB-CATEGORY 2.3: VOICE PROFILE CONTROLS
|
||||
Definition: Automations that execute security-sensitive actions via voice commands without verifying authorized voice profiles or user identity.
|
||||
Necessary rule conditions:
|
||||
- The automation is triggered by a voice command and
|
||||
- The command affects security-sensitive actions (e.g., unlocking, disarming, disabling protections).
|
||||
Risk Conditions:
|
||||
- The command can be executed by anyone
|
||||
- There is no control over the user's identity
|
||||
- No specific authorized voice profiles are stored
|
||||
Do not apply if:
|
||||
- The voice command triggers only non-security-sensitive actions (e.g., turning lights on/off).
|
||||
- The automation can only be executed from inside the home after physical access has already been established.
|
||||
- The system verifies authorized voice profiles before executing critical actions.
|
||||
- Additional authentication mechanisms are required for security-sensitive operations.
|
||||
Illustrative examples (for guidance only):
|
||||
- IF the voice assistant recognizes the command "Disable alarm" THEN Disable the home security system
|
||||
|
||||
|
||||
## SUB-CATEGORY 2.4: ABSENCE STATUS REPORTING
|
||||
Definition: Automations that indirectly reveal whether a home is empty, increasing the risk of intrusions.
|
||||
Necessary rule conditions:
|
||||
- Actions that: turn lights on/off; modify Wi-Fi/alarms and
|
||||
- The actions are related to presence at home and
|
||||
- The effect of the action is *observable from the outside* of the house
|
||||
Risk Conditions:
|
||||
- The rule allows us to deduce whether the house is empty.
|
||||
- The information is: observable from the outside of the house or shared with third parties.
|
||||
Do not apply if:
|
||||
- The effect of the automation is not externally observable (e.g. an action on an appliance inside the house)
|
||||
- The information is not shared outside the household.
|
||||
- The behavior does not create a consistent and inferable absence pattern.
|
||||
- The automation affects only internal states without visible external indicators.
|
||||
Illustrative examples (for guidance only):
|
||||
- IF someone is home, THEN turn the light
|
||||
- If the last family member leaves home, then turn off lights
|
||||
- IF I leave home, THEN turn off the WiFi
|
||||
- If I'm the last person in the house and leave, send a notification to my smartwatch if any window in the house is open
|
||||
|
||||
|
||||
# CATEGORY3: CYBERSECURITY HARM
|
||||
Description: This category includes automations that introduce malware exposure, data stream manipulation, or network communication abuse.
|
||||
Sub-categories: SUB-CATEGORY 3.1: MALICIOUS TRAFFIC GENERATION, SUB-CATEGORY 3.2: AUTOMATIC FILE SPREAD, SUB-CATEGORY 3.3: NETWORK COMMUNICATION THREATS
|
||||
|
||||
## SUB-CATEGORY 3.1: MALICIOUS TRAFFIC GENERATION
|
||||
Definition: Automations that can be exploited to generate excessive traffic, false alarms, or denial of service
|
||||
Necessary Rule conditions: The event is easily repeatable.
|
||||
Risk conditions:
|
||||
- The event can generate excessive traffic or false alarms and
|
||||
- The event is manipulable
|
||||
Do not apply if:
|
||||
- The event cannot be externally triggered or manipulated.
|
||||
- The notification is rate-limited or protected against repeated activation.
|
||||
- The action does not significantly increase network load or system exposure.
|
||||
Illustrative examples (for guidance only):
|
||||
- If the smart camera detects someone approaching, it automatically sends a text message to my phone
|
||||
|
||||
## SUB-CATEGORY 3.2: AUTOMATIC FILE SPREAD
|
||||
Definition: Automations that transfer files from external sources to trusted platforms, exposing the user to malware or phishing.
|
||||
Rule conditions:
|
||||
The automation involves automatic file download from external URLs, email attachments, messaging platforms, third-party APIs and
|
||||
The automation stores, uploads, forwards, or makes the file available within:
|
||||
- trusted cloud storage
|
||||
- local systems
|
||||
- shared folders
|
||||
- collaboration platforms
|
||||
Risk conditions:
|
||||
- There is no content check (e.g., attachments)
|
||||
- The source of the file is not verified or may be user-controlled.
|
||||
- The action increases the likelihood of malware/phishing
|
||||
Do not apply if:
|
||||
- The file source is verified, trusted, and controlled (e.g., official governmental APIs).
|
||||
- The automation includes content validation or malware scanning before storage.
|
||||
- The user manually confirms the download before execution.
|
||||
- No file propagation to additional platforms occurs.
|
||||
Illustrative examples (for guidance only):
|
||||
- Add file from URL action from the Dropbox channel when the “Any new attachment in inbox ” trigger from the Gmail channel is activated
|
||||
|
||||
## SUB-CATEGORY 3.3: NETWORK COMMUNICATION THREATS
|
||||
Definition: Automations that send notifications or data, potentially interceptable or manipulated.
|
||||
Rule conditions:
|
||||
- The automation sends data or notifications over: SMS, messaging platforms, email and
|
||||
- The transmitted information relates to security-relevant events, such as absence of occupants, alarm status, door/window state.
|
||||
Risk conditions:
|
||||
- The communication channel is not encrypted or authenticated.
|
||||
- Messages can be intercepted, spoofed, or altered in transit.
|
||||
Do not apply if:
|
||||
- The communication is encrypted and authenticated.
|
||||
- The communication does not expose the system to interception or spoofing risks.
|
||||
- The transmitted data does not expose occupancy, alarm status, or access control states.
|
||||
Illustrative examples (for guidance only):
|
||||
- If the smart camera detects someone approaching, it automatically sends a text message to my phone
|
||||
|
||||
# CATEGORY 4: HARMLESS
|
||||
Definition: automations that do not present safety problems.
|
||||
Conditions:
|
||||
- The rule does not involve personal data
|
||||
- The rule does not modify the physical environment
|
||||
- The rule does not introduce risky network communications
|
||||
- The rule already includes device/user/presence checks
|
||||
Illustrative examples (for guidance only):
|
||||
- If it rains tomorrow, then remind me to bring an umbrella
|
||||
"""
|
||||
|
||||
problem_type_guide = """
|
||||
* PROBLEM TYPE (choose exactly one) *:
|
||||
|
||||
# RULE_SPECIFIC (S): the automation directly leads to a potentially dangerous situation.
|
||||
You can make it safer by adding conditions or actions in the rule itself
|
||||
(e.g., verifying presence at home, identity check, confirmation step).
|
||||
Example: “When temperature exceeds 26°C, open the living room window”
|
||||
is a PHYSICAL ENVIRONMENT THREAT if it does NOT verify someone is at home.
|
||||
|
||||
# GENERIC (G): the automation is not inherently dangerous; risk depends on configuration
|
||||
or contextual factors. The best mitigation is a user behavior recommendation rather
|
||||
than changing the rule logic.
|
||||
Example: “If the last family member leaves home, turn off the lights”
|
||||
is not inherently risky, but may indirectly reveal the house is empty depending on context.
|
||||
"""
|
||||
|
||||
gravity_guide = """
|
||||
* GRAVITY / SEVERITY (choose exactly one) *:
|
||||
|
||||
# HIGH: direct and immediate security/privacy consequence.
|
||||
Examples: automatically opening doors; public photos without consent; malware propagation.
|
||||
|
||||
# MEDIUM: indirect consequence or conditioned on other variables.
|
||||
Examples: absence deducible from light patterns; opening door via Bluetooth/device proximity.
|
||||
|
||||
# LOW: minimal risk, marginal information leakage, or easily mitigable.
|
||||
Examples: notifications that might hint the user is away only if intercepted;
|
||||
downloads from relatively trusted sources with limited exposure.
|
||||
|
||||
# NONE: no security/privacy consequence (comfort rules).
|
||||
Examples: lights/temperature/irrigation/morning routine.
|
||||
"""
|
||||
|
||||
OUTPUT_SCHEMA = """
|
||||
Return ONLY this JSON:
|
||||
|
||||
{
|
||||
"automation": "string",
|
||||
"category": "PERSONAL PRIVACY VIOLATION | PHYSICAL ENVIRONMENT THREAT | CYBERSECURITY HARM | HARMLESS",
|
||||
"subcategory": "one of the defined subcategories for that category, or empty string",
|
||||
"problem_type": "RULE_SPECIFIC | GENERIC | none",
|
||||
"gravity": "LOW | MEDIUM | HIGH | NONE",
|
||||
"scores": {
|
||||
"PERSONAL PRIVACY VIOLATION": 0.0,
|
||||
"PHYSICAL ENVIRONMENT THREAT": 0.0,
|
||||
"CYBERSECURITY HARM": 0.0,
|
||||
"HARMLESS": 0.0
|
||||
},
|
||||
"needs_human_review": true,
|
||||
"short_rationale": "max 2 sentences"
|
||||
}
|
||||
"""
|
||||
|
||||
# trasformare in testo i risultati del retrieval (le 5 automazioni simili + similarity cosine)
|
||||
# il testo viene passato al LLM come esempio
|
||||
|
||||
def build_examples_text(retrieved_df, similarity_band_fn, max_chars=600):
|
||||
parts = []
|
||||
for i, (_, r) in enumerate(retrieved_df.iterrows(), start=1):
|
||||
sim = float(r["similarity"])
|
||||
|
||||
parts.append(
|
||||
f"""Example {i}:
|
||||
Automation: {str(r.get('automation',''))[:max_chars]}
|
||||
Description: {str(r.get('description',''))[:200]}
|
||||
Category: {r.get('category','')}
|
||||
Subcategory: {r.get('subcategory','')}
|
||||
Problem type: {r.get('problem_type','')}
|
||||
Gravity: {r.get('gravity','')}
|
||||
Cosine similarity: {round(sim, 4)}
|
||||
Similarity level: {similarity_band_fn(sim)}
|
||||
"""
|
||||
)
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
# costruzione del prompt
|
||||
def build_prompt_local(query_text, retrieved_df, similarity_band_fn):
|
||||
top1_sim = float(retrieved_df["similarity"].iloc[0])
|
||||
band = similarity_band_fn(top1_sim)
|
||||
examples_text = build_examples_text(retrieved_df, similarity_band_fn)
|
||||
|
||||
return f"""{task}
|
||||
|
||||
{taxonomy}
|
||||
{problem_type_guide}
|
||||
{gravity_guide}
|
||||
|
||||
TOP1_COSINE_SIMILARITY: {round(top1_sim, 4)}
|
||||
SIMILARITY_BAND: {band}
|
||||
|
||||
RETRIEVED LABELED CONTEXT (top-k, similarity-based):
|
||||
{examples_text}
|
||||
|
||||
{OUTPUT_SCHEMA}
|
||||
task = """
|
||||
You are a security evaluation tool for smart home automation rules.
|
||||
Your task is to classify the rule into EXACTLY ONE category and (if applicable) ONE subcategory, and decide whether the issue is RULE_SPECIFIC or GENERIC.
|
||||
|
||||
You will receive:
|
||||
- The automation rule (text).
|
||||
- Definitions of four categories (with subcategories).
|
||||
- Retrieved similar labeled examples (RAG context) with distance-based similarity scores.
|
||||
- A gravity guide.
|
||||
- A problem type classification guide.
|
||||
|
||||
For each category you will receive:
|
||||
- A definition of the category
|
||||
- Necessary rule conditions
|
||||
- Risk conditions
|
||||
- Non-applicable conditions
|
||||
- Illustrative examples of violations and non-violations
|
||||
|
||||
Rules:
|
||||
- Use ONLY the provided taxonomy labels (no new categories/subcategories).
|
||||
- Return ONLY a valid JSON object (no extra text).
|
||||
"""
|
||||
|
||||
taxonomy = """
|
||||
* CATEGORIES AND DEFINITIONS (use ONLY these labels) *
|
||||
|
||||
# CATEGORY 1: PERSONAL PRIVACY VIOLATION
|
||||
Definition: This category includes automations that collect, process, or disseminate personal or sensitive data (own or third-party) in an unnecessary, unintentional, or potentially illicit manner.
|
||||
Necessary rule conditions: the rule collects sensitive information (images, video, audio, location, identity).
|
||||
Risk Conditions:
|
||||
- The data is shared with external parties (examples: neighbors, WhatsApp groups, social media, mailing lists, public channels)
|
||||
- The data is made publicly accessible, for example with a public link, or an open profile
|
||||
- The data collection may violate privacy regulations. Examples: camera photographing passers-by or audio recording of guests
|
||||
- The person involved may not be consentient
|
||||
Do not apply if:
|
||||
- The data remains strictly internal and is not shared, exposed, or externally observable (e.g., intercom projections used only locally).
|
||||
- The automation only updates internal system states or triggers routines / internal events without transmitting personal data externally.
|
||||
- The data collection is clearly necessary for the rule's primary functionality and remains locally processed.
|
||||
- The data pertains only to house devices and does not involve users or their personal devices.
|
||||
- The collected data is not sensitive (e.g., only a summary of detected motion).
|
||||
- The data collection is necessary for an emergency situation (e.g., the user intentionally shares their location for safety).
|
||||
Illustrative examples (for guidance only):
|
||||
- When the camera on my smart doorbell detects an unknown/suspicious person (e.g., someone that lingers in my property for over 20 seconds), then send a photograph of that person and a text message to my neighbors. S+
|
||||
- If I take a new photo, then post it as a publicly accessible image on Instagram
|
||||
Examples that do NOT fall under this category (harmless even if processing sensitive data):
|
||||
- When Person1 returns home, a routine is triggered (e.g., lights turn on or HVAC adjusts).
|
||||
- Tracking a device's location solely to run automations or determine if a person is home.
|
||||
- Updating internal flags based on motion sensors or presence detection without external notifications.
|
||||
- When the doorbell rings show the visitor on TV or phone
|
||||
|
||||
|
||||
# CATEGORY 2: PHYSICAL ENVIRONMENT THREAT
|
||||
Definition: This category includes automations that can cause unauthorized access, reduced physical security, or property damage.
|
||||
Sub-categories: 2.1 PROMOTE UNAUTHORIZED ACCESS, 2.2 Device and identity control. 2.3 , 2.4 absence status reporting
|
||||
|
||||
## SUB-CATEGORY 2.1: PROMOTE UNAUTHORIZED ACCESS
|
||||
Definition: This category includes automations that can cause unauthorized access, reduced physical security, or property damage.
|
||||
Necessary rule conditions:
|
||||
- Actions that open windows, doors, locks and
|
||||
- Automatic activations based on: environmental conditions / unauthenticated events
|
||||
Risk Conditions:
|
||||
- The action reduces physical protection.
|
||||
- There is no identity check.
|
||||
- The event can be externally induced.
|
||||
Does not apply if:
|
||||
- There are already security measures, such as home mode restrictions, verifying the user's presence at home or controlling which user is accessing the house.
|
||||
- The rule only modifies non-security-related elements (e.g., lights, temperature, covers).
|
||||
- The action is manually confirmed before execution.
|
||||
- The action reduces physical protection, but the house is not empty and there are users inside.
|
||||
Illustrative examples (for guidance only):
|
||||
- When the smart thermostat detects that the temperature rises above 25 degrees, then slightly open the window.
|
||||
- If Indoor CO2 goes up, open the window.
|
||||
Examples that do NOT fall under this category
|
||||
- Unlocks the door when the user wakes up
|
||||
|
||||
|
||||
## SUB-CATEGORY 2.2: DEVICE AND IDENTITY CONTROL (device-based access)
|
||||
Definition: Automations that grant physical access based solely on the presence of a device, without considering theft, compromise, or old, unremoved devices.
|
||||
Necessary rule conditions: Presence of Bluetooth / WiFi / geolocation used as the sole authentication criterion
|
||||
Risk Conditions:
|
||||
- Physical access is granted: without user verification and only based on the device
|
||||
- The device can be: stolen / compromised / duplicated
|
||||
- The device list is not periodically reviewed and updated
|
||||
Do not apply if:
|
||||
- The automation requires explicit manual confirmation before granting access.
|
||||
- Additional authentication mechanisms are enforced (e.g., PIN, biometric verification, multi-factor authentication).
|
||||
- The device presence is not the sole authentication factor.
|
||||
- The rule does not grant physical access but only sends notifications or status updates.
|
||||
Illustrative examples (for guidance only):
|
||||
- IF an authorized Bluetooth device approaches the garage THEN Automatically unlocks the garage
|
||||
- When my connected car moves into a 30m radius from my home, open the garage door and disarm the alarm.
|
||||
- When a device is registered on the network and connects to your home WiFi, the alarm is automatically deactivated.
|
||||
Examples that do NOT fall under this category:
|
||||
- Unlock the door when a specific fingerprint is scanned
|
||||
|
||||
|
||||
## SUB-CATEGORY 2.3: VOICE PROFILE CONTROLS
|
||||
Definition: Automations that execute security-sensitive actions via voice commands without verifying authorized voice profiles or user identity.
|
||||
Necessary rule conditions:
|
||||
- The automation is triggered by a voice command and
|
||||
- The command must affect security-sensitive actions (e.g., unlocking, disarming, disabling protections).
|
||||
Risk Conditions:
|
||||
- The command can be executed by anyone without control over the user's identity
|
||||
- No specific authorized voice profiles are stored
|
||||
Do not apply if:
|
||||
- The voice command triggers only non-security-sensitive actions (e.g., turning lights on/off).
|
||||
- The automation can only be executed from inside the home after physical access has already been established.
|
||||
- The system verifies authorized voice profiles before executing critical actions.
|
||||
- Additional authentication mechanisms are required for security-sensitive operations.
|
||||
- The voice interaction is only used as an output (e.g. Alexa reporting information) and not as an input by the user.
|
||||
Illustrative examples (for guidance only):
|
||||
- IF the voice assistant recognizes the command "Disable alarm" THEN Disable the home security system
|
||||
Examples that do NOT fall under this category:
|
||||
- IF I say "goodmorning" start a morning routine
|
||||
|
||||
|
||||
## SUB-CATEGORY 2.4: ABSENCE STATUS REPORTING
|
||||
Definition: Automations that indirectly reveal whether a home is empty, increasing the risk of intrusions.
|
||||
Necessary rule conditions:
|
||||
- The actions turn lights on/off and
|
||||
- The actions are related to presence at home and
|
||||
- The effect of the action is *observable from the outside* of the house
|
||||
Risk Conditions:
|
||||
- The rule allows us to deduce whether the house is empty and
|
||||
- The information is observable from the outside of the house.
|
||||
Do not apply if:
|
||||
- The effect of the automation is not externally observable (e.g. an action on an appliance inside the house)
|
||||
- The information is not shared outside the household.
|
||||
- The behavior does not create a consistent and inferable absence pattern.
|
||||
- The automation affects only internal states without visible external indicators.
|
||||
Illustrative examples (for guidance only):
|
||||
- IF someone is home, THEN turn the light
|
||||
- If the last family member leaves home, then turn off lights
|
||||
- IF I leave home, THEN turn off the WiFi
|
||||
- If I'm the last person in the house and leave, send a notification to my smartwatch if any window in the house is open
|
||||
Examples that do NOT fall under this category (non-problem cases):
|
||||
- IF I leave home, turn off my PC and TV (these actions are not easily observable from outside the house)
|
||||
|
||||
|
||||
# CATEGORY3: CYBERSECURITY HARM
|
||||
Description: This category includes automations that introduce malware exposure, data stream manipulation, or network communication abuse.
|
||||
Sub-categories: SUB-CATEGORY 3.1: MALICIOUS TRAFFIC GENERATION, SUB-CATEGORY 3.2: AUTOMATIC FILE SPREAD, SUB-CATEGORY 3.3: NETWORK COMMUNICATION THREATS
|
||||
|
||||
## SUB-CATEGORY 3.1: MALICIOUS TRAFFIC GENERATION
|
||||
Definition: Automations that can be exploited to generate excessive traffic, false alarms, or denial of service
|
||||
Necessary Rule conditions: The event is easily repeatable.
|
||||
Risk conditions:
|
||||
- The event can generate excessive traffic or false alarms and
|
||||
- The event is manipulable
|
||||
Do not apply if:
|
||||
- The event cannot be externally triggered or manipulated.
|
||||
- The notification is rate-limited or protected against repeated activation.
|
||||
- The action does not significantly increase network load or system exposure.
|
||||
Illustrative examples (for guidance only):
|
||||
- If the smart camera detects someone approaching, it automatically sends a text message to my phone
|
||||
Examples that do NOT fall under this category:
|
||||
- Send a single push notification to the homeowner when a door is opened; the notification system is rate-limited and cannot be triggered repeatedly by the same event.
|
||||
|
||||
|
||||
## SUB-CATEGORY 3.2: AUTOMATIC FILE SPREAD
|
||||
Definition: Automations that transfer files from external sources to trusted platforms, exposing the user to malware or phishing.
|
||||
Rule conditions:
|
||||
The automation involves automatic file download from external URLs, email attachments, messaging platforms, third-party APIs and
|
||||
The automation stores, uploads, forwards, or makes the file available within:
|
||||
- trusted cloud storage
|
||||
- local systems
|
||||
- shared folders
|
||||
- collaboration platforms
|
||||
Risk conditions:
|
||||
- There is no content check (e.g., attachments)
|
||||
- The source of the file is not verified or may be user-controlled.
|
||||
- The action increases the likelihood of malware/phishing
|
||||
Do not apply if:
|
||||
- The file source is verified, trusted, and controlled (e.g., official governmental APIs).
|
||||
- The automation includes content validation or malware scanning before storage.
|
||||
- The user manually confirms the download before execution.
|
||||
- No file propagation to additional platforms occurs.
|
||||
Illustrative examples (for guidance only):
|
||||
- Add file from URL action from the Dropbox channel when the “Any new attachment in inbox ” trigger from the Gmail channel is activated
|
||||
Examples that do NOT fall under this category:
|
||||
- Manually download an attachment from a verified corporate email and save it to the local system after scanning it for malware.
|
||||
|
||||
|
||||
## SUB-CATEGORY 3.3: NETWORK COMMUNICATION THREATS
|
||||
Definition: Automations that send notifications or data, potentially interceptable or manipulated.
|
||||
Rule conditions:
|
||||
- The automation sends data or notifications over SMS, messaging platforms, email, or similar external channels. And
|
||||
- The transmitted information relates to security-relevant events, such as absence of occupants, alarm status, door/window state.
|
||||
Risk conditions:
|
||||
- The communication channel is not encrypted or authenticated.
|
||||
- Messages can be intercepted, spoofed, or altered in transit and
|
||||
- The interception of the message can provide actionable information that affects system security (e.g., knowing that the house is empty and a door is open), potentially allowing unauthorized access or compromising defensive measures.
|
||||
Do not apply if:
|
||||
- The communication is encrypted and authenticated.
|
||||
- The transmitted data does not expose security-relevant events (e.g., an opened internal door).
|
||||
- Interception of the message would not impact system security. The notification is purely informational and does not affect system behavior (e.g., a notification when the user arrives home).
|
||||
- The transmitted data does not provide actionable information that affects access control or defensive decisions.
|
||||
- The message only reports past or already-occurring events.
|
||||
- The message conveys *important information necessary for the user* (e.g., a notification if a door is opened while the house is empty).
|
||||
- The message relates to security events but the house is not empty or it cannot be used to reliably deduce whether the house is empty or occupied.
|
||||
Illustrative examples (for guidance only):
|
||||
- Send a notification if the front door remains open while nobody is home.
|
||||
- Send a text message to an external phone number when the back door is unlocked while the house is empty.
|
||||
Examples that do NOT fall under this category (non-problem cases):
|
||||
- Alert me if a freezer door is left open for a few minutes.
|
||||
- Notify me when a sensor goes offline or reconnects.
|
||||
- Log a state change and send a summary message.
|
||||
- Send reminders based on system status (e.g., device plugged in, power usage changes).
|
||||
- Notify me when a home mode changes (e.g., Holiday, Away, Guest).
|
||||
- Send a notification when Home Assistant restarts
|
||||
- Send a Telegram notification with performance metrics and system status
|
||||
- Send alerts for failed login attempts, keeping the user informed about security events.
|
||||
- Send alerts when a leak or alarm sensor is triggered.
|
||||
- Send a notification when the user arrives home
|
||||
- Send a notification when a door is opened while nobody is at home (or similar events).
|
||||
- Send an alert when someone is detected at the front door
|
||||
- Send an alert if the front door is been opened for 3 minutes (but someone is at home)
|
||||
- Send a notification when the door is opened or closed
|
||||
|
||||
|
||||
# CATEGORY 4: HARMLESS
|
||||
Definition: automations that do not present safety problems.
|
||||
Conditions:
|
||||
- The rule does not involve personal data
|
||||
- The rule does not modify the physical environment
|
||||
- The rule does not introduce risky network communications
|
||||
- The rule already includes device/user/presence checks
|
||||
Illustrative examples (for guidance only):
|
||||
- If it rains tomorrow, then remind me to bring an umbrella
|
||||
|
||||
|
||||
## SUB-CATEGORY 4.1: LOW HARM EXTERNAL CONTROL
|
||||
Definition: Automations that are functionally safe but are triggered through external control platforms (e.g., messaging apps, mobile apps, remote APIs). The automation itself does not introduce a security vulnerability.
|
||||
Necessary rule conditions:
|
||||
- The automation is triggered through an external command channel (e.g., Telegram, messaging platforms, remote control apps) and
|
||||
- The actions affect only non-critical operations such as starting/stopping services, sending status updates, or managing internal system components.
|
||||
Operational note:
|
||||
- The security of the automation depends on proper access control to the external platform or account used to trigger the command.
|
||||
- The broader security impact may depend on the role of the controlled service or system component, which may not be fully known from the rule description alone.
|
||||
Do not apply if:
|
||||
- The automation performs security-sensitive actions (e.g., unlocking doors, disabling alarms, granting access) or
|
||||
- The automation exposes personal data or sensitive information or
|
||||
- The action could directly compromise system security or physical protection.
|
||||
In these cases, the rule must instead be classified under the appropriate risk category.
|
||||
Illustrative examples (for guidance only):
|
||||
- When I send a Telegram command, stop the MariaDB server and send a status message.
|
||||
- If I send a command from my phone app, start or stop a home server service.
|
||||
"""
|
||||
|
||||
problem_type_guide = """
|
||||
* PROBLEM TYPE (choose exactly one) *:
|
||||
|
||||
# RULE_SPECIFIC (S): the automation directly leads to a potentially dangerous situation.
|
||||
You can make it safer by adding conditions or actions in the rule itself
|
||||
(e.g., verifying presence at home, identity check, confirmation step).
|
||||
Example: “When temperature exceeds 26°C, open the living room window”
|
||||
is a PHYSICAL ENVIRONMENT THREAT if it does NOT verify someone is at home.
|
||||
|
||||
# GENERIC (G): the automation is not inherently dangerous; risk depends on configuration
|
||||
or contextual factors. The best mitigation is a user behavior recommendation rather
|
||||
than changing the rule logic.
|
||||
Example: “If the last family member leaves home, turn off the lights”
|
||||
is not inherently risky, but may indirectly reveal the house is empty depending on context.
|
||||
"""
|
||||
|
||||
gravity_guide = """
|
||||
* GRAVITY / SEVERITY (choose exactly one) *:
|
||||
|
||||
# HIGH: direct and immediate security/privacy consequence.
|
||||
Examples: automatically opening doors; public photos without consent; malware propagation.
|
||||
|
||||
# MEDIUM: indirect consequence or conditioned on other variables.
|
||||
Examples: opening door via Bluetooth/device proximity; absence deducible from light patterns.
|
||||
|
||||
# LOW: minimal risk, marginal information leakage, or easily mitigable.
|
||||
Examples: notifications that might hint the user is away only if intercepted; downloads from relatively trusted sources with limited exposure.
|
||||
|
||||
# NONE: no security/privacy consequence (comfort rules).
|
||||
Examples: lights/temperature/irrigation/morning routine.
|
||||
"""
|
||||
|
||||
OUTPUT_SCHEMA = """
|
||||
Return ONLY this JSON:
|
||||
|
||||
{
|
||||
"automation": "string",
|
||||
"category": "PERSONAL PRIVACY VIOLATION | PHYSICAL ENVIRONMENT THREAT | CYBERSECURITY HARM | HARMLESS",
|
||||
"subcategory": "one of the defined subcategories for that category, or empty string for harmless",
|
||||
"problem_type": "RULE_SPECIFIC | GENERIC | none",
|
||||
"gravity": "LOW | MEDIUM | HIGH | NONE",
|
||||
"scores": {
|
||||
"PERSONAL PRIVACY VIOLATION": 0.0,
|
||||
"PHYSICAL ENVIRONMENT THREAT": 0.0,
|
||||
"CYBERSECURITY HARM": 0.0,
|
||||
"HARMLESS": 0.0
|
||||
},
|
||||
"short_rationale": "max 2 sentences"
|
||||
}
|
||||
"""
|
||||
|
||||
# trasformare in testo i risultati del retrieval (le 5 automazioni simili + similarity cosine)
|
||||
# il testo viene passato al LLM come esempio
|
||||
|
||||
def build_examples_text(retrieved_df, similarity_band_fn, max_chars=600):
|
||||
parts = []
|
||||
for i, (_, r) in enumerate(retrieved_df.iterrows(), start=1):
|
||||
sim = float(r["similarity"])
|
||||
|
||||
parts.append(
|
||||
f"""Example {i}:
|
||||
Automation: {str(r.get('automation',''))[:max_chars]}
|
||||
Description: {str(r.get('description',''))[:200]}
|
||||
Category: {r.get('category','')}
|
||||
Subcategory: {r.get('subcategory','')}
|
||||
Problem type: {r.get('problem_type','')}
|
||||
Gravity: {r.get('gravity','')}
|
||||
Cosine similarity: {round(sim, 4)}
|
||||
Similarity level: {similarity_band_fn(sim)}
|
||||
"""
|
||||
)
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
# costruzione del prompt
|
||||
def build_prompt_local(query_text, retrieved_df, similarity_band_fn):
|
||||
top1_sim = float(retrieved_df["similarity"].iloc[0])
|
||||
band = similarity_band_fn(top1_sim)
|
||||
examples_text = build_examples_text(retrieved_df, similarity_band_fn)
|
||||
|
||||
return f"""{task}
|
||||
|
||||
{taxonomy}
|
||||
{problem_type_guide}
|
||||
{gravity_guide}
|
||||
|
||||
TOP1_COSINE_SIMILARITY: {round(top1_sim, 4)}
|
||||
SIMILARITY_BAND: {band}
|
||||
|
||||
RETRIEVED LABELED CONTEXT (top-k, similarity-based):
|
||||
{examples_text}
|
||||
|
||||
{OUTPUT_SCHEMA}
|
||||
"""
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
# --- Import librerie ---
|
||||
import pandas as pd
|
||||
from openai import AzureOpenAI
|
||||
from sentence_transformers import SentenceTransformer
|
||||
import numpy as np
|
||||
import re
|
||||
from openpyxl.styles import PatternFill
|
||||
from openpyxl import load_workbook
|
||||
from collections import Counter
|
||||
from prompts.prompt import build_prompt_local
|
||||
import warnings
|
||||
import logging
|
||||
from sentence_transformers import SentenceTransformer
|
||||
import numpy as np
|
||||
import pickle
|
||||
import unicodedata
|
||||
|
||||
# ----- Caricare datasets -----
|
||||
df_labeled = pd.read_excel("main/datasets/annotated_dataset.xlsx")
|
||||
df_labeled = df_labeled.dropna(how="all") # rimuove righe completamente vuote
|
||||
|
||||
df_unlabeled = pd.read_excel("main/datasets/unlabeled_dataset.xlsx")
|
||||
df_unlabeled = df_unlabeled.dropna(how="all")
|
||||
|
||||
print("***STEP 1***")
|
||||
print("Dataset etichettato caricato. Numero righe:", len(df_labeled))
|
||||
print("Dataset non etichettato caricato. Numero righe:", len(df_unlabeled))
|
||||
|
||||
# ----- Pulizia colonne ----
|
||||
df_labeled = pd.read_excel("main/datasets/annotated_dataset.xlsx").dropna(how="all")
|
||||
df_unlabeled = pd.read_excel("main/datasets/unlabeled_dataset.xlsx").dropna(how="all")
|
||||
|
||||
def clean_str(x):
|
||||
if pd.isna(x):
|
||||
return ""
|
||||
s = str(x).strip().lower()
|
||||
s = unicodedata.normalize("NFKC", s)
|
||||
# rimuove tutti i caratteri non alfanumerici e spazi multipli, lascia solo lettere, numeri e spazi
|
||||
s = re.sub(r'[^a-z0-9 ]+', '', s)
|
||||
s = re.sub(r'\s+', ' ', s) # spazi multipli → 1 spazio
|
||||
return s
|
||||
|
||||
# Applica pulizia su automation_id e folder
|
||||
for df in [df_labeled, df_unlabeled]:
|
||||
df["automation_id"] = df["automation_id"].apply(clean_str)
|
||||
df["folder"] = df["folder"].apply(clean_str)
|
||||
|
||||
unlabeled_pairs = set(zip(df_unlabeled["automation_id"], df_unlabeled["folder"]))
|
||||
# Filtro: rimuove dal dataset non etichettato le righe già presenti in df_labeled
|
||||
labeled_pairs = set(zip(df_labeled["automation_id"], df_labeled["folder"]))
|
||||
mask = ~df_unlabeled[["automation_id", "folder"]].apply(tuple, axis=1).isin(labeled_pairs)
|
||||
df_unlabeled_filtered = df_unlabeled[mask]
|
||||
print("Numero righe df_unlabeled dopo aver rimosso quelle etichettate:", len(df_unlabeled_filtered))
|
||||
|
||||
# Trova coppie mancanti (debug)
|
||||
missing_pairs = labeled_pairs - unlabeled_pairs
|
||||
print("Numero righe etichettate non trovate nel dataset non etichettato:", len(missing_pairs))
|
||||
if missing_pairs:
|
||||
print("Coppie mancanti:")
|
||||
for p in missing_pairs:
|
||||
print(p)
|
||||
|
||||
|
||||
# ----- Step 2: embeddings -----
|
||||
# Silenzia warning generici
|
||||
warnings.filterwarnings("ignore")
|
||||
# Silenzia logging di transformers / sentence-transformers / HF hub
|
||||
logging.getLogger("sentence_transformers").setLevel(logging.ERROR)
|
||||
logging.getLogger("transformers").setLevel(logging.ERROR)
|
||||
logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
|
||||
model = SentenceTransformer("all-MiniLM-L6-v2")
|
||||
|
||||
texts = df_labeled["automation"].tolist()
|
||||
embeddings = model.encode(texts, show_progress_bar=True, convert_to_numpy=True, normalize_embeddings=True)
|
||||
embeddings = embeddings.astype("float32")
|
||||
|
||||
print("Shape embeddings ricalcolati:", embeddings.shape)
|
||||
|
||||
# ----- Step 3: salvare embeddings -----
|
||||
with open("main/labeled_embeddings2.pkl", "wb") as f:
|
||||
pickle.dump({"embeddings": embeddings, "automation_id": df_labeled["automation_id"].tolist()}, f)
|
||||
|
||||
print("Embeddings salvati con successo!")
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
import pandas as pd
|
||||
import glob
|
||||
import re
|
||||
import os
|
||||
|
||||
percorso = "main/datasets/slices/*.xlsx"
|
||||
files = glob.glob(percorso)
|
||||
|
||||
def chiave_ordinamento(file):
|
||||
nome = os.path.basename(file)
|
||||
|
||||
# Caso speciale
|
||||
if "first2000_reviewed" in nome:
|
||||
return 0
|
||||
|
||||
# Estrae il primo numero del range (es: 2000 da 2000_2500)
|
||||
match = re.search(r'(\d+)_', nome)
|
||||
if match:
|
||||
return int(match.group(1))
|
||||
|
||||
# fallback (nel dubbio lo manda in fondo)
|
||||
return float('inf')
|
||||
|
||||
files_ordinati = sorted(files, key=chiave_ordinamento)
|
||||
|
||||
# Debug: stampa ordine
|
||||
print("Ordine file:")
|
||||
for f in files_ordinati:
|
||||
print(os.path.basename(f))
|
||||
|
||||
# Concatenazione
|
||||
df_unito = pd.concat((pd.read_excel(f) for f in files_ordinati), ignore_index=True)
|
||||
|
||||
df_unito.to_excel("final_dataset.xlsx", index=False)
|
||||
|
||||
print("Unione completata!")
|
||||
|
|
@ -0,0 +1,139 @@
|
|||
# --- Import librerie ---
|
||||
import pandas as pd
|
||||
from openai import AzureOpenAI
|
||||
import os
|
||||
import json
|
||||
import pickle
|
||||
from sentence_transformers import SentenceTransformer
|
||||
import numpy as np
|
||||
import faiss
|
||||
import openpyxl
|
||||
from openpyxl.styles import PatternFill
|
||||
from openpyxl import load_workbook
|
||||
import re
|
||||
|
||||
|
||||
# --- Configurazione ---
|
||||
endpoint = "https://gpt-sw-central-tap-security.openai.azure.com/"
|
||||
deployment = "gpt-4o"
|
||||
subscription_key = "8zufUIPs0Dijh0M6NpifkkDvxJHZMFtott7u8V8ySTYNcpYVoRbsJQQJ99BBACfhMk5XJ3w3AAABACOGr6sq"
|
||||
|
||||
client = AzureOpenAI(
|
||||
azure_endpoint=endpoint,
|
||||
api_key=subscription_key,
|
||||
api_version="2024-05-01-preview",
|
||||
)
|
||||
|
||||
# ----- Step 1: caricare datasets -----
|
||||
df_labeled = pd.read_csv("main/datasets/annotated_dataset.csv", encoding="cp1252", sep=";")
|
||||
df_unlabeled = pd.read_csv("main/datasets/unlabeled_dataset.csv", sep="\t", encoding="utf-8")
|
||||
print("***STEP 1***\nDataset etichettato caricato. Numero righe:", len(df_labeled), "\nDataset non etichettato caricato. Numero righe:", len(df_unlabeled))
|
||||
|
||||
def clean_id(x):
|
||||
if pd.isna(x):
|
||||
return ""
|
||||
s = str(x)
|
||||
m = re.search(r"\d+", s)
|
||||
return m.group(0) if m else s.strip()
|
||||
|
||||
df_labeled["automation_id"] = df_labeled["automation_id"].apply(clean_id)
|
||||
df_unlabeled["automation_id"] = df_unlabeled["automation_id"].apply(clean_id)
|
||||
df_labeled["folder"] = df_labeled["folder"].astype(str).str.strip()
|
||||
df_unlabeled["folder"] = df_unlabeled["folder"].astype(str).str.strip()
|
||||
|
||||
labeled_pairs = set(zip(df_labeled["automation_id"], df_labeled["folder"]))
|
||||
df_unlabeled_filtered = df_unlabeled[
|
||||
~df_unlabeled.apply(lambda row: (row["automation_id"], row["folder"]) in labeled_pairs, axis=1)
|
||||
]
|
||||
|
||||
|
||||
# Step 3: embeddings ---
|
||||
print("\n***Step 3 ***\nEmbeddings")
|
||||
model = SentenceTransformer("all-MiniLM-L6-v2")
|
||||
texts = df_labeled['automation'].astype(str).tolist()
|
||||
|
||||
with open("main/labeled_embeddings.pkl", "rb") as f:
|
||||
data = pickle.load(f)
|
||||
|
||||
embeddings = data['embeddings']
|
||||
print("Shape embeddings:", embeddings.shape)
|
||||
|
||||
|
||||
# ----- Step4: Creazione indice FAISS ---
|
||||
dimension = embeddings.shape[1]
|
||||
index = faiss.IndexFlatL2(dimension) # indice L2 (distanza Euclidea)
|
||||
index.add(embeddings)
|
||||
print(f"\n***Step 4: Indice FAISS creato***. \nNumero di vettori nell'indice: {index.ntotal}")
|
||||
|
||||
faiss.normalize_L2(embeddings)
|
||||
dimension = embeddings.shape[1]
|
||||
index = faiss.IndexFlatIP(dimension)
|
||||
index.add(embeddings)
|
||||
|
||||
|
||||
# Prova con le prima 50 automazioni non annotate
|
||||
k = 5
|
||||
output_rows = []
|
||||
df_sample = df_unlabeled.head(50)
|
||||
|
||||
for i, row in df_sample.iterrows():
|
||||
query_text = str(row["human_like"])
|
||||
|
||||
# Calcolo embedding della nuova automazione
|
||||
query_emb = model.encode([query_text], convert_to_numpy=True).astype("float32")
|
||||
|
||||
# Recupera indici dei k vicini più prossimi
|
||||
distances, indices = index.search(query_emb, k)
|
||||
|
||||
# Estrae automazioni simili dal DataFrame
|
||||
for rank in range(k):
|
||||
idx = indices[0][rank]
|
||||
distance = distances[0][rank]
|
||||
confidence = 1 / (1 + float(distance))
|
||||
|
||||
retrieved_row = df_labeled.iloc[idx]
|
||||
|
||||
output_rows.append({
|
||||
"automazione da etichettare": query_text,
|
||||
"rank": rank + 1,
|
||||
"automazione simile": retrieved_row["automation"],
|
||||
"categoria automazione simile": retrieved_row["category"],
|
||||
"distanza": distance,
|
||||
"confidence": round(confidence, 4)
|
||||
})
|
||||
|
||||
# Creazione DataFrame risultati
|
||||
df_results = pd.DataFrame(output_rows)
|
||||
output_path = "main/datasets/similarity_analysis.xlsx"
|
||||
df_results.to_excel(output_path, index=False)
|
||||
|
||||
wb = load_workbook(output_path)
|
||||
ws = wb.active
|
||||
|
||||
distanza_col_idx = None
|
||||
for idx, cell in enumerate(ws[1], start=1):
|
||||
if cell.value == "distanza":
|
||||
distanza_col_idx = idx
|
||||
break
|
||||
|
||||
if distanza_col_idx is None:
|
||||
raise ValueError("Colonna 'distanza' non trovata!")
|
||||
|
||||
# Applichiamo i colori in base al valore
|
||||
for row in ws.iter_rows(min_row=2, max_row=ws.max_row, min_col=distanza_col_idx, max_col=distanza_col_idx):
|
||||
cell = row[0]
|
||||
try:
|
||||
val = float(cell.value)
|
||||
if val < 0.5:
|
||||
color = "90EE90" # verde chiaro
|
||||
elif val < 1.0:
|
||||
color = "FFFF00" # giallo
|
||||
else:
|
||||
color = "FF6347" # rosso
|
||||
cell.fill = PatternFill(start_color=color, end_color=color, fill_type="solid")
|
||||
except:
|
||||
continue
|
||||
|
||||
# Salva il file direttamente con colori applicati
|
||||
wb.save(output_path)
|
||||
print(f"Excel salvato in {output_path}")
|
||||
Binary file not shown.
|
|
@ -0,0 +1,35 @@
|
|||
# --- Import librerie ---
|
||||
import pandas as pd
|
||||
from openai import AzureOpenAI
|
||||
import os
|
||||
import json
|
||||
from sentence_transformers import SentenceTransformer
|
||||
import numpy as np
|
||||
|
||||
base_folder = r"C:\Users\Arianna\Desktop\AutomationDataset"
|
||||
dataset_rows = []
|
||||
subfolders = [f for f in os.listdir(base_folder) if os.path.isdir(os.path.join(base_folder, f))]
|
||||
for subfolder in subfolders:
|
||||
subfolder_path = os.path.join(base_folder, subfolder)
|
||||
#Cerco file "automation-descriptions" (json)
|
||||
for file_name in os.listdir(subfolder_path):
|
||||
if "automation-descriptions" in file_name and file_name.endswith(".json"):
|
||||
file_path = os.path.join(subfolder_path, file_name)
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
try:
|
||||
data = json.load(f)
|
||||
for entry in data:
|
||||
# Prendo solo il campo human_like
|
||||
row = {
|
||||
"automation_id": entry.get("id", ""),
|
||||
"human_like": entry.get("result", {}).get("human_like", ""),
|
||||
"folder": subfolder
|
||||
}
|
||||
dataset_rows.append(row)
|
||||
except Exception as e:
|
||||
print(f"Errore nel file {file_path}: {e}")
|
||||
|
||||
df_unlabeled = pd.DataFrame(dataset_rows)
|
||||
path = r"C:\Users\Arianna\Desktop\secureTAP+\main\datasets\unlabeled_dataset2.xlsx"
|
||||
df_unlabeled.to_excel(path, index=False)
|
||||
|
||||
Loading…
Reference in New Issue