From bb18bc6e74ca957b8014284899e920c605afd2b4 Mon Sep 17 00:00:00 2001 From: nicola leonardi Date: Thu, 30 Jul 2026 10:47:56 +0200 Subject: [PATCH] browser-use e graph --- scripts/browser-use/webagent-ollama.py | 76 ++ .../browser-use/webagent-openai-full-log.py | 318 ++++++++ scripts/browser-use/webagent-openai.py | 75 ++ scripts/costruzione_grafo/hag_queries.py | 740 +++++++++++++----- .../heterogeneous_ax_graph.py | 28 +- 5 files changed, 1040 insertions(+), 197 deletions(-) create mode 100644 scripts/browser-use/webagent-ollama.py create mode 100644 scripts/browser-use/webagent-openai-full-log.py create mode 100644 scripts/browser-use/webagent-openai.py diff --git a/scripts/browser-use/webagent-ollama.py b/scripts/browser-use/webagent-ollama.py new file mode 100644 index 0000000..e77d3c2 --- /dev/null +++ b/scripts/browser-use/webagent-ollama.py @@ -0,0 +1,76 @@ +#pip install browser-use langchain-openai langchain-ollama playwright +#playwright install + +"""import asyncio +from browser_use import Agent +from langchain_ollama import ChatOllama + +async def run_local_agent(): + # Inizializziamo il modello locale tramite Ollama + # Impostiamo num_ctx a 32000 perché l'albero visivo/DOM consuma molti token + llm = ChatOllama( + + model="gemma4:e2b", + num_ctx=32000, + temperature=0.0 + ) + + agent = Agent( + task="Naviga su wikipedia, cerca la pagina dell'intelligenza artificiale e dimmi l'anno di nascita del termine.",#"Vai su google.it, cerca 'meteo Roma' e leggi la temperatura attuale.", + llm=llm + ) + + result = await agent.run(max_steps=20) + print("\n[Risultato Finale Ollama]:\n", result) + +if __name__ == "__main__": + asyncio.run(run_local_agent())""" + +import asyncio +from browser_use import Agent, ChatOpenAI +from langchain_ollama import ChatOllama + +async def run_remote_ollama_agent(): + # Poiché l'istanza Ollama è protetta da un Bearer Token, usiamo la classe ChatOpenAI + # che è nativamente progettata per gestire base_url custom ed header di autenticazione. + llm = ChatOpenAI( + model="ministral-3:3b",#"gemma3:4b",#"gemma4:e4b",#"ministral-3:3b",#"gemma4:e2b", # Inserisci il nome esatto del modello presente sul tuo server remoto + base_url="https://vgpu.hiis.cloud.isti.cnr.it/v1", # L'endpoint v1 del tuo proxy/gateway + api_key="", # Il tuo token di autenticazione va inserito qui + + # Parametri critici per far funzionare i modelli Open Source locali con browser-use: + temperature=0.0, + + + + # Sovrascriviamo il comportamento di default dicendo al client OpenAI + # di effettuare la chiamata POST puntando alla rotta esatta richiesta dal tuo server + default_headers={ + "Authorization": "Bearer your_api_key" + }, + + ) + + # FORZATURA DIRETTA: Iniettiamo num_ctx direttamente nei model_kwargs dell'oggetto appena creato. + # Questo bypassa i controlli del costruttore __init__() e scrive direttamente nella configurazione di rete. + llm.model_kwargs = { #SEMBRA CHE NON SI APPLICHI + "extra_body": { + "num_ctx": 32000 + } + } + + + # Inizializzazione dell'agente + agent = Agent( + #task="Naviga su wikipedia, cerca la pagina dell'intelligenza artificiale e dimmi l'anno di nascita del termine.",#"Vai su google.it, cerca 'meteo Roma' e leggi la temperatura attuale.", + task="Open the URL https://github.com/rasbt and check if the page offers an alternative function to the interactive map of content in the year (contribution heatmap data visualization). 'Equivalent' is to be interpreted according to the WCAG Success Criterion 2.5.8 Target Size (Minimum): 'The function can be achieved through a different control on the same page that meets this criterion' - Answer YES or NO and justify your answer.", + + llm=llm + ) + + # Esecuzione del workflow + result = await agent.run(max_steps=20) + print("\n[Risultato Finale]:\n", result) + +if __name__ == "__main__": + asyncio.run(run_remote_ollama_agent()) \ No newline at end of file diff --git a/scripts/browser-use/webagent-openai-full-log.py b/scripts/browser-use/webagent-openai-full-log.py new file mode 100644 index 0000000..5efbf4f --- /dev/null +++ b/scripts/browser-use/webagent-openai-full-log.py @@ -0,0 +1,318 @@ + +#pip install browser-use langchain-openai langchain-ollama playwright +#playwright install + + ## comandi da CLI +""" +browser-use open https://example.com # Navigate to URL +browser-use state # See clickable elements +browser-use click 5 # Click element by index +browser-use type "Hello" # Type text +browser-use screenshot page.png # Take screenshot +browser-use close # Close browser +""" + + + +import asyncio +import aiofiles +import os +import logging +import sys +import re +from datetime import datetime +from browser_use import Agent, ChatAzureOpenAI, Browser +import base64, os, json + + +from browser_use.browser import BrowserProfile, BrowserSession +from browser_use.browser.profile import ViewportSize + +# highlight_elements goes on BrowserProfile, not Browser +browser_profile = BrowserProfile( + headless=False, #Visible window opens on screen + highlight_elements=True, # draws colored bounding boxes on interactive elements + # dom_highlight_elements=True, # alternative: DOM-based highlighting (takes priority if both set) + + window_size=ViewportSize(width=900, height=1100), # bigger window to compensate for OS chrome + viewport=ViewportSize(width=768, height=1024), # ✅ forces exact page content size +) + +browser_session = BrowserSession(browser_profile=browser_profile) + + + + + +# ─── LOGGING SETUP ──────────────────────────────────────────────────────────── + +os.environ["BROWSER_USE_LOGGING_LEVEL"] = "debug" + + +# ── BBox tracker — captures the count from the log message ────────────────── +class BBoxTracker: + """Intercepts the BBox filtering log line and stores the latest count.""" + last_excluded: int = 0 + history: list[dict] = [] + + def record(self, step: int, count: int): + self.last_excluded = count + self.history.append({"step": step, "excluded_nodes": count}) + +bbox_tracker = BBoxTracker() +_BBOX_RE = re.compile(r"BBox filtering excluded (\d+) nodes", re.IGNORECASE) + +class BBoxInterceptHandler(logging.Handler): + """Silently watches every log record for the BBox line.""" + def emit(self, record: logging.LogRecord): + msg = record.getMessage() + m = _BBOX_RE.search(msg) + if m: + bbox_tracker.last_excluded = int(m.group(1)) + +# Colored formatter +class ColoredFormatter(logging.Formatter): + COLORS = { + logging.DEBUG: "\033[36m", + logging.INFO: "\033[32m", + logging.WARNING: "\033[33m", + logging.ERROR: "\033[31m", + logging.CRITICAL: "\033[35m", + } + RESET = "\033[0m" + BOLD = "\033[1m" + + def format(self, record): + color = self.COLORS.get(record.levelno, self.RESET) + record.levelname = f"{color}{self.BOLD}{record.levelname:<8}{self.RESET}" + record.name = f"\033[90m{record.name}{self.RESET}" + return super().format(record) + +console_handler = logging.StreamHandler(sys.stdout) +console_handler.setLevel(logging.DEBUG) +console_handler.setFormatter(ColoredFormatter( + fmt="%(asctime)s │ %(levelname)s │ %(name)s\n └─ %(message)s\n", + datefmt="%H:%M:%S", +)) + +root_logger = logging.getLogger() +root_logger.setLevel(logging.DEBUG) +root_logger.handlers.clear() +root_logger.addHandler(console_handler) +root_logger.addHandler(BBoxInterceptHandler()) # ← silent tracker + +for noisy in ("httpx", "httpcore", "playwright", "urllib3", "asyncio"): + logging.getLogger(noisy).setLevel(logging.WARNING) + +logger = logging.getLogger("agent_runner") + +# ─── STEP CALLBACKS ─────────────────────────────────────────────────────────── + +step_count = 0 +step_start_time: datetime | None = None + + +os.makedirs("./screenshots_on_step", exist_ok=True) + +import asyncio +import aiofiles +import base64 +import io +import os +from PIL import Image, ImageDraw, ImageFont + + +async def on_step(state, model_output, step_num): + if not state.screenshot: + return + + img = Image.open(io.BytesIO(base64.b64decode(state.screenshot))) + draw = ImageDraw.Draw(img) + + selector_map = state.dom_state.selector_map + colors = ["#FF0000", "#FF6600", "#0066FF", "#009900", "#9900CC"] + + for idx, node in selector_map.items(): + # ✅ node IS the EnhancedDOMTreeNode directly — no .original_node + rect = node.absolute_position + if rect is None: + continue + + x, y, w, h = rect.x, rect.y, rect.width, rect.height + if w <= 0 or h <= 0: + continue + + color = colors[idx % len(colors)] + draw.rectangle([x, y, x + w, y + h], outline=color, width=2) + + label = str(idx) + draw.rectangle([x, y - 14, x + len(label) * 8 + 4, y], fill=color) + draw.text((x + 2, y - 13), label, fill="white") + + path = f"./screenshots_on_step/step_{step_num:03d}.png" + buf = io.BytesIO() + img.save(buf, format="PNG") + async with aiofiles.open(path, "wb") as f: + await f.write(buf.getvalue()) + + print(f"[step {step_num}] annotated: {path} ({len(selector_map)} elements) | {state.url}") + +async def on_step_start(agent): + global step_count, step_start_time + step_count += 1 + step_start_time = datetime.now() + bbox_tracker.last_excluded = 0 # reset for this step + print(f"\n\033[1;34m{'━'*60}\033[0m") + print(f"\033[1;34m 🚀 STEP {step_count} — {step_start_time.strftime('%H:%M:%S')}\033[0m") + print(f"\033[1;34m{'━'*60}\033[0m") + +async def on_step_end(agent): + global step_start_time + elapsed = (datetime.now() - step_start_time).total_seconds() if step_start_time else 0 + + # record bbox stats for this step + bbox_tracker.record(step_count, bbox_tracker.last_excluded) + + history = agent.history.history + last = history[-1] if history else None + + url = "N/A" + if last and last.state and hasattr(last.state, "url"): + url = last.state.url + + actions = "N/A" + if last and last.model_output and hasattr(last.model_output, "action"): + acts = last.model_output.action + if isinstance(acts, list): + actions = ", ".join( + type(a).__name__ if not isinstance(a, dict) else list(a.keys())[0] + for a in acts + ) + else: + actions = str(acts) + + result_text = "N/A" + if last and last.result: + results = last.result if isinstance(last.result, list) else [last.result] + parts = [] + for r in results: + if hasattr(r, "extracted_content") and r.extracted_content: + parts.append(f"📄 {str(r.extracted_content)[:120]}") + elif hasattr(r, "error") and r.error: + parts.append(f"❌ {str(r.error)[:120]}") + result_text = " | ".join(parts) if parts else "✔ done" + + # BBox colour: green if low, yellow if medium, red if high + bbox_n = bbox_tracker.last_excluded + if bbox_n < 20: + bbox_color = "\033[32m" + elif bbox_n < 60: + bbox_color = "\033[33m" + else: + bbox_color = "\033[31m" + + print(f"\033[32m ✅ STEP {step_count} COMPLETE ({elapsed:.2f}s)\033[0m") + print(f"\033[90m URL :\033[0m {url}") + print(f"\033[90m Actions:\033[0m {actions}") + print(f"\033[90m Result :\033[0m {result_text}") + print(f"\033[90m BBox :\033[0m {bbox_color}{bbox_n} nodes pruned (redundant children of links/buttons)\033[0m") + print(f"\033[1;34m{'━'*60}\033[0m\n") + + +# ✅ highlight_elements draws bounding boxes from DOM on the live browser +#browser = Browser( +# headless=False, +# highlight_elements=True, +#) + +# ─── MAIN ───────────────────────────────────────────────────────────────────── + +async def run_azure_agent(): + logger.info("Initializing Azure OpenAI LLM...") + + llm = ChatAzureOpenAI( + model="gpt-4o", + azure_endpoint="https://hiis-accessibility-fonderia.cognitiveservices.azure.com/", + azure_deployment="gpt-4o", + api_version="2025-01-01-preview", + api_key="" + ) + + agent = Agent( + + + #task=( + # "Open the URL https://github.com/rasbt and check if the page offers " + # "an alternative function to the interactive map of content in the year " + # "(contribution heatmap data visualization). 'Equivalent' is to be interpreted " + # "according to the WCAG Success Criterion 2.5.8 Target Size (Minimum): " + # "'The function can be achieved through a different control on the same page " + # "that meets this criterion' - Answer YES or NO and justify your answer." + #) + + task=( + "Open the URL https://github.com/rasbt and check if the page offers " + "an 'Equivalent' function to the ”Contribution graph cells”: The dense grid of small squares in " + "the '1,535 contributions in the last year' heatmap calendar." + "'Equivalent' is to be interpreted according to the WCAG Success Criterion 2.5.8 Target Size (Minimum): " + "'The function can be achieved through a different control on the same page " + "that meets this criterion'. - Answer YES or NO and justify your answer." + ), + llm=llm, + #on_step_start=on_step_start, + #on_step_end=on_step_end, + save_conversation_path="./logs/session.json",# all screenshots embedded here + save_conversation_path_encoding="utf-8", # default + # Optional: keep full history (don't prune steps) + max_history_items=None, # None = keep ALL steps in memory + use_vision=True, + browser_session=browser_session, #browser # pass session, not Browser() + vision_detail_level="high", # 'low', 'high', or 'auto' + generate_gif="./logs/session.gif", + register_new_step_callback=on_step, # real-time callback + ) + + print(f"\n\033[1;35m{'═'*60}\033[0m") + print(f"\033[1;35m 🤖 BROWSER-USE AGENT STARTING\033[0m") + print(f"\033[1;35m{'═'*60}\033[0m\n") + + result = await agent.run(max_steps=20) + #await browser.close() + + """ + #### non va ### + import os + import base64 + import json + os.makedirs("screenshots_from_results", exist_ok=True) + for i, shot in enumerate(result.screenshots()): + # screenshots() returns base64 strings + img_bytes = base64.b64decode(shot) + path = f"screenshots_from_results/step_{i+1:02d}.png" + with open(path, "wb") as f: + f.write(img_bytes) + print(f"Saved {path}") + """ + + + + # ── BBox summary across all steps ────────────────────────────────────── + print(f"\n\033[1;33m{'═'*60}\033[0m") + print(f"\033[1;33m 📦 BBOX FILTERING SUMMARY\033[0m") + print(f"\033[1;33m{'═'*60}\033[0m") + total = sum(s["excluded_nodes"] for s in bbox_tracker.history) + for s in bbox_tracker.history: + bar = "█" * min(s["excluded_nodes"] // 5, 30) + print(f" Step {s['step']:>2} │ {s['excluded_nodes']:>4} nodes {bar}") + print(f" {'─'*40}") + print(f" Total pruned across all steps: {total} nodes") + print(f"\033[1;33m{'═'*60}\033[0m\n") + + print(f"\n\033[1;35m{'═'*60}\033[0m") + print(f"\033[1;35m 🏁 FINAL RESULT\033[0m") + print(f"\033[1;35m{'═'*60}\033[0m") + print(result) + + +if __name__ == "__main__": + asyncio.run(run_azure_agent()) \ No newline at end of file diff --git a/scripts/browser-use/webagent-openai.py b/scripts/browser-use/webagent-openai.py new file mode 100644 index 0000000..953ca30 --- /dev/null +++ b/scripts/browser-use/webagent-openai.py @@ -0,0 +1,75 @@ + +#pip install browser-use langchain-openai langchain-ollama playwright +#playwright install + + ## comandi da CLI +""" +browser-use open https://example.com # Navigate to URL +browser-use state # See clickable elements +browser-use click 5 # Click element by index +browser-use type "Hello" # Type text +browser-use screenshot page.png # Take screenshot +browser-use close # Close browser +""" + + +""" +import asyncio +import os +from browser_use import Agent +from langchain_openai import AzureChatOpenAI + +async def run_azure_agent(): + # Inizializziamo il modello Azure OpenAI tramite LangChain + # Sostituisci "il-tuo-deployment-name" con il nome esatto del deployment su Azure (es. gpt-4o) + llm = AzureChatOpenAI( + azure_deployment="gpt-4o", + openai_api_version=os.getenv("AZURE_OPENAI_API_VERSION", "2025-01-01-preview"), + azure_endpoint=os.getenv("AZURE_OPENAI_ENDPOINT","https://hiis-accessibility-fonderia.cognitiveservices.azure.com"), + api_key=os.getenv("AZURE_OPENAI_API_KEY","4lwGUwrx7jsqdxESGBpN9wYYyLNsxzC2s8ZLQlZPCQUayDWuDo3NJQQJ99BKACfhMk5XJ3w3AAAAACOGs2uw") + ) + + # Definiamo l'agente con il modello Azure + agent = Agent( + task="Naviga su wikipedia, cerca la pagina dell'intelligenza artificiale e dimmi l'anno di nascita del termine.", + llm=llm + ) + + # Eseguiamo l'agente + result = await agent.run() + print("\n[Risultato Finale Azure OpenAI]:\n", result) + +if __name__ == "__main__": + asyncio.run(run_azure_agent()) +""" + + + +import asyncio +import os +# IMPORTANTE: Importiamo ChatAzureOpenAI direttamente da browser_use! +from browser_use import Agent, ChatAzureOpenAI + +async def run_azure_agent(): + # browser-use gestisce internamente la classe per evitare l'errore di Pydantic + llm = ChatAzureOpenAI( + model="gpt-4o", + azure_endpoint="https://hiis-accessibility-fonderia.cognitiveservices.azure.com/", + azure_deployment="gpt-4o", + api_version="2025-01-01-preview", + api_key="" # Sostituisci con la tua chiave Azure reale + ) + + # Configurazione dell'agente + agent = Agent( + #task="apri l'url https://github.com/rasbt e verifica se la pagina presenta una funzione alternativa alla mappa interattiva dei contenuti nell'anno (contribution heatmap data visualization). 'Equivalente' è da interpretare secondo la definizione WCAG Sucess Criterion 2.5.8 Target Size (Minimum): 'The function can be achieved through a different control on the same page that meets this criterion' - Rispondi SI o No e motiva la tua risposta",#"Naviga su wikipedia, cerca la pagina dell'intelligenza artificiale e dimmi l'anno di nascita del termine.", + task="Open the URL https://github.com/rasbt and check if the page offers an alternative function to the interactive map of content in the year (contribution heatmap data visualization). 'Equivalent' is to be interpreted according to the WCAG Success Criterion 2.5.8 Target Size (Minimum): 'The function can be achieved through a different control on the same page that meets this criterion' - Answer YES or NO and justify your answer.", + llm=llm + ) + + # Esecuzione del workflow + result = await agent.run() + print("\n[Risultato Finale Azure OpenAI]:\n", result) + +if __name__ == "__main__": + asyncio.run(run_azure_agent()) \ No newline at end of file diff --git a/scripts/costruzione_grafo/hag_queries.py b/scripts/costruzione_grafo/hag_queries.py index 82f23fe..62014c9 100644 --- a/scripts/costruzione_grafo/hag_queries.py +++ b/scripts/costruzione_grafo/hag_queries.py @@ -1,4 +1,3 @@ -from __future__ import annotations """ hag_queries.py ============== @@ -32,9 +31,7 @@ Or from the command line (requires a saved JSON graph): python hag_queries.py hag.json --verbose python hag_queries.py hag.json --query Q1.1 python hag_queries.py hag.json --report issues.json -""" -""" Architettura del modulo Ogni query è una funzione autonoma che restituisce una lista di Issue — un dataclass con query_id, title, wcag, severity, layer, node_id, detail e un dizionario extra per dati specifici. La severità è calcolata in modo contestuale (non fissa) — ad esempio Q1.2 diventa critical se il nodo visivo senza AX occupa più del 10% della viewport. Le query sono organizzate in quattro gap, coerenti con la MVAR discussa prima: @@ -44,11 +41,13 @@ Le query sono organizzate in quattro gap, coerenti con la MVAR discussa prima: Δ3 — DOM - Visual (Q3.x): elementi visivamente prominenti il cui DOM node non porta semantica (div/span senza ruolo ARIA). """ - +from __future__ import annotations import json +import math import sys from dataclasses import dataclass, field, asdict +from statistics import harmonic_mean from typing import Any import networkx as nx @@ -105,78 +104,189 @@ def _dom_neighbors_via_cross(HAG: nx.MultiDiGraph, dom_node: str) -> list[str]: return _neighbors_by_relation(HAG, dom_node, "dom_to_ax") +def _dom_parent(HAG: nx.MultiDiGraph, dom_node: str) -> Optional[str]: + """Strict parent lookup (child --hasParent--> parent), no reverse edges.""" + for _, v, d in HAG.out_edges(dom_node, data=True): + if d.get("relation") == "hasParent": + return v + return None + + +def _dom_children(HAG: nx.MultiDiGraph, dom_node: str) -> list[str]: + """Strict children lookup (parent --hasChild--> child), no reverse edges.""" + return [v for _, v, d in HAG.out_edges(dom_node, data=True) + if d.get("relation") == "hasChild"] + + +def _is_aria_expanded_false(attrs: dict) -> bool: + val = attrs.get("aria_expanded") + return val is not None and str(val).strip().lower() == "false" + + +def _inside_collapsed_disclosure( + HAG: nx.MultiDiGraph, dom_node: str, max_depth: int = 6 +) -> bool: + """ + Heuristic for the "modal / dropdown menu" case: a DOM element can be + legitimately absent from the accessibility tree because it lives inside + a disclosure widget (dropdown menu, accordion, modal, …) that is + currently collapsed/closed. Browsers commonly drop such subtrees from + the AX tree entirely and only expose them once the trigger is activated + (aria-expanded="true"). + + We walk up the DOM ancestor chain and, at each level, check both the + ancestor itself and its direct children (covers the common pattern + where the trigger — e.g. a