GUI Agent¶

"""
Set-of-Mark: 在截图上标注可交互元素的编号
这使得LLM可以通过编号引用具体元素
"""

from PIL import Image, ImageDraw, ImageFont
import json

class SetOfMarkAnnotator:
    """
    SoM标注器：在截图上标注UI元素
    """

    def __init__(self):
        self.elements = []

    def annotate_screenshot(self, screenshot_path: str,
                            elements: list) -> Image.Image:
        """
        在截图上标注可交互元素

        Args:
            screenshot_path: 截图路径
            elements: UI元素列表, 每个元素包含:
                - bbox: [x1, y1, x2, y2] 边界框
                - type: 元素类型 (button/input/link/...)
                - text: 元素文本
        Returns:
            标注后的图片
        """
        img = Image.open(screenshot_path)
        draw = ImageDraw.Draw(img)

        self.elements = elements
        colors = {
            'button': '#FF4444',
            'input': '#44FF44',
            'link': '#4444FF',
            'text': '#FFFF44',
            'other': '#FF44FF'
        }

        for i, elem in enumerate(elements):
            x1, y1, x2, y2 = elem['bbox']
            color = colors.get(elem.get('type', 'other'), '#FFFFFF')

            # 绘制边界框
            draw.rectangle([x1, y1, x2, y2], outline=color, width=2)

            # 绘制编号标签
            label = f"[{i}]"
            draw.rectangle([x1, y1-18, x1+30, y1], fill=color)
            draw.text((x1+2, y1-16), label, fill='white')

        return img

    def get_element_description(self) -> str:
        """生成元素描述文本，供LLM理解"""
        desc = "可交互元素列表:\n"
        for i, elem in enumerate(self.elements):
            desc += f"  [{i}] {elem.get('type', 'unknown')}: "
            desc += f"\"{elem.get('text', '')}\" "
            desc += f"位置: {elem['bbox']}\n"
        return desc

# 使用示例
annotator = SetOfMarkAnnotator()
elements = [
    {"bbox": [100, 200, 250, 240], "type": "input", "text": "搜索框"},
    {"bbox": [260, 200, 340, 240], "type": "button", "text": "搜索"},
    {"bbox": [100, 260, 300, 290], "type": "link", "text": "高级搜索选项"},
]

desc = annotator.get_element_description()
print(desc)

"""
UI Grounding: 将自然语言描述映射到具体的UI元素坐标

方法1: 基于OCR + 空间推理
方法2: 基于多模态LLM直接预测坐标
方法3: 基于专用grounding模型 (如SeeClick, OS-Atlas)
"""

class UIGrounding:
    """UI元素定位"""

    @staticmethod
    def ocr_based_grounding(screenshot, target_text: str,
                             ocr_results: list) -> dict:
        """
        基于OCR的元素定位
        1. OCR识别所有文字
        2. 匹配目标文字
        3. 返回对应位置
        """
        for result in ocr_results:
            if target_text.lower() in result['text'].lower():
                return {
                    'found': True,
                    'bbox': result['bbox'],
                    'confidence': result['confidence'],
                    'center': (
                        (result['bbox'][0] + result['bbox'][2]) / 2,
                        (result['bbox'][1] + result['bbox'][3]) / 2
                    )
                }
        return {'found': False}

    @staticmethod
    def build_grounding_prompt(task: str, screenshot_desc: str) -> str:
        """
        构造多模态LLM的grounding prompt
        让LLM直接输出要操作的坐标
        """
        prompt = f"""你是一个GUI操作助手。根据用户任务和屏幕截图，确定需要操作的UI元素位置。

当前屏幕内容:
{screenshot_desc}

用户任务: {task}

请回答:
1. 需要操作的元素是什么？
2. 该元素的坐标位置 [x, y]（屏幕中心为参考点）
3. 需要执行的操作类型（click/type/scroll）
4. 如果是输入操作，输入的内容是什么？

以JSON格式回答:
{{"element": "...", "position": [x, y], "action": "...", "value": "..."}}
"""
        return prompt

print("UI Grounding核心挑战:")
print("1. 小图标/小按钮的精确定位")
print("2. 动态内容（如弹窗、加载中状态）")
print("3. 多语言界面的OCR准确性")
print("4. 相似元素的区分（多个按钮/链接）")

from dataclasses import dataclass
from enum import Enum
from typing import Optional, Tuple

class ActionType(Enum):
    """GUI Agent标准动作类型"""
    CLICK = "click"              # 单击
    DOUBLE_CLICK = "double_click" # 双击
    RIGHT_CLICK = "right_click"  # 右键
    TYPE = "type"                # 输入文字
    PRESS = "press"              # 按键（如Enter, Tab）
    HOTKEY = "hotkey"            # 快捷键（如Ctrl+C）
    SCROLL = "scroll"            # 滚动
    DRAG = "drag"                # 拖拽
    WAIT = "wait"                # 等待
    SCREENSHOT = "screenshot"    # 截图（用于观察）
    DONE = "done"                # 任务完成
    FAIL = "fail"                # 放弃/失败

@dataclass
class GUIAction:
    """标准化GUI动作"""
    action_type: ActionType
    position: Optional[Tuple[int, int]] = None     # (x, y) 坐标
    text: Optional[str] = None                      # 输入文本
    key: Optional[str] = None                       # 按键名
    direction: Optional[str] = None                 # 滚动方向
    end_position: Optional[Tuple[int, int]] = None  # 拖拽终点
    element_id: Optional[int] = None                # SoM元素编号

    def to_dict(self):
        return {k: v for k, v in self.__dict__.items() if v is not None}  # 字典推导+__dict__：过滤None值序列化对象属性

    def __str__(self):
        if self.action_type == ActionType.CLICK:
            if self.position is None:
                return "Click (no position)"
            return f"Click at ({self.position[0]}, {self.position[1]})"
        elif self.action_type == ActionType.TYPE:
            pos_str = f"({self.position[0]}, {self.position[1]})" if self.position else "(no position)"
            return f"Type '{self.text}' at {pos_str}"
        elif self.action_type == ActionType.PRESS:
            return f"Press {self.key}"
        elif self.action_type == ActionType.SCROLL:
            return f"Scroll {self.direction}"
        elif self.action_type == ActionType.DONE:
            return "Task completed"
        return f"{self.action_type.value}: {self.to_dict()}"

# 示例动作序列
actions = [
    GUIAction(ActionType.CLICK, position=(150, 220)),
    GUIAction(ActionType.TYPE, position=(150, 220), text="Python GUI Agent"),
    GUIAction(ActionType.PRESS, key="Enter"),
    GUIAction(ActionType.WAIT),
    GUIAction(ActionType.CLICK, position=(200, 350), element_id=3),
    GUIAction(ActionType.DONE),
]

print("示例动作序列:")
for i, action in enumerate(actions):
    print(f"  Step {i+1}: {action}")

import re
import json

class ActionParser:
    """
    从LLM输出解析GUI动作

    支持两种格式:
    1. JSON格式: {"action": "click", "position": [100, 200]}
    2. 自然语言格式: "Click on the search button at (100, 200)"
    """

    @staticmethod
    def parse_json(llm_output: str) -> GUIAction:
        """解析JSON格式的动作"""
        # 提取JSON
        json_match = re.search(r'\{[^{}]+\}', llm_output)
        if not json_match:
            raise ValueError("No JSON found in output")

        try:
            data = json.loads(json_match.group())
        except json.JSONDecodeError as e:
            raise ValueError(f"Invalid JSON in LLM output: {e}")

        action_map = {
            'click': ActionType.CLICK,
            'type': ActionType.TYPE,
            'press': ActionType.PRESS,
            'scroll': ActionType.SCROLL,
            'done': ActionType.DONE,
        }

        action_type = action_map.get(data.get('action', ''), ActionType.CLICK)
        position = tuple(data['position']) if 'position' in data else None

        return GUIAction(
            action_type=action_type,
            position=position,
            text=data.get('text'),
            key=data.get('key'),
            element_id=data.get('element_id'),
        )

    @staticmethod
    def parse_natural_language(text: str) -> GUIAction:
        """解析自然语言描述的动作"""
        text = text.lower()

        # 提取坐标
        coord_match = re.search(r'\((\d+),\s*(\d+)\)', text)
        position = tuple(map(int, coord_match.groups())) if coord_match else None  # map将字符串坐标转为整数元组，匹配失败返回None

        if 'click' in text:
            return GUIAction(ActionType.CLICK, position=position)
        elif 'type' in text or 'input' in text:
            # 提取输入文本
            input_match = re.search(r"['\"](.+?)['\"]", text)
            input_text = input_match.group(1) if input_match else ""
            return GUIAction(ActionType.TYPE, position=position, text=input_text)
        elif 'scroll' in text:
            direction = 'down' if 'down' in text else 'up'
            return GUIAction(ActionType.SCROLL, direction=direction)
        elif 'done' in text or 'finish' in text:
            return GUIAction(ActionType.DONE)

        return GUIAction(ActionType.CLICK, position=position)

# 测试解析
parser = ActionParser()

json_output = '我需要点击搜索按钮 {"action": "click", "position": [260, 220], "element_id": 1}'
action = parser.parse_json(json_output)
print(f"JSON解析: {action}")

nl_output = "Click on the search box at (150, 220)"
action = parser.parse_natural_language(nl_output)
print(f"NL解析: {action}")

import time
from abc import ABC, abstractmethod
from typing import List, Dict, Any

class GUIAgent:
    """
    通用GUI Agent框架

    核心循环: Observe → Think → Act → Observe → ...
    """

    def __init__(self, llm_client, max_steps: int = 20):
        self.llm_client = llm_client  # LLM调用接口
        self.max_steps = max_steps
        self.action_history: List[Dict] = []
        self.observation_history: List[str] = []

    def run(self, task: str) -> Dict[str, Any]:
        """
        执行GUI任务

        Args:
            task: 用户的自然语言任务描述
        Returns:
            执行结果字典
        """
        print(f"📋 任务: {task}")
        print("=" * 60)

        for step in range(self.max_steps):
            print(f"\n--- Step {step + 1}/{self.max_steps} ---")

            # 1. Observe: 获取当前屏幕状态
            observation = self._observe()
            self.observation_history.append(observation)
            print(f"🔍 观察: {observation[:100]}...")

            # 2. Think: 让LLM决定下一步动作
            action = self._think(task, observation)
            print(f"🧠 决策: {action}")

            # 3. 检查是否完成
            if action.action_type == ActionType.DONE:
                print("\n✅ 任务完成!")
                return {"status": "success", "steps": step + 1,
                        "history": self.action_history}

            if action.action_type == ActionType.FAIL:
                print("\n❌ 任务失败")
                return {"status": "failed", "steps": step + 1,
                        "history": self.action_history}

            # 4. Act: 执行动作
            self._act(action)
            self.action_history.append({
                "step": step + 1,
                "action": str(action),
                "observation": observation[:200]
            })

            # 等待UI响应
            time.sleep(0.5)

        print("\n⚠️ 达到最大步数限制")
        return {"status": "max_steps", "steps": self.max_steps,
                "history": self.action_history}

    def _observe(self) -> str:
        """
        获取当前屏幕状态
        实际实现中：截图 + OCR + 元素检测
        """
        # 这里是框架代码，实际需要：
        # 1. 截图: pyautogui.screenshot()
        # 2. OCR: 识别文字
        # 3. UI检测: 识别可交互元素
        # 4. SoM标注: 给元素编号
        return "当前屏幕状态描述（实际为截图+标注）"

    def _think(self, task: str, observation: str) -> GUIAction:
        """
        调用LLM进行决策
        """
        # 构造prompt
        prompt = self._build_prompt(task, observation)

        # 调用LLM（伪代码）
        # response = self.llm_client.chat(prompt, images=[screenshot])
        # action = ActionParser.parse_json(response)

        # 示例返回
        return GUIAction(ActionType.CLICK, position=(100, 200))

    def _act(self, action: GUIAction):
        """
        执行GUI操作
        实际使用 pyautogui 或 playwright
        """
        # import pyautogui
        if action.action_type == ActionType.CLICK:
            # pyautogui.click(action.position[0], action.position[1])
            print(f"  ⚡ 执行点击: {action.position}")
        elif action.action_type == ActionType.TYPE:
            # pyautogui.click(action.position[0], action.position[1])
            # pyautogui.typewrite(action.text)
            print(f"  ⚡ 输入文本: '{action.text}'")
        elif action.action_type == ActionType.SCROLL:
            # pyautogui.scroll(-3 if action.direction == 'down' else 3)
            print(f"  ⚡ 滚动: {action.direction}")

    def _build_prompt(self, task: str, observation: str) -> str:
        """构造决策prompt"""
        history_text = ""
        for h in self.action_history[-5:]:  # 最近5步历史
            history_text += f"  Step {h['step']}: {h['action']}\n"

        prompt = f"""你是一个GUI自动化Agent。你需要通过操作图形界面来完成用户任务。

## 用户任务
{task}

## 操作历史
{history_text if history_text else "  (这是第一步)"}

## 当前屏幕
{observation}

## 可用动作
- click(x, y): 点击指定坐标
- type(x, y, text): 在指定位置输入文本
- press(key): 按下键盘键
- scroll(direction): 上下滚动
- done(): 任务已完成
- fail(): 无法完成任务

## 请决定下一步动作
分析当前屏幕并选择最合适的操作。以JSON格式回答:
{{"thought": "分析推理过程", "action": "click/type/press/scroll/done/fail", "position": [x, y], "text": "可选输入文本"}}
"""
        return prompt

# 框架使用示例
print("GUI Agent框架示例:")
agent = GUIAgent(llm_client=None, max_steps=10)
print("Agent已创建，核心循环: Observe → Think → Act")
print("实际运行需要: LLM API + 屏幕操作库(pyautogui/playwright)")

"""
Web GUI Agent: 基于Playwright实现Web自动化

Playwright优势:
- 支持Chrome/Firefox/Safari
- 内置等待机制
- 可获取DOM和截图
- 支持无头模式
"""

# pip install playwright
# python -m playwright install

class WebGUIAgent:
    """
    Web自动化Agent
    结合Playwright操作 + LLM决策
    """

    def __init__(self, llm_client=None, headless: bool = False):
        self.llm_client = llm_client
        self.headless = headless
        self.browser = None
        self.page = None

    async def setup(self):
        """初始化浏览器"""
        from playwright.async_api import async_playwright

        pw = await async_playwright().start()
        self.browser = await pw.chromium.launch(headless=self.headless)
        self.page = await self.browser.new_page()

    async def navigate(self, url: str):
        """导航到指定URL"""
        await self.page.goto(url, wait_until='networkidle')

    async def observe(self) -> dict:
        """
        获取页面状态
        """
        # 1. 截图
        screenshot = await self.page.screenshot()

        # 2. 获取可交互元素
        elements = await self.page.evaluate("""() => {
            const interactable = document.querySelectorAll(
                'a, button, input, select, textarea, [role="button"], [onclick]'
            );
            return Array.from(interactable).map((el, idx) => {
                const rect = el.getBoundingClientRect();
                return {
                    id: idx,
                    tag: el.tagName.toLowerCase(),
                    type: el.type || '',
                    text: (el.textContent || el.value || el.placeholder || '').trim().slice(0, 50),
                    bbox: [rect.x, rect.y, rect.x + rect.width, rect.y + rect.height],
                    visible: rect.width > 0 && rect.height > 0
                };
            }).filter(el => el.visible);
        }""")

        # 3. 获取页面标题和URL
        title = await self.page.title()
        url = self.page.url

        return {
            "screenshot": screenshot,
            "elements": elements,
            "title": title,
            "url": url,
            "element_description": self._format_elements(elements)
        }

    def _format_elements(self, elements: list) -> str:
        """格式化元素描述"""
        desc = []
        for el in elements[:30]:  # 限制数量
            text = el['text'][:30] if el['text'] else '(empty)'
            desc.append(f"  [{el['id']}] <{el['tag']}> {text}")
        return "\n".join(desc)

    async def execute_action(self, action: dict):
        """
        执行Web操作
        """
        action_type = action.get('action')

        if action_type == 'click':
            element_id = action.get('element_id')
            if element_id is not None:
                elements = await self.page.query_selector_all(
                    'a, button, input, select, textarea, [role="button"], [onclick]'
                )
                if element_id < len(elements):
                    await elements[element_id].click()
            elif 'position' in action:
                x, y = action['position']
                await self.page.mouse.click(x, y)

        elif action_type == 'type':
            element_id = action.get('element_id')
            text = action.get('text', '')
            elements = await self.page.query_selector_all('input, textarea')
            if element_id is not None and element_id < len(elements):
                await elements[element_id].fill(text)

        elif action_type == 'press':
            key = action.get('key', 'Enter')
            await self.page.keyboard.press(key)

        elif action_type == 'scroll':
            direction = action.get('direction', 'down')
            delta = 300 if direction == 'down' else -300
            await self.page.mouse.wheel(0, delta)

        # 等待页面响应
        await self.page.wait_for_load_state('networkidle')

    async def cleanup(self):
        """关闭浏览器"""
        if self.browser:
            await self.browser.close()

# 使用示例（伪代码）
print("""
Web Agent 使用示例:

async def main():
    agent = WebGUIAgent(llm_client=openai_client)
    await agent.setup()
    await agent.navigate("https://www.google.com")

    # 观察-决策-执行循环
    observation = await agent.observe()

    # LLM决策
    action = llm_decide(observation)

    # 执行
    await agent.execute_action(action)

    await agent.cleanup()
""")

class DOMSimplifier:
    """
    DOM简化器

    问题: 完整DOM可能有几万个节点，超过LLM上下文限制
    解决: 智能过滤和压缩
    """

    @staticmethod
    def simplify_html(html: str, max_tokens: int = 4000) -> str:
        """
        简化HTML，保留关键交互元素
        """
        from html.parser import HTMLParser

        # 保留的标签
        keep_tags = {
            'a', 'button', 'input', 'select', 'option', 'textarea',
            'form', 'h1', 'h2', 'h3', 'h4', 'p', 'label', 'img',
            'nav', 'main', 'article', 'section', 'div'  # 结构性标签
        }

        # 移除的属性（减少噪音）
        remove_attrs = {
            'style', 'class', 'data-', 'aria-', 'tabindex',
            'onclick', 'onchange'  # 事件处理器
        }

        # 简化逻辑（基于规则）
        simplified = []
        lines = html.split('\n')
        for line in lines:
            line = line.strip()
            if not line:
                continue
            # 检查是否包含有用标签
            for tag in keep_tags:
                if f'<{tag}' in line.lower() or f'</{tag}>' in line.lower():
                    # 清理属性
                    simplified.append(line[:200])  # 限制行长度
                    break

        result = '\n'.join(simplified)

        # 截断到token限制
        if len(result) > max_tokens * 4:  # 粗略估算
            result = result[:max_tokens * 4] + "\n... (truncated)"

        return result

    @staticmethod
    def extract_accessibility_tree(page_content: str) -> str:
        """
        提取Accessibility Tree（更结构化的表示）

        Accessibility Tree比DOM更精炼:
        - 只包含对用户可见/可交互的元素
        - 包含语义角色(role)信息
        - 自然层级结构
        """
        # 示例Accessibility Tree结构
        example = """
[document] Google
  [navigation] 主导航
    [link] Gmail
    [link] 图片
    [button] 登录
  [main]
    [img] Google Logo
    [textbox] 搜索框 (focused)
    [button] Google 搜索
    [button] 手气不错
  [footer]
    [link] 关于Google
    [link] 隐私
        """
        return example

print("DOM简化策略:")
print("1. 只保留可交互元素 (input/button/link)")
print("2. 移除样式相关属性 (class/style)")
print("3. 保留语义信息 (aria-label/placeholder)")
print("4. 使用Accessibility Tree替代原始DOM")
print("5. 限制深度和节点数量")

"""
桌面GUI Agent: 操作任何桌面应用
代表项目: Microsoft UFO, Claude Computer Use
"""

class DesktopAgent:
    """
    桌面自动化Agent骨架

    核心依赖:
    - pyautogui: 鼠标/键盘控制
    - Pillow: 截图处理
    - pytesseract: OCR文字识别
    """

    def __init__(self, llm_client=None):
        self.llm_client = llm_client
        self.screen_width = 1920   # 示例分辨率
        self.screen_height = 1080

    def take_screenshot(self):
        """截取全屏或区域截图"""
        # import pyautogui
        # screenshot = pyautogui.screenshot()
        # return screenshot
        pass

    def find_element_by_text(self, text: str):
        """基于OCR找到文字位置"""
        # import pytesseract
        # screenshot = self.take_screenshot()
        # data = pytesseract.image_to_data(screenshot, output_type=pytesseract.Output.DICT)
        #
        # for i, word in enumerate(data['text']):
        #     if text.lower() in word.lower():
        #         x = data['left'][i] + data['width'][i] // 2
        #         y = data['top'][i] + data['height'][i] // 2
        #         return (x, y)
        pass

    def execute(self, action: GUIAction):
        """在操作系统级别执行操作"""
        # import pyautogui
        #
        # if action.action_type == ActionType.CLICK:
        #     pyautogui.click(*action.position)
        # elif action.action_type == ActionType.TYPE:
        #     pyautogui.click(*action.position)
        #     pyautogui.typewrite(action.text, interval=0.05)
        # elif action.action_type == ActionType.HOTKEY:
        #     pyautogui.hotkey(*action.key.split('+'))
        # elif action.action_type == ActionType.SCROLL:
        #     delta = -3 if action.direction == 'down' else 3
        #     pyautogui.scroll(delta)
        pass

print("""
桌面Agent典型应用场景：

1. 办公自动化
   - "打开Excel，在A1输入今天日期，保存为report.xlsx"
   - "打开邮箱，给张三发一封会议通知"

2. 软件测试
   - "打开计算器，验证1+1=2"
   - "在Photoshop中打开图片，调整亮度+20"

3. 数据录入
   - "从PDF表格中提取数据，填入ERP系统"
   - "批量处理100张发票的信息录入"

4. 跨应用工作流
   - "从浏览器下载文件→解压→导入到数据库"
""")

class MobileGUIAction:
    """移动端GUI动作扩展"""

    ACTIONS = {
        "tap": "单击",
        "long_press": "长按",
        "swipe_up": "上滑",
        "swipe_down": "下滑",
        "swipe_left": "左滑",
        "swipe_right": "右滑",
        "type": "输入文本",
        "back": "返回",
        "home": "回到主屏",
        "app_switch": "切换应用",
        "scroll_to": "滚动到元素",
    }

print("""
移动端GUI Agent代表工作：

1. AppAgent (2023)
   - 多模态LLM + Android UI层级
   - 通过探索学习APP操作方式
   - 建立操作知识库

2. CogAgent (2023)
   - 高分辨率视觉编码
   - 端到端的UI理解和操作

3. DigiRL (2024)
   - 使用RL训练移动端Agent
   - 在真实Android环境中在线学习

4. UI-TARS (2025)
   - 统一的视觉-动作模型
   - 支持Web/桌面/移动端
""")

class GUIAgentEvaluator:
    """GUI Agent评测器"""

    @staticmethod
    def compute_metrics(predictions, ground_truths):
        """
        计算GUI Agent评测指标

        核心指标:
        1. 任务成功率 (Task Success Rate)
        2. 步骤效率 (Step Efficiency)
        3. 元素定位精度 (Grounding Accuracy)
        """
        metrics = {}

        # 1. 任务成功率：完成目标的比例
        successes = sum(1 for p, g in zip(predictions, ground_truths)
                       if p['status'] == 'success')
        metrics['success_rate'] = successes / len(predictions)

        # 2. 步骤效率：实际步数/最优步数
        efficiencies = []
        for p, g in zip(predictions, ground_truths):
            if p['status'] == 'success' and g['optimal_steps'] > 0:
                efficiency = g['optimal_steps'] / p['steps']
                efficiencies.append(min(efficiency, 1.0))
        metrics['step_efficiency'] = (sum(efficiencies) / len(efficiencies)
                                      if efficiencies else 0)

        # 3. 元素定位精度（IoU）
        ious = []
        for p, g in zip(predictions, ground_truths):
            if 'bbox' in p and 'bbox' in g:
                iou = GUIAgentEvaluator._compute_iou(p['bbox'], g['bbox'])
                ious.append(iou)
        metrics['grounding_accuracy'] = (sum(ious) / len(ious)
                                         if ious else 0)

        return metrics

    @staticmethod
    def _compute_iou(box1, box2):
        """计算两个边界框的IoU"""
        x1 = max(box1[0], box2[0])
        y1 = max(box1[1], box2[1])
        x2 = min(box1[2], box2[2])
        y2 = min(box1[3], box2[3])

        intersection = max(0, x2 - x1) * max(0, y2 - y1)

        area1 = (box1[2] - box1[0]) * (box1[3] - box1[1])
        area2 = (box2[2] - box2[0]) * (box2[3] - box2[1])

        union = area1 + area2 - intersection
        return intersection / union if union > 0 else 0

# 示例评测
predictions = [
    {"status": "success", "steps": 5, "bbox": [100, 100, 200, 200]},
    {"status": "success", "steps": 8, "bbox": [150, 150, 250, 250]},
    {"status": "failed", "steps": 20, "bbox": [300, 300, 400, 400]},
]
ground_truths = [
    {"optimal_steps": 4, "bbox": [100, 100, 200, 200]},
    {"optimal_steps": 3, "bbox": [145, 148, 255, 252]},
    {"optimal_steps": 5, "bbox": [310, 305, 395, 395]},
]

metrics = GUIAgentEvaluator.compute_metrics(predictions, ground_truths)
print("GUI Agent评测结果:")
for k, v in metrics.items():
    print(f"  {k}: {v:.4f}")