Gemini 2.5 Computer Use 终极指南：从入门到实作，打造你的 AI 自动化助理

pip install google-genai playwright
playwright install chromium

from playwright.sync_api import sync_playwright

# 1. 设定目标环境的萤幕尺寸
SCREEN_WIDTH = 1440
SCREEN_HEIGHT = 900

# 2. 启动 Playwright 浏览器
# 在生产环境中，请使用沙盒环境
playwright = sync_playwright().start()
# 设定 headless=False 可以在萤幕上看到 AI 的操作过程
browser = playwright.chromium.launch(headless=False)

# 3. 建立具有指定尺寸的上下文和页面
context = browser.new_context(
    viewport={"width": SCREEN_WIDTH, "height": SCREEN_HEIGHT}
)
page = context.new_page()

# 4. 导航到一个初始页面开始任务
page.goto("https://www.google.com")

print("浏览器已初始化，准备开始任务。")

# 辅助函式：将模型回传的正规化座标 (0-999) 转换为实际像素座标
def denormalize_x(x, screen_width):
    return int(x / 1000 * screen_width)

def denormalize_y(y, screen_height):
    return int(y / 1000 * screen_height)

# 辅助函式：执行模型回传的 function calls
def execute_function_calls(candidate, page, screen_width, screen_height):
    results = []
    for part in candidate.content.parts:
        if not part.function_call:
            continue
        
        fname = part.function_call.name
        args = part.function_call.args
        print(f"-> 正在执行: {fname}，参数: {args}")

        try:
            if fname == "click_at":
                x = denormalize_x(args["x"], screen_width)
                y = denormalize_y(args["y"], screen_height)
                page.mouse.click(x, y)
            elif fname == "type_text_at":
                x = denormalize_x(args["x"], screen_width)
                y = denormalize_y(args["y"], screen_height)
                page.mouse.click(x, y)
                page.keyboard.type(args["text"])
                if args.get("press_enter", False):
                    page.keyboard.press("Enter")
            # ... 在此处实现其他支援的动作 ...
            else:
                 print(f"警告: 未实现的函式 {fname}")
            
            # 等待页面加载
            page.wait_for_load_state(timeout=5000)
            results.append((fname, {"status": "success"}))
        except Exception as e:
            print(f"错误: 执行 {fname} 时发生错误: {e}")
            results.append((fname, {"error": str(e)}))
            
    return results

# 辅助函式：撷取新的萤幕状态并打包成回传格式
def get_function_responses(page, results):
    screenshot_bytes = page.screenshot(type="png")
    current_url = page.url
    
    function_responses = []
    for name, result in results:
        response_data = {"url": current_url, **result}
        function_responses.append({
            "name": name,
            "response": response_data,
            "screenshot": screenshot_bytes
        })
    return function_responses

# (请确保您已设定好您的 genai API 金钥)
import google.genai as genai
from google.genai import types

# --- 完整的代理回圈主程式 ---
try:
    # 1. 设定模型
    model = genai.GenerativeModel(
        'gemini-2.5-computer-use-preview-10-2025',
        tools=[types.Tool(
            computer_use=types.ComputerUse(
                environment=types.Environment.ENVIRONMENT_BROWSER
            )
        )]
    )
    chat = model.start_chat()

    # 2. 初始化任务
    USER_PROMPT = "前往 aistudio.google.com 并搜寻关于 agents 的文件"
    print(f"目标: {USER_PROMPT}")
    
    initial_screenshot = page.screenshot(type="png")
    
    # 3. 进入代理回圈
    turn_limit = 10
    for i in range(turn_limit):
        print(f"\n--- 第 {i+1} 轮 ---")
        
        # 发送请求给模型
        if i == 0:
            response = chat.send_message(
                [USER_PROMPT, initial_screenshot],
            )
        else:
            response = chat.send_message(function_responses)
            
        # 接收并执行模型的回应
        candidate = response.candidates[0]
        
        if not any(part.function_call for part in candidate.content.parts):
            print("代理完成:", candidate.content.parts[0].text)
            break
        
        results = execute_function_calls(candidate, page, SCREEN_WIDTH, SCREEN_HEIGHT)
        
        # 撷取新状态并准备下一轮
        print("-> 正在撷取新状态...")
        function_responses = get_function_responses(page, results)
        
finally:
    # 清理资源
    print("\n任务结束，关闭浏览器...")
    browser.close()
    playwright.stop()