Gemini 2.5 Computer Use 終極指南：從入門到實作，打造你的 AI 自動化助理

pip install google-genai playwright
playwright install chromium

from playwright.sync_api import sync_playwright

# 1. 設定目標環境的螢幕尺寸
SCREEN_WIDTH = 1440
SCREEN_HEIGHT = 900

# 2. 啟動 Playwright 瀏覽器
# 在生產環境中，請使用沙盒環境
playwright = sync_playwright().start()
# 設定 headless=False 可以在螢幕上看到 AI 的操作過程
browser = playwright.chromium.launch(headless=False)

# 3. 建立具有指定尺寸的上下文和頁面
context = browser.new_context(
    viewport={"width": SCREEN_WIDTH, "height": SCREEN_HEIGHT}
)
page = context.new_page()

# 4. 導航到一個初始頁面開始任務
page.goto("https://www.google.com")

print("瀏覽器已初始化，準備開始任務。")

# 輔助函式：將模型回傳的正規化座標 (0-999) 轉換為實際像素座標
def denormalize_x(x, screen_width):
    return int(x / 1000 * screen_width)

def denormalize_y(y, screen_height):
    return int(y / 1000 * screen_height)

# 輔助函式：執行模型回傳的 function calls
def execute_function_calls(candidate, page, screen_width, screen_height):
    results = []
    for part in candidate.content.parts:
        if not part.function_call:
            continue
        
        fname = part.function_call.name
        args = part.function_call.args
        print(f"-> 正在執行: {fname}，參數: {args}")

        try:
            if fname == "click_at":
                x = denormalize_x(args["x"], screen_width)
                y = denormalize_y(args["y"], screen_height)
                page.mouse.click(x, y)
            elif fname == "type_text_at":
                x = denormalize_x(args["x"], screen_width)
                y = denormalize_y(args["y"], screen_height)
                page.mouse.click(x, y)
                page.keyboard.type(args["text"])
                if args.get("press_enter", False):
                    page.keyboard.press("Enter")
            # ... 在此處實現其他支援的動作 ...
            else:
                 print(f"警告: 未實現的函式 {fname}")
            
            # 等待頁面加載
            page.wait_for_load_state(timeout=5000)
            results.append((fname, {"status": "success"}))
        except Exception as e:
            print(f"錯誤: 執行 {fname} 時發生錯誤: {e}")
            results.append((fname, {"error": str(e)}))
            
    return results

# 輔助函式：擷取新的螢幕狀態並打包成回傳格式
def get_function_responses(page, results):
    screenshot_bytes = page.screenshot(type="png")
    current_url = page.url
    
    function_responses = []
    for name, result in results:
        response_data = {"url": current_url, **result}
        function_responses.append({
            "name": name,
            "response": response_data,
            "screenshot": screenshot_bytes
        })
    return function_responses

# (請確保您已設定好您的 genai API 金鑰)
import google.genai as genai
from google.genai import types

# --- 完整的代理迴圈主程式 ---
try:
    # 1. 設定模型
    model = genai.GenerativeModel(
        'gemini-2.5-computer-use-preview-10-2025',
        tools=[types.Tool(
            computer_use=types.ComputerUse(
                environment=types.Environment.ENVIRONMENT_BROWSER
            )
        )]
    )
    chat = model.start_chat()

    # 2. 初始化任務
    USER_PROMPT = "前往 aistudio.google.com 並搜尋關於 agents 的文件"
    print(f"目標: {USER_PROMPT}")
    
    initial_screenshot = page.screenshot(type="png")
    
    # 3. 進入代理迴圈
    turn_limit = 10
    for i in range(turn_limit):
        print(f"\n--- 第 {i+1} 輪 ---")
        
        # 發送請求給模型
        if i == 0:
            response = chat.send_message(
                [USER_PROMPT, initial_screenshot],
            )
        else:
            response = chat.send_message(function_responses)
            
        # 接收並執行模型的回應
        candidate = response.candidates[0]
        
        if not any(part.function_call for part in candidate.content.parts):
            print("代理完成:", candidate.content.parts[0].text)
            break
        
        results = execute_function_calls(candidate, page, SCREEN_WIDTH, SCREEN_HEIGHT)
        
        # 擷取新狀態並準備下一輪
        print("-> 正在擷取新狀態...")
        function_responses = get_function_responses(page, results)
        
finally:
    # 清理資源
    print("\n任務結束，關閉瀏覽器...")
    browser.close()
    playwright.stop()