feat: add screen annotation and rpa agent demo

agent/config.example.toml

@@ -0,0 +1,13 @@
+# Copy this file to config.toml and fill in your API key.
+[volcengine]
+base_url = "https://ark.cn-beijing.volces.com/api/v3"
+api_key = "YOUR_ARK_API_KEY"
+model = "doubao-seed-2-0-lite-260428"
+
+[agent]
+app_data_dir = "~/Library/Application Support/com.tauri.dev"
+regions_path = "data/regions/wechat.json"
+screenshot_path = "data/screenshots/WeChat.jpg"
+task_output_path = "data/tasks/latest_task.json"
+task_history_dir = "data/tasks/history"
+dry_run = true

agent/config.go

@@ -0,0 +1,80 @@
+package main
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/pelletier/go-toml/v2"
+)
+
+func loadConfig(path string) (Config, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return Config{}, err
+	}
+
+	var config Config
+	if err := toml.Unmarshal(data, &config); err != nil {
+		return Config{}, err
+	}
+
+	applyConfigDefaults(&config)
+	if config.Volcengine.BaseURL == "" || config.Volcengine.APIKey == "" || config.Volcengine.Model == "" {
+		return Config{}, errors.New("missing volcengine base_url/api_key/model in config.toml")
+	}
+
+	appDataDir, err := expandPath(config.Agent.AppDataDir)
+	if err != nil {
+		return Config{}, err
+	}
+	config.Agent.AppDataDir = appDataDir
+
+	return config, nil
+}
+
+func applyConfigDefaults(config *Config) {
+	if config.Agent.AppDataDir == "" {
+		config.Agent.AppDataDir = "~/Library/Application Support/com.tauri.dev"
+	}
+	if config.Agent.RegionsPath == "" {
+		config.Agent.RegionsPath = "data/regions/wechat.json"
+	}
+	if config.Agent.ScreenshotPath == "" {
+		config.Agent.ScreenshotPath = "data/screenshots/WeChat.jpg"
+	}
+	if config.Agent.TaskOutputPath == "" {
+		config.Agent.TaskOutputPath = "data/tasks/latest_task.json"
+	}
+	if config.Agent.TaskHistoryDir == "" {
+		config.Agent.TaskHistoryDir = "data/tasks/history"
+	}
+	config.Agent.DryRun = true
+}
+
+func expandPath(path string) (string, error) {
+	if path == "" {
+		return "", nil
+	}
+
+	if path == "~" || strings.HasPrefix(path, "~/") {
+		home, err := os.UserHomeDir()
+		if err != nil {
+			return "", err
+		}
+		if path == "~" {
+			return home, nil
+		}
+		return filepath.Join(home, path[2:]), nil
+	}
+
+	return path, nil
+}
+
+func appDataPath(config Config, relativePath string) string {
+	if filepath.IsAbs(relativePath) {
+		return relativePath
+	}
+	return filepath.Join(config.Agent.AppDataDir, relativePath)
+}

agent/go.mod

@@ -1,3 +1,5 @@
 module agent
 
 go 1.26.1
+
+require github.com/pelletier/go-toml/v2 v2.2.4

agent/go.sum

@@ -0,0 +1,2 @@
+github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
+github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=

agent/image.go

@@ -0,0 +1,15 @@
+package main
+
+import (
+	"encoding/base64"
+	"os"
+)
+
+func loadImageDataURL(path string) (string, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return "", err
+	}
+
+	return "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString(data), nil
+}

agent/llm.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+)
+
+func requestLLM(config Config, prompt string, imageDataURL string) (string, error) {
+	url := strings.TrimRight(config.Volcengine.BaseURL, "/") + "/chat/completions"
+	body := openAIRequest{
+		Model: config.Volcengine.Model,
+		Messages: []openAIMessage{
+			{
+				Role: "user",
+				Content: []openAIContent{
+					{Type: "text", Text: prompt},
+					{Type: "image_url", ImageURL: &openAIImageURL{URL: imageDataURL}},
+				},
+			},
+		},
+		Temperature: 0.2,
+	}
+
+	data, err := json.Marshal(body)
+	if err != nil {
+		return "", err
+	}
+
+	request, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(data))
+	if err != nil {
+		return "", err
+	}
+	request.Header.Set("Authorization", "Bearer "+config.Volcengine.APIKey)
+	request.Header.Set("Content-Type", "application/json")
+
+	client := http.Client{Timeout: 90 * time.Second}
+	response, err := client.Do(request)
+	if err != nil {
+		return "", err
+	}
+	defer response.Body.Close()
+
+	responseBody, err := io.ReadAll(response.Body)
+	if err != nil {
+		return "", err
+	}
+
+	if response.StatusCode < 200 || response.StatusCode >= 300 {
+		return "", fmt.Errorf("LLM request failed: status=%d body=%s", response.StatusCode, string(responseBody))
+	}
+
+	var result openAIResponse
+	if err := json.Unmarshal(responseBody, &result); err != nil {
+		return "", err
+	}
+	if result.Error != nil && result.Error.Message != "" {
+		return "", errors.New(result.Error.Message)
+	}
+	if len(result.Choices) == 0 || result.Choices[0].Message.Content == "" {
+		return "", errors.New("LLM returned empty content")
+	}
+
+	return result.Choices[0].Message.Content, nil
+}

agent/logger.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+)
+
+func logJSON(level, message string) {
+	entry := LogEntry{
+		Level:     level,
+		Message:   message,
+		Timestamp: time.Now().Format("15:04:05"),
+	}
+
+	data, err := json.Marshal(entry)
+	if err != nil {
+		fmt.Printf(`{"level":"error","message":%q,"timestamp":"%s"}`+"\n", err.Error(), time.Now().Format("15:04:05"))
+		return
+	}
+
+	fmt.Println(string(data))
+}
+
+func logInfo(message string) {
+	logJSON("info", message)
+}
+
+func logThink(message string) {
+	logJSON("think", message)
+}
+
+func logWarning(message string) {
+	logJSON("warning", message)
+}
+
+func logError(message string) {
+	logJSON("error", message)
+}
+
+func logTask(message string) {
+	logJSON("task", message)
+}

agent/main.go

@@ -1,58 +1,90 @@
 package main
 
 import (
-	"encoding/json"
 	"fmt"
-	"sync"
+	"os"
-	"time"
 )
 
-var waitGroup sync.WaitGroup
-
-type logEntry struct {
-	Level     string `json:"level"`
-	Message   string `json:"message"`
-	Timestamp string `json:"timestamp"`
-}
-
-// 创建一个函数，启动一个协程，每隔600ms输出一次日志信息到命令行界面
-func logToConsole() {
-	defer waitGroup.Done()
-	for {
-		time.Sleep(600 * time.Millisecond)
-		fmt.Println(randomLogMsg())
-	}
-}
-
-// 生成随机日志信息，日志信息从以下列表中随机选择 日志格式为json格式，格式{"level": "info", "message": "正在处理用户请求..."}，日志级别有info、debug、error,warning, think五种，日志信息有正在处理用户请求...、正在连接服务器...、正在加载数据...、正在分析用户输入...、正在生成回复...、正在优化模型...、正在更新知识库...等
-func randomLogMsg() string {
-	logMsgs := []logEntry{
-		{Level: "info", Message: "正在处理用户请求..."},
-		{Level: "debug", Message: "正在连接服务器..."},
-		{Level: "error", Message: "正在加载数据..."},
-		{Level: "warning", Message: "正在分析用户输入..."},
-		{Level: "think", Message: "正在生成回复..."},
-		{Level: "info", Message: "正在优化模型..."},
-		{Level: "info", Message: "正在更新知识库..."},
-	}
-	logMsg := logMsgs[time.Now().UnixNano()%int64(len(logMsgs))]
-	logMsg.Timestamp = time.Now().Format("15:04:05")
-
-	data, err := json.Marshal(logMsg)
-	if err != nil {
-		return fmt.Sprintf(`{"level":"error","message":"%s","timestamp":"%s"}`, err.Error(), time.Now().Format("15:04:05"))
-	}
-
-	return string(data)
-}
-
 func main() {
-	waitGroup = sync.WaitGroup{}
+	if err := runOnce(); err != nil {
-	// 启动日志协程
+		logError(err.Error())
-	waitGroup.Add(1)
+		os.Exit(1)
-	go logToConsole()
+	}
-	// 等待协程main执行完成
+}
-	time.Sleep(50 * time.Second)
+
-	waitGroup.Wait()
+func runOnce() error {
-
+	logInfo("Go RPA agent demo 启动")
+
+	config, err := loadConfig("config.toml")
+	if err != nil {
+		return fmt.Errorf("读取 config.toml 失败：%w", err)
+	}
+	logInfo("已读取 config.toml，模型=" + config.Volcengine.Model)
+
+	regionsPath := appDataPath(config, config.Agent.RegionsPath)
+	screenshotPath := appDataPath(config, config.Agent.ScreenshotPath)
+	annotation, err := loadAnnotation(regionsPath)
+	if err != nil {
+		return fmt.Errorf("读取标注文件失败：%w", err)
+	}
+	summaries := summarizeRegions(annotation.Regions)
+	summaries, inferenceWarnings := inferMissingRegionTypes(summaries)
+	logInfo(fmt.Sprintf("已读取标注文件，regions=%d", len(summaries)))
+	for _, warning := range inferenceWarnings {
+		logWarning(warning)
+	}
+	if err := validateRequiredRegions(summaries); err != nil {
+		return err
+	}
+
+	for _, summary := range summaries {
+		logThink(fmt.Sprintf("区域识别：%s center=[%.1f, %.1f]", summary.Type, summary.CenterScreen[0], summary.CenterScreen[1]))
+	}
+
+	imageDataURL, err := loadImageDataURL(screenshotPath)
+	if err != nil {
+		return fmt.Errorf("读取截图失败：%w", err)
+	}
+	logInfo("已读取截图 WeChat.jpg")
+
+	prompt, err := buildPrompt(summaries)
+	if err != nil {
+		return fmt.Errorf("构造 prompt 失败：%w", err)
+	}
+
+	logThink("正在请求豆包多模态模型识别聊天内容并生成任务计划")
+	rawResponse, err := requestLLM(config, prompt, imageDataURL)
+	if err != nil {
+		return fmt.Errorf("请求 LLM 失败：%w", err)
+	}
+
+	task, _, err := parseTaskFromLLM(rawResponse)
+	if err != nil {
+		_ = saveRawResponse(config, rawResponse)
+		return fmt.Errorf("解析 LLM task JSON 失败，已保存 latest_task_raw.txt：%w", err)
+	}
+
+	task = finalizeTask(task, summaries, config, regionsPath, screenshotPath)
+	logJSON("summary", fmt.Sprintf(
+		"LLM 摘要：联系人=%s；最新消息=%s；回复=%s；confidence=%.2f",
+		task.LLMSummary.ContactName,
+		task.LLMSummary.LatestUserMessage,
+		task.LLMSummary.ReplyText,
+		task.Confidence,
+	))
+	logTask(fmt.Sprintf("已生成任务 %s，type=%s，actions=%d，dry_run=%v", task.TaskID, task.TaskType, len(task.Actions), config.Agent.DryRun))
+
+	for _, action := range task.Actions {
+		logTask(fmt.Sprintf("动作记录：%s %s status=%s target=%s", action.ActionID, action.Name, action.Status, action.TargetRegion))
+	}
+
+	latestPath, historyPath, err := saveTaskFiles(task, config)
+	if err != nil {
+		return fmt.Errorf("保存 task 文件失败：%w", err)
+	}
+	logInfo("已保存 latest task: " + latestPath)
+	logInfo("已保存 history task: " + historyPath)
+	logInfo("Go RPA agent demo 完成，未真实执行鼠标键盘")
+
+	return nil
 }

agent/planner.go

@@ -0,0 +1,87 @@
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"strings"
+)
+
+func buildPrompt(summaries []RegionSummary) (string, error) {
+	regionsJSON, err := json.MarshalIndent(summaries, "", "  ")
+	if err != nil {
+		return "", err
+	}
+
+	return `你是微信 RPA 多模态任务规划器。
+
+请根据图片和标注区域完成：
+1. 识别当前聊天记录。
+2. 识别联系人或会话对象，如果无法识别则填 unknown。
+3. 判断是否需要回复。
+4. 如果需要回复，生成 reply_text。
+5. 生成 dry-run 动作计划。
+
+限制：
+- 只能返回合法 JSON，不要 Markdown，不要解释。
+- 不要编造坐标。
+- 点击坐标只能使用我提供的区域中心点。
+- 如果区域 type 是 custom，请重点参考 description 判断该区域的用途和可执行动作。
+- description 是用户标注时填写的用途说明，优先级高于模型自行猜测。
+- 微信发送消息使用键入回车键，不要规划点击发送按钮。
+- action.type 只能是 click、type_text、press_enter、scroll、wait、noop。
+- 当前是 dry_run，只生成计划，不要假设动作已执行。
+- 如果无需回复，task_type 使用 wechat_observe 或 noop，并给出 observations。
+
+标注区域如下：
+` + string(regionsJSON) + `
+
+请严格返回如下 JSON 结构：
+{
+  "task_id": "",
+  "task_name": "识别微信聊天并生成回复",
+  "task_type": "wechat_reply",
+  "task_status": "planned",
+  "confidence": 0.0,
+  "llm_summary": {
+    "contact_name": "unknown",
+    "chat_summary": "",
+    "latest_user_message": "",
+    "reply_text": ""
+  },
+  "actions": [
+    {"action_id":"action_001","type":"click","name":"点击输入框","status":"pending","target_region":"input_box"},
+    {"action_id":"action_002","type":"type_text","name":"输入回复内容","status":"pending","text":""},
+    {"action_id":"action_003","type":"press_enter","name":"按回车发送","status":"pending"}
+  ]
+}`, nil
+}
+
+func extractJSON(text string) (string, error) {
+	trimmed := strings.TrimSpace(text)
+	trimmed = strings.TrimPrefix(trimmed, "```json")
+	trimmed = strings.TrimPrefix(trimmed, "```")
+	trimmed = strings.TrimSuffix(trimmed, "```")
+	trimmed = strings.TrimSpace(trimmed)
+
+	start := strings.Index(trimmed, "{")
+	end := strings.LastIndex(trimmed, "}")
+	if start < 0 || end < start {
+		return "", errors.New("no JSON object found in LLM response")
+	}
+
+	return trimmed[start : end+1], nil
+}
+
+func parseTaskFromLLM(raw string) (Task, string, error) {
+	jsonText, err := extractJSON(raw)
+	if err != nil {
+		return Task{}, "", err
+	}
+
+	var task Task
+	if err := json.Unmarshal([]byte(jsonText), &task); err != nil {
+		return Task{}, jsonText, err
+	}
+
+	return task, jsonText, nil
+}

agent/regions.go

@@ -0,0 +1,204 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"math"
+	"os"
+	"strings"
+)
+
+func loadAnnotation(path string) (AnnotationFile, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return AnnotationFile{}, err
+	}
+
+	var annotation AnnotationFile
+	if err := json.Unmarshal(data, &annotation); err != nil {
+		return AnnotationFile{}, err
+	}
+
+	return annotation, nil
+}
+
+func summarizeRegions(regions []Region) []RegionSummary {
+	summaries := make([]RegionSummary, 0, len(regions))
+	for _, region := range regions {
+		bboxSource := region.BBoxSource
+		if isZeroBox(bboxSource) {
+			scaleFactor := region.ScaleFactor
+			if scaleFactor == 0 {
+				scaleFactor = 1
+			}
+			bboxSource = [4]float64{
+				region.BBoxImage[0] / scaleFactor,
+				region.BBoxImage[1] / scaleFactor,
+				region.BBoxImage[2] / scaleFactor,
+				region.BBoxImage[3] / scaleFactor,
+			}
+		}
+
+		summaries = append(summaries, RegionSummary{
+			Name:         region.Name,
+			Description:  region.Description,
+			Type:         region.Type,
+			BBoxScreen:   region.BBoxScreen,
+			BBoxSource:   bboxSource,
+			CenterScreen: centerOf(region.BBoxScreen),
+			CenterSource: centerOf(bboxSource),
+		})
+	}
+	return summaries
+}
+
+func centerOf(box [4]float64) [2]float64 {
+	return [2]float64{(box[0] + box[2]) / 2, (box[1] + box[3]) / 2}
+}
+
+func regionByType(summaries []RegionSummary) map[string]RegionSummary {
+	index := make(map[string]RegionSummary, len(summaries))
+	for _, summary := range summaries {
+		if _, exists := index[summary.Type]; !exists {
+			index[summary.Type] = summary
+		}
+	}
+	return index
+}
+
+func validateRequiredRegions(summaries []RegionSummary) error {
+	index := regionByType(summaries)
+	missing := make([]string, 0)
+	for _, regionType := range []string{"chat_content", "input_box"} {
+		if _, exists := index[regionType]; !exists {
+			missing = append(missing, regionType)
+		}
+	}
+	if len(missing) > 0 {
+		return fmt.Errorf("missing required regions: %v", missing)
+	}
+	return nil
+}
+
+func inferMissingRegionTypes(summaries []RegionSummary) ([]RegionSummary, []string) {
+	index := regionByType(summaries)
+	warnings := make([]string, 0)
+	customs := make([]RegionSummary, 0)
+	for _, summary := range summaries {
+		if inferredType := inferTypeFromDescription(summary.Description); inferredType != "" {
+			if _, exists := index[inferredType]; !exists {
+				inferred := summary
+				inferred.Type = inferredType
+				inferred.Name = inferred.Name + "（按描述推断）"
+				summaries = append(summaries, inferred)
+				index[inferredType] = inferred
+				warnings = append(warnings, fmt.Sprintf("区域 %q 已按描述推断为 %s", summary.Name, inferredType))
+			}
+		}
+
+		if summary.Type == "custom" {
+			customs = append(customs, summary)
+		}
+	}
+	if len(customs) == 0 {
+		return summaries, warnings
+	}
+
+	if _, exists := index["contact_list"]; !exists {
+		best := customs[0]
+		for _, candidate := range customs[1:] {
+			if candidate.CenterScreen[0] < best.CenterScreen[0] && boxHeight(candidate.BBoxScreen) > boxHeight(best.BBoxScreen)*0.5 {
+				best = candidate
+			}
+		}
+		best.Type = "contact_list"
+		best.Name = best.Name + "（推断联系人区域）"
+		summaries = append(summaries, best)
+		index[best.Type] = best
+		warnings = append(warnings, "未标注 contact_list，已按最左侧大区域推断")
+	}
+
+	if _, exists := index["input_box"]; !exists {
+		best := customs[0]
+		for _, candidate := range customs[1:] {
+			if candidate.CenterScreen[1] > best.CenterScreen[1] || nearlyEqual(candidate.CenterScreen[1], best.CenterScreen[1]) && boxArea(candidate.BBoxScreen) > boxArea(best.BBoxScreen) {
+				best = candidate
+			}
+		}
+		best.Type = "input_box"
+		best.Name = best.Name + "（推断输入框区域）"
+		summaries = append(summaries, best)
+		index[best.Type] = best
+		warnings = append(warnings, "未标注 input_box，已按底部区域推断")
+	}
+
+	if _, exists := index["chat_content"]; !exists {
+		input := index["input_box"]
+		contact := index["contact_list"]
+		best := customs[0]
+		bestScore := math.Inf(-1)
+		for _, candidate := range customs {
+			if sameBox(candidate.BBoxScreen, input.BBoxScreen) || sameBox(candidate.BBoxScreen, contact.BBoxScreen) {
+				continue
+			}
+			score := boxArea(candidate.BBoxScreen) - math.Abs(candidate.CenterScreen[1]-input.CenterScreen[1])*0.2
+			if score > bestScore {
+				best = candidate
+				bestScore = score
+			}
+		}
+		best.Type = "chat_content"
+		best.Name = best.Name + "（推断聊天内容区域）"
+		summaries = append(summaries, best)
+		index[best.Type] = best
+		warnings = append(warnings, "未标注 chat_content，已按大面积内容区推断")
+	}
+
+	return summaries, warnings
+}
+
+func sameBox(a, b [4]float64) bool {
+	return nearlyEqual(a[0], b[0]) && nearlyEqual(a[1], b[1]) && nearlyEqual(a[2], b[2]) && nearlyEqual(a[3], b[3])
+}
+
+func isZeroBox(box [4]float64) bool {
+	return box[0] == 0 && box[1] == 0 && box[2] == 0 && box[3] == 0
+}
+
+func boxWidth(box [4]float64) float64 {
+	return math.Abs(box[2] - box[0])
+}
+
+func boxHeight(box [4]float64) float64 {
+	return math.Abs(box[3] - box[1])
+}
+
+func boxArea(box [4]float64) float64 {
+	return boxWidth(box) * boxHeight(box)
+}
+
+func nearlyEqual(a, b float64) bool {
+	return math.Abs(a-b) < 8
+}
+
+func inferTypeFromDescription(description string) string {
+	text := strings.ToLower(description)
+	if text == "" {
+		return ""
+	}
+
+	if strings.Contains(text, "chat_content") || strings.Contains(text, "聊天") || strings.Contains(text, "消息") || strings.Contains(text, "记录") {
+		return "chat_content"
+	}
+	if strings.Contains(text, "input_box") || strings.Contains(text, "输入") || strings.Contains(text, "文本框") || strings.Contains(text, "编辑框") {
+		return "input_box"
+	}
+	if strings.Contains(text, "send_button") || strings.Contains(text, "发送") || strings.Contains(text, "send") {
+		return "send_button"
+	}
+	if strings.Contains(text, "contact_list") || strings.Contains(text, "联系人") || strings.Contains(text, "会话列表") || strings.Contains(text, "好友") {
+		return "contact_list"
+	}
+
+	return ""
+}

agent/task.go

@@ -0,0 +1,162 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+var allowedTaskTypes = map[string]bool{
+	"wechat_reply":   true,
+	"wechat_observe": true,
+	"noop":           true,
+}
+
+var allowedActionTypes = map[string]bool{
+	"click":       true,
+	"type_text":   true,
+	"press_enter": true,
+	"scroll":      true,
+	"wait":        true,
+	"noop":        true,
+}
+
+func finalizeTask(task Task, summaries []RegionSummary, config Config, regionsPath string, screenshotPath string) Task {
+	now := time.Now()
+	if task.TaskID == "" {
+		task.TaskID = "task_" + now.Format("20060102_150405")
+	}
+	if task.TaskName == "" {
+		task.TaskName = "识别微信聊天并生成回复"
+	}
+	if task.TaskType == "" || !allowedTaskTypes[task.TaskType] {
+		task.TaskType = "wechat_reply"
+	}
+	if task.TaskStatus == "" {
+		task.TaskStatus = "planned"
+	}
+	if config.Agent.DryRun {
+		task.TaskStatus = "dry_run"
+	}
+	if task.CreatedAt == "" {
+		task.CreatedAt = now.Format(time.RFC3339)
+	}
+	task.Source = TaskSource{RegionsPath: regionsPath, ScreenshotPath: screenshotPath}
+	task.Regions = summaries
+
+	regionIndex := regionByType(summaries)
+	if len(task.Actions) == 0 {
+		task.Actions = defaultActions(task.LLMSummary.ReplyText)
+	}
+
+	for index := range task.Actions {
+		action := &task.Actions[index]
+		if action.ActionID == "" {
+			action.ActionID = fmt.Sprintf("action_%03d", index+1)
+		}
+		if action.Name == "" {
+			action.Name = action.Type
+		}
+		if !allowedActionTypes[action.Type] {
+			action.Type = "noop"
+			action.Reason = "unsupported action type replaced by noop"
+		}
+		if action.Type == "click" || action.Type == "scroll" {
+			if summary, exists := regionIndex[action.TargetRegion]; exists {
+				point := summary.CenterScreen
+				action.Point = &point
+			} else if action.Type == "click" {
+				action.Type = "noop"
+				action.Reason = "target_region not found"
+			}
+		}
+		if action.Type == "type_text" && len([]rune(action.Text)) > 500 {
+			runes := []rune(action.Text)
+			action.Text = string(runes[:500])
+		}
+		if config.Agent.DryRun {
+			action.Status = "skipped"
+			if action.Reason == "" {
+				action.Reason = "dry_run enabled"
+			}
+		} else if action.Status == "" {
+			action.Status = "pending"
+		}
+
+		task.ExecutionLog = append(task.ExecutionLog, ExecutionLog{
+			ActionID: action.ActionID,
+			Status:   action.Status,
+			Message:  executionMessage(*action),
+			Time:     time.Now().Format(time.RFC3339),
+		})
+	}
+
+	return task
+}
+
+func defaultActions(replyText string) []Action {
+	if strings.TrimSpace(replyText) == "" {
+		return []Action{{ActionID: "action_001", Type: "noop", Name: "无需回复", Status: "pending"}}
+	}
+
+	return []Action{
+		{ActionID: "action_001", Type: "click", Name: "点击输入框", Status: "pending", TargetRegion: "input_box"},
+		{ActionID: "action_002", Type: "type_text", Name: "输入回复内容", Status: "pending", Text: replyText},
+		{ActionID: "action_003", Type: "press_enter", Name: "按回车发送", Status: "pending"},
+	}
+}
+
+func executionMessage(action Action) string {
+	switch action.Type {
+	case "click":
+		return fmt.Sprintf("dry_run: 跳过点击 %s", action.TargetRegion)
+	case "type_text":
+		return "dry_run: 跳过输入回复内容"
+	case "press_enter":
+		return "dry_run: 跳过按回车发送"
+	case "scroll":
+		return fmt.Sprintf("dry_run: 跳过滑动 %s delta_y=%d", action.TargetRegion, action.DeltaY)
+	case "wait":
+		return fmt.Sprintf("dry_run: 跳过等待 %dms", action.DurationMs)
+	default:
+		return "dry_run: noop"
+	}
+}
+
+func saveTaskFiles(task Task, config Config) (string, string, error) {
+	latestPath := appDataPath(config, config.Agent.TaskOutputPath)
+	historyDir := appDataPath(config, config.Agent.TaskHistoryDir)
+	if err := os.MkdirAll(filepath.Dir(latestPath), 0755); err != nil {
+		return "", "", err
+	}
+	if err := os.MkdirAll(historyDir, 0755); err != nil {
+		return "", "", err
+	}
+
+	data, err := json.MarshalIndent(task, "", "  ")
+	if err != nil {
+		return "", "", err
+	}
+
+	if err := os.WriteFile(latestPath, data, 0644); err != nil {
+		return "", "", err
+	}
+
+	historyPath := filepath.Join(historyDir, task.TaskID+".json")
+	if err := os.WriteFile(historyPath, data, 0644); err != nil {
+		return "", "", err
+	}
+
+	return latestPath, historyPath, nil
+}
+
+func saveRawResponse(config Config, raw string) error {
+	path := appDataPath(config, "data/tasks/latest_task_raw.txt")
+	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+		return err
+	}
+	return os.WriteFile(path, []byte(raw), 0644)
+}

agent/types.go

@@ -0,0 +1,140 @@
+package main
+
+type Config struct {
+	Volcengine VolcengineConfig `toml:"volcengine"`
+	Agent      AgentConfig      `toml:"agent"`
+}
+
+type VolcengineConfig struct {
+	BaseURL string `toml:"base_url"`
+	APIKey  string `toml:"api_key"`
+	Model   string `toml:"model"`
+}
+
+type AgentConfig struct {
+	AppDataDir     string `toml:"app_data_dir"`
+	RegionsPath    string `toml:"regions_path"`
+	ScreenshotPath string `toml:"screenshot_path"`
+	TaskOutputPath string `toml:"task_output_path"`
+	TaskHistoryDir string `toml:"task_history_dir"`
+	DryRun         bool   `toml:"dry_run"`
+}
+
+type LogEntry struct {
+	Level     string `json:"level"`
+	Message   string `json:"message"`
+	Timestamp string `json:"timestamp"`
+}
+
+type AnnotationFile struct {
+	App              string          `json:"app"`
+	ScreenshotPath   string          `json:"screenshotPath"`
+	ScreenshotWidth  int             `json:"screenshotWidth"`
+	ScreenshotHeight int             `json:"screenshotHeight"`
+	ScaleFactor      float64         `json:"scaleFactor"`
+	Source           jsonRawOptional `json:"source,omitempty"`
+	Regions          []Region        `json:"regions"`
+	CreatedAt        string          `json:"createdAt"`
+	UpdatedAt        string          `json:"updatedAt"`
+}
+
+type jsonRawOptional map[string]any
+
+type Region struct {
+	ID          string     `json:"id"`
+	Name        string     `json:"name"`
+	Description string     `json:"description"`
+	Type        string     `json:"type"`
+	BBoxImage   [4]float64 `json:"bbox_image"`
+	BBoxSource  [4]float64 `json:"bbox_source,omitempty"`
+	BBoxScreen  [4]float64 `json:"bbox_screen"`
+	ScaleFactor float64    `json:"scaleFactor"`
+}
+
+type RegionSummary struct {
+	Name         string     `json:"name"`
+	Description  string     `json:"description,omitempty"`
+	Type         string     `json:"type"`
+	BBoxScreen   [4]float64 `json:"bbox_screen"`
+	BBoxSource   [4]float64 `json:"bbox_source,omitempty"`
+	CenterScreen [2]float64 `json:"center_screen"`
+	CenterSource [2]float64 `json:"center_source,omitempty"`
+}
+
+type Task struct {
+	TaskID       string          `json:"task_id"`
+	TaskName     string          `json:"task_name"`
+	TaskType     string          `json:"task_type"`
+	TaskStatus   string          `json:"task_status"`
+	CreatedAt    string          `json:"created_at"`
+	Source       TaskSource      `json:"source"`
+	Confidence   float64         `json:"confidence"`
+	LLMSummary   LLMSummary      `json:"llm_summary"`
+	Regions      []RegionSummary `json:"regions"`
+	Actions      []Action        `json:"actions"`
+	ExecutionLog []ExecutionLog  `json:"execution_log"`
+}
+
+type TaskSource struct {
+	RegionsPath    string `json:"regions_path"`
+	ScreenshotPath string `json:"screenshot_path"`
+}
+
+type LLMSummary struct {
+	ContactName       string `json:"contact_name"`
+	ChatSummary       string `json:"chat_summary"`
+	LatestUserMessage string `json:"latest_user_message"`
+	ReplyText         string `json:"reply_text"`
+}
+
+type Action struct {
+	ActionID     string      `json:"action_id"`
+	Type         string      `json:"type"`
+	Name         string      `json:"name"`
+	Status       string      `json:"status"`
+	TargetRegion string      `json:"target_region,omitempty"`
+	Point        *[2]float64 `json:"point,omitempty"`
+	Text         string      `json:"text,omitempty"`
+	DeltaY       int         `json:"delta_y,omitempty"`
+	DurationMs   int         `json:"duration_ms,omitempty"`
+	Reason       string      `json:"reason,omitempty"`
+}
+
+type ExecutionLog struct {
+	ActionID string `json:"action_id"`
+	Status   string `json:"status"`
+	Message  string `json:"message"`
+	Time     string `json:"time"`
+}
+
+type openAIRequest struct {
+	Model       string          `json:"model"`
+	Messages    []openAIMessage `json:"messages"`
+	Temperature float64         `json:"temperature"`
+}
+
+type openAIMessage struct {
+	Role    string          `json:"role"`
+	Content []openAIContent `json:"content"`
+}
+
+type openAIContent struct {
+	Type     string          `json:"type"`
+	Text     string          `json:"text,omitempty"`
+	ImageURL *openAIImageURL `json:"image_url,omitempty"`
+}
+
+type openAIImageURL struct {
+	URL string `json:"url"`
+}
+
+type openAIResponse struct {
+	Choices []struct {
+		Message struct {
+			Content string `json:"content"`
+		} `json:"message"`
+	} `json:"choices"`
+	Error *struct {
+		Message string `json:"message"`
+	} `json:"error,omitempty"`
+}

package-lock.json

@@ -8,6 +8,7 @@
       "name": "wechat-ai-ui",
       "version": "0.1.0",
       "dependencies": {
+        "@radix-ui/react-select": "^2.3.0",
         "@tauri-apps/api": "^2.11.0",
         "@vitejs/plugin-react": "^5.0.0",
         "autoprefixer": "^10.4.20",
@@ -713,6 +714,44 @@
         "node": ">=18"
       }
     },
+    "node_modules/@floating-ui/core": {
+      "version": "1.7.5",
+      "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.7.5.tgz",
+      "integrity": "sha512-1Ih4WTWyw0+lKyFMcBHGbb5U5FtuHJuujoyyr5zTaWS5EYMeT6Jb2AuDeftsCsEuchO+mM2ij5+q9crhydzLhQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@floating-ui/utils": "^0.2.11"
+      }
+    },
+    "node_modules/@floating-ui/dom": {
+      "version": "1.7.6",
+      "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.7.6.tgz",
+      "integrity": "sha512-9gZSAI5XM36880PPMm//9dfiEngYoC6Am2izES1FF406YFsjvyBMmeJ2g4SAju3xWwtuynNRFL2s9hgxpLI5SQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@floating-ui/core": "^1.7.5",
+        "@floating-ui/utils": "^0.2.11"
+      }
+    },
+    "node_modules/@floating-ui/react-dom": {
+      "version": "2.1.8",
+      "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.1.8.tgz",
+      "integrity": "sha512-cC52bHwM/n/CxS87FH0yWdngEZrjdtLW/qVruo68qg+prK7ZQ4YGdut2GyDVpoGeAYe/h899rVeOVm6Oi40k2A==",
+      "license": "MIT",
+      "dependencies": {
+        "@floating-ui/dom": "^1.7.6"
+      },
+      "peerDependencies": {
+        "react": ">=16.8.0",
+        "react-dom": ">=16.8.0"
+      }
+    },
+    "node_modules/@floating-ui/utils": {
+      "version": "0.2.11",
+      "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.11.tgz",
+      "integrity": "sha512-RiB/yIh78pcIxl6lLMG0CgBXAZ2Y0eVHqMPYugu+9U0AeT6YBeiJpf7lbdJNIugFP5SIjwNRgo4DhR1Qxi26Gg==",
+      "license": "MIT"
+    },
     "node_modules/@jridgewell/gen-mapping": {
       "version": "0.3.13",
       "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz",
@@ -793,6 +832,526 @@
         "node": ">= 8"
       }
     },
+    "node_modules/@radix-ui/number": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@radix-ui/number/-/number-1.1.2.tgz",
+      "integrity": "sha512-ceTwaxc4I5IOi97DgCotl3pqiyRGvffcc0oOsE2dQYaJOFIDsDt4VWG6xEbg1QePv9QWausCEIppud/tJ1wNig==",
+      "license": "MIT"
+    },
+    "node_modules/@radix-ui/primitive": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.4.tgz",
+      "integrity": "sha512-7AdCK9PQyiljKoBDbN8OuctCbd/esdwZPQ8RtOE3SsyQtUpiPb+ND75q0jEhC1m1ecBI0MFNeLJvwIh9iKHRcQ==",
+      "license": "MIT"
+    },
+    "node_modules/@radix-ui/react-arrow": {
+      "version": "1.1.9",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-arrow/-/react-arrow-1.1.9.tgz",
+      "integrity": "sha512-yqHW5WQ/cTpU/un7dqqIKNy2iRU8BC0JB78PEzTfCCYvZu1U6W9KwObAniMk9nhSfyotKPQTYaUD/HB0f5muig==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-primitive": "2.1.5"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-collection": {
+      "version": "1.1.9",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-collection/-/react-collection-1.1.9.tgz",
+      "integrity": "sha512-zuSVi7ziP7uQRqc+yGxsKJfNkdyHv3ZKDaHe0gzg4dRgws96TPKWIiz84tVHP4GEcEl8bC0mdt17NkcxaJHmaQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-compose-refs": "1.1.3",
+        "@radix-ui/react-context": "1.1.4",
+        "@radix-ui/react-primitive": "2.1.5",
+        "@radix-ui/react-slot": "1.2.5"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-compose-refs": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.3.tgz",
+      "integrity": "sha512-rYOP8OMnuuPMQF1uhPVlGNcCDlkokKqGFE3JcxFViIkAXP7EvFWUliJAstrapypaBLJNHbZL6jGhbVDGTwmVhA==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-context": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.1.4.tgz",
+      "integrity": "sha512-QwH4PO5urrbO+FaGd5Aglg+YJgWTyyuZ3g/6mKvsqraLkglDdckw9JafgL5McL5VEJ6EPNduPaT3ZE9BttDAqg==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-direction": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-direction/-/react-direction-1.1.2.tgz",
+      "integrity": "sha512-C3vFhbyi4SW3PmbAi6Awpu4OzJtd0MxGurvSsYtr7p7nM8RNB3VAF3CUmnp2j50knpkrRcB7+ycVXzgLgF6yNA==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-dismissable-layer": {
+      "version": "1.1.12",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.1.12.tgz",
+      "integrity": "sha512-MhoruH6xEzsbvOmo4TNgMfmtvRGyDZw4MDSdf4ybMHfezjqwzv6hyd4lsMzBp8K9Sn6sGzCF62x1I7BYUECXOg==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/primitive": "1.1.4",
+        "@radix-ui/react-compose-refs": "1.1.3",
+        "@radix-ui/react-primitive": "2.1.5",
+        "@radix-ui/react-use-callback-ref": "1.1.2",
+        "@radix-ui/react-use-escape-keydown": "1.1.2"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-focus-guards": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-guards/-/react-focus-guards-1.1.4.tgz",
+      "integrity": "sha512-cot/aB/mOm0IYVYTTmQcEEK1M48lZWi8FlYe5nDPQQ8NYZUlXEFgncJ9p2Kzer3RKSrY7cTTpEMLZKNo9QoP5Q==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-focus-scope": {
+      "version": "1.1.9",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-scope/-/react-focus-scope-1.1.9.tgz",
+      "integrity": "sha512-9Se8t+Zry+1rEOL7Y6l/4ANYU/TOtAtf8O2fKdwLltcaMcm6kOqYGbzO4tMFQ0bvzO920pRAoHpFZ4W85S3keQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-compose-refs": "1.1.3",
+        "@radix-ui/react-primitive": "2.1.5",
+        "@radix-ui/react-use-callback-ref": "1.1.2"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-id": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.1.2.tgz",
+      "integrity": "sha512-orBC88futVpqCmhX1p4cvquNHsELQ+w+vBJnuj3ftETI5bJb0bZn3Tqu3SWN2IOcPycTnMGnhwoermvISt72sA==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-use-layout-effect": "1.1.2"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-popper": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-popper/-/react-popper-1.3.0.tgz",
+      "integrity": "sha512-9PB589e1aWZbrlFUHdz6WiPCL+xLZHQFX7oibqG/6Q0SwOkxDyQX9W/cyPa+sAPPKuC8cpLCpRczE5a/1DiwVQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@floating-ui/react-dom": "^2.0.0",
+        "@radix-ui/react-arrow": "1.1.9",
+        "@radix-ui/react-compose-refs": "1.1.3",
+        "@radix-ui/react-context": "1.1.4",
+        "@radix-ui/react-primitive": "2.1.5",
+        "@radix-ui/react-use-callback-ref": "1.1.2",
+        "@radix-ui/react-use-layout-effect": "1.1.2",
+        "@radix-ui/react-use-rect": "1.1.2",
+        "@radix-ui/react-use-size": "1.1.2",
+        "@radix-ui/rect": "1.1.2"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-portal": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.1.11.tgz",
+      "integrity": "sha512-UEytdjgEh2tJGgD/gZK4FUx6t1rNIlM3U0DENhSrG7I75FGm1DnaDuVUWF1pWAWUwGmn1sCJ1VGHn8LhN1aTOw==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-primitive": "2.1.5",
+        "@radix-ui/react-use-layout-effect": "1.1.2"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-presence": {
+      "version": "1.1.6",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.1.6.tgz",
+      "integrity": "sha512-zdTk4PlUO0E18HnZ3wYbW0KkJJxWCdiNYp6g6X1PtONFhxVkg01vliTJAmwIszU6mHiyBOoW9P0rAugl5/hULQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-use-layout-effect": "1.1.2"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-primitive": {
+      "version": "2.1.5",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.1.5.tgz",
+      "integrity": "sha512-zifXeB8Y88qCYx8PLZ5oQb32KwZub+s925mMoZsBBq9KUQqWKkREubTfs6ASjRPPBe7Jt9O8OHH89+95VG+grA==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-slot": "1.2.5"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-select": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-select/-/react-select-2.3.0.tgz",
+      "integrity": "sha512-mENc7WpJvJcW8hlMpzfFcHcEhTvYS5JMBmi9HVC1Q00uhBwML086MHYUV8QQdQv6lcu0Wg8dzd1RB8AFADcG/g==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/number": "1.1.2",
+        "@radix-ui/primitive": "1.1.4",
+        "@radix-ui/react-collection": "1.1.9",
+        "@radix-ui/react-compose-refs": "1.1.3",
+        "@radix-ui/react-context": "1.1.4",
+        "@radix-ui/react-direction": "1.1.2",
+        "@radix-ui/react-dismissable-layer": "1.1.12",
+        "@radix-ui/react-focus-guards": "1.1.4",
+        "@radix-ui/react-focus-scope": "1.1.9",
+        "@radix-ui/react-id": "1.1.2",
+        "@radix-ui/react-popper": "1.3.0",
+        "@radix-ui/react-portal": "1.1.11",
+        "@radix-ui/react-presence": "1.1.6",
+        "@radix-ui/react-primitive": "2.1.5",
+        "@radix-ui/react-slot": "1.2.5",
+        "@radix-ui/react-use-callback-ref": "1.1.2",
+        "@radix-ui/react-use-controllable-state": "1.2.3",
+        "@radix-ui/react-use-layout-effect": "1.1.2",
+        "@radix-ui/react-use-previous": "1.1.2",
+        "@radix-ui/react-visually-hidden": "1.2.5",
+        "aria-hidden": "^1.2.4",
+        "react-remove-scroll": "^2.7.2"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-slot": {
+      "version": "1.2.5",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.2.5.tgz",
+      "integrity": "sha512-rCMO3QsIVKv5JTY5CVbo2MvO77SpEqqYc8AvRE7OWqRDOIqAKjsp+DrmnY9uc8NPdxB5E2z47HTYGeE2+NTptg==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-compose-refs": "1.1.3"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-use-callback-ref": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.1.2.tgz",
+      "integrity": "sha512-xCso9j1/u8sEgP1RNHjFrXJLApL8LiqOkI1R4ywuN00rxWdYg4oQXuwKLS3i0j5NWLromUD27/4nlxj2UFVvIw==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-use-controllable-state": {
+      "version": "1.2.3",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.2.3.tgz",
+      "integrity": "sha512-PLzC90MS+ReootmjC597dvopoelpZ8Q61HJkDXZSExitIq7PL55vHNnesAHwguHK0aPfBnpdNzQtv1uliaqQrA==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-use-effect-event": "0.0.3",
+        "@radix-ui/react-use-layout-effect": "1.1.2"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-use-effect-event": {
+      "version": "0.0.3",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-effect-event/-/react-use-effect-event-0.0.3.tgz",
+      "integrity": "sha512-6c8ZqvPTWILEKnyVkP53EGRCcpnJiKTC21sS/6R1GF5xKyHJJWQEPfkqlcgUkdRQivd6tb23abUwe4ngWmY0JA==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-use-layout-effect": "1.1.2"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-use-escape-keydown": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-escape-keydown/-/react-use-escape-keydown-1.1.2.tgz",
+      "integrity": "sha512-2uVLvLjgO7NZCWw01/FdqRwmA42J0BcjPMUCA+koFEOAb+zjqIP7SiFz/7zWPrKnVmSqr76Omq2ALyCuX4dhLw==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-use-callback-ref": "1.1.2"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-use-layout-effect": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.1.2.tgz",
+      "integrity": "sha512-jrBWOxZITuGcnjRCM2t2U5ZPkCLxD+Ym6DjfssS5haTj2iiak/DOb64JeN6OdLfLgptb6/e2kKR+ZuTrGoZTPA==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-use-previous": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-previous/-/react-use-previous-1.1.2.tgz",
+      "integrity": "sha512-IGBQPtRFdhN6MQ8dbegVmBq1LVZluya3F1jWY+puIcQC3MHctRwTDSBWCkL/3ZcnMJLTMJ++Z+ktmvg0F89iCw==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-use-rect": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-rect/-/react-use-rect-1.1.2.tgz",
+      "integrity": "sha512-d8a+bBY/FxikNPlgJJoaBHZX+zKVbWHYJGTLnLvveQgFSTntkGdEKv3JDtHrMS0DNYpllz2nRsTLGLKYttbpmw==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/rect": "1.1.2"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-use-size": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-size/-/react-use-size-1.1.2.tgz",
+      "integrity": "sha512-giWQp+4mxjBPt4KZ0MmyuykFNWfbDxKt4x+fPkRYmgRFJSbCZFzUglvMb/Kjn38tm10YP4ufiQZDx3zna4LU6w==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-use-layout-effect": "1.1.2"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-visually-hidden": {
+      "version": "1.2.5",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-visually-hidden/-/react-visually-hidden-1.2.5.tgz",
+      "integrity": "sha512-tPcHNI3FajdDBFpl/Ez1m2WL0ufJqBKyHxMDBvKitopamK36WwBGOMicuMEZKkM5Wce41QxUyv6BsiqfrWBiGg==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-primitive": "2.1.5"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/rect": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@radix-ui/rect/-/rect-1.1.2.tgz",
+      "integrity": "sha512-xnXE7wG13PI+cxieVssYXlQJuYVRhH9NBoxt3KNwzghDIA69GMm7d4wXRouHIYjE+KvS6U/MsMO73NdS2MH9ZA==",
+      "license": "MIT"
+    },
     "node_modules/@rolldown/pluginutils": {
       "version": "1.0.0-rc.3",
       "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.3.tgz",
@@ -1443,6 +2002,18 @@
       "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==",
       "license": "MIT"
     },
+    "node_modules/aria-hidden": {
+      "version": "1.2.6",
+      "resolved": "https://registry.npmjs.org/aria-hidden/-/aria-hidden-1.2.6.tgz",
+      "integrity": "sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA==",
+      "license": "MIT",
+      "dependencies": {
+        "tslib": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
     "node_modules/autoprefixer": {
       "version": "10.5.0",
       "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.5.0.tgz",
@@ -1657,6 +2228,12 @@
         }
       }
     },
+    "node_modules/detect-node-es": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz",
+      "integrity": "sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ==",
+      "license": "MIT"
+    },
     "node_modules/didyoumean": {
       "version": "1.2.2",
       "resolved": "https://registry.npmjs.org/didyoumean/-/didyoumean-1.2.2.tgz",
@@ -1828,6 +2405,15 @@
         "node": ">=6.9.0"
       }
     },
+    "node_modules/get-nonce": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/get-nonce/-/get-nonce-1.0.1.tgz",
+      "integrity": "sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/glob-parent": {
       "version": "6.0.2",
       "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
@@ -2325,6 +2911,75 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/react-remove-scroll": {
+      "version": "2.7.2",
+      "resolved": "https://registry.npmjs.org/react-remove-scroll/-/react-remove-scroll-2.7.2.tgz",
+      "integrity": "sha512-Iqb9NjCCTt6Hf+vOdNIZGdTiH1QSqr27H/Ek9sv/a97gfueI/5h1s3yRi1nngzMUaOOToin5dI1dXKdXiF+u0Q==",
+      "license": "MIT",
+      "dependencies": {
+        "react-remove-scroll-bar": "^2.3.7",
+        "react-style-singleton": "^2.2.3",
+        "tslib": "^2.1.0",
+        "use-callback-ref": "^1.3.3",
+        "use-sidecar": "^1.1.3"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/react-remove-scroll-bar": {
+      "version": "2.3.8",
+      "resolved": "https://registry.npmjs.org/react-remove-scroll-bar/-/react-remove-scroll-bar-2.3.8.tgz",
+      "integrity": "sha512-9r+yi9+mgU33AKcj6IbT9oRCO78WriSj6t/cF8DWBZJ9aOGPOTEDvdUDz1FwKim7QXWwmHqtdHnRJfhAxEG46Q==",
+      "license": "MIT",
+      "dependencies": {
+        "react-style-singleton": "^2.2.2",
+        "tslib": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/react-style-singleton": {
+      "version": "2.2.3",
+      "resolved": "https://registry.npmjs.org/react-style-singleton/-/react-style-singleton-2.2.3.tgz",
+      "integrity": "sha512-b6jSvxvVnyptAiLjbkWLE/lOnR4lfTtDAl+eUC7RZy+QQWc6wRzIV2CE6xBuMmDxc2qIihtDCZD5NPOFl7fRBQ==",
+      "license": "MIT",
+      "dependencies": {
+        "get-nonce": "^1.0.0",
+        "tslib": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/read-cache": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz",
@@ -2623,6 +3278,12 @@
       "integrity": "sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==",
       "license": "Apache-2.0"
     },
+    "node_modules/tslib": {
+      "version": "2.8.1",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
+      "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
+      "license": "0BSD"
+    },
     "node_modules/update-browserslist-db": {
       "version": "1.2.3",
       "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz",
@@ -2653,6 +3314,49 @@
         "browserslist": ">= 4.21.0"
       }
     },
+    "node_modules/use-callback-ref": {
+      "version": "1.3.3",
+      "resolved": "https://registry.npmjs.org/use-callback-ref/-/use-callback-ref-1.3.3.tgz",
+      "integrity": "sha512-jQL3lRnocaFtu3V00JToYz/4QkNWswxijDaCVNZRiRTO3HQDLsdu1ZtmIUvV4yPp+rvWm5j0y0TG/S61cuijTg==",
+      "license": "MIT",
+      "dependencies": {
+        "tslib": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/use-sidecar": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/use-sidecar/-/use-sidecar-1.1.3.tgz",
+      "integrity": "sha512-Fedw0aZvkhynoPYlA5WXrMCAMm+nSWdZt6lzJQ7Ok8S6Q+VsHmHpRWndVRJ8Be0ZbkfPc5LRYH+5XrzXcEeLRQ==",
+      "license": "MIT",
+      "dependencies": {
+        "detect-node-es": "^1.1.0",
+        "tslib": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/util-deprecate": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",

package.json

@@ -10,6 +10,7 @@
     "preview": "vite preview --host 127.0.0.1"
   },
   "dependencies": {
+    "@radix-ui/react-select": "^2.3.0",
     "@tauri-apps/api": "^2.11.0",
     "@vitejs/plugin-react": "^5.0.0",
     "autoprefixer": "^10.4.20",

src-tauri/Cargo.toml

@@ -21,6 +21,9 @@ tauri-build = { version = "2.6.2", features = [] }
 serde_json = "1.0"
 serde = { version = "1.0", features = ["derive"] }
 log = "0.4"
-tauri = { version = "2.11.2", features = ["macos-private-api"] }
+image = { version = "0.25", default-features = false, features = ["jpeg"] }
+screenshots = "0.8"
+tauri = { version = "2.11.2", features = ["protocol-asset", "macos-private-api"] }
 tauri-plugin-log = "2"
 tauri-plugin-shell = "2.3.5"
+xcap = "0.6"

src-tauri/capabilities/default.json

@@ -9,6 +9,7 @@
   "permissions": [
     "core:default",
     "core:window:allow-close",
+    "core:window:allow-set-fullscreen",
     "core:window:allow-start-dragging"
   ]
 }

src-tauri/src/lib.rs

@@ -1,10 +1,15 @@
+use screenshots::Screen;
 use serde::{Deserialize, Serialize};
+use std::fs;
+use std::path::PathBuf;
 use std::sync::Mutex;
+use std::time::Instant;
 use tauri::{Emitter, Manager, WebviewUrl, WebviewWindowBuilder};
 use tauri_plugin_shell::{
     process::{CommandChild, CommandEvent},
     ShellExt,
 };
+use xcap::{Monitor, Window};
 
 struct AgentProcess(Mutex<Option<CommandChild>>);
 
@@ -15,6 +20,131 @@ struct AgentLog {
     timestamp: Option<String>,
 }
 
+#[derive(Clone, Serialize)]
+#[serde(rename_all = "camelCase")]
+struct CaptureResult {
+    screenshot_path: String,
+    screenshot_width: u32,
+    screenshot_height: u32,
+    scale_factor: f64,
+    source: Option<CaptureSource>,
+    screen_list_ms: u128,
+    capture_ms: u128,
+    save_ms: u128,
+    total_ms: u128,
+}
+
+#[derive(Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+struct CaptureSource {
+    id: String,
+    kind: String,
+    label: String,
+    app_name: Option<String>,
+    title: Option<String>,
+    pid: Option<u32>,
+    x: i32,
+    y: i32,
+    width: u32,
+    height: u32,
+    scale_factor: f64,
+}
+
+#[derive(Clone, Deserialize, Serialize)]
+struct Region {
+    id: String,
+    name: String,
+    description: Option<String>,
+    #[serde(rename = "type")]
+    region_type: String,
+    bbox_image: [f64; 4],
+    bbox_source: Option<[f64; 4]>,
+    bbox_screen: [f64; 4],
+    #[serde(rename = "scaleFactor")]
+    scale_factor: f64,
+}
+
+#[derive(Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+struct AnnotationFile {
+    app: String,
+    screenshot_path: String,
+    screenshot_width: u32,
+    screenshot_height: u32,
+    scale_factor: f64,
+    source: Option<CaptureSource>,
+    regions: Vec<Region>,
+    created_at: String,
+    updated_at: String,
+}
+
+fn data_dir(app: &tauri::AppHandle) -> Result<PathBuf, String> {
+    let dir = app
+        .path()
+        .app_data_dir()
+        .map_err(|error| error.to_string())?
+        .join("data");
+    fs::create_dir_all(&dir).map_err(|error| error.to_string())?;
+    Ok(dir)
+}
+
+fn regions_path(app: &tauri::AppHandle) -> Result<PathBuf, String> {
+    let dir = data_dir(app)?.join("regions");
+    fs::create_dir_all(&dir).map_err(|error| error.to_string())?;
+    Ok(dir.join("wechat.json"))
+}
+
+fn screenshot_path(app: &tauri::AppHandle) -> Result<PathBuf, String> {
+    let screenshot_dir = data_dir(app)?.join("screenshots");
+    fs::create_dir_all(&screenshot_dir).map_err(|error| error.to_string())?;
+    Ok(screenshot_dir.join("WeChat.jpg"))
+}
+
+fn save_capture_image(
+    width: u32,
+    height: u32,
+    rgba_bytes: Vec<u8>,
+    screenshot_path: PathBuf,
+    source: Option<CaptureSource>,
+    default_scale_factor: f64,
+    screen_list_ms: u128,
+    capture_ms: u128,
+    total_started_at: Instant,
+) -> Result<CaptureResult, String> {
+    let save_started_at = Instant::now();
+    let rgba_image = image::RgbaImage::from_raw(width, height, rgba_bytes)
+        .ok_or_else(|| "invalid screenshot buffer".to_string())?;
+    let rgb_image = image::DynamicImage::ImageRgba8(rgba_image).to_rgb8();
+    let mut file = fs::File::create(&screenshot_path).map_err(|error| error.to_string())?;
+    let mut encoder = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut file, 88);
+    encoder
+        .encode_image(&rgb_image)
+        .map_err(|error| error.to_string())?;
+    let save_ms = save_started_at.elapsed().as_millis();
+    let total_ms = total_started_at.elapsed().as_millis();
+    let scale_factor = source
+        .as_ref()
+        .map(|item| item.scale_factor)
+        .unwrap_or(default_scale_factor);
+
+    println!(
+        "capture_source timing: screen_list={}ms capture={}ms save={}ms total={}ms size={}x{} scale={}",
+        screen_list_ms, capture_ms, save_ms, total_ms, width, height, scale_factor
+    );
+
+    Ok(CaptureResult {
+        screenshot_path: screenshot_path.to_string_lossy().to_string(),
+        screenshot_width: width,
+        screenshot_height: height,
+        scale_factor,
+        source,
+        screen_list_ms,
+        capture_ms,
+        save_ms,
+        total_ms,
+    })
+}
+
 fn emit_agent_log(app: &tauri::AppHandle, level: &str, message: impl Into<String>) {
     let _ = app.emit(
         "agent-log",
@@ -110,6 +240,246 @@ fn stop_agent(app: tauri::AppHandle, process: tauri::State<AgentProcess>) -> Res
     Ok(())
 }
 
+#[tauri::command]
+async fn capture_screen(app: tauri::AppHandle) -> Result<CaptureResult, String> {
+    let screenshot_path = screenshot_path(&app)?;
+
+    tauri::async_runtime::spawn_blocking(move || {
+        let total_started_at = Instant::now();
+        let screen_list_started_at = Instant::now();
+        let screen = Screen::all()
+            .map_err(|error| error.to_string())?
+            .into_iter()
+            .next()
+            .ok_or_else(|| "no screen found".to_string())?;
+        let screen_list_ms = screen_list_started_at.elapsed().as_millis();
+
+        let capture_started_at = Instant::now();
+        let image = screen.capture().map_err(|error| error.to_string())?;
+        let capture_ms = capture_started_at.elapsed().as_millis();
+        let width = image.width();
+        let height = image.height();
+        save_capture_image(
+            width,
+            height,
+            image.into_raw(),
+            screenshot_path,
+            None,
+            screen.display_info.scale_factor as f64,
+            screen_list_ms,
+            capture_ms,
+            total_started_at,
+        )
+    })
+    .await
+    .map_err(|error| error.to_string())?
+}
+
+#[tauri::command]
+async fn list_capture_sources() -> Result<Vec<CaptureSource>, String> {
+    tauri::async_runtime::spawn_blocking(move || {
+        let mut sources = Vec::new();
+
+        for (index, monitor) in Monitor::all()
+            .map_err(|error| error.to_string())?
+            .into_iter()
+            .enumerate()
+        {
+            let width = monitor.width().map_err(|error| error.to_string())?;
+            let height = monitor.height().map_err(|error| error.to_string())?;
+            let x = monitor.x().map_err(|error| error.to_string())?;
+            let y = monitor.y().map_err(|error| error.to_string())?;
+            let scale_factor = monitor.scale_factor().map_err(|error| error.to_string())? as f64;
+
+            sources.push(CaptureSource {
+                id: format!("display:{index}"),
+                kind: "display".to_string(),
+                label: format!("桌面 {} · {}x{}", index + 1, width, height),
+                app_name: None,
+                title: None,
+                pid: None,
+                x,
+                y,
+                width,
+                height,
+                scale_factor,
+            });
+        }
+
+        if let Ok(windows) = Window::all() {
+            for window in windows {
+                let width = match window.width() {
+                    Ok(width) => width,
+                    Err(_) => continue,
+                };
+                let height = match window.height() {
+                    Ok(height) => height,
+                    Err(_) => continue,
+                };
+
+                if window.is_minimized().unwrap_or(true) || width < 80 || height < 80 {
+                    continue;
+                }
+
+                let app_name = window.app_name().unwrap_or_default();
+                let title = window.title().unwrap_or_default();
+                if app_name.is_empty() && title.is_empty() {
+                    continue;
+                }
+
+                sources.push(CaptureSource {
+                    id: format!("window:{}", window.id().map_err(|error| error.to_string())?),
+                    kind: "window".to_string(),
+                    label: format!("{} · {}", app_name, title),
+                    app_name: Some(app_name),
+                    title: Some(title),
+                    pid: window.pid().ok(),
+                    x: window.x().unwrap_or(0),
+                    y: window.y().unwrap_or(0),
+                    width,
+                    height,
+                    scale_factor: 1.0,
+                });
+            }
+        }
+
+        Ok(sources)
+    })
+    .await
+    .map_err(|error| error.to_string())?
+}
+
+#[tauri::command]
+async fn capture_source(app: tauri::AppHandle, source_id: String) -> Result<CaptureResult, String> {
+    let screenshot_path = screenshot_path(&app)?;
+
+    tauri::async_runtime::spawn_blocking(move || {
+        let total_started_at = Instant::now();
+        let screen_list_started_at = Instant::now();
+
+        if let Some(index_text) = source_id.strip_prefix("display:") {
+            let index = index_text
+                .parse::<usize>()
+                .map_err(|error| error.to_string())?;
+            let monitors = Monitor::all().map_err(|error| error.to_string())?;
+            let monitor = monitors
+                .into_iter()
+                .nth(index)
+                .ok_or_else(|| "display source not found".to_string())?;
+            let screen_list_ms = screen_list_started_at.elapsed().as_millis();
+            let width = monitor.width().map_err(|error| error.to_string())?;
+            let height = monitor.height().map_err(|error| error.to_string())?;
+            let scale_factor = monitor.scale_factor().map_err(|error| error.to_string())? as f64;
+            let source = CaptureSource {
+                id: source_id,
+                kind: "display".to_string(),
+                label: format!("桌面 {} · {}x{}", index + 1, width, height),
+                app_name: None,
+                title: None,
+                pid: None,
+                x: monitor.x().map_err(|error| error.to_string())?,
+                y: monitor.y().map_err(|error| error.to_string())?,
+                width,
+                height,
+                scale_factor,
+            };
+
+            let capture_started_at = Instant::now();
+            let image = monitor.capture_image().map_err(|error| error.to_string())?;
+            let capture_ms = capture_started_at.elapsed().as_millis();
+            let image_width = image.width();
+            let image_height = image.height();
+            return save_capture_image(
+                image_width,
+                image_height,
+                image.into_raw(),
+                screenshot_path,
+                Some(source.clone()),
+                source.scale_factor,
+                screen_list_ms,
+                capture_ms,
+                total_started_at,
+            );
+        }
+
+        if let Some(window_id_text) = source_id.strip_prefix("window:") {
+            let window_id = window_id_text
+                .parse::<u32>()
+                .map_err(|error| error.to_string())?;
+            let windows = Window::all().map_err(|error| error.to_string())?;
+            let window = windows
+                .into_iter()
+                .find(|item| item.id().ok() == Some(window_id))
+                .ok_or_else(|| "window source not found".to_string())?;
+            let screen_list_ms = screen_list_started_at.elapsed().as_millis();
+            let window_width = window.width().map_err(|error| error.to_string())?;
+            let window_height = window.height().map_err(|error| error.to_string())?;
+            let app_name = window.app_name().unwrap_or_default();
+            let title = window.title().unwrap_or_default();
+            let capture_started_at = Instant::now();
+            let image = window.capture_image().map_err(|error| error.to_string())?;
+            let capture_ms = capture_started_at.elapsed().as_millis();
+            let image_width = image.width();
+            let image_height = image.height();
+            let scale_factor = if window_width > 0 {
+                image_width as f64 / window_width as f64
+            } else {
+                1.0
+            };
+            let source = CaptureSource {
+                id: source_id,
+                kind: "window".to_string(),
+                label: format!("{} · {}", app_name, title),
+                app_name: Some(app_name),
+                title: Some(title),
+                pid: window.pid().ok(),
+                x: window.x().unwrap_or(0),
+                y: window.y().unwrap_or(0),
+                width: window_width,
+                height: window_height,
+                scale_factor,
+            };
+
+            return save_capture_image(
+                image_width,
+                image_height,
+                image.into_raw(),
+                screenshot_path,
+                Some(source.clone()),
+                source.scale_factor,
+                screen_list_ms,
+                capture_ms,
+                total_started_at,
+            );
+        }
+
+        Err("unsupported capture source".to_string())
+    })
+    .await
+    .map_err(|error| error.to_string())?
+}
+
+#[tauri::command]
+fn save_regions(app: tauri::AppHandle, annotation: AnnotationFile) -> Result<String, String> {
+    let path = regions_path(&app)?;
+    let json = serde_json::to_string_pretty(&annotation).map_err(|error| error.to_string())?;
+    fs::write(&path, json).map_err(|error| error.to_string())?;
+    Ok(path.to_string_lossy().to_string())
+}
+
+#[tauri::command]
+fn load_regions(app: tauri::AppHandle) -> Result<Option<AnnotationFile>, String> {
+    let path = regions_path(&app)?;
+    if !path.exists() {
+        return Ok(None);
+    }
+
+    let json = fs::read_to_string(&path).map_err(|error| error.to_string())?;
+    serde_json::from_str(&json)
+        .map(Some)
+        .map_err(|error| error.to_string())
+}
+
 #[tauri::command]
 fn open_popup_window(app: tauri::AppHandle, route: String) -> Result<(), String> {
     let route = if route.starts_with('/') {
@@ -125,6 +495,7 @@ fn open_popup_window(app: tauri::AppHandle, route: String) -> Result<(), String>
     }
 
     let title = match route.as_str() {
+        "/annotate" => "屏幕区域标注",
         "/window/settings" => "设置",
         "/window/engine-logs" => "分身引擎日志",
         "/window/knowledge-create" => "新增知识库",
@@ -135,7 +506,7 @@ fn open_popup_window(app: tauri::AppHandle, route: String) -> Result<(), String>
         _ => "二级窗口",
     };
 
-    WebviewWindowBuilder::new(
+    let mut builder = WebviewWindowBuilder::new(
         &app,
         label,
         WebviewUrl::App(format!("index.html#{route}").into()),
@@ -144,9 +515,13 @@ fn open_popup_window(app: tauri::AppHandle, route: String) -> Result<(), String>
     .inner_size(1080.0, 800.0)
     .decorations(false)
     .transparent(true)
-    .resizable(true)
+    .resizable(true);
-    .build()
+
-    .map_err(|error| error.to_string())?;
+    if route == "/annotate" {
+        builder = builder.maximized(true).fullscreen(true);
+    }
+
+    builder.build().map_err(|error| error.to_string())?;
 
     Ok(())
 }
@@ -167,7 +542,12 @@ pub fn run() {
             Ok(())
         })
         .invoke_handler(tauri::generate_handler![
+            capture_source,
+            capture_screen,
+            list_capture_sources,
+            load_regions,
             open_popup_window,
+            save_regions,
             start_agent,
             stop_agent
         ])

src-tauri/tauri.conf.json

@@ -25,7 +25,13 @@
       }
     ],
     "security": {
-      "csp": null
+      "csp": null,
+      "assetProtocol": {
+        "enable": true,
+        "scope": [
+          "$APPDATA/data/screenshots/**"
+        ]
+      }
     }
   },
   "bundle": {

src/App.jsx

@@ -1,5 +1,6 @@
 import { useEffect, useState } from "react";
 import MainShell from "./components/MainShell";
+import AnnotationPage from "./pages/AnnotationPage";
 import SecondaryWindow from "./windows/SecondaryWindow";
 
 function App() {
@@ -15,6 +16,17 @@ function App() {
     localStorage.setItem("theme", theme);
   }, [theme]);
 
+  useEffect(() => {
+    const onStorage = (event) => {
+      if (event.key === "theme" && event.newValue) {
+        setTheme(event.newValue);
+      }
+    };
+
+    window.addEventListener("storage", onStorage);
+    return () => window.removeEventListener("storage", onStorage);
+  }, []);
+
   useEffect(() => {
     const onHashChange = () => setRoute(window.location.hash.replace("#", "") || "/");
     window.addEventListener("hashchange", onHashChange);
@@ -22,10 +34,13 @@ function App() {
   }, []);
 
   const isWindowRoute = route.startsWith("/window/");
+  const isAnnotationRoute = route === "/annotate";
 
   return (
     <div className="min-h-dvh overflow-hidden rounded-[12px] bg-page text-text">
-      {isWindowRoute ? (
+      {isAnnotationRoute ? (
+        <AnnotationPage />
+      ) : isWindowRoute ? (
         <SecondaryWindow
           route={route}
           theme={theme}

src/components/ui/select.jsx

@@ -0,0 +1,92 @@
+import * as SelectPrimitive from "@radix-ui/react-select";
+import { Check, ChevronDown, ChevronUp } from "lucide-react";
+
+function cn(...classes) {
+  return classes.filter(Boolean).join(" ");
+}
+
+const Select = SelectPrimitive.Root;
+const SelectGroup = SelectPrimitive.Group;
+const SelectValue = SelectPrimitive.Value;
+
+function SelectTrigger({ className, children, ...props }) {
+  return (
+    <SelectPrimitive.Trigger
+      className={cn(
+        "flex h-[42px] w-full items-center justify-between rounded-[11px] border border-lineStrong bg-surface px-3 text-[13px] font-semibold text-text outline-none transition placeholder:text-subtle focus:border-primary disabled:cursor-not-allowed disabled:bg-surface2 disabled:text-muted",
+        className,
+      )}
+      {...props}
+    >
+      {children}
+      <SelectPrimitive.Icon asChild>
+        <ChevronDown className="h-4 w-4 shrink-0 text-muted" />
+      </SelectPrimitive.Icon>
+    </SelectPrimitive.Trigger>
+  );
+}
+
+function SelectScrollUpButton({ className, ...props }) {
+  return (
+    <SelectPrimitive.ScrollUpButton
+      className={cn("flex cursor-default items-center justify-center py-1 text-muted", className)}
+      {...props}
+    >
+      <ChevronUp className="h-4 w-4" />
+    </SelectPrimitive.ScrollUpButton>
+  );
+}
+
+function SelectScrollDownButton({ className, ...props }) {
+  return (
+    <SelectPrimitive.ScrollDownButton
+      className={cn("flex cursor-default items-center justify-center py-1 text-muted", className)}
+      {...props}
+    >
+      <ChevronDown className="h-4 w-4" />
+    </SelectPrimitive.ScrollDownButton>
+  );
+}
+
+function SelectContent({ className, children, position = "popper", ...props }) {
+  return (
+    <SelectPrimitive.Portal>
+      <SelectPrimitive.Content
+        className={cn(
+          "relative z-50 max-h-72 min-w-[8rem] overflow-hidden rounded-[12px] border border-line bg-surface text-text shadow-[0_18px_52px_rgba(0,0,0,.18)] data-[state=open]:animate-in data-[state=closed]:animate-out",
+          position === "popper" && "data-[side=bottom]:translate-y-1 data-[side=top]:-translate-y-1",
+          className,
+        )}
+        position={position}
+        {...props}
+      >
+        <SelectScrollUpButton />
+        <SelectPrimitive.Viewport className={cn("p-1", position === "popper" && "min-w-[var(--radix-select-trigger-width)]")}>
+          {children}
+        </SelectPrimitive.Viewport>
+        <SelectScrollDownButton />
+      </SelectPrimitive.Content>
+    </SelectPrimitive.Portal>
+  );
+}
+
+function SelectItem({ className, children, ...props }) {
+  return (
+    <SelectPrimitive.Item
+      className={cn(
+        "relative flex h-9 w-full cursor-default select-none items-center rounded-[9px] py-1.5 pl-8 pr-2 text-[13px] font-semibold outline-none transition focus:bg-surface2 data-[disabled]:pointer-events-none data-[disabled]:opacity-50",
+        className,
+      )}
+      {...props}
+    >
+      <span className="absolute left-2 flex h-4 w-4 items-center justify-center">
+        <SelectPrimitive.ItemIndicator>
+          <Check className="h-4 w-4 text-primary" />
+        </SelectPrimitive.ItemIndicator>
+      </span>
+      <SelectPrimitive.ItemText>{children}</SelectPrimitive.ItemText>
+    </SelectPrimitive.Item>
+  );
+}
+
+export { Select, SelectContent, SelectGroup, SelectItem, SelectTrigger, SelectValue };

src/pages/AnnotationPage.jsx

@@ -0,0 +1,525 @@
+import { useEffect, useMemo, useRef, useState } from "react";
+import { convertFileSrc, invoke } from "@tauri-apps/api/core";
+import { getCurrentWindow } from "@tauri-apps/api/window";
+import { Check, MousePointer2, Trash2, X } from "lucide-react";
+import { Select, SelectContent, SelectGroup, SelectItem, SelectTrigger, SelectValue } from "../components/ui/select";
+
+const regionTypes = [
+  { value: "contact_list", label: "联系人区域" },
+  { value: "chat_content", label: "聊天内容区域" },
+  { value: "input_box", label: "输入框区域" },
+  { value: "send_button", label: "发送按钮区域" },
+  { value: "custom", label: "自定义区域" },
+];
+
+const emptyCapture = {
+  screenshotPath: "",
+  screenshotWidth: 0,
+  screenshotHeight: 0,
+  scaleFactor: window.devicePixelRatio || 1,
+};
+
+function clamp(value, min, max) {
+  return Math.min(Math.max(value, min), max);
+}
+
+function normalizeBox(start, end) {
+  return [
+    Math.min(start.x, end.x),
+    Math.min(start.y, end.y),
+    Math.max(start.x, end.x),
+    Math.max(start.y, end.y),
+  ];
+}
+
+function roundCoord(value) {
+  return Math.round(value * 100) / 100;
+}
+
+function toScreenBox(box, capture) {
+  const scaleFactor = capture.scaleFactor || 1;
+  const sourceX = capture.source?.x || 0;
+  const sourceY = capture.source?.y || 0;
+  const sourceBox = toSourceBox(box, scaleFactor);
+
+  return [
+    roundCoord(sourceX + sourceBox[0]),
+    roundCoord(sourceY + sourceBox[1]),
+    roundCoord(sourceX + sourceBox[2]),
+    roundCoord(sourceY + sourceBox[3]),
+  ];
+}
+
+function toSourceBox(box, scaleFactor) {
+  return box.map((value) => roundCoord(value / scaleFactor));
+}
+
+function createRegion(box, capture) {
+  const bboxImage = box.map(roundCoord);
+  const type = "custom";
+
+  return {
+    id: crypto.randomUUID(),
+    name: `区域 ${new Date().toLocaleTimeString("zh-CN", { hour12: false })}`,
+    type,
+    description: "",
+    bbox_image: bboxImage,
+    bbox_source: toSourceBox(bboxImage, capture.scaleFactor || 1),
+    bbox_screen: toScreenBox(bboxImage, capture),
+    scaleFactor: capture.scaleFactor || 1,
+  };
+}
+
+function normalizeRegion(region) {
+  return {
+    ...region,
+    description: region.description || "",
+    bbox_source: region.bbox_source || region.bbox_image.map((value) => roundCoord(value / (region.scaleFactor || 1))),
+  };
+}
+
+function formatCaptureTiming(capture, frontendMs) {
+  const totalMs = capture.totalMs ?? Math.round(frontendMs);
+
+  return [
+    "拖拽截图区域创建矩形框",
+    `总耗时 ${totalMs}ms`,
+    `枚举屏幕 ${capture.screenListMs ?? "-"}ms`,
+    `截屏 ${capture.captureMs ?? "-"}ms`,
+    `保存 ${capture.saveMs ?? "-"}ms`,
+    `前端等待 ${Math.round(frontendMs)}ms`,
+  ].join(" · ");
+}
+
+export default function AnnotationPage() {
+  const [capture, setCapture] = useState(emptyCapture);
+  const [regions, setRegions] = useState([]);
+  const [selectedId, setSelectedId] = useState(null);
+  const [drawStart, setDrawStart] = useState(null);
+  const [drawEnd, setDrawEnd] = useState(null);
+  const [status, setStatus] = useState("正在准备截图...");
+  const [saving, setSaving] = useState(false);
+  const imageRef = useRef(null);
+
+  const selectedRegion = regions.find((region) => region.id === selectedId) || null;
+  const screenshotSrc = capture.screenshotPath ? convertFileSrc(capture.screenshotPath) : "";
+  const tempBox = drawStart && drawEnd ? normalizeBox(drawStart, drawEnd) : null;
+
+  const typeLabelMap = useMemo(
+    () => Object.fromEntries(regionTypes.map((type) => [type.value, type.label])),
+    [],
+  );
+
+  useEffect(() => {
+    let cancelled = false;
+
+    async function init() {
+      const stored = sessionStorage.getItem("pendingAnnotationCapture");
+      if (stored) {
+        const nextCapture = JSON.parse(stored);
+        if (!cancelled) {
+          setCapture(nextCapture);
+          setStatus("拖拽截图区域创建矩形框");
+          if (window.__TAURI_INTERNALS__) {
+            await getCurrentWindow().setFullscreen(true);
+          }
+        }
+        return;
+      }
+
+      const sharedCapture = localStorage.getItem("annotationCaptureResult");
+      if (sharedCapture) {
+        const nextCapture = JSON.parse(sharedCapture);
+        if (!cancelled) {
+          setCapture(nextCapture);
+          setStatus("拖拽截图区域创建矩形框");
+        }
+        return;
+      }
+
+      if (localStorage.getItem("annotationCaptureStatus") === "pending") {
+        setStatus("正在截取屏幕，请稍候...");
+        const startedAt = performance.now();
+        const nextCapture = await new Promise((resolve, reject) => {
+          const readResult = () => {
+            const result = localStorage.getItem("annotationCaptureResult");
+            if (result) {
+              resolve(JSON.parse(result));
+              return true;
+            }
+
+            const error = localStorage.getItem("annotationCaptureError");
+            if (error) {
+              reject(new Error(error));
+              return true;
+            }
+
+            return false;
+          };
+
+          if (readResult()) return;
+
+          const timer = window.setInterval(() => {
+            if (readResult()) window.clearInterval(timer);
+          }, 100);
+
+          window.addEventListener("storage", () => readResult(), { once: true });
+        });
+
+        if (!cancelled) {
+          sessionStorage.setItem("pendingAnnotationCapture", JSON.stringify(nextCapture));
+          setCapture(nextCapture);
+          setStatus(formatCaptureTiming(nextCapture, performance.now() - startedAt));
+        }
+        return;
+      }
+
+      if (!window.__TAURI_INTERNALS__) {
+        setStatus("浏览器预览模式无法截屏，请在 Tauri 应用中使用");
+        return;
+      }
+
+      setStatus("正在截取屏幕，请稍候...");
+      const startedAt = performance.now();
+      const nextCapture = await invoke("capture_screen");
+      if (!cancelled) {
+        sessionStorage.setItem("pendingAnnotationCapture", JSON.stringify(nextCapture));
+        setCapture(nextCapture);
+        setStatus(formatCaptureTiming(nextCapture, performance.now() - startedAt));
+        await getCurrentWindow().setFullscreen(true);
+      }
+    }
+
+    init().catch((error) => {
+      setStatus(`截屏失败：${String(error)}`);
+    });
+
+    return () => {
+      cancelled = true;
+    };
+  }, []);
+
+  useEffect(() => {
+    async function loadSaved() {
+      if (!window.__TAURI_INTERNALS__) return;
+
+      const saved = await invoke("load_regions");
+      if (saved?.regions?.length) {
+        setRegions(saved.regions.map(normalizeRegion));
+        setSelectedId(saved.regions[0].id);
+      }
+    }
+
+    loadSaved().catch(() => {});
+  }, []);
+
+  function getImagePoint(event) {
+    const rect = imageRef.current?.getBoundingClientRect();
+    if (!rect || !capture.screenshotWidth || !capture.screenshotHeight) return null;
+
+    const displayX = clamp(event.clientX - rect.left, 0, rect.width);
+    const displayY = clamp(event.clientY - rect.top, 0, rect.height);
+
+    return {
+      x: (displayX * capture.screenshotWidth) / rect.width,
+      y: (displayY * capture.screenshotHeight) / rect.height,
+    };
+  }
+
+  function handlePointerDown(event) {
+    if (event.button !== 0) return;
+    const point = getImagePoint(event);
+    if (!point) return;
+
+    event.currentTarget.setPointerCapture(event.pointerId);
+    setDrawStart(point);
+    setDrawEnd(point);
+    setSelectedId(null);
+  }
+
+  function handlePointerMove(event) {
+    if (!drawStart) return;
+    const point = getImagePoint(event);
+    if (point) setDrawEnd(point);
+  }
+
+  function handlePointerUp(event) {
+    if (!drawStart || !drawEnd) return;
+
+    if (event.currentTarget.hasPointerCapture(event.pointerId)) {
+      event.currentTarget.releasePointerCapture(event.pointerId);
+    }
+
+    const box = normalizeBox(drawStart, drawEnd);
+    const width = box[2] - box[0];
+    const height = box[3] - box[1];
+    setDrawStart(null);
+    setDrawEnd(null);
+
+    if (width < 6 || height < 6) return;
+
+    const region = createRegion(box, capture);
+    setRegions((current) => [...current, region]);
+    setSelectedId(region.id);
+  }
+
+  function handlePointerCancel(event) {
+    if (event.currentTarget.hasPointerCapture(event.pointerId)) {
+      event.currentTarget.releasePointerCapture(event.pointerId);
+    }
+    setDrawStart(null);
+    setDrawEnd(null);
+  }
+
+  function updateSelected(patch) {
+    setRegions((current) =>
+      current.map((region) => (region.id === selectedId ? { ...region, ...patch } : region)),
+    );
+  }
+
+  function deleteSelected() {
+    if (!selectedId) return;
+    setRegions((current) => current.filter((region) => region.id !== selectedId));
+    setSelectedId(null);
+  }
+
+  async function exitAnnotation() {
+    sessionStorage.removeItem("pendingAnnotationCapture");
+    localStorage.removeItem("annotationCaptureStatus");
+    localStorage.removeItem("annotationCaptureResult");
+    localStorage.removeItem("annotationCaptureError");
+    localStorage.removeItem("annotationCaptureSource");
+    if (window.__TAURI_INTERNALS__) {
+      const currentWindow = getCurrentWindow();
+      if (currentWindow.label !== "main") {
+        await currentWindow.close();
+        return;
+      }
+
+      await currentWindow.setFullscreen(false);
+    }
+    window.location.hash = "/";
+  }
+
+  async function save() {
+    if (!window.__TAURI_INTERNALS__) {
+      setStatus("浏览器预览模式无法保存到本地文件");
+      return;
+    }
+
+    setSaving(true);
+    const now = new Date().toISOString();
+    const existing = await invoke("load_regions").catch(() => null);
+    const annotation = {
+      app: "wechat",
+      screenshotPath: capture.screenshotPath,
+      screenshotWidth: capture.screenshotWidth,
+      screenshotHeight: capture.screenshotHeight,
+      scaleFactor: capture.scaleFactor || 1,
+      source: capture.source || null,
+      regions,
+      createdAt: existing?.createdAt || now,
+      updatedAt: now,
+    };
+
+    try {
+      const path = await invoke("save_regions", { annotation });
+      sessionStorage.setItem("lastRegionsPath", path);
+      setStatus(`已保存：${path}`);
+      await exitAnnotation();
+    } catch (error) {
+      setStatus(`保存失败：${String(error)}`);
+    } finally {
+      setSaving(false);
+    }
+  }
+
+  return (
+    <div className="flex h-dvh w-full overflow-hidden bg-page text-text">
+      <aside className="flex w-[320px] shrink-0 flex-col border-r border-line bg-surface p-4 shadow-2xl" onPointerDown={(event) => event.stopPropagation()}>
+        <div>
+          <div className="flex items-center gap-2 text-[13px] font-black text-primary">
+            <MousePointer2 size={16} />
+            屏幕区域标注
+          </div>
+          <h1 className="mt-2 text-[20px] font-extrabold">微信 RPA 坐标配置</h1>
+          <p className="mt-2 text-[12px] leading-5 text-muted">{status}</p>
+        </div>
+
+        <div className="mt-4 grid grid-cols-2 gap-2 rounded-[12px] border border-line bg-surface2 p-3 text-[11px] text-muted">
+          <span>截图像素</span>
+          <span className="text-right text-text">{capture.screenshotWidth} x {capture.screenshotHeight}</span>
+          <span>ScaleFactor</span>
+          <span className="text-right text-text">{capture.scaleFactor || 1}</span>
+          <span>区域数量</span>
+          <span className="text-right text-text">{regions.length}</span>
+          <span>来源</span>
+          <span className="truncate text-right text-text">{capture.source?.kind === "window" ? "窗口" : "桌面"}</span>
+        </div>
+
+        <div className="mt-4 space-y-3 rounded-[14px] border border-line bg-surface2 p-3">
+          <label className="block text-[12px] font-bold text-muted">区域名称</label>
+          <input
+            className="input-like"
+            value={selectedRegion?.name || ""}
+            onChange={(event) => updateSelected({ name: event.target.value })}
+            disabled={!selectedRegion}
+            placeholder="先拖拽创建一个区域"
+          />
+
+          <label className="block text-[12px] font-bold text-muted">区域类型</label>
+          <Select
+            value={selectedRegion?.type || "custom"}
+            onValueChange={(value) => updateSelected({ type: value })}
+            disabled={!selectedRegion}
+          >
+            <SelectTrigger>
+              <SelectValue placeholder="选择区域类型" />
+            </SelectTrigger>
+            <SelectContent>
+              <SelectGroup>
+                {regionTypes.map((type) => (
+                  <SelectItem key={type.value} value={type.value}>{type.label}</SelectItem>
+                ))}
+              </SelectGroup>
+            </SelectContent>
+          </Select>
+
+          <label className="block text-[12px] font-bold text-muted">区域用途描述</label>
+          <textarea
+            className="editor min-h-[84px]"
+            value={selectedRegion?.description || ""}
+            onChange={(event) => updateSelected({ description: event.target.value })}
+            disabled={!selectedRegion}
+            placeholder="例如：这是微信聊天消息列表，用于识别最近一条用户消息；这是发送按钮，用于点击发送。自定义区域请尽量描述它的用途和可执行动作。"
+          />
+
+          {selectedRegion ? (
+            <div className="rounded-[10px] bg-page p-2 font-mono text-[11px] leading-5 text-muted">
+              <div>image: [{selectedRegion.bbox_image.join(", ")}]</div>
+              <div>screen: [{selectedRegion.bbox_screen.join(", ")}]</div>
+            </div>
+          ) : null}
+
+          <button className="btn-danger h-9 w-full gap-2" onClick={deleteSelected} disabled={!selectedRegion}>
+            <Trash2 size={15} />
+            删除当前框
+          </button>
+        </div>
+
+        <div className="mt-4 min-h-0 flex-1 overflow-y-auto pr-1">
+          <div className="mb-2 text-[12px] font-black text-muted">区域列表</div>
+          <div className="space-y-2">
+            {regions.map((region, index) => (
+              <button
+                key={region.id}
+                className={`w-full rounded-[12px] border p-3 text-left transition ${
+                  region.id === selectedId
+                    ? "border-primary bg-primary/15"
+                    : "border-line bg-surface hover:bg-surface2"
+                }`}
+                onClick={() => setSelectedId(region.id)}
+              >
+                <div className="flex items-center justify-between gap-2">
+                  <span className="text-[13px] font-extrabold">{index + 1}. {region.name}</span>
+                  <span className="rounded-full border border-line bg-surface2 px-2 py-0.5 text-[10px] text-muted">{typeLabelMap[region.type]}</span>
+                </div>
+                {region.description ? (
+                  <div className="mt-1 line-clamp-2 text-[11px] leading-4 text-muted">{region.description}</div>
+                ) : null}
+                <div className="mt-1 font-mono text-[10px] text-subtle">[{region.bbox_screen.join(", ")}]</div>
+              </button>
+            ))}
+          </div>
+        </div>
+
+        <div className="mt-4 grid grid-cols-2 gap-3">
+          <button className="btn-secondary h-10 gap-2" onClick={exitAnnotation}>
+            <X size={16} />
+            取消
+          </button>
+          <button className="btn-primary h-10 gap-2" onClick={save} disabled={saving || !capture.screenshotPath}>
+            <Check size={16} />
+            {saving ? "保存中" : "保存"}
+          </button>
+        </div>
+      </aside>
+
+      <main className="flex min-w-0 flex-1 items-center justify-center overflow-auto bg-surface2 p-6">
+        <div className="relative max-h-full max-w-full select-none shadow-[0_20px_80px_rgba(0,0,0,.55)]">
+          {screenshotSrc ? (
+            <>
+              <img
+                ref={imageRef}
+                src={screenshotSrc}
+                alt="当前屏幕截图"
+                className="block max-h-[calc(100dvh-48px)] max-w-full object-contain"
+                draggable={false}
+                onLoad={() => setStatus("拖拽截图区域创建矩形框")}
+                onError={() => setStatus(`截图图片加载失败：${capture.screenshotPath}`)}
+              />
+              <svg
+                className="absolute inset-0 h-full w-full touch-none"
+                viewBox={`0 0 ${capture.screenshotWidth} ${capture.screenshotHeight}`}
+                preserveAspectRatio="none"
+                onPointerDown={handlePointerDown}
+                onPointerMove={handlePointerMove}
+                onPointerUp={handlePointerUp}
+                onPointerCancel={handlePointerCancel}
+              >
+                <rect width={capture.screenshotWidth} height={capture.screenshotHeight} fill="transparent" />
+                {regions.map((region) => {
+                  const [x1, y1, x2, y2] = region.bbox_image;
+                  const selected = region.id === selectedId;
+                  return (
+                    <g key={region.id} onPointerDown={(event) => { event.stopPropagation(); setSelectedId(region.id); }}>
+                      <rect
+                        x={x1}
+                        y={y1}
+                        width={x2 - x1}
+                        height={y2 - y1}
+                        fill={selected ? "rgba(7,193,96,0.28)" : "rgba(45,140,255,0.18)"}
+                        stroke={selected ? "#07c160" : "#2d8cff"}
+                        strokeWidth={selected ? 5 : 3}
+                        vectorEffect="non-scaling-stroke"
+                      />
+                      <text
+                        x={x1 + 8}
+                        y={Math.max(y1 + 24, 24)}
+                        fill="#fff"
+                        stroke="rgba(0,0,0,.7)"
+                        strokeWidth="4"
+                        paintOrder="stroke"
+                        fontSize="18"
+                        fontWeight="800"
+                      >
+                        {region.name}
+                      </text>
+                    </g>
+                  );
+                })}
+                {tempBox ? (
+                  <rect
+                    x={tempBox[0]}
+                    y={tempBox[1]}
+                    width={tempBox[2] - tempBox[0]}
+                    height={tempBox[3] - tempBox[1]}
+                    fill="rgba(7,193,96,0.2)"
+                    stroke="#07c160"
+                    strokeDasharray="10 8"
+                    strokeWidth="4"
+                    vectorEffect="non-scaling-stroke"
+                  />
+                ) : null}
+              </svg>
+            </>
+          ) : (
+            <div className="grid h-[360px] w-[640px] place-items-center rounded-[20px] border border-line bg-surface text-muted">
+              {status}
+            </div>
+          )}
+        </div>
+      </main>
+    </div>
+  );
+}

src/pages/ClonePage.jsx

@@ -1,7 +1,7 @@
 import { useEffect, useState } from "react";
 import { invoke } from "@tauri-apps/api/core";
 import { listen } from "@tauri-apps/api/event";
-import { Power, PowerOff, Settings } from "lucide-react";
+import { Crosshair, Power, PowerOff, Settings } from "lucide-react";
 import StatusPill from "../components/StatusPill";
 import Terminal from "../components/Terminal";
 import { openWindow } from "../utils/navigation";
@@ -9,6 +9,11 @@ import { openWindow } from "../utils/navigation";
 export default function ClonePage() {
   const [engineEnabled, setEngineEnabled] = useState(false);
   const [agentLogs, setAgentLogs] = useState([]);
+  const [regionsFile, setRegionsFile] = useState(null);
+  const [annotating, setAnnotating] = useState(false);
+  const [sourcePickerOpen, setSourcePickerOpen] = useState(false);
+  const [captureSources, setCaptureSources] = useState([]);
+  const [sourceStatus, setSourceStatus] = useState("");
 
   useEffect(() => {
     if (!window.__TAURI_INTERNALS__) return undefined;
@@ -33,6 +38,70 @@ export default function ClonePage() {
     };
   }, []);
 
+  useEffect(() => {
+    loadSavedRegions();
+  }, []);
+
+  async function loadSavedRegions() {
+    if (!window.__TAURI_INTERNALS__) return;
+
+    try {
+      const saved = await invoke("load_regions");
+      setRegionsFile(saved);
+    } catch (error) {
+      setAgentLogs((currentLogs) => [
+        ...currentLogs,
+        { level: "error", message: `读取标注失败：${String(error)}` },
+      ]);
+    }
+  }
+
+  async function startAnnotation() {
+    if (!window.__TAURI_INTERNALS__) {
+      window.location.hash = "/annotate";
+      return;
+    }
+
+    setSourcePickerOpen(true);
+    setSourceStatus("正在读取桌面和窗口列表...");
+    try {
+      const sources = await invoke("list_capture_sources");
+      setCaptureSources(sources);
+      setSourceStatus(sources.length ? "请选择一个桌面或应用窗口" : "未找到可用来源");
+    } catch (error) {
+      setCaptureSources([]);
+      setSourceStatus(`读取来源失败：${String(error)}`);
+    }
+  }
+
+  function beginAnnotationWithSource(source) {
+    setAnnotating(true);
+    sessionStorage.removeItem("pendingAnnotationCapture");
+    localStorage.removeItem("annotationCaptureResult");
+    localStorage.removeItem("annotationCaptureError");
+    localStorage.setItem("annotationCaptureSource", JSON.stringify(source));
+    localStorage.setItem("annotationCaptureStatus", "pending");
+    setSourcePickerOpen(false);
+
+    if (window.__TAURI_INTERNALS__) {
+      invoke("capture_source", { sourceId: source.id })
+        .then((capture) => {
+          localStorage.setItem("annotationCaptureResult", JSON.stringify(capture));
+          localStorage.setItem("annotationCaptureStatus", "done");
+        })
+        .catch((error) => {
+          localStorage.setItem("annotationCaptureError", String(error));
+          localStorage.setItem("annotationCaptureStatus", "error");
+        });
+
+      openWindow("/annotate").finally(() => setAnnotating(false));
+      return;
+    }
+
+    window.location.hash = "/annotate";
+    setAnnotating(false);
+  }
+
   async function toggleAgent() {
     if (!window.__TAURI_INTERNALS__) {
       setEngineEnabled((enabled) => !enabled);
@@ -77,6 +146,64 @@ export default function ClonePage() {
           <button className="icon-action shrink-0" onClick={() => openWindow("/window/settings")} aria-label="设置"><Settings size={18} strokeWidth={2.5} /></button>
         </div>
       </div>
+      <div className="panel p-4">
+        <div className="flex items-center justify-between gap-3">
+          <div>
+            <h3 className="text-[16px] font-extrabold">屏幕区域标注</h3>
+            <p className="mt-1 text-[12px] text-muted">截取当前屏幕，配置联系人、聊天内容、输入框和发送按钮坐标。</p>
+          </div>
+          <button className="btn-primary shrink-0 gap-2" onClick={startAnnotation} disabled={annotating}>
+            <Crosshair size={16} strokeWidth={2.6} />
+            {annotating ? "截屏中" : "开始标注"}
+          </button>
+        </div>
+        <div className="mt-3 space-y-2">
+          {regionsFile?.regions?.length ? (
+            regionsFile.regions.map((region) => (
+              <div key={region.id} className="list-card !rounded-[12px] !p-3">
+                <div className="min-w-0">
+                  <div className="truncate text-[13px] font-extrabold">{region.name}</div>
+                  <div className="mt-1 font-mono text-[11px] text-muted">screen [{region.bbox_screen.join(", ")}]</div>
+                </div>
+                <span className="tag shrink-0">{region.type}</span>
+              </div>
+            ))
+          ) : (
+            <div className="rounded-[12px] border border-dashed border-line p-3 text-center text-[12px] text-muted">暂无已保存区域</div>
+          )}
+        </div>
+      </div>
+      {sourcePickerOpen ? (
+        <div className="fixed inset-0 z-50 grid place-items-center bg-black/35 px-5">
+          <div className="panel max-h-[78dvh] w-full max-w-[440px] overflow-hidden p-4">
+            <div className="flex items-start justify-between gap-3">
+              <div>
+                <h3 className="text-[17px] font-extrabold">选择标注来源 Demo</h3>
+                <p className="mt-1 text-[12px] text-muted">可以选择某个桌面，或某个可见应用窗口。</p>
+              </div>
+              <button className="btn-ghost" onClick={() => setSourcePickerOpen(false)}>关闭</button>
+            </div>
+            <div className="mt-3 rounded-[12px] bg-surface2 px-3 py-2 text-[12px] text-muted">{sourceStatus}</div>
+            <div className="mt-3 max-h-[54dvh] space-y-2 overflow-y-auto pr-1">
+              {captureSources.map((source) => (
+                <button
+                  key={source.id}
+                  className="list-card w-full !rounded-[12px] text-left transition hover:border-primary"
+                  onClick={() => beginAnnotationWithSource(source)}
+                >
+                  <div className="min-w-0">
+                    <div className="truncate text-[13px] font-extrabold">{source.label}</div>
+                    <div className="mt-1 text-[11px] text-muted">
+                      {source.kind === "display" ? "桌面" : "窗口"} · {source.width}x{source.height} · x:{source.x} y:{source.y}
+                    </div>
+                  </div>
+                  <span className="tag shrink-0">{source.kind === "display" ? "桌面" : "窗口"}</span>
+                </button>
+              ))}
+            </div>
+          </div>
+        </div>
+      ) : null}
       <Terminal title="运行日志" lines={agentLogs} onClear={() => setAgentLogs([])} />
     </section>
   );