diff --git a/.gitignore b/.gitignore
index 3327715..0a77644 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,6 +7,15 @@ src-tauri/gen/
agent/agent
agent/agent-*
+agent/config.toml
+
+wechat_vision/venv/
+wechat_vision/__pycache__/
+wechat_vision/ouptsw/
+wechat_vision/test/
+
+.codegraph/daemon.pid
+.vscode/
.DS_Store
*.log
diff --git a/agent/ax_probe.go b/agent/ax_probe.go
new file mode 100644
index 0000000..5c04464
--- /dev/null
+++ b/agent/ax_probe.go
@@ -0,0 +1,252 @@
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+ "sort"
+ "strings"
+)
+
+type AXProbeResult struct {
+ AppName string `json:"app_name"`
+ PID int `json:"pid"`
+ Nodes []AXNode `json:"nodes"`
+}
+
+type AXNode struct {
+ Role string `json:"role"`
+ Title string `json:"title"`
+ Value string `json:"value"`
+ Description string `json:"description"`
+ Frame [4]float64 `json:"frame"`
+ Depth int `json:"depth"`
+}
+
+func runAXProbe() error {
+ logInfo("macOS Accessibility 微信 UI 树探测启动")
+
+ config, err := loadConfig("config.toml")
+ if err != nil {
+ return fmt.Errorf("读取 config.toml 失败：%w", err)
+ }
+
+ regionsPath := appDataPath(config, config.Agent.RegionsPath)
+ annotation, err := loadAnnotation(regionsPath)
+ if err != nil {
+ return fmt.Errorf("读取标注文件失败：%w", err)
+ }
+ summaries, warnings := inferMissingRegionTypes(summarizeRegions(annotation.Regions))
+ for _, warning := range warnings {
+ logWarning(warning)
+ }
+
+ probe, err := probeWeChatAXTree()
+ if err != nil {
+ return err
+ }
+ logInfo(fmt.Sprintf("AX 节点总数=%d pid=%d app=%s", len(probe.Nodes), probe.PID, probe.AppName))
+
+ textNodes := usefulAXTextNodes(probe.Nodes)
+ logInfo(fmt.Sprintf("可读文本/按钮节点=%d", len(textNodes)))
+ for index, node := range firstNodes(textNodes, 80) {
+ logJSON("ax", fmt.Sprintf("#%02d depth=%d role=%s frame=[%.0f,%.0f,%.0f,%.0f] text=%s", index+1, node.Depth, node.Role, node.Frame[0], node.Frame[1], node.Frame[2], node.Frame[3], axNodeText(node)))
+ }
+
+ if chat, ok := regionByType(summaries)["chat_content"]; ok {
+ chatNodes := nodesInsideBox(textNodes, chat.BBoxScreen)
+ sort.SliceStable(chatNodes, func(i, j int) bool {
+ if chatNodes[i].Frame[1] == chatNodes[j].Frame[1] {
+ return chatNodes[i].Frame[0] < chatNodes[j].Frame[0]
+ }
+ return chatNodes[i].Frame[1] < chatNodes[j].Frame[1]
+ })
+ logInfo(fmt.Sprintf("chat_content 区域内可读节点=%d", len(chatNodes)))
+ for index, node := range firstNodes(chatNodes, 80) {
+ logJSON("ax-chat", fmt.Sprintf("#%02d role=%s frame=[%.0f,%.0f,%.0f,%.0f] text=%s", index+1, node.Role, node.Frame[0], node.Frame[1], node.Frame[2], node.Frame[3], axNodeText(node)))
+ }
+ } else {
+ logWarning("未找到 chat_content 标注区域，跳过区域过滤")
+ }
+
+ return nil
+}
+
+func probeWeChatAXTree() (AXProbeResult, error) {
+ script := `
+import AppKit
+import ApplicationServices
+import Foundation
+
+struct Node: Encodable {
+ let role: String
+ let title: String
+ let value: String
+ let description: String
+ let frame: [Double]
+ let depth: Int
+}
+
+struct Result: Encodable {
+ let app_name: String
+ let pid: Int
+ let nodes: [Node]
+}
+
+func stringAttr(_ element: AXUIElement, _ key: String) -> String {
+ var value: CFTypeRef?
+ let error = AXUIElementCopyAttributeValue(element, key as CFString, &value)
+ if error != .success || value == nil { return "" }
+ if let str = value as? String { return clean(str) }
+ if let attr = value as? NSAttributedString { return clean(attr.string) }
+ return ""
+}
+
+func clean(_ value: String) -> String {
+ return value
+ .replacingOccurrences(of: "\n", with: " ")
+ .replacingOccurrences(of: "\r", with: " ")
+ .replacingOccurrences(of: "\t", with: " ")
+}
+
+func frameOf(_ element: AXUIElement) -> [Double] {
+ var positionValue: CFTypeRef?
+ var sizeValue: CFTypeRef?
+ var point = CGPoint.zero
+ var size = CGSize.zero
+ if AXUIElementCopyAttributeValue(element, kAXPositionAttribute as CFString, &positionValue) == .success,
+ let positionValue,
+ CFGetTypeID(positionValue) == AXValueGetTypeID() {
+ AXValueGetValue(positionValue as! AXValue, .cgPoint, &point)
+ }
+ if AXUIElementCopyAttributeValue(element, kAXSizeAttribute as CFString, &sizeValue) == .success,
+ let sizeValue,
+ CFGetTypeID(sizeValue) == AXValueGetTypeID() {
+ AXValueGetValue(sizeValue as! AXValue, .cgSize, &size)
+ }
+ return [Double(point.x), Double(point.y), Double(size.width), Double(size.height)]
+}
+
+func childrenOf(_ element: AXUIElement) -> [AXUIElement] {
+ var value: CFTypeRef?
+ let error = AXUIElementCopyAttributeValue(element, kAXChildrenAttribute as CFString, &value)
+ if error != .success || value == nil { return [] }
+ return value as? [AXUIElement] ?? []
+}
+
+func walk(_ element: AXUIElement, depth: Int, nodes: inout [Node], visited: inout Int) {
+ if visited > 2500 || depth > 20 { return }
+ visited += 1
+ let node = Node(
+ role: stringAttr(element, kAXRoleAttribute),
+ title: stringAttr(element, kAXTitleAttribute),
+ value: stringAttr(element, kAXValueAttribute),
+ description: stringAttr(element, kAXDescriptionAttribute),
+ frame: frameOf(element),
+ depth: depth
+ )
+ nodes.append(node)
+ for child in childrenOf(element) {
+ walk(child, depth: depth + 1, nodes: &nodes, visited: &visited)
+ }
+}
+
+let trusted = AXIsProcessTrustedWithOptions([kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true] as CFDictionary)
+if !trusted {
+ fputs("Accessibility permission is not granted\n", stderr)
+ exit(10)
+}
+
+let apps = NSWorkspace.shared.runningApplications.filter { app in
+ let name = app.localizedName ?? ""
+ return name == "微信" || name == "WeChat" || name.lowercased().contains("wechat")
+}
+
+guard let app = apps.first else {
+ fputs("WeChat app not found\n", stderr)
+ exit(2)
+}
+
+app.activate(options: [.activateIgnoringOtherApps])
+usleep(300000)
+
+let root = AXUIElementCreateApplication(app.processIdentifier)
+var nodes: [Node] = []
+var visited = 0
+walk(root, depth: 0, nodes: &nodes, visited: &visited)
+let result = Result(app_name: app.localizedName ?? "", pid: Int(app.processIdentifier), nodes: nodes)
+let data = try JSONEncoder().encode(result)
+print(String(data: data, encoding: .utf8)!)
+`
+
+ output, err := runSwift(script)
+ if err != nil {
+ return AXProbeResult{}, fmt.Errorf("AX probe failed: %w", err)
+ }
+
+ var result AXProbeResult
+ if err := json.Unmarshal([]byte(output), &result); err != nil {
+ preview := output
+ if len(preview) > 300 {
+ preview = preview[:300]
+ }
+ return AXProbeResult{}, fmt.Errorf("parse AX JSON failed: %w output_prefix=%q", err, preview)
+ }
+ return result, nil
+}
+
+func usefulAXTextNodes(nodes []AXNode) []AXNode {
+ result := make([]AXNode, 0)
+ seen := map[string]bool{}
+ for _, node := range nodes {
+ text := strings.TrimSpace(axNodeText(node))
+ if text == "" {
+ continue
+ }
+ if node.Frame[2] <= 0 || node.Frame[3] <= 0 {
+ continue
+ }
+ key := fmt.Sprintf("%s|%s|%.0f|%.0f|%.0f|%.0f", node.Role, text, node.Frame[0], node.Frame[1], node.Frame[2], node.Frame[3])
+ if seen[key] {
+ continue
+ }
+ seen[key] = true
+ result = append(result, node)
+ }
+ sort.SliceStable(result, func(i, j int) bool {
+ if result[i].Frame[1] == result[j].Frame[1] {
+ return result[i].Frame[0] < result[j].Frame[0]
+ }
+ return result[i].Frame[1] < result[j].Frame[1]
+ })
+ return result
+}
+
+func axNodeText(node AXNode) string {
+ parts := []string{node.Value, node.Title, node.Description}
+ for _, part := range parts {
+ part = strings.TrimSpace(part)
+ if part != "" {
+ return part
+ }
+ }
+ return ""
+}
+
+func nodesInsideBox(nodes []AXNode, box [4]float64) []AXNode {
+ result := make([]AXNode, 0)
+ for _, node := range nodes {
+ cx := node.Frame[0] + node.Frame[2]/2
+ cy := node.Frame[1] + node.Frame[3]/2
+ if cx >= box[0] && cx <= box[2] && cy >= box[1] && cy <= box[3] {
+ result = append(result, node)
+ }
+ }
+ return result
+}
+
+func firstNodes(nodes []AXNode, limit int) []AXNode {
+ if len(nodes) <= limit {
+ return nodes
+ }
+ return nodes[:limit]
+}
diff --git a/agent/chat_sync.go b/agent/chat_sync.go
new file mode 100644
index 0000000..405e668
--- /dev/null
+++ b/agent/chat_sync.go
@@ -0,0 +1,274 @@
+package main
+
+import (
+ "crypto/sha1"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "time"
+)
+
+func runChatSync() error {
+ logInfo("聊天记录同步 demo 启动")
+
+ config, err := loadConfig("config.toml")
+ if err != nil {
+ return fmt.Errorf("读取 config.toml 失败：%w", err)
+ }
+
+ regionsPath := appDataPath(config, config.Agent.RegionsPath)
+ screenshotPath := appDataPath(config, config.Agent.ScreenshotPath)
+ archivePath := appDataPath(config, config.Agent.ChatRecordsPath)
+
+ annotation, err := loadAnnotation(regionsPath)
+ if err != nil {
+ return fmt.Errorf("读取标注文件失败：%w", err)
+ }
+ summaries, warnings := inferMissingRegionTypes(summarizeRegions(annotation.Regions))
+ for _, warning := range warnings {
+ logWarning(warning)
+ }
+ if err := validateChatSyncRegions(summaries); err != nil {
+ return err
+ }
+
+ archive, existed, err := loadChatArchive(archivePath)
+ if err != nil {
+ return err
+ }
+ if !existed || len(archive.Messages) == 0 {
+ logInfo("本地没有聊天记录，先滚动到聊天记录顶部")
+ if err := scrollChatToTop(config, summaries, screenshotPath); err != nil {
+ return err
+ }
+ } else {
+ logInfo(fmt.Sprintf("已读取本地聊天记录 messages=%d，将从当前视图继续向下读取", len(archive.Messages)))
+ }
+
+ if archive.App == "" {
+ archive.App = "wechat"
+ archive.CreatedAt = time.Now().Format(time.RFC3339)
+ }
+ archive.Source = TaskSource{RegionsPath: regionsPath, ScreenshotPath: screenshotPath}
+
+ seenNoNewPages := 0
+ for pageIndex := 0; pageIndex < config.Agent.ChatSyncMaxPages; pageIndex++ {
+ window, err := findWeChatWindow()
+ if err != nil {
+ return err
+ }
+ if err := captureWindow(window, screenshotPath); err != nil {
+ return err
+ }
+
+ result, err := extractChatPage(config, summaries, screenshotPath)
+ if err != nil {
+ return err
+ }
+ if result.ContactName != "" && archive.Contact == "" {
+ archive.Contact = result.ContactName
+ }
+
+ newCount := appendChatMessages(&archive, result.Messages, pageIndex)
+ archive.Pages = append(archive.Pages, ChatPage{
+ PageIndex: pageIndex,
+ Direction: "down",
+ NewCount: newCount,
+ SeenAt: time.Now().Format(time.RFC3339),
+ })
+ logTask(fmt.Sprintf("聊天页 page=%d 提取 messages=%d new=%d summary=%s", pageIndex+1, len(result.Messages), newCount, result.PageSummary))
+
+ if newCount == 0 {
+ seenNoNewPages++
+ } else {
+ seenNoNewPages = 0
+ }
+ if seenNoNewPages >= 2 {
+ logInfo("连续页面没有新增聊天记录，停止同步")
+ break
+ }
+
+ if pageIndex < config.Agent.ChatSyncMaxPages-1 {
+ if err := scrollChat(config, summaries, -absInt(config.Agent.ScrollDeltaY)); err != nil {
+ return err
+ }
+ time.Sleep(800 * time.Millisecond)
+ }
+ }
+
+ archive.UpdatedAt = time.Now().Format(time.RFC3339)
+ if err := saveChatArchive(archivePath, archive); err != nil {
+ return err
+ }
+ logInfo(fmt.Sprintf("聊天记录同步完成，messages=%d path=%s", len(archive.Messages), archivePath))
+
+ return nil
+}
+
+func validateChatSyncRegions(summaries []RegionSummary) error {
+ index := regionByType(summaries)
+ if _, exists := index["chat_content"]; !exists {
+ return fmt.Errorf("missing required region: chat_content")
+ }
+ return nil
+}
+
+func scrollChatToTop(config Config, summaries []RegionSummary, screenshotPath string) error {
+ for index := 0; index < config.Agent.ChatSyncTopScrolls; index++ {
+ if err := scrollChat(config, summaries, absInt(config.Agent.ScrollDeltaY)); err != nil {
+ return err
+ }
+ logTask(fmt.Sprintf("向上滚动到历史顶部 round=%d/%d", index+1, config.Agent.ChatSyncTopScrolls))
+ time.Sleep(350 * time.Millisecond)
+ }
+
+ window, err := findWeChatWindow()
+ if err != nil {
+ return err
+ }
+ return captureWindow(window, screenshotPath)
+}
+
+func scrollChat(config Config, summaries []RegionSummary, deltaY int) error {
+ chat, ok := regionByType(summaries)["chat_content"]
+ if !ok {
+ return fmt.Errorf("chat_content region not found")
+ }
+ window, err := findWeChatWindow()
+ if err != nil {
+ return err
+ }
+ pointX := window.X + chat.CenterSource[0]
+ pointY := window.Y + chat.CenterSource[1]
+ logTask(fmt.Sprintf("滚动聊天区域 point=[%.1f, %.1f] delta_y=%d", pointX, pointY, deltaY))
+ return scrollAt(pointX, pointY, deltaY)
+}
+
+func extractChatPage(config Config, summaries []RegionSummary, screenshotPath string) (ChatExtractResult, error) {
+ imageDataURL, err := loadImageDataURL(screenshotPath)
+ if err != nil {
+ return ChatExtractResult{}, err
+ }
+ prompt, err := buildChatExtractPrompt(summaries)
+ if err != nil {
+ return ChatExtractResult{}, err
+ }
+ logThink("正在用豆包多模态抽取当前聊天区域消息")
+ raw, err := requestLLM(config, prompt, imageDataURL)
+ if err != nil {
+ return ChatExtractResult{}, err
+ }
+ jsonText, err := extractJSON(raw)
+ if err != nil {
+ _ = saveRawResponse(config, raw)
+ return ChatExtractResult{}, err
+ }
+ var result ChatExtractResult
+ if err := json.Unmarshal([]byte(jsonText), &result); err != nil {
+ _ = saveRawResponse(config, raw)
+ return ChatExtractResult{}, err
+ }
+ return result, nil
+}
+
+func buildChatExtractPrompt(summaries []RegionSummary) (string, error) {
+ regionsJSON, err := json.MarshalIndent(summaries, "", " ")
+ if err != nil {
+ return "", err
+ }
+ return `你是微信聊天记录抽取器。
+
+请只识别截图中 chat_content 标注区域里的聊天消息，不要识别联系人列表、输入框或其它区域。
+
+要求：
+- 按消息在页面中的从上到下顺序输出。
+- sender 可填 "me"、"other" 或能识别到的昵称；无法判断填 "unknown"。
+- role 可填 "me"、"other"、"system"。
+- time 如果画面中没有明确时间则留空字符串。
+- content 必须是消息文本，不能编造。
+- 只能返回合法 JSON，不要 Markdown，不要解释。
+
+标注区域：
+` + string(regionsJSON) + `
+
+返回结构：
+{
+ "contact_name": "unknown",
+ "page_summary": "当前页聊天内容摘要",
+ "messages": [
+ {"sender":"other","role":"other","time":"","content":"消息文本"}
+ ]
+}`, nil
+}
+
+func loadChatArchive(path string) (ChatArchive, bool, error) {
+ data, err := os.ReadFile(path)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return ChatArchive{}, false, nil
+ }
+ return ChatArchive{}, false, err
+ }
+ var archive ChatArchive
+ if err := json.Unmarshal(data, &archive); err != nil {
+ return ChatArchive{}, false, err
+ }
+ return archive, true, nil
+}
+
+func saveChatArchive(path string, archive ChatArchive) error {
+ if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+ return err
+ }
+ data, err := json.MarshalIndent(archive, "", " ")
+ if err != nil {
+ return err
+ }
+ return os.WriteFile(path, data, 0644)
+}
+
+func appendChatMessages(archive *ChatArchive, messages []ChatMessage, pageIndex int) int {
+ existing := make(map[string]bool, len(archive.Messages))
+ for _, message := range archive.Messages {
+ existing[message.ID] = true
+ }
+
+ newCount := 0
+ for _, message := range messages {
+ message.Content = strings.TrimSpace(message.Content)
+ if message.Content == "" {
+ continue
+ }
+ message.PageIndex = pageIndex
+ message.SeenAt = time.Now().Format(time.RFC3339)
+ message.ID = chatMessageID(message)
+ if existing[message.ID] {
+ continue
+ }
+ existing[message.ID] = true
+ archive.Messages = append(archive.Messages, message)
+ newCount++
+ }
+ return newCount
+}
+
+func chatMessageID(message ChatMessage) string {
+ key := strings.Join([]string{
+ strings.TrimSpace(message.Sender),
+ strings.TrimSpace(message.Role),
+ strings.TrimSpace(message.Time),
+ strings.TrimSpace(message.Content),
+ }, "|")
+ sum := sha1.Sum([]byte(key))
+ return hex.EncodeToString(sum[:])
+}
+
+func absInt(value int) int {
+ if value < 0 {
+ return -value
+ }
+ return value
+}
diff --git a/agent/config.example.toml b/agent/config.example.toml
index f82ee6a..6c2a729 100644
--- a/agent/config.example.toml
+++ b/agent/config.example.toml
@@ -11,3 +11,9 @@ screenshot_path = "data/screenshots/WeChat.jpg"
task_output_path = "data/tasks/latest_task.json"
task_history_dir = "data/tasks/history"
dry_run = true
+observe_mode = "normal" # normal | scroll_first
+max_scrolls = 1
+scroll_delta_y = 6
+chat_records_path = "data/chats/wechat_chat_records.json"
+chat_sync_max_pages = 3
+chat_sync_top_scrolls = 8
diff --git a/agent/config.go b/agent/config.go
index b73b5a5..297f1b6 100644
--- a/agent/config.go
+++ b/agent/config.go
@@ -50,6 +50,24 @@ func applyConfigDefaults(config *Config) {
if config.Agent.TaskHistoryDir == "" {
config.Agent.TaskHistoryDir = "data/tasks/history"
}
+ if config.Agent.ObserveMode == "" {
+ config.Agent.ObserveMode = "normal"
+ }
+ if config.Agent.MaxScrolls <= 0 {
+ config.Agent.MaxScrolls = 1
+ }
+ if config.Agent.ScrollDeltaY == 0 {
+ config.Agent.ScrollDeltaY = 6
+ }
+ if config.Agent.ChatRecordsPath == "" {
+ config.Agent.ChatRecordsPath = "data/chats/wechat_chat_records.json"
+ }
+ if config.Agent.ChatSyncMaxPages <= 0 {
+ config.Agent.ChatSyncMaxPages = 3
+ }
+ if config.Agent.ChatSyncTopScrolls <= 0 {
+ config.Agent.ChatSyncTopScrolls = 8
+ }
config.Agent.DryRun = true
}
diff --git a/agent/executor.go b/agent/executor.go
new file mode 100644
index 0000000..57c1ff7
--- /dev/null
+++ b/agent/executor.go
@@ -0,0 +1,178 @@
+package main
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "math"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "runtime"
+ "strings"
+ "time"
+)
+
+func maybeScrollBeforeObserve(config Config, summaries []RegionSummary, screenshotPath string) error {
+ if config.Agent.ObserveMode != "scroll_first" {
+ return nil
+ }
+ if runtime.GOOS != "darwin" {
+ return fmt.Errorf("observe_mode=scroll_first currently supports macOS only")
+ }
+
+ chat, ok := regionByType(summaries)["chat_content"]
+ if !ok {
+ return fmt.Errorf("cannot scroll before observe: chat_content region not found")
+ }
+
+ maxScrolls := config.Agent.MaxScrolls
+ if maxScrolls <= 0 {
+ maxScrolls = 1
+ }
+
+ for index := 0; index < maxScrolls; index++ {
+ window, err := findWeChatWindow()
+ if err != nil {
+ return err
+ }
+
+ pointX := window.X + chat.CenterSource[0]
+ pointY := window.Y + chat.CenterSource[1]
+ logTask(fmt.Sprintf("观察前滚动聊天区域：round=%d point=[%.1f, %.1f] delta_y=%d window=[%.0f, %.0f, %.0f, %.0f]", index+1, pointX, pointY, config.Agent.ScrollDeltaY, window.X, window.Y, window.Width, window.Height))
+ if err := scrollAt(pointX, pointY, config.Agent.ScrollDeltaY); err != nil {
+ return err
+ }
+ time.Sleep(700 * time.Millisecond)
+
+ if err := captureWindow(window, screenshotPath); err != nil {
+ return err
+ }
+ logInfo("已滚动并覆盖更新截图 WeChat.jpg")
+ }
+
+ return nil
+}
+
+func findWeChatWindow() (WindowBounds, error) {
+ script := `
+import CoreGraphics
+import Foundation
+
+let options = CGWindowListOption(arrayLiteral: [.optionOnScreenOnly, .excludeDesktopElements])
+guard let infos = CGWindowListCopyWindowInfo(options, kCGNullWindowID) as? [[String: Any]] else {
+ exit(1)
+}
+
+for info in infos {
+ let owner = info[kCGWindowOwnerName as String] as? String ?? ""
+ let title = info[kCGWindowName as String] as? String ?? ""
+ let layer = info[kCGWindowLayer as String] as? Int ?? -1
+ if layer == 0 && (owner.contains("微信") || owner.lowercased().contains("wechat") || title == "微信") {
+ if let bounds = info[kCGWindowBounds as String] as? [String: Any],
+ let x = bounds["X"] as? Double,
+ let y = bounds["Y"] as? Double,
+ let w = bounds["Width"] as? Double,
+ let h = bounds["Height"] as? Double {
+ let id = info[kCGWindowNumber as String] as? Int ?? 0
+ let result: [String: Any] = ["x": x, "y": y, "width": w, "height": h, "owner": owner, "title": title, "id": id]
+ let data = try! JSONSerialization.data(withJSONObject: result)
+ print(String(data: data, encoding: .utf8)!)
+ exit(0)
+ }
+ }
+}
+
+exit(2)
+`
+
+ output, err := runSwift(script)
+ if err != nil {
+ return WindowBounds{}, fmt.Errorf("find WeChat window failed: %w", err)
+ }
+
+ var raw struct {
+ X float64 `json:"x"`
+ Y float64 `json:"y"`
+ Width float64 `json:"width"`
+ Height float64 `json:"height"`
+ Owner string `json:"owner"`
+ Title string `json:"title"`
+ ID int `json:"id"`
+ }
+ if err := json.Unmarshal([]byte(output), &raw); err != nil {
+ return WindowBounds{}, err
+ }
+
+ return WindowBounds(raw), nil
+}
+
+func scrollAt(x float64, y float64, deltaY int) error {
+ if deltaY == 0 {
+ deltaY = 6
+ }
+
+ script := fmt.Sprintf(`
+import CoreGraphics
+import Foundation
+import AppKit
+
+NSWorkspace.shared.runningApplications
+ .first { $0.localizedName == "微信" || $0.localizedName == "WeChat" }?
+ .activate(options: [.activateIgnoringOtherApps])
+usleep(300000)
+
+let point = CGPoint(x: %.3f, y: %.3f)
+let move = CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left)
+move?.post(tap: .cghidEventTap)
+usleep(80000)
+let down = CGEvent(mouseEventSource: nil, mouseType: .leftMouseDown, mouseCursorPosition: point, mouseButton: .left)
+let up = CGEvent(mouseEventSource: nil, mouseType: .leftMouseUp, mouseCursorPosition: point, mouseButton: .left)
+down?.post(tap: .cghidEventTap)
+usleep(60000)
+up?.post(tap: .cghidEventTap)
+usleep(120000)
+
+let delta = Int32(%d)
+for _ in 0..<8 {
+ let lineScroll = CGEvent(scrollWheelEvent2Source: nil, units: .line, wheelCount: 1, wheel1: delta, wheel2: 0, wheel3: 0)
+ lineScroll?.post(tap: .cghidEventTap)
+ usleep(45000)
+}
+
+for _ in 0..<4 {
+ let pixelScroll = CGEvent(scrollWheelEvent2Source: nil, units: .pixel, wheelCount: 1, wheel1: delta * 80, wheel2: 0, wheel3: 0)
+ pixelScroll?.post(tap: .cghidEventTap)
+ usleep(45000)
+}
+`, x, y, deltaY)
+
+ _, err := runSwift(script)
+ return err
+}
+
+func captureWindow(window WindowBounds, screenshotPath string) error {
+ if err := os.MkdirAll(filepath.Dir(screenshotPath), 0755); err != nil {
+ return err
+ }
+
+ rect := fmt.Sprintf("%d,%d,%d,%d", int(math.Round(window.X)), int(math.Round(window.Y)), int(math.Round(window.Width)), int(math.Round(window.Height)))
+ command := exec.Command("screencapture", "-x", "-R", rect, screenshotPath)
+ if output, err := command.CombinedOutput(); err != nil {
+ return fmt.Errorf("screencapture failed: %w output=%s", err, string(output))
+ }
+
+ return nil
+}
+
+func runSwift(script string) (string, error) {
+ command := exec.Command("swift", "-")
+ command.Stdin = strings.NewReader(script)
+ var stderr bytes.Buffer
+ command.Stderr = &stderr
+ output, err := command.Output()
+ if err != nil {
+ return "", fmt.Errorf("%w stderr=%s", err, stderr.String())
+ }
+ return strings.TrimSpace(string(output)), nil
+}
diff --git a/agent/main.go b/agent/main.go
index 126a1a9..3c06aa7 100644
--- a/agent/main.go
+++ b/agent/main.go
@@ -6,6 +6,22 @@ import (
)
func main() {
+ if len(os.Args) > 1 && os.Args[1] == "ax-probe" {
+ if err := runAXProbe(); err != nil {
+ logError(err.Error())
+ os.Exit(1)
+ }
+ return
+ }
+
+ if len(os.Args) > 1 && os.Args[1] == "sync-chat" {
+ if err := runChatSync(); err != nil {
+ logError(err.Error())
+ os.Exit(1)
+ }
+ return
+ }
+
if err := runOnce(); err != nil {
logError(err.Error())
os.Exit(1)
@@ -41,6 +57,10 @@ func runOnce() error {
logThink(fmt.Sprintf("区域识别：%s center=[%.1f, %.1f]", summary.Type, summary.CenterScreen[0], summary.CenterScreen[1]))
}
+ if err := maybeScrollBeforeObserve(config, summaries, screenshotPath); err != nil {
+ return fmt.Errorf("滚动获取更多聊天记录失败：%w", err)
+ }
+
imageDataURL, err := loadImageDataURL(screenshotPath)
if err != nil {
return fmt.Errorf("读取截图失败：%w", err)
diff --git a/agent/planner.go b/agent/planner.go
index 0fd1cfe..e6c42bc 100644
--- a/agent/planner.go
+++ b/agent/planner.go
@@ -19,7 +19,8 @@ func buildPrompt(summaries []RegionSummary) (string, error) {
2. 识别联系人或会话对象，如果无法识别则填 unknown。
3. 判断是否需要回复。
4. 如果需要回复，生成 reply_text。
-5. 生成 dry-run 动作计划。
+5. 如果聊天上下文不足以判断是否回复，可以规划滚动聊天区域查看更多记录。
+6. 生成 dry-run 动作计划。
限制：
- 只能返回合法 JSON，不要 Markdown，不要解释。
@@ -28,7 +29,13 @@ func buildPrompt(summaries []RegionSummary) (string, error) {
- 如果区域 type 是 custom，请重点参考 description 判断该区域的用途和可执行动作。
- description 是用户标注时填写的用途说明，优先级高于模型自行猜测。
- 微信发送消息使用键入回车键，不要规划点击发送按钮。
-- action.type 只能是 click、type_text、press_enter、scroll、wait、noop。
+- 如果当前聊天内容不足以判断是否回复，可以返回 scroll 动作。
+- scroll 只能作用于 chat_content 区域。
+- delta_y < 0 表示向上滚动查看更多历史消息；delta_y > 0 表示向下滚动回到最新消息。
+- 每次最多返回一个 scroll 动作，abs(delta_y) 不要超过 5。
+- 如果滚动后需要重新识别，请追加 observe_again 动作。
+- 不要在同一个任务里同时包含 scroll 和 type_text/press_enter；上下文不足时先滚动观察，不要直接回复。
+- action.type 只能是 click、type_text、press_enter、scroll、observe_again、wait、noop。
- 当前是 dry_run，只生成计划，不要假设动作已执行。
- 如果无需回复，task_type 使用 wechat_observe 或 noop，并给出 observations。
diff --git a/agent/task.go b/agent/task.go
index c97aa13..b172b2f 100644
--- a/agent/task.go
+++ b/agent/task.go
@@ -16,12 +16,13 @@ var allowedTaskTypes = map[string]bool{
}
var allowedActionTypes = map[string]bool{
- "click": true,
- "type_text": true,
- "press_enter": true,
- "scroll": true,
- "wait": true,
- "noop": true,
+ "click": true,
+ "type_text": true,
+ "press_enter": true,
+ "observe_again": true,
+ "scroll": true,
+ "wait": true,
+ "noop": true,
}
func finalizeTask(task Task, summaries []RegionSummary, config Config, regionsPath string, screenshotPath string) Task {
@@ -68,11 +69,26 @@ func finalizeTask(task Task, summaries []RegionSummary, config Config, regionsPa
if summary, exists := regionIndex[action.TargetRegion]; exists {
point := summary.CenterScreen
action.Point = &point
- } else if action.Type == "click" {
+ } else if action.Type == "click" || action.Type == "scroll" {
action.Type = "noop"
action.Reason = "target_region not found"
}
}
+ if action.Type == "scroll" {
+ if action.TargetRegion != "chat_content" {
+ action.Type = "noop"
+ action.Reason = "scroll only allowed on chat_content"
+ }
+ if action.DeltaY > 5 {
+ action.DeltaY = 5
+ }
+ if action.DeltaY < -5 {
+ action.DeltaY = -5
+ }
+ if action.DeltaY == 0 {
+ action.DeltaY = -3
+ }
+ }
if action.Type == "type_text" && len([]rune(action.Text)) > 500 {
runes := []rune(action.Text)
action.Text = string(runes[:500])
@@ -119,6 +135,8 @@ func executionMessage(action Action) string {
return "dry_run: 跳过按回车发送"
case "scroll":
return fmt.Sprintf("dry_run: 跳过滑动 %s delta_y=%d", action.TargetRegion, action.DeltaY)
+ case "observe_again":
+ return "dry_run: 跳过重新截图并识别"
case "wait":
return fmt.Sprintf("dry_run: 跳过等待 %dms", action.DurationMs)
default:
diff --git a/agent/types.go b/agent/types.go
index dd70ed3..98800b2 100644
--- a/agent/types.go
+++ b/agent/types.go
@@ -12,12 +12,28 @@ type VolcengineConfig struct {
}
type AgentConfig struct {
- AppDataDir string `toml:"app_data_dir"`
- RegionsPath string `toml:"regions_path"`
- ScreenshotPath string `toml:"screenshot_path"`
- TaskOutputPath string `toml:"task_output_path"`
- TaskHistoryDir string `toml:"task_history_dir"`
- DryRun bool `toml:"dry_run"`
+ AppDataDir string `toml:"app_data_dir"`
+ RegionsPath string `toml:"regions_path"`
+ ScreenshotPath string `toml:"screenshot_path"`
+ TaskOutputPath string `toml:"task_output_path"`
+ TaskHistoryDir string `toml:"task_history_dir"`
+ DryRun bool `toml:"dry_run"`
+ ObserveMode string `toml:"observe_mode"`
+ MaxScrolls int `toml:"max_scrolls"`
+ ScrollDeltaY int `toml:"scroll_delta_y"`
+ ChatRecordsPath string `toml:"chat_records_path"`
+ ChatSyncMaxPages int `toml:"chat_sync_max_pages"`
+ ChatSyncTopScrolls int `toml:"chat_sync_top_scrolls"`
+}
+
+type WindowBounds struct {
+ X float64
+ Y float64
+ Width float64
+ Height float64
+ Owner string
+ Title string
+ ID int
}
type LogEntry struct {
@@ -138,3 +154,36 @@ type openAIResponse struct {
Message string `json:"message"`
} `json:"error,omitempty"`
}
+
+type ChatArchive struct {
+ App string `json:"app"`
+ Contact string `json:"contact"`
+ Source TaskSource `json:"source"`
+ Messages []ChatMessage `json:"messages"`
+ Pages []ChatPage `json:"pages"`
+ CreatedAt string `json:"created_at"`
+ UpdatedAt string `json:"updated_at"`
+}
+
+type ChatMessage struct {
+ ID string `json:"id"`
+ Sender string `json:"sender"`
+ Role string `json:"role"`
+ Time string `json:"time,omitempty"`
+ Content string `json:"content"`
+ PageIndex int `json:"page_index"`
+ SeenAt string `json:"seen_at"`
+}
+
+type ChatPage struct {
+ PageIndex int `json:"page_index"`
+ Direction string `json:"direction"`
+ NewCount int `json:"new_count"`
+ SeenAt string `json:"seen_at"`
+}
+
+type ChatExtractResult struct {
+ ContactName string `json:"contact_name"`
+ PageSummary string `json:"page_summary"`
+ Messages []ChatMessage `json:"messages"`
+}
diff --git a/wechat_vision/app.py b/wechat_vision/app.py
new file mode 100644
index 0000000..c556100
--- /dev/null
+++ b/wechat_vision/app.py
@@ -0,0 +1,163 @@
+from pathlib import Path
+
+import numpy as np
+import onnxruntime as ort
+from PIL import Image, ImageDraw
+
+
+MODEL_PATH = Path(__file__).with_name("best.onnx")
+TEST_DIR = Path(__file__).with_name("test")
+OUTPUT_DIR = Path(__file__).with_name("ouptsw")
+IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".bmp", ".webp"}
+CONFIDENCE_THRESHOLD = 0.1
+IOU_THRESHOLD = 0.5
+
+
+def _dtype(onnx_type):
+ if onnx_type == "tensor(float)":
+ return np.float32
+ if onnx_type == "tensor(float16)":
+ return np.float16
+ if onnx_type == "tensor(double)":
+ return np.float64
+ if onnx_type in {"tensor(int64)", "tensor(uint64)"}:
+ return np.int64
+ if onnx_type in {"tensor(int32)", "tensor(uint32)"}:
+ return np.int32
+ raise ValueError(f"Unsupported input type: {onnx_type}")
+
+
+def _shape(input_meta):
+ return [dim if isinstance(dim, int) else 1 for dim in input_meta.shape]
+
+
+def _iou(box, boxes):
+ x1 = np.maximum(box[0], boxes[:, 0])
+ y1 = np.maximum(box[1], boxes[:, 1])
+ x2 = np.minimum(box[2], boxes[:, 2])
+ y2 = np.minimum(box[3], boxes[:, 3])
+
+ intersection = np.maximum(0, x2 - x1) * np.maximum(0, y2 - y1)
+ box_area = np.maximum(0, box[2] - box[0]) * np.maximum(0, box[3] - box[1])
+ boxes_area = np.maximum(0, boxes[:, 2] - boxes[:, 0]) * np.maximum(
+ 0,
+ boxes[:, 3] - boxes[:, 1],
+ )
+ return intersection / np.maximum(box_area + boxes_area - intersection, 1e-6)
+
+
+def _nms(detections):
+ detections = detections[detections[:, 4] >= CONFIDENCE_THRESHOLD]
+ if len(detections) == 0:
+ return detections
+
+ detections = detections[np.argsort(detections[:, 4])[::-1]]
+ kept = []
+ while len(detections) > 0:
+ best = detections[0]
+ kept.append(best)
+ if len(detections) == 1:
+ break
+ detections = detections[1:][_iou(best[:4], detections[1:, :4]) < IOU_THRESHOLD]
+
+ return np.array(kept)
+
+
+def _preprocess(image, input_shape):
+ _, _, height, width = input_shape
+ resized = image.resize((width, height))
+ array = np.asarray(resized, dtype=np.float32) / 255.0
+ return np.transpose(array, (2, 0, 1))[None]
+
+
+def _scale_detections(detections, original_size, input_shape):
+ _, _, input_height, input_width = input_shape
+ original_width, original_height = original_size
+ scaled = detections.copy()
+ scaled[:, [0, 2]] *= original_width / input_width
+ scaled[:, [1, 3]] *= original_height / input_height
+ scaled[:, [0, 2]] = np.clip(scaled[:, [0, 2]], 0, original_width)
+ scaled[:, [1, 3]] = np.clip(scaled[:, [1, 3]], 0, original_height)
+ return scaled
+
+
+def _draw_detections(image, detections):
+ draw = ImageDraw.Draw(image)
+ for detection in detections:
+ x1, y1, x2, y2, score, class_id = detection
+ cx = (x1 + x2) / 2
+ cy = (y1 + y2) / 2
+ label = f"({cx:.1f}, {cy:.1f}) {score:.2f}"
+
+ draw.rectangle((x1, y1, x2, y2), outline="red", width=3)
+ draw.ellipse((cx - 5, cy - 5, cx + 5, cy + 5), fill="yellow", outline="red")
+ draw.line((cx - 10, cy, cx + 10, cy), fill="red", width=2)
+ draw.line((cx, cy - 10, cx, cy + 10), fill="red", width=2)
+ draw.text((x1, max(0, y1 - 14)), label, fill="yellow")
+
+
+def main():
+ session = ort.InferenceSession(str(MODEL_PATH))
+ input_meta = session.get_inputs()[0]
+ input_shape = _shape(input_meta)
+
+ print(f"model: {MODEL_PATH}")
+ print(f"providers: {session.get_providers()}")
+
+ print("inputs:")
+ feeds = {}
+ for input_meta in session.get_inputs():
+ print(
+ f"- name={input_meta.name}, type={input_meta.type}, "
+ f"shape={input_meta.shape}"
+ )
+ feeds[input_meta.name] = np.zeros(
+ _shape(input_meta),
+ dtype=_dtype(input_meta.type),
+ )
+
+ print("outputs:")
+ for output_meta in session.get_outputs():
+ print(
+ f"- name={output_meta.name}, type={output_meta.type}, "
+ f"shape={output_meta.shape}"
+ )
+
+ outputs = session.run(None, feeds)
+ print("dummy inference: ok")
+ for index, output in enumerate(outputs):
+ print(f"- output[{index}]: shape={output.shape}, dtype={output.dtype}")
+
+ OUTPUT_DIR.mkdir(exist_ok=True)
+ result_lines = ["image,class_id,score,center_x,center_y,x1,y1,x2,y2"]
+
+ for image_path in sorted(TEST_DIR.iterdir()):
+ if image_path.suffix.lower() not in IMAGE_SUFFIXES:
+ continue
+
+ image = Image.open(image_path).convert("RGB")
+ tensor = _preprocess(image, input_shape)
+ detections = session.run(None, {input_meta.name: tensor})[0][0]
+ detections = _scale_detections(_nms(detections), image.size, input_shape)
+
+ annotated = image.copy()
+ _draw_detections(annotated, detections)
+ output_path = OUTPUT_DIR / image_path.name
+ annotated.save(output_path)
+
+ print(f"{image_path.name}: {len(detections)} target(s) -> {output_path}")
+ for detection in detections:
+ x1, y1, x2, y2, score, class_id = detection
+ cx = (x1 + x2) / 2
+ cy = (y1 + y2) / 2
+ print(f" center=({cx:.1f}, {cy:.1f}), score={score:.3f}")
+ result_lines.append(
+ f"{image_path.name},{int(class_id)},{score:.6f},{cx:.2f},{cy:.2f},"
+ f"{x1:.2f},{y1:.2f},{x2:.2f},{y2:.2f}"
+ )
+
+ (OUTPUT_DIR / "coordinates.csv").write_text("\n".join(result_lines) + "\n")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/wechat_vision/best.onnx b/wechat_vision/best.onnx
new file mode 100644
index 0000000..4e3a7b5
Binary files /dev/null and b/wechat_vision/best.onnx differ
diff --git a/wechat_vision/ffmpeg_realtime_detect.py b/wechat_vision/ffmpeg_realtime_detect.py
new file mode 100644
index 0000000..b0bfae8
--- /dev/null
+++ b/wechat_vision/ffmpeg_realtime_detect.py
@@ -0,0 +1,898 @@
+import argparse
+import json
+import signal
+import subprocess
+import sys
+import threading
+import time
+import webbrowser
+from datetime import datetime, timezone
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from io import BytesIO
+from pathlib import Path
+
+import numpy as np
+import onnxruntime as ort
+from PIL import Image, ImageDraw
+
+
+BASE_DIR = Path(__file__).resolve().parent
+MODEL_PATH = BASE_DIR / "best.onnx"
+DEBUG_DIR = BASE_DIR / "ouptsw" / "monitor"
+VISUALIZE_DIR = BASE_DIR / "ouptsw" / "realtime_view"
+CLASS_NAMES = {
+ 0: "input_box",
+}
+IOU_THRESHOLD = 0.5
+
+
+class StreamState:
+ def __init__(self):
+ self.condition = threading.Condition()
+ self.jpeg = None
+ self.event = None
+ self.sequence = 0
+
+ def update(self, jpeg, event):
+ with self.condition:
+ self.jpeg = jpeg
+ self.event = event
+ self.sequence += 1
+ self.condition.notify_all()
+
+ def snapshot(self):
+ with self.condition:
+ return self.jpeg, self.event, self.sequence
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(
+ description="Realtime FFmpeg screen capture + ONNX detection demo."
+ )
+ parser.add_argument("--model", default=str(MODEL_PATH), help="ONNX model path")
+ parser.add_argument(
+ "--input",
+ default="3:none",
+ help="FFmpeg avfoundation input, e.g. '3:none'. Use --list-devices to inspect.",
+ )
+ parser.add_argument("--fps", type=float, default=2.0, help="capture FPS")
+ parser.add_argument("--confidence", type=float, default=0.1, help="minimum score")
+ parser.add_argument("--target-class", type=int, default=0, help="target class id")
+ parser.add_argument(
+ "--max-frames",
+ type=int,
+ default=0,
+ help="stop after N frames; 0 means run until interrupted",
+ )
+ parser.add_argument(
+ "--emit-empty",
+ action="store_true",
+ help="emit no_detection events when target is not found",
+ )
+ parser.add_argument(
+ "--debug-dir",
+ default="",
+ help="save annotated frames into this directory",
+ )
+ parser.add_argument(
+ "--debug-every",
+ type=int,
+ default=0,
+ help="save one debug image every N frames; 0 disables debug images",
+ )
+ parser.add_argument(
+ "--visualize-dir",
+ default="",
+ help="write latest.jpg/latest.json/index.html for browser visualization",
+ )
+ parser.add_argument(
+ "--visualize-every",
+ type=int,
+ default=1,
+ help="update visualization every N frames when --visualize-dir is enabled",
+ )
+ parser.add_argument(
+ "--open-visualizer",
+ action="store_true",
+ help="open the visualization HTML in the default browser",
+ )
+ parser.add_argument(
+ "--serve",
+ action="store_true",
+ help="serve a realtime MJPEG stream and dashboard over HTTP",
+ )
+ parser.add_argument("--host", default="127.0.0.1", help="HTTP server host")
+ parser.add_argument("--port", type=int, default=8765, help="HTTP server port")
+ parser.add_argument(
+ "--display",
+ action="store_true",
+ help="open an ffplay window with the annotated realtime video",
+ )
+ parser.add_argument(
+ "--display-title",
+ default="WeChat Vision ONNX Preview",
+ help="ffplay window title when --display is enabled",
+ )
+ parser.add_argument(
+ "--display-width",
+ type=int,
+ default=720,
+ help="ffplay window width when --display is enabled",
+ )
+ parser.add_argument(
+ "--display-height",
+ type=int,
+ default=450,
+ help="ffplay window height when --display is enabled",
+ )
+ parser.add_argument(
+ "--display-left",
+ type=int,
+ default=0,
+ help="ffplay window left position when --display is enabled",
+ )
+ parser.add_argument(
+ "--display-top",
+ type=int,
+ default=0,
+ help="ffplay window top position when --display is enabled",
+ )
+ parser.add_argument(
+ "--list-devices",
+ action="store_true",
+ help="list FFmpeg avfoundation devices and exit",
+ )
+ return parser.parse_args()
+
+
+def log_error(message):
+ print(message, file=sys.stderr, flush=True)
+
+
+def utc_now():
+ return datetime.now(timezone.utc).isoformat()
+
+
+def input_shape(input_meta):
+ return [dim if isinstance(dim, int) else 1 for dim in input_meta.shape]
+
+
+def preprocess(image, shape):
+ _, _, height, width = shape
+ resized = image.convert("RGB").resize((width, height))
+ array = np.asarray(resized, dtype=np.float32) / 255.0
+ return np.transpose(array, (2, 0, 1))[None]
+
+
+def iou(box, boxes):
+ x1 = np.maximum(box[0], boxes[:, 0])
+ y1 = np.maximum(box[1], boxes[:, 1])
+ x2 = np.minimum(box[2], boxes[:, 2])
+ y2 = np.minimum(box[3], boxes[:, 3])
+ intersection = np.maximum(0, x2 - x1) * np.maximum(0, y2 - y1)
+ box_area = np.maximum(0, box[2] - box[0]) * np.maximum(0, box[3] - box[1])
+ boxes_area = np.maximum(0, boxes[:, 2] - boxes[:, 0]) * np.maximum(
+ 0,
+ boxes[:, 3] - boxes[:, 1],
+ )
+ return intersection / np.maximum(box_area + boxes_area - intersection, 1e-6)
+
+
+def nms(detections, confidence):
+ detections = detections[detections[:, 4] >= confidence]
+ if len(detections) == 0:
+ return detections
+
+ detections = detections[np.argsort(detections[:, 4])[::-1]]
+ kept = []
+ while len(detections) > 0:
+ best = detections[0]
+ kept.append(best)
+ if len(detections) == 1:
+ break
+ detections = detections[1:][iou(best[:4], detections[1:, :4]) < IOU_THRESHOLD]
+ return np.array(kept)
+
+
+def scale_detections(detections, original_size, shape):
+ if len(detections) == 0:
+ return detections
+
+ _, _, input_height, input_width = shape
+ original_width, original_height = original_size
+ scaled = detections.copy()
+ scaled[:, [0, 2]] *= original_width / input_width
+ scaled[:, [1, 3]] *= original_height / input_height
+ scaled[:, [0, 2]] = np.clip(scaled[:, [0, 2]], 0, original_width)
+ scaled[:, [1, 3]] = np.clip(scaled[:, [1, 3]], 0, original_height)
+ return scaled
+
+
+def detect(session, input_meta, image, target_class, confidence):
+ shape = input_shape(input_meta)
+ tensor = preprocess(image, shape)
+ raw = session.run(None, {input_meta.name: tensor})[0][0]
+ detections = raw[raw[:, 5].astype(int) == target_class]
+ detections = nms(detections, confidence)
+ return scale_detections(detections, image.size, shape)
+
+
+def emit(event):
+ print(json.dumps(event, ensure_ascii=False, separators=(",", ":")), flush=True)
+
+
+def write_json_atomic(path, data):
+ path.parent.mkdir(parents=True, exist_ok=True)
+ temp_path = path.with_suffix(path.suffix + ".tmp")
+ temp_path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
+ temp_path.replace(path)
+
+
+def save_image_atomic(image, path):
+ path.parent.mkdir(parents=True, exist_ok=True)
+ temp_path = path.with_suffix(path.suffix + ".tmp")
+ image.save(temp_path, format="JPEG", quality=85)
+ temp_path.replace(path)
+
+
+def image_to_jpeg_bytes(image):
+ buffer = BytesIO()
+ image.save(buffer, format="JPEG", quality=85)
+ return buffer.getvalue()
+
+
+def detection_event(detection, frame_index, frame_size, class_name):
+ x1, y1, x2, y2, score, class_id = detection
+ cx = (x1 + x2) / 2
+ cy = (y1 + y2) / 2
+ return {
+ "type": "detection",
+ "timestamp": utc_now(),
+ "frame_index": frame_index,
+ "class_id": int(class_id),
+ "class_name": class_name,
+ "confidence": round(float(score), 6),
+ "bbox_screen": [round(float(x1), 2), round(float(y1), 2), round(float(x2), 2), round(float(y2), 2)],
+ "center_screen": [round(float(cx), 2), round(float(cy), 2)],
+ "frame_size": [frame_size[0], frame_size[1]],
+ }
+
+
+def no_detection_event(frame_index, frame_size, target_class, class_name):
+ return {
+ "type": "no_detection",
+ "timestamp": utc_now(),
+ "frame_index": frame_index,
+ "class_id": target_class,
+ "class_name": class_name,
+ "frame_size": [frame_size[0], frame_size[1]],
+ }
+
+
+def save_debug_image(image, detections, path, class_name):
+ debug = draw_detections(image, detections, class_name)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ debug.save(path)
+
+
+def draw_detections(image, detections, class_name):
+ debug = image.convert("RGB")
+ draw = ImageDraw.Draw(debug)
+ for detection in detections:
+ x1, y1, x2, y2, score, _ = detection
+ cx = (x1 + x2) / 2
+ cy = (y1 + y2) / 2
+ draw.rectangle((x1, y1, x2, y2), outline="red", width=3)
+ draw.ellipse((cx - 5, cy - 5, cx + 5, cy + 5), fill="yellow", outline="red")
+ draw.text(
+ (x1, max(0, y1 - 14)),
+ f"{class_name} ({cx:.1f}, {cy:.1f}) {score:.2f}",
+ fill="yellow",
+ )
+ return debug
+
+
+def draw_frame_info(image, event):
+ draw = ImageDraw.Draw(image)
+ frame_index = event.get("frame_index", "-")
+ timestamp = event.get("timestamp", "")
+ text = f"frame={frame_index} {timestamp}"
+ draw.rectangle((12, 12, 620, 52), fill=(0, 0, 0))
+ draw.text((22, 22), text, fill=(80, 255, 120))
+ return image
+
+
+def write_visualizer_html(path):
+ path.parent.mkdir(parents=True, exist_ok=True)
+ html = """
+
+
+
+
+ WeChat Vision Realtime
+
+
+
+
+
![latest detection frame]()
+
+
+
+
+ waiting for frames
+
+
+
+
+
+
+
+
+