wechat_ai/agent/executor.go

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"math"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"
)

func maybeScrollBeforeObserve(config Config, summaries []RegionSummary, screenshotPath string) error {
	if config.Agent.ObserveMode != "scroll_first" {
		return nil
	}
	if runtime.GOOS != "darwin" {
		return fmt.Errorf("observe_mode=scroll_first currently supports macOS only")
	}

	chat, ok := regionByType(summaries)["chat_content"]
	if !ok {
		return fmt.Errorf("cannot scroll before observe: chat_content region not found")
	}

	maxScrolls := config.Agent.MaxScrolls
	if maxScrolls <= 0 {
		maxScrolls = 1
	}

	for index := 0; index < maxScrolls; index++ {
		window, err := findWeChatWindow()
		if err != nil {
			return err
		}

		pointX := window.X + chat.CenterSource[0]
		pointY := window.Y + chat.CenterSource[1]
		logTask(fmt.Sprintf("观察前滚动聊天区域：round=%d point=[%.1f, %.1f] delta_y=%d window=[%.0f, %.0f, %.0f, %.0f]", index+1, pointX, pointY, config.Agent.ScrollDeltaY, window.X, window.Y, window.Width, window.Height))
		if err := scrollAt(pointX, pointY, config.Agent.ScrollDeltaY); err != nil {
			return err
		}
		time.Sleep(700 * time.Millisecond)

		if err := captureWindow(window, screenshotPath); err != nil {
			return err
		}
		logInfo("已滚动并覆盖更新截图 WeChat.jpg")
	}

	return nil
}

func findWeChatWindow() (WindowBounds, error) {
	script := `
import CoreGraphics
import Foundation

let options = CGWindowListOption(arrayLiteral: [.optionOnScreenOnly, .excludeDesktopElements])
guard let infos = CGWindowListCopyWindowInfo(options, kCGNullWindowID) as? [[String: Any]] else {
  exit(1)
}

for info in infos {
  let owner = info[kCGWindowOwnerName as String] as? String ?? ""
  let title = info[kCGWindowName as String] as? String ?? ""
  let layer = info[kCGWindowLayer as String] as? Int ?? -1
  if layer == 0 && (owner.contains("微信") || owner.lowercased().contains("wechat") || title == "微信") {
    if let bounds = info[kCGWindowBounds as String] as? [String: Any],
       let x = bounds["X"] as? Double,
       let y = bounds["Y"] as? Double,
       let w = bounds["Width"] as? Double,
       let h = bounds["Height"] as? Double {
      let id = info[kCGWindowNumber as String] as? Int ?? 0
      let result: [String: Any] = ["x": x, "y": y, "width": w, "height": h, "owner": owner, "title": title, "id": id]
      let data = try! JSONSerialization.data(withJSONObject: result)
      print(String(data: data, encoding: .utf8)!)
      exit(0)
    }
  }
}

exit(2)
`

	output, err := runSwift(script)
	if err != nil {
		return WindowBounds{}, fmt.Errorf("find WeChat window failed: %w", err)
	}

	var raw struct {
		X      float64 `json:"x"`
		Y      float64 `json:"y"`
		Width  float64 `json:"width"`
		Height float64 `json:"height"`
		Owner  string  `json:"owner"`
		Title  string  `json:"title"`
		ID     int     `json:"id"`
	}
	if err := json.Unmarshal([]byte(output), &raw); err != nil {
		return WindowBounds{}, err
	}

	return WindowBounds(raw), nil
}

func scrollAt(x float64, y float64, deltaY int) error {
	if deltaY == 0 {
		deltaY = 6
	}

	script := fmt.Sprintf(`
import CoreGraphics
import Foundation
import AppKit

NSWorkspace.shared.runningApplications
  .first { $0.localizedName == "微信" || $0.localizedName == "WeChat" }?
  .activate(options: [.activateIgnoringOtherApps])
usleep(300000)

let point = CGPoint(x: %.3f, y: %.3f)
let move = CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left)
move?.post(tap: .cghidEventTap)
usleep(80000)
let down = CGEvent(mouseEventSource: nil, mouseType: .leftMouseDown, mouseCursorPosition: point, mouseButton: .left)
let up = CGEvent(mouseEventSource: nil, mouseType: .leftMouseUp, mouseCursorPosition: point, mouseButton: .left)
down?.post(tap: .cghidEventTap)
usleep(60000)
up?.post(tap: .cghidEventTap)
usleep(120000)

let delta = Int32(%d)
for _ in 0..<8 {
  let lineScroll = CGEvent(scrollWheelEvent2Source: nil, units: .line, wheelCount: 1, wheel1: delta, wheel2: 0, wheel3: 0)
  lineScroll?.post(tap: .cghidEventTap)
  usleep(45000)
}

for _ in 0..<4 {
  let pixelScroll = CGEvent(scrollWheelEvent2Source: nil, units: .pixel, wheelCount: 1, wheel1: delta * 80, wheel2: 0, wheel3: 0)
  pixelScroll?.post(tap: .cghidEventTap)
  usleep(45000)
}
`, x, y, deltaY)

	_, err := runSwift(script)
	return err
}

func captureWindow(window WindowBounds, screenshotPath string) error {
	if err := os.MkdirAll(filepath.Dir(screenshotPath), 0755); err != nil {
		return err
	}

	rect := fmt.Sprintf("%d,%d,%d,%d", int(math.Round(window.X)), int(math.Round(window.Y)), int(math.Round(window.Width)), int(math.Round(window.Height)))
	command := exec.Command("screencapture", "-x", "-R", rect, screenshotPath)
	if output, err := command.CombinedOutput(); err != nil {
		return fmt.Errorf("screencapture failed: %w output=%s", err, string(output))
	}

	return nil
}

func runSwift(script string) (string, error) {
	command := exec.Command("swift", "-")
	command.Stdin = strings.NewReader(script)
	var stderr bytes.Buffer
	command.Stderr = &stderr
	output, err := command.Output()
	if err != nil {
		return "", fmt.Errorf("%w stderr=%s", err, stderr.String())
	}
	return strings.TrimSpace(string(output)), nil
}