Add WeChat vision streaming workbench

.codegraph/.gitignore

@@ -1,16 +1,5 @@
-# CodeGraph data files
+# CodeGraph data files — local to each machine, not for committing.
-# These are local to each machine and should not be committed
+# Ignore everything in .codegraph/ except this file itself, so transient
-
+# files (the database, daemon.pid, sockets, logs) never show up in git.
-# Database
+*
-*.db
+!.gitignore
-*.db-wal
-*.db-shm
-
-# Cache
-cache/
-
-# Logs
-*.log
-
-# Hook markers
-.dirty

.codegraph/daemon.pid

@@ -1,6 +0,0 @@
-{
-  "pid": 30162,
-  "version": "0.9.7",
-  "socketPath": "/Users/wang/agent_dev/wechat_ai/.codegraph/daemon.sock",
-  "startedAt": 1781241939180
-}

.gitignore

@@ -14,6 +14,9 @@ wechat_vision/__pycache__/
 wechat_vision/ouptsw/
 wechat_vision/test/
 
+vision_test/.venv/
+
+demos/go-ffmpeg-wechat-stream/go-ffmpeg-wechat-stream
 .codegraph/daemon.pid
 .vscode/
 

agent/main.go

@@ -6,6 +6,14 @@ import (
 )
 
 func main() {
+	if len(os.Args) > 1 && os.Args[1] == "vision-stream" {
+		if err := runVisionStream(os.Args[2:]); err != nil {
+			logError(err.Error())
+			os.Exit(1)
+		}
+		return
+	}
+
 	if len(os.Args) > 1 && os.Args[1] == "ax-probe" {
 		if err := runAXProbe(); err != nil {
 			logError(err.Error())

agent/vision_stream.go

@@ -0,0 +1,668 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"image"
+	_ "image/jpeg"
+	"io"
+	"math"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+)
+
+type visionFrameState struct {
+	condition   *sync.Cond
+	jpeg        []byte
+	frameWidth  int
+	frameHeight int
+	sequence    int64
+	frameIndex  int64
+	updatedAt   time.Time
+	target      previewTarget
+	annotation  annotationMeta
+	event       map[string]any
+}
+
+type annotationMeta struct {
+	Width       int     `json:"width"`
+	Height      int     `json:"height"`
+	ScaleFactor float64 `json:"scaleFactor"`
+}
+
+type captureSource struct {
+	ID          string  `json:"id"`
+	Kind        string  `json:"kind"`
+	Label       string  `json:"label"`
+	AppName     string  `json:"appName"`
+	Title       string  `json:"title"`
+	PID         int     `json:"pid"`
+	X           float64 `json:"x"`
+	Y           float64 `json:"y"`
+	Width       float64 `json:"width"`
+	Height      float64 `json:"height"`
+	ScaleFactor float64 `json:"scaleFactor"`
+}
+
+type previewTarget struct {
+	Kind     string
+	Label    string
+	WindowID int
+	Rect     WindowBounds
+}
+
+func newVisionFrameState(target previewTarget) *visionFrameState {
+	return &visionFrameState{
+		condition: sync.NewCond(&sync.Mutex{}),
+		target:    target,
+		event:     map[string]any{"type": "waiting"},
+	}
+}
+
+func (state *visionFrameState) update(jpeg []byte) {
+	frameWidth, frameHeight := jpegSize(jpeg)
+	state.condition.L.Lock()
+	state.jpeg = append(state.jpeg[:0], jpeg...)
+	if frameWidth > 0 && frameHeight > 0 {
+		state.frameWidth = frameWidth
+		state.frameHeight = frameHeight
+	}
+	state.sequence++
+	state.frameIndex++
+	state.updatedAt = time.Now().UTC()
+	state.event = map[string]any{
+		"type":         "preview_frame",
+		"timestamp":    state.updatedAt.Format(time.RFC3339Nano),
+		"frame_index":  state.frameIndex,
+		"target_kind":  state.target.Kind,
+		"target_label": state.target.Label,
+	}
+	state.condition.Broadcast()
+	state.condition.L.Unlock()
+}
+
+func (state *visionFrameState) updateAnnotation(annotation annotationMeta) {
+	state.condition.L.Lock()
+	state.annotation = annotation
+	state.condition.L.Unlock()
+}
+
+func (state *visionFrameState) updateStatus(event map[string]any, target previewTarget) {
+	state.condition.L.Lock()
+	state.updatedAt = time.Now().UTC()
+	state.event = event
+	state.target = target
+	state.condition.Broadcast()
+	state.condition.L.Unlock()
+}
+
+func (state *visionFrameState) snapshot() ([]byte, map[string]any, int64, time.Time, int, int, annotationMeta) {
+	state.condition.L.Lock()
+	defer state.condition.L.Unlock()
+	jpeg := append([]byte(nil), state.jpeg...)
+	event := make(map[string]any, len(state.event))
+	for key, value := range state.event {
+		event[key] = value
+	}
+	return jpeg, event, state.sequence, state.updatedAt, state.frameWidth, state.frameHeight, state.annotation
+}
+
+func (state *visionFrameState) waitForNext(lastSequence int64) ([]byte, int64) {
+	state.condition.L.Lock()
+	defer state.condition.L.Unlock()
+	for state.sequence == lastSequence {
+		state.condition.Wait()
+	}
+	return append([]byte(nil), state.jpeg...), state.sequence
+}
+
+func runVisionStream(args []string) error {
+	flags := flag.NewFlagSet("vision-stream", flag.ContinueOnError)
+	host := flags.String("host", "127.0.0.1", "HTTP server host")
+	port := flags.Int("port", 8765, "HTTP server port")
+	input := flags.String("input", "3:none", "FFmpeg avfoundation input for experiments")
+	fps := flags.Float64("fps", 30, "capture FPS")
+	cropWechat := flags.Bool("crop-wechat", true, "capture only the WeChat window")
+	regionsPath := flags.String("regions", defaultRegionsPath(), "annotation regions JSON path")
+	if err := flags.Parse(args); err != nil {
+		return err
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	state := newVisionFrameState(previewTarget{Kind: "waiting", Label: "等待标注来源"})
+	server := &http.Server{
+		Addr:              fmt.Sprintf("%s:%d", *host, *port),
+		Handler:           visionHTTPHandler(state),
+		ReadHeaderTimeout: 5 * time.Second,
+	}
+
+	go func() {
+		<-ctx.Done()
+		shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 2*time.Second)
+		defer shutdownCancel()
+		_ = server.Shutdown(shutdownCtx)
+	}()
+
+	errCh := make(chan error, 2)
+	go func() {
+		errCh <- server.ListenAndServe()
+	}()
+	go func() {
+		errCh <- runVisionCapture(ctx, state, *input, *fps, *cropWechat, *regionsPath)
+	}()
+
+	logInfo(fmt.Sprintf("Go annotation source stream started http://%s:%d/stream.mjpg regions=%s fps=%.1f", *host, *port, *regionsPath, *fps))
+
+	for err := range errCh {
+		if err == nil || err == http.ErrServerClosed || ctx.Err() != nil {
+			return nil
+		}
+		cancel()
+		return err
+	}
+
+	return nil
+}
+
+func visionHTTPHandler(state *visionFrameState) http.Handler {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) {
+		writer.Header().Set("Content-Type", "text/html; charset=utf-8")
+		_, _ = io.WriteString(writer, `<html><body style="margin:0;background:#050505;color:#d7ffe4;font-family:monospace"><img src="/stream.mjpg" style="width:100vw;height:100vh;object-fit:contain" /></body></html>`)
+	})
+	mux.HandleFunc("/latest.json", func(writer http.ResponseWriter, request *http.Request) {
+		_, event, sequence, updatedAt, frameWidth, frameHeight, annotation := state.snapshot()
+		payload := map[string]any{
+			"source":     "go-window-capture",
+			"sequence":   sequence,
+			"updated_at": updatedAt.Format(time.RFC3339Nano),
+			"event":      event,
+			"frame": map[string]any{
+				"width":  frameWidth,
+				"height": frameHeight,
+			},
+			"annotation": annotation,
+			"target": map[string]any{
+				"kind":      state.target.Kind,
+				"label":     state.target.Label,
+				"window_id": state.target.WindowID,
+				"rect":      state.target.Rect,
+			},
+		}
+		if updatedAt.IsZero() {
+			payload["updated_at"] = time.Now().UTC().Format(time.RFC3339Nano)
+		}
+		writer.Header().Set("Content-Type", "application/json; charset=utf-8")
+		writer.Header().Set("Cache-Control", "no-store")
+		_ = json.NewEncoder(writer).Encode(payload)
+	})
+	mux.HandleFunc("/frame.jpg", func(writer http.ResponseWriter, request *http.Request) {
+		jpeg, _, sequence, _, _, _, _ := state.snapshot()
+		if len(jpeg) == 0 {
+			http.Error(writer, "frame not ready", http.StatusServiceUnavailable)
+			return
+		}
+		writer.Header().Set("Content-Type", "image/jpeg")
+		writer.Header().Set("Cache-Control", "no-store")
+		writer.Header().Set("X-Frame-Sequence", fmt.Sprintf("%d", sequence))
+		_, _ = writer.Write(jpeg)
+	})
+	mux.HandleFunc("/stream.mjpg", func(writer http.ResponseWriter, request *http.Request) {
+		flusher, ok := writer.(http.Flusher)
+		if !ok {
+			http.Error(writer, "streaming unsupported", http.StatusInternalServerError)
+			return
+		}
+
+		writer.Header().Set("Content-Type", "multipart/x-mixed-replace; boundary=frame")
+		writer.Header().Set("Cache-Control", "no-store")
+		lastSequence := int64(-1)
+		for request.Context().Err() == nil {
+			jpeg, sequence := state.waitForNext(lastSequence)
+			lastSequence = sequence
+			if len(jpeg) == 0 {
+				continue
+			}
+			_, err := fmt.Fprintf(writer, "--frame\r\nContent-Type: image/jpeg\r\nContent-Length: %d\r\n\r\n", len(jpeg))
+			if err != nil {
+				return
+			}
+			if _, err := writer.Write(jpeg); err != nil {
+				return
+			}
+			if _, err := writer.Write([]byte("\r\n")); err != nil {
+				return
+			}
+			flusher.Flush()
+		}
+	})
+	return mux
+}
+
+func runVisionCapture(ctx context.Context, state *visionFrameState, input string, fps float64, cropWechat bool, regionsPath string) error {
+	for ctx.Err() == nil {
+		target, annotation, err := previewTargetFromAnnotation(regionsPath)
+		if err != nil {
+			message := "等待标注来源窗口出现：" + err.Error()
+			logWarning(message)
+			state.updateStatus(map[string]any{
+				"type":      "source_waiting",
+				"timestamp": time.Now().UTC().Format(time.RFC3339Nano),
+				"message":   message,
+			}, previewTarget{Kind: "waiting", Label: "等待标注来源"})
+			select {
+			case <-ctx.Done():
+				return nil
+			case <-time.After(2 * time.Second):
+				continue
+			}
+		}
+
+		state.updateAnnotation(annotation)
+		return runResolvedVisionCapture(ctx, state, input, fps, cropWechat, target)
+	}
+	return nil
+}
+
+func runResolvedVisionCapture(ctx context.Context, state *visionFrameState, input string, fps float64, cropWechat bool, target previewTarget) error {
+	if target.Kind == "window" {
+		logInfo(fmt.Sprintf("using macOS window-only capture window_id=%d", target.WindowID))
+		return runScreencaptureMJPEG(ctx, state, fps, &target.Rect, target.WindowID)
+	}
+
+	if target.Kind == "display" {
+		logInfo(fmt.Sprintf("using macOS display rectangle capture rect=[%.0f,%.0f,%.0f,%.0f]", target.Rect.X, target.Rect.Y, target.Rect.Width, target.Rect.Height))
+		return runScreencaptureMJPEG(ctx, state, fps, &target.Rect, 0)
+	}
+
+	window, err := maybeFindWeChatWindow(cropWechat)
+	if err != nil {
+		return err
+	}
+	logInfo("using macOS window-only capture for fallback WeChat preview")
+	return runScreencaptureMJPEG(ctx, state, fps, window, window.ID)
+
+	// Keep the FFmpeg path below as an experiment. It captures the screen and crops
+	// by rectangle, so covered windows can leak into the preview. Tauri preview uses
+	// screencapture -l instead, which captures the WeChat window itself.
+	ffmpegErr := make(chan error, 1)
+	ffmpegCtx, cancelFFmpeg := context.WithCancel(ctx)
+	go func() {
+		ffmpegErr <- runFFmpegMJPEG(ffmpegCtx, state, input, fps, window)
+	}()
+
+	timer := time.NewTimer(3 * time.Second)
+	defer timer.Stop()
+	select {
+	case <-ctx.Done():
+		cancelFFmpeg()
+		return nil
+	case err := <-ffmpegErr:
+		cancelFFmpeg()
+		if err != nil {
+			logWarning("FFmpeg exited before first frame, fallback to screencapture: " + err.Error())
+		}
+		return runScreencaptureMJPEG(ctx, state, fps, window, window.ID)
+	case <-timer.C:
+		_, _, sequence, _, _, _, _ := state.snapshot()
+		if sequence > 0 {
+			logInfo("FFmpeg produced frames, keep using FFmpeg capture")
+			return <-ffmpegErr
+		}
+		cancelFFmpeg()
+		logWarning("FFmpeg produced no frames in 3s, fallback to screencapture")
+		return runScreencaptureMJPEG(ctx, state, fps, window, window.ID)
+	}
+}
+
+func maybeFindWeChatWindow(cropWechat bool) (*WindowBounds, error) {
+	if !cropWechat {
+		return nil, nil
+	}
+	bounds, err := findWeChatWindow()
+	if err != nil {
+		return nil, fmt.Errorf("find WeChat window failed, fallback to full screen capture: %w", err)
+	}
+	logInfo(fmt.Sprintf(
+		"WeChat window found owner=%s title=%s id=%d rect=[%.0f,%.0f,%.0f,%.0f]",
+		bounds.Owner,
+		bounds.Title,
+		bounds.ID,
+		bounds.X,
+		bounds.Y,
+		bounds.Width,
+		bounds.Height,
+	))
+	return &bounds, nil
+}
+
+func defaultRegionsPath() string {
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return "data/regions/wechat.json"
+	}
+	return filepath.Join(home, "Library", "Application Support", "com.tauri.dev", "data", "regions", "wechat.json")
+}
+
+func previewTargetFromAnnotation(path string) (previewTarget, annotationMeta, error) {
+	annotation, err := loadAnnotation(path)
+	if err != nil {
+		return previewTarget{}, annotationMeta{}, fmt.Errorf("load annotation source failed: %w", err)
+	}
+	meta := annotationMeta{
+		Width:       annotation.ScreenshotWidth,
+		Height:      annotation.ScreenshotHeight,
+		ScaleFactor: annotation.ScaleFactor,
+	}
+
+	source := captureSourceFromAnnotation(annotation.Source)
+	if source.Kind == "" {
+		target, err := fallbackWeChatPreviewTarget()
+		return target, meta, err
+	}
+
+	if source.Kind == "window" {
+		window, err := findWindowForSource(source)
+		if err != nil {
+			return previewTarget{}, meta, err
+		}
+		return previewTarget{
+			Kind:     "window",
+			Label:    source.Label,
+			WindowID: window.ID,
+			Rect:     window,
+		}, meta, nil
+	}
+
+	if source.Kind == "display" {
+		return previewTarget{
+			Kind:  "display",
+			Label: source.Label,
+			Rect: WindowBounds{
+				X:      source.X,
+				Y:      source.Y,
+				Width:  source.Width,
+				Height: source.Height,
+				Owner:  "display",
+				Title:  source.Label,
+			},
+		}, meta, nil
+	}
+
+	return previewTarget{}, meta, fmt.Errorf("unsupported annotation source kind: %s", source.Kind)
+}
+
+func fallbackWeChatPreviewTarget() (previewTarget, error) {
+	window, err := findWeChatWindow()
+	if err != nil {
+		return previewTarget{}, err
+	}
+	return previewTarget{
+		Kind:     "window",
+		Label:    "微信",
+		WindowID: window.ID,
+		Rect:     window,
+	}, nil
+}
+
+func captureSourceFromAnnotation(raw jsonRawOptional) captureSource {
+	data, err := json.Marshal(raw)
+	if err != nil {
+		return captureSource{}
+	}
+	var source captureSource
+	if err := json.Unmarshal(data, &source); err != nil {
+		return captureSource{}
+	}
+	return source
+}
+
+func findWindowForSource(source captureSource) (WindowBounds, error) {
+	appName := strings.ReplaceAll(source.AppName, "\\", "\\\\")
+	appName = strings.ReplaceAll(appName, `"`, `\"`)
+	title := strings.ReplaceAll(source.Title, "\\", "\\\\")
+	title = strings.ReplaceAll(title, `"`, `\"`)
+
+	script := fmt.Sprintf(`
+import CoreGraphics
+import Foundation
+
+let targetOwner = "%s"
+let targetTitle = "%s"
+let options = CGWindowListOption(arrayLiteral: [.optionOnScreenOnly, .excludeDesktopElements])
+guard let infos = CGWindowListCopyWindowInfo(options, kCGNullWindowID) as? [[String: Any]] else { exit(1) }
+
+func printWindow(_ info: [String: Any]) {
+  let owner = info[kCGWindowOwnerName as String] as? String ?? ""
+  let title = info[kCGWindowName as String] as? String ?? ""
+  let bounds = info[kCGWindowBounds as String] as? [String: Any] ?? [:]
+  let x = bounds["X"] as? Double ?? 0
+  let y = bounds["Y"] as? Double ?? 0
+  let w = bounds["Width"] as? Double ?? 0
+  let h = bounds["Height"] as? Double ?? 0
+  let id = info[kCGWindowNumber as String] as? Int ?? 0
+  let result: [String: Any] = ["x": x, "y": y, "width": w, "height": h, "owner": owner, "title": title, "id": id]
+  let data = try! JSONSerialization.data(withJSONObject: result)
+  print(String(data: data, encoding: .utf8)!)
+}
+
+for info in infos {
+  let owner = info[kCGWindowOwnerName as String] as? String ?? ""
+  let title = info[kCGWindowName as String] as? String ?? ""
+  let layer = info[kCGWindowLayer as String] as? Int ?? -1
+  if layer == 0 && owner == targetOwner && title == targetTitle { printWindow(info); exit(0) }
+}
+
+for info in infos {
+  let owner = info[kCGWindowOwnerName as String] as? String ?? ""
+  let title = info[kCGWindowName as String] as? String ?? ""
+  let layer = info[kCGWindowLayer as String] as? Int ?? -1
+  if layer == 0 && targetOwner != "" && owner.contains(targetOwner) { printWindow(info); exit(0) }
+}
+
+exit(2)
+`, appName, title)
+
+	output, err := runSwift(script)
+	if err != nil {
+		return WindowBounds{}, fmt.Errorf("find annotation window source failed app=%q title=%q: %w", source.AppName, source.Title, err)
+	}
+	var window WindowBounds
+	if err := json.Unmarshal([]byte(output), &window); err != nil {
+		return WindowBounds{}, err
+	}
+	return window, nil
+}
+
+func runFFmpegMJPEG(ctx context.Context, state *visionFrameState, input string, fps float64, window *WindowBounds) error {
+	args := []string{
+		"-hide_banner",
+		"-loglevel", "error",
+		"-f", "avfoundation",
+		"-framerate", fmt.Sprintf("%.3f", fps),
+		"-pixel_format", "bgr0",
+		"-i", input,
+	}
+	if window != nil {
+		args = append(args, "-vf", ffmpegCropFilter(*window))
+	}
+	args = append(args,
+		"-f", "image2pipe",
+		"-vcodec", "mjpeg",
+		"-",
+	)
+	logInfo("FFmpeg command: ffmpeg " + strings.Join(args, " "))
+	command := exec.CommandContext(ctx, "ffmpeg", args...)
+	stdout, err := command.StdoutPipe()
+	if err != nil {
+		return err
+	}
+	stderr, err := command.StderrPipe()
+	if err != nil {
+		return err
+	}
+
+	if err := command.Start(); err != nil {
+		return fmt.Errorf("启动 FFmpeg 失败：%w", err)
+	}
+
+	go drainFFmpegStderr(stderr)
+
+	readErr := readJPEGFrames(stdout, state.update)
+	waitErr := command.Wait()
+	if ctx.Err() != nil {
+		return nil
+	}
+	if readErr != nil {
+		return readErr
+	}
+	if waitErr != nil {
+		return fmt.Errorf("FFmpeg 退出：%w", waitErr)
+	}
+	return nil
+}
+
+func runScreencaptureMJPEG(ctx context.Context, state *visionFrameState, fps float64, window *WindowBounds, windowID int) error {
+	if window == nil {
+		return errors.New("screencapture requires a capture target")
+	}
+	interval := time.Duration(float64(time.Second) / math.Max(fps, 1))
+	if interval < 80*time.Millisecond {
+		interval = 80 * time.Millisecond
+	}
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	tempDir, err := os.MkdirTemp("", "wechat-go-vision-*")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tempDir)
+	imagePath := filepath.Join(tempDir, "frame.jpg")
+	rect := captureRect(*window)
+	if windowID > 0 {
+		logInfo(fmt.Sprintf("window-only screencapture started window_id=%d", windowID))
+	} else {
+		logInfo("rectangle screencapture started rect=" + rect)
+	}
+	for ctx.Err() == nil {
+		args := []string{"-x"}
+		if windowID > 0 {
+			args = append(args, "-l", fmt.Sprintf("%d", windowID))
+		} else {
+			args = append(args, "-R", rect)
+		}
+		args = append(args, imagePath)
+		command := exec.CommandContext(ctx, "screencapture", args...)
+		if output, err := command.CombinedOutput(); err != nil {
+			if ctx.Err() != nil {
+				return nil
+			}
+			logWarning(fmt.Sprintf("screencapture failed: %v output=%s", err, string(output)))
+		} else if jpeg, err := os.ReadFile(imagePath); err != nil {
+			logWarning("read screencapture frame failed: " + err.Error())
+		} else if len(jpeg) > 0 {
+			state.update(jpeg)
+		}
+
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-ticker.C:
+		}
+	}
+	return nil
+}
+
+func ffmpegCropFilter(window WindowBounds) string {
+	width := int(math.Max(1, math.Round(window.Width)))
+	height := int(math.Max(1, math.Round(window.Height)))
+	x := int(math.Max(0, math.Round(window.X)))
+	y := int(math.Max(0, math.Round(window.Y)))
+	return fmt.Sprintf("crop=%d:%d:%d:%d", width, height, x, y)
+}
+
+func captureRect(window WindowBounds) string {
+	x := int(math.Round(window.X))
+	y := int(math.Round(window.Y))
+	width := int(math.Round(window.Width))
+	height := int(math.Round(window.Height))
+	if x < 0 {
+		width += x
+		x = 0
+	}
+	if y < 0 {
+		height += y
+		y = 0
+	}
+	if width < 1 {
+		width = 1
+	}
+	if height < 1 {
+		height = 1
+	}
+	return fmt.Sprintf("%d,%d,%d,%d", x, y, width, height)
+}
+
+func readJPEGFrames(reader io.Reader, onFrame func([]byte)) error {
+	buffer := make([]byte, 0, 1024*1024)
+	chunk := make([]byte, 64*1024)
+	for {
+		readCount, err := reader.Read(chunk)
+		if readCount > 0 {
+			buffer = append(buffer, chunk[:readCount]...)
+			for {
+				start := bytes.Index(buffer, []byte{0xff, 0xd8})
+				if start < 0 {
+					if len(buffer) > 1024*1024 {
+						buffer = buffer[len(buffer)-2:]
+					}
+					break
+				}
+				endOffset := bytes.Index(buffer[start+2:], []byte{0xff, 0xd9})
+				if endOffset < 0 {
+					if start > 0 {
+						buffer = buffer[start:]
+					}
+					break
+				}
+				end := start + 2 + endOffset + 2
+				frame := append([]byte(nil), buffer[start:end]...)
+				buffer = buffer[end:]
+				onFrame(frame)
+			}
+		}
+		if err != nil {
+			if err == io.EOF {
+				return nil
+			}
+			return err
+		}
+	}
+}
+
+func jpegSize(jpeg []byte) (int, int) {
+	config, _, err := image.DecodeConfig(bytes.NewReader(jpeg))
+	if err != nil {
+		return 0, 0
+	}
+	return config.Width, config.Height
+}
+
+func drainFFmpegStderr(reader io.Reader) {
+	scanner := bufio.NewScanner(reader)
+	for scanner.Scan() {
+		logWarning("ffmpeg: " + scanner.Text())
+	}
+}

demos/go-ffmpeg-wechat-stream/README.md

@@ -0,0 +1,38 @@
+# Go FFmpeg WeChat Stream Demo
+
+Standalone demo: Go finds the visible WeChat window, starts FFmpeg `avfoundation` screen capture, crops to the WeChat window rectangle, and serves an MJPEG stream locally.
+
+If FFmpeg screen capture does not produce frames, the demo automatically falls back to macOS window-only capture with `screencapture -l <window_id>`. This captures the WeChat window itself, so other windows covering it should not appear in the stream.
+
+Run:
+
+```bash
+cd demos/go-ffmpeg-wechat-stream
+go run .
+```
+
+Open:
+
+```text
+http://127.0.0.1:8765/
+```
+
+Options:
+
+```bash
+go run . -input 3:none -fps 15 -port 8765 -open=true -wait=true
+```
+
+Notes:
+
+- macOS only.
+- Requires FFmpeg installed and available in `PATH`.
+- Requires Screen Recording permission for the terminal/app running the demo.
+- WeChat must be visible on screen.
+- By default the demo waits until a visible WeChat window is found. Use `-wait=false` to fail immediately.
+- FFmpeg `avfoundation` is screen-based, so crop mode can include windows covering WeChat. The fallback `screencapture -l` path is window-based and is closer to Tencent Meeting style app-window sharing.
+- `-input` is the FFmpeg `avfoundation` screen device. List devices with:
+
+```bash
+ffmpeg -hide_banner -f avfoundation -list_devices true -i ""
+```

demos/go-ffmpeg-wechat-stream/go.mod

@@ -0,0 +1,3 @@
+module go-ffmpeg-wechat-stream
+
+go 1.22

demos/go-ffmpeg-wechat-stream/main.go

@@ -0,0 +1,470 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"math"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+	"time"
+)
+
+type windowBounds struct {
+	X      float64 `json:"x"`
+	Y      float64 `json:"y"`
+	Width  float64 `json:"width"`
+	Height float64 `json:"height"`
+	Owner  string  `json:"owner"`
+	Title  string  `json:"title"`
+	ID     int     `json:"id"`
+}
+
+type streamState struct {
+	condition *sync.Cond
+	jpeg      []byte
+	sequence  int64
+	window    windowBounds
+	startedAt time.Time
+	updatedAt time.Time
+}
+
+func newStreamState(window windowBounds) *streamState {
+	return &streamState{
+		condition: sync.NewCond(&sync.Mutex{}),
+		window:    window,
+		startedAt: time.Now(),
+	}
+}
+
+func (state *streamState) update(jpeg []byte) {
+	state.condition.L.Lock()
+	state.jpeg = append(state.jpeg[:0], jpeg...)
+	state.sequence++
+	state.updatedAt = time.Now()
+	state.condition.Broadcast()
+	state.condition.L.Unlock()
+}
+
+func (state *streamState) snapshot() (int64, time.Time, windowBounds) {
+	state.condition.L.Lock()
+	defer state.condition.L.Unlock()
+	return state.sequence, state.updatedAt, state.window
+}
+
+func (state *streamState) waitNext(lastSequence int64) ([]byte, int64) {
+	state.condition.L.Lock()
+	defer state.condition.L.Unlock()
+	for state.sequence == lastSequence {
+		state.condition.Wait()
+	}
+	return append([]byte(nil), state.jpeg...), state.sequence
+}
+
+func main() {
+	if runtime.GOOS != "darwin" {
+		log.Fatal("this demo currently supports macOS only because it uses avfoundation + CoreGraphics")
+	}
+
+	var host string
+	var port int
+	var input string
+	var fps float64
+	var openBrowser bool
+	var waitWindow bool
+	flag.StringVar(&host, "host", "127.0.0.1", "HTTP server host")
+	flag.IntVar(&port, "port", 8765, "HTTP server port")
+	flag.StringVar(&input, "input", "3:none", "FFmpeg avfoundation input, e.g. 3:none")
+	flag.Float64Var(&fps, "fps", 15, "capture frame rate")
+	flag.BoolVar(&openBrowser, "open", true, "open browser automatically")
+	flag.BoolVar(&waitWindow, "wait", true, "wait until a visible WeChat window is found")
+	flag.Parse()
+
+	window, err := waitForWeChatWindow(waitWindow)
+	if err != nil {
+		log.Fatalf("find WeChat window failed: %v", err)
+	}
+	log.Printf("WeChat window: owner=%q title=%q id=%d rect=[%.0f,%.0f,%.0f,%.0f]", window.Owner, window.Title, window.ID, window.X, window.Y, window.Width, window.Height)
+
+	state := newStreamState(window)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	go runCapture(ctx, state, input, fps, window)
+
+	server := &http.Server{
+		Addr:              fmt.Sprintf("%s:%d", host, port),
+		Handler:           handler(state),
+		ReadHeaderTimeout: 5 * time.Second,
+	}
+
+	url := fmt.Sprintf("http://%s:%d/", host, port)
+	log.Printf("Serving %s", url)
+	if openBrowser {
+		go func() {
+			time.Sleep(500 * time.Millisecond)
+			_ = exec.Command("open", url).Start()
+		}()
+	}
+
+	if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+		log.Fatal(err)
+	}
+}
+
+func runCapture(ctx context.Context, state *streamState, input string, fps float64, window windowBounds) {
+	ffmpegCtx, cancelFFmpeg := context.WithCancel(ctx)
+	ffmpegErr := make(chan error, 1)
+	go func() {
+		ffmpegErr <- runFFmpeg(ffmpegCtx, state, input, fps, window)
+	}()
+
+	timer := time.NewTimer(3 * time.Second)
+	defer timer.Stop()
+	select {
+	case <-ctx.Done():
+		cancelFFmpeg()
+		return
+	case err := <-ffmpegErr:
+		cancelFFmpeg()
+		log.Printf("FFmpeg stopped before first frame, fallback to screencapture: %v", err)
+	case <-timer.C:
+		sequence, _, _ := state.snapshot()
+		if sequence > 0 {
+			log.Printf("FFmpeg is producing frames")
+			if err := <-ffmpegErr; err != nil && ctx.Err() == nil {
+				log.Printf("FFmpeg stopped: %v", err)
+			}
+			return
+		}
+		cancelFFmpeg()
+		log.Printf("FFmpeg produced no frames in 3s, fallback to window-only screencapture")
+	}
+
+	if err := runScreencapture(ctx, state, fps, window); err != nil && ctx.Err() == nil {
+		log.Printf("screencapture stopped: %v", err)
+	}
+}
+
+func waitForWeChatWindow(wait bool) (windowBounds, error) {
+	for {
+		window, err := findWeChatWindow()
+		if err == nil {
+			return window, nil
+		}
+		if !wait {
+			return windowBounds{}, err
+		}
+		log.Printf("waiting for visible WeChat window: %v", err)
+		time.Sleep(2 * time.Second)
+	}
+}
+
+func findWeChatWindow() (windowBounds, error) {
+	script := `
+import CoreGraphics
+import Foundation
+
+let options = CGWindowListOption(arrayLiteral: [.optionOnScreenOnly, .excludeDesktopElements])
+guard let infos = CGWindowListCopyWindowInfo(options, kCGNullWindowID) as? [[String: Any]] else {
+  exit(1)
+}
+
+for info in infos {
+  let owner = info[kCGWindowOwnerName as String] as? String ?? ""
+  let title = info[kCGWindowName as String] as? String ?? ""
+  let layer = info[kCGWindowLayer as String] as? Int ?? -1
+  if layer == 0 && (owner.contains("微信") || owner.lowercased().contains("wechat") || title == "微信") {
+    if let bounds = info[kCGWindowBounds as String] as? [String: Any],
+       let x = bounds["X"] as? Double,
+       let y = bounds["Y"] as? Double,
+       let w = bounds["Width"] as? Double,
+       let h = bounds["Height"] as? Double {
+      let id = info[kCGWindowNumber as String] as? Int ?? 0
+      let result: [String: Any] = ["x": x, "y": y, "width": w, "height": h, "owner": owner, "title": title, "id": id]
+      let data = try! JSONSerialization.data(withJSONObject: result)
+      print(String(data: data, encoding: .utf8)!)
+      exit(0)
+    }
+  }
+}
+
+exit(2)
+`
+	command := exec.Command("swift", "-")
+	command.Stdin = strings.NewReader(script)
+	output, err := command.Output()
+	if err != nil {
+		return windowBounds{}, err
+	}
+
+	var window windowBounds
+	if err := json.Unmarshal(output, &window); err != nil {
+		return windowBounds{}, err
+	}
+	return window, nil
+}
+
+func runFFmpeg(ctx context.Context, state *streamState, input string, fps float64, window windowBounds) error {
+	args := []string{
+		"-hide_banner",
+		"-loglevel", "warning",
+		"-f", "avfoundation",
+		"-framerate", fmt.Sprintf("%.3f", fps),
+		"-pixel_format", "bgr0",
+		"-i", input,
+		"-vf", cropFilter(window),
+		"-f", "image2pipe",
+		"-vcodec", "mjpeg",
+		"-q:v", "5",
+		"-",
+	}
+	log.Printf("FFmpeg command: ffmpeg %s", strings.Join(args, " "))
+
+	command := exec.CommandContext(ctx, "ffmpeg", args...)
+	stdout, err := command.StdoutPipe()
+	if err != nil {
+		return err
+	}
+	stderr, err := command.StderrPipe()
+	if err != nil {
+		return err
+	}
+
+	if err := command.Start(); err != nil {
+		return err
+	}
+	go logLines("ffmpeg", stderr)
+
+	readErr := readJPEGFrames(stdout, state.update)
+	waitErr := command.Wait()
+	if ctx.Err() != nil {
+		return nil
+	}
+	if readErr != nil {
+		return readErr
+	}
+	return waitErr
+}
+
+func runScreencapture(ctx context.Context, state *streamState, fps float64, window windowBounds) error {
+	interval := time.Duration(float64(time.Second) / math.Max(fps, 1))
+	if interval < 80*time.Millisecond {
+		interval = 80 * time.Millisecond
+	}
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+
+	tempDir, err := os.MkdirTemp("", "go-ffmpeg-wechat-stream-*")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tempDir)
+
+	framePath := filepath.Join(tempDir, "frame.jpg")
+	rect := captureRect(window)
+	log.Printf("window-only screencapture fallback started window_id=%d rect=%s", window.ID, rect)
+	for ctx.Err() == nil {
+		command := exec.CommandContext(ctx, "screencapture", "-x", "-l", fmt.Sprintf("%d", window.ID), framePath)
+		if output, err := command.CombinedOutput(); err != nil {
+			if ctx.Err() != nil {
+				return nil
+			}
+			log.Printf("screencapture failed: %v output=%s", err, string(output))
+		} else if jpeg, err := os.ReadFile(framePath); err != nil {
+			log.Printf("read frame failed: %v", err)
+		} else if len(jpeg) > 0 {
+			state.update(jpeg)
+		}
+
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-ticker.C:
+		}
+	}
+	return nil
+}
+
+func captureRect(window windowBounds) string {
+	x := int(math.Round(window.X))
+	y := int(math.Round(window.Y))
+	width := int(math.Round(window.Width))
+	height := int(math.Round(window.Height))
+	if x < 0 {
+		width += x
+		x = 0
+	}
+	if y < 0 {
+		height += y
+		y = 0
+	}
+	if width < 1 {
+		width = 1
+	}
+	if height < 1 {
+		height = 1
+	}
+	return fmt.Sprintf("%d,%d,%d,%d", x, y, width, height)
+}
+
+func cropFilter(window windowBounds) string {
+	width := int(math.Max(1, math.Round(window.Width)))
+	height := int(math.Max(1, math.Round(window.Height)))
+	x := int(math.Max(0, math.Round(window.X)))
+	y := int(math.Max(0, math.Round(window.Y)))
+	return fmt.Sprintf("crop=%d:%d:%d:%d", width, height, x, y)
+}
+
+func readJPEGFrames(reader io.Reader, onFrame func([]byte)) error {
+	buffer := make([]byte, 0, 1024*1024)
+	chunk := make([]byte, 64*1024)
+	for {
+		readCount, err := reader.Read(chunk)
+		if readCount > 0 {
+			buffer = append(buffer, chunk[:readCount]...)
+			for {
+				start := bytes.Index(buffer, []byte{0xff, 0xd8})
+				if start < 0 {
+					if len(buffer) > 1024*1024 {
+						buffer = buffer[len(buffer)-2:]
+					}
+					break
+				}
+				endOffset := bytes.Index(buffer[start+2:], []byte{0xff, 0xd9})
+				if endOffset < 0 {
+					if start > 0 {
+						buffer = buffer[start:]
+					}
+					break
+				}
+				end := start + 2 + endOffset + 2
+				onFrame(append([]byte(nil), buffer[start:end]...))
+				buffer = buffer[end:]
+			}
+		}
+		if err != nil {
+			if err == io.EOF {
+				return nil
+			}
+			return err
+		}
+	}
+}
+
+func handler(state *streamState) http.Handler {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) {
+		writer.Header().Set("Content-Type", "text/html; charset=utf-8")
+		_, _ = io.WriteString(writer, `<!doctype html>
+<html>
+<head>
+  <meta charset="utf-8" />
+  <title>Go FFmpeg WeChat Stream</title>
+  <style>
+    html, body { margin: 0; height: 100%; background: #050505; color: #d7ffe4; font-family: ui-monospace, SFMono-Regular, Menlo, monospace; }
+    .wrap { height: 100%; display: grid; grid-template-rows: auto 1fr; }
+    .bar { display: flex; gap: 16px; align-items: center; padding: 10px 14px; background: #101510; border-bottom: 1px solid #263526; font-size: 13px; }
+    img { width: 100%; height: 100%; object-fit: contain; background: #000; }
+  </style>
+</head>
+<body>
+  <div class="wrap">
+    <div class="bar"><strong>Go FFmpeg WeChat Stream</strong><span id="status">connecting...</span></div>
+    <img src="/stream.mjpg" />
+  </div>
+  <script>
+    let prevSeq = null, prevTime = performance.now();
+    async function tick() {
+      try {
+        const res = await fetch('/latest.json?t=' + Date.now(), {cache: 'no-store'});
+        const data = await res.json();
+        const now = performance.now();
+        let fps = '--';
+        if (prevSeq !== null && data.sequence >= prevSeq) fps = ((data.sequence - prevSeq) / ((now - prevTime) / 1000)).toFixed(1);
+        prevSeq = data.sequence;
+        prevTime = now;
+        document.getElementById('status').textContent = 'seq=' + data.sequence + ' fps=' + fps + ' window=' + data.window.width + 'x' + data.window.height;
+      } catch (err) {
+        document.getElementById('status').textContent = 'waiting for stream';
+      }
+    }
+    tick(); setInterval(tick, 1000);
+  </script>
+</body>
+</html>`)
+	})
+	mux.HandleFunc("/latest.json", func(writer http.ResponseWriter, request *http.Request) {
+		sequence, updatedAt, window := state.snapshot()
+		writer.Header().Set("Content-Type", "application/json; charset=utf-8")
+		writer.Header().Set("Cache-Control", "no-store")
+		_ = json.NewEncoder(writer).Encode(map[string]any{
+			"source":     "go-ffmpeg-demo",
+			"sequence":   sequence,
+			"updated_at": updatedAt.Format(time.RFC3339Nano),
+			"window":     window,
+		})
+	})
+	mux.HandleFunc("/stream.mjpg", func(writer http.ResponseWriter, request *http.Request) {
+		flusher, ok := writer.(http.Flusher)
+		if !ok {
+			http.Error(writer, "streaming unsupported", http.StatusInternalServerError)
+			return
+		}
+		writer.Header().Set("Content-Type", "multipart/x-mixed-replace; boundary=frame")
+		writer.Header().Set("Cache-Control", "no-store")
+		lastSequence := int64(-1)
+		for request.Context().Err() == nil {
+			jpeg, sequence := state.waitNext(lastSequence)
+			lastSequence = sequence
+			if len(jpeg) == 0 {
+				continue
+			}
+			if _, err := fmt.Fprintf(writer, "--frame\r\nContent-Type: image/jpeg\r\nContent-Length: %d\r\n\r\n", len(jpeg)); err != nil {
+				return
+			}
+			if _, err := writer.Write(jpeg); err != nil {
+				return
+			}
+			if _, err := writer.Write([]byte("\r\n")); err != nil {
+				return
+			}
+			flusher.Flush()
+		}
+	})
+	return mux
+}
+
+func logLines(prefix string, reader io.Reader) {
+	buffer := make([]byte, 4096)
+	var line []byte
+	for {
+		count, err := reader.Read(buffer)
+		if count > 0 {
+			line = append(line, buffer[:count]...)
+			for {
+				index := bytes.IndexByte(line, '\n')
+				if index < 0 {
+					break
+				}
+				log.Printf("%s: %s", prefix, strings.TrimSpace(string(line[:index])))
+				line = line[index+1:]
+			}
+		}
+		if err != nil {
+			if len(line) > 0 {
+				log.Printf("%s: %s", prefix, strings.TrimSpace(string(line)))
+			}
+			return
+		}
+	}
+}

src-tauri/src/lib.rs

@@ -2,6 +2,7 @@ use screenshots::Screen;
 use serde::{Deserialize, Serialize};
 use std::fs;
 use std::path::PathBuf;
+use std::process::Command;
 use std::sync::Mutex;
 use std::time::Instant;
 use tauri::{Emitter, Manager, WebviewUrl, WebviewWindowBuilder};
@@ -12,6 +13,7 @@ use tauri_plugin_shell::{
 use xcap::{Monitor, Window};
 
 struct AgentProcess(Mutex<Option<CommandChild>>);
+struct VisionStreamProcess(Mutex<Option<CommandChild>>);
 
 #[derive(Clone, Deserialize, Serialize)]
 struct AgentLog {
@@ -240,6 +242,110 @@ fn stop_agent(app: tauri::AppHandle, process: tauri::State<AgentProcess>) -> Res
     Ok(())
 }
 
+#[tauri::command]
+fn start_vision_stream(
+    app: tauri::AppHandle,
+    process: tauri::State<VisionStreamProcess>,
+) -> Result<(), String> {
+    let mut process_guard = process.0.lock().map_err(|error| error.to_string())?;
+    if process_guard.is_some() {
+        return Ok(());
+    }
+
+    stop_legacy_python_vision_stream();
+
+    let (mut rx, child) = app
+        .shell()
+        .sidecar("agent")
+        .map_err(|error| error.to_string())?
+        .args(["vision-stream"])
+        .spawn()
+        .map_err(|error| error.to_string())?;
+
+    *process_guard = Some(child);
+
+    let app_handle = app.clone();
+    tauri::async_runtime::spawn(async move {
+        while let Some(event) = rx.recv().await {
+            match event {
+                CommandEvent::Stdout(line) => {
+                    emit_agent_line(
+                        &app_handle,
+                        "info",
+                        String::from_utf8_lossy(&line).trim().to_string(),
+                    );
+                }
+                CommandEvent::Stderr(line) => {
+                    emit_agent_line(
+                        &app_handle,
+                        "error",
+                        String::from_utf8_lossy(&line).trim().to_string(),
+                    );
+                }
+                CommandEvent::Error(error) => {
+                    emit_agent_log(
+                        &app_handle,
+                        "error",
+                        format!("vision stream error: {error}"),
+                    );
+                }
+                CommandEvent::Terminated(payload) => {
+                    emit_agent_log(
+                        &app_handle,
+                        "warning",
+                        format!("vision stream exited with code {:?}", payload.code),
+                    );
+                    if let Ok(mut process_guard) =
+                        app_handle.state::<VisionStreamProcess>().0.lock()
+                    {
+                        *process_guard = None;
+                    }
+                    break;
+                }
+                _ => {}
+            }
+        }
+    });
+
+    Ok(())
+}
+
+fn stop_legacy_python_vision_stream() {
+    let Ok(cwd) = std::env::current_dir() else {
+        return;
+    };
+    let candidates = [
+        cwd.join("wechat_vision"),
+        cwd.join("..").join("wechat_vision"),
+        cwd.join("..").join("..").join("wechat_vision"),
+    ];
+
+    for dir in candidates {
+        let pid_file = dir.join("ouptsw").join("wechat_algorithm_live.pid");
+        let stop_script = dir.join("stop_wechat_algorithm_live.sh");
+        if pid_file.exists() && stop_script.exists() {
+            let _ = Command::new("bash")
+                .arg(stop_script)
+                .current_dir(dir)
+                .output();
+            return;
+        }
+    }
+}
+
+#[tauri::command]
+fn stop_vision_stream(
+    _app: tauri::AppHandle,
+    process: tauri::State<VisionStreamProcess>,
+) -> Result<(), String> {
+    let mut process_guard = process.0.lock().map_err(|error| error.to_string())?;
+    if let Some(child) = process_guard.take() {
+        child.kill().map_err(|error| error.to_string())?;
+    }
+
+    Ok(())
+}
+
 #[tauri::command]
 async fn capture_screen(app: tauri::AppHandle) -> Result<CaptureResult, String> {
     let screenshot_path = screenshot_path(&app)?;
@@ -490,6 +596,12 @@ fn open_popup_window(app: tauri::AppHandle, route: String) -> Result<(), String>
     let label = format!("popup-{}", route.trim_start_matches('/').replace('/', "-"));
 
     if let Some(window) = app.get_webview_window(&label) {
+        if route == "/window/engine-workbench" {
+            window
+                .set_fullscreen(false)
+                .map_err(|error| error.to_string())?;
+            window.maximize().map_err(|error| error.to_string())?;
+        }
         window.set_focus().map_err(|error| error.to_string())?;
         return Ok(());
     }
@@ -497,6 +609,7 @@ fn open_popup_window(app: tauri::AppHandle, route: String) -> Result<(), String>
     let title = match route.as_str() {
         "/annotate" => "屏幕区域标注",
         "/window/settings" => "设置",
+        "/window/engine-workbench" => "微信引擎工作台",
         "/window/engine-logs" => "分身引擎日志",
         "/window/knowledge-create" => "新增知识库",
         "/window/knowledge-detail" => "知识库详情",
@@ -519,6 +632,11 @@ fn open_popup_window(app: tauri::AppHandle, route: String) -> Result<(), String>
 
     if route == "/annotate" {
         builder = builder.maximized(true).fullscreen(true);
+    } else if route == "/window/engine-workbench" {
+        builder = builder
+            .inner_size(1440.0, 920.0)
+            .maximized(true)
+            .fullscreen(false);
     }
 
     builder.build().map_err(|error| error.to_string())?;
@@ -530,6 +648,7 @@ fn open_popup_window(app: tauri::AppHandle, route: String) -> Result<(), String>
 pub fn run() {
     tauri::Builder::default()
         .manage(AgentProcess(Mutex::new(None)))
+        .manage(VisionStreamProcess(Mutex::new(None)))
         .plugin(tauri_plugin_shell::init())
         .setup(|app| {
             if cfg!(debug_assertions) {
@@ -549,6 +668,8 @@ pub fn run() {
             open_popup_window,
             save_regions,
             start_agent,
+            start_vision_stream,
+            stop_vision_stream,
             stop_agent
         ])
         .run(tauri::generate_context!())

src/components/Terminal.jsx

@@ -27,7 +27,7 @@ export default function Terminal({ title, lines, tall = false, onClear }) {
   };
 
   return (
-    <div className={`terminal ${tall ? "basis-[650px]" : "basis-[250px]"} flex min-h-0 flex-1 flex-col`}>
+    <div className={`terminal ${tall ? "min-h-[520px] basis-[650px]" : "min-h-[220px] basis-[250px]"} flex flex-1 flex-col`}>
       <div className="mb-3 flex shrink-0 items-center justify-between">
         <h3 className="text-[13px] font-black text-[#d7ffe4]">{title}</h3>
         <button className="cursor-pointer border-0 bg-transparent p-0 text-[11px] font-bold text-[#7ea58b]" onClick={onClear}>清理日志</button>

src/components/TitleBar.jsx

@@ -2,7 +2,7 @@ import { LogOut, RotateCcw } from "lucide-react";
 import ThemeSwitch from "./ThemeSwitch";
 import { closeWindow, startWindowDrag } from "../utils/navigation";
 
-export default function TitleBar({ title, theme, setTheme, compact = false }) {
+export default function TitleBar({ title, theme, setTheme, compact = false, onClose }) {
   return (
     <header className="glass-bar relative z-10 flex items-center justify-between gap-4 border-b border-line px-5 py-2">
       <div
@@ -24,7 +24,7 @@ export default function TitleBar({ title, theme, setTheme, compact = false }) {
             <button className="icon-btn" aria-label="退出"><LogOut size={16} strokeWidth={2.5} /></button>
           </>
         ) : (
-          <button className="btn-secondary h-9 px-3" onClick={closeWindow}>关闭</button>
+          <button className="btn-secondary h-9 px-3" onClick={onClose || closeWindow}>关闭</button>
         )}
       </div>
     </header>

src/pages/ClonePage.jsx

@@ -8,8 +8,8 @@ import { openWindow } from "../utils/navigation";
 
 export default function ClonePage() {
   const [engineEnabled, setEngineEnabled] = useState(false);
+  const [engineBusy, setEngineBusy] = useState(false);
   const [agentLogs, setAgentLogs] = useState([]);
-  const [regionsFile, setRegionsFile] = useState(null);
   const [annotating, setAnnotating] = useState(false);
   const [sourcePickerOpen, setSourcePickerOpen] = useState(false);
   const [captureSources, setCaptureSources] = useState([]);
@@ -20,15 +20,18 @@ export default function ClonePage() {
 
     let unlisten;
     let cancelled = false;
-
+    listen("engine-state-changed", (event) => {
-    listen("agent-log", (event) => {
+      setEngineEnabled(Boolean(event.payload?.enabled));
-      setAgentLogs((currentLogs) => [...currentLogs.slice(-300), event.payload]);
+      setEngineBusy(false);
+      setAgentLogs((currentLogs) => [
+        ...currentLogs.slice(-300),
+        { level: "warning", message: "工作台已关闭，引擎状态已同步" },
+      ]);
     }).then((cleanup) => {
       if (cancelled) {
         cleanup();
         return;
       }
-
       unlisten = cleanup;
     });
 
@@ -38,24 +41,6 @@ export default function ClonePage() {
     };
   }, []);
 
-  useEffect(() => {
-    loadSavedRegions();
-  }, []);
-
-  async function loadSavedRegions() {
-    if (!window.__TAURI_INTERNALS__) return;
-
-    try {
-      const saved = await invoke("load_regions");
-      setRegionsFile(saved);
-    } catch (error) {
-      setAgentLogs((currentLogs) => [
-        ...currentLogs,
-        { level: "error", message: `读取标注失败：${String(error)}` },
-      ]);
-    }
-  }
-
   async function startAnnotation() {
     if (!window.__TAURI_INTERNALS__) {
       window.location.hash = "/annotate";
@@ -103,24 +88,50 @@ export default function ClonePage() {
   }
 
   async function toggleAgent() {
+    if (engineBusy) return;
+
     if (!window.__TAURI_INTERNALS__) {
       setEngineEnabled((enabled) => !enabled);
       return;
     }
 
+    setEngineBusy(true);
     try {
       if (engineEnabled) {
         await invoke("stop_agent");
+        await invoke("stop_vision_stream").catch((error) => {
+          setAgentLogs((currentLogs) => [
+            ...currentLogs,
+            { level: "warning", message: `停止视觉流失败：${String(error)}` },
+          ]);
+        });
         setEngineEnabled(false);
+        setAgentLogs((currentLogs) => [
+          ...currentLogs.slice(-300),
+          { level: "warning", message: "引擎已停用，暂未读取 Rust 日志" },
+        ]);
       } else {
-        await invoke("start_agent");
+        await invoke("start_vision_stream");
+        await invoke("start_agent").catch((error) => {
+          setAgentLogs((currentLogs) => [
+            ...currentLogs.slice(-300),
+            { level: "warning", message: `agent 启动失败，预览已继续启动：${String(error)}` },
+          ]);
+        });
         setEngineEnabled(true);
+        setAgentLogs((currentLogs) => [
+          ...currentLogs.slice(-300),
+          { level: "info", message: "引擎已启动，暂未读取 Rust 日志" },
+        ]);
+        await openWindow("/window/engine-workbench");
       }
     } catch (error) {
       setAgentLogs((currentLogs) => [
         ...currentLogs,
         { level: "error", message: String(error) },
       ]);
+    } finally {
+      setEngineBusy(false);
     }
   }
 
@@ -139,9 +150,10 @@ export default function ClonePage() {
           <button
             className={`${engineEnabled ? "btn-danger" : "btn-primary"} w-full gap-2`}
             onClick={toggleAgent}
+            disabled={engineBusy}
           >
             {engineEnabled ? <PowerOff size={16} strokeWidth={2.6} /> : <Power size={16} strokeWidth={2.6} />}
-            {engineEnabled ? "停用引擎" : "启动引擎"}
+            {engineBusy ? "处理中" : engineEnabled ? "停用引擎" : "启动引擎"}
           </button>
           <button className="icon-action shrink-0" onClick={() => openWindow("/window/settings")} aria-label="设置"><Settings size={18} strokeWidth={2.5} /></button>
         </div>
@@ -157,21 +169,6 @@ export default function ClonePage() {
             {annotating ? "截屏中" : "开始标注"}
           </button>
         </div>
-        <div className="mt-3 space-y-2">
-          {regionsFile?.regions?.length ? (
-            regionsFile.regions.map((region) => (
-              <div key={region.id} className="list-card !rounded-[12px] !p-3">
-                <div className="min-w-0">
-                  <div className="truncate text-[13px] font-extrabold">{region.name}</div>
-                  <div className="mt-1 font-mono text-[11px] text-muted">screen [{region.bbox_screen.join(", ")}]</div>
-                </div>
-                <span className="tag shrink-0">{region.type}</span>
-              </div>
-            ))
-          ) : (
-            <div className="rounded-[12px] border border-dashed border-line p-3 text-center text-[12px] text-muted">暂无已保存区域</div>
-          )}
-        </div>
       </div>
       {sourcePickerOpen ? (
         <div className="fixed inset-0 z-50 grid place-items-center bg-black/35 px-5">

src/windows/EngineWorkbench.jsx

@@ -0,0 +1,274 @@
+import { useEffect, useMemo, useRef, useState } from "react";
+import { invoke } from "@tauri-apps/api/core";
+import { Bot, SendHorizontal, UserRound } from "lucide-react";
+
+const MONITOR_ENDPOINT = "http://127.0.0.1:8765";
+
+function messageLabel(role) {
+  if (role === "me") return "我";
+  if (role === "user") return "客户";
+  return "未知";
+}
+
+export default function EngineWorkbench() {
+  const [monitorEvent, setMonitorEvent] = useState(null);
+  const [streamUrl, setStreamUrl] = useState(`${MONITOR_ENDPOINT}/frame.jpg?t=${Date.now()}`);
+  const [streamFps, setStreamFps] = useState(0);
+  const [streamError, setStreamError] = useState("");
+  const [annotation, setAnnotation] = useState(null);
+  const [frameMeta, setFrameMeta] = useState(null);
+  const [videoBox, setVideoBox] = useState({ width: 0, height: 0 });
+  const [question, setQuestion] = useState("");
+  const [qaMessages, setQaMessages] = useState([]);
+  const sequenceRef = useRef({ sequence: null, time: 0 });
+  const streamRefreshedRef = useRef(false);
+
+  useEffect(() => {
+    if (!window.__TAURI_INTERNALS__) return;
+    invoke("start_vision_stream").catch((error) => {
+      setStreamError(`预览服务启动失败：${String(error)}`);
+    });
+    invoke("load_regions")
+      .then((saved) => setAnnotation(saved || null))
+      .catch(() => setAnnotation(null));
+  }, []);
+
+  useEffect(() => {
+    const updateBox = () => {
+      const element = document.getElementById("wechat-preview-frame");
+      if (!element) return;
+      const rect = element.getBoundingClientRect();
+      setVideoBox({ width: rect.width, height: rect.height });
+    };
+    updateBox();
+    window.addEventListener("resize", updateBox);
+    return () => window.removeEventListener("resize", updateBox);
+  }, []);
+
+  useEffect(() => {
+    let cancelled = false;
+
+    async function pollMonitor() {
+      try {
+        const response = await fetch(`${MONITOR_ENDPOINT}/latest.json?t=${Date.now()}`, { cache: "no-store" });
+        const data = await response.json();
+        if (!cancelled) {
+          if (data.source !== "go-window-capture") {
+            setMonitorEvent(null);
+            setStreamFps(0);
+            setStreamError(`预览服务来源异常：${data.source || "unknown"}`);
+            return;
+          }
+          if (data.event?.type === "source_waiting") {
+            setMonitorEvent(null);
+            setStreamFps(0);
+            setStreamError(data.event.message || "等待标注来源窗口出现...");
+            setFrameMeta(data.frame || null);
+            return;
+          }
+          setStreamError("");
+          setFrameMeta(data.frame || null);
+          const now = performance.now();
+          const previous = sequenceRef.current;
+          if (typeof data.sequence === "number" && typeof previous.sequence === "number" && previous.time) {
+            const elapsedSeconds = Math.max((now - previous.time) / 1000, 0.001);
+            setStreamFps(Math.max(0, (data.sequence - previous.sequence) / elapsedSeconds));
+          }
+          sequenceRef.current = { sequence: data.sequence, time: now };
+          setMonitorEvent(data.event?.type === "preview_frame" ? data.event : null);
+          if (typeof data.sequence === "number" && data.sequence > 0) {
+            streamRefreshedRef.current = true;
+            setStreamUrl(`${MONITOR_ENDPOINT}/frame.jpg?seq=${data.sequence}&t=${Date.now()}`);
+          }
+        }
+      } catch {
+        if (!cancelled) {
+          setMonitorEvent(null);
+          setStreamError("等待预览服务启动...");
+        }
+      }
+    }
+
+    pollMonitor();
+    const timer = window.setInterval(pollMonitor, 500);
+    return () => {
+      cancelled = true;
+      window.clearInterval(timer);
+    };
+  }, []);
+
+  const latestLlm = monitorEvent?.latest_llm || monitorEvent?.llm_chat_reading || null;
+  const visibleMessages = useMemo(() => latestLlm?.visible_messages || [], [latestLlm]);
+  const latestUserMessage = latestLlm?.latest_user_message || "等待识别当前聊天记录";
+  const replyText = latestLlm?.reply_text || "暂无建议回复";
+  const overlay = useMemo(() => {
+    const frameWidth = frameMeta?.width || annotation?.screenshotWidth;
+    const frameHeight = frameMeta?.height || annotation?.screenshotHeight;
+    if (!frameWidth || !frameHeight || !videoBox.width || !videoBox.height) return null;
+    const scale = Math.min(videoBox.width / frameWidth, videoBox.height / frameHeight);
+    const width = frameWidth * scale;
+    const height = frameHeight * scale;
+    return {
+      left: (videoBox.width - width) / 2,
+      top: (videoBox.height - height) / 2,
+      scale,
+      frameWidth,
+      frameHeight,
+    };
+  }, [annotation, frameMeta, videoBox]);
+
+  function regionStyle(region) {
+    if (!overlay || !region.bbox_image) return null;
+    const sourceWidth = annotation?.screenshotWidth || overlay.frameWidth;
+    const sourceHeight = annotation?.screenshotHeight || overlay.frameHeight;
+    const [ix1, iy1, ix2, iy2] = region.bbox_image;
+    const x1 = (ix1 * overlay.frameWidth) / sourceWidth;
+    const y1 = (iy1 * overlay.frameHeight) / sourceHeight;
+    const x2 = (ix2 * overlay.frameWidth) / sourceWidth;
+    const y2 = (iy2 * overlay.frameHeight) / sourceHeight;
+    return {
+      left: overlay.left + x1 * overlay.scale,
+      top: overlay.top + y1 * overlay.scale,
+      width: Math.max(1, (x2 - x1) * overlay.scale),
+      height: Math.max(1, (y2 - y1) * overlay.scale),
+    };
+  }
+
+  function submitQuestion(event) {
+    event.preventDefault();
+    const trimmed = question.trim();
+    if (!trimmed) return;
+
+    const contextLines = visibleMessages.map((message) => `${messageLabel(message.role)}：${message.content}`).join("\n");
+    const answer = contextLines
+      ? `基于当前已识别聊天记录：\n${contextLines}\n\n建议回复：${replyText}`
+      : "当前还没有读取到聊天记录。请先确认监控服务已启动，并等待检测到新消息后再询问。";
+
+    setQaMessages((current) => [
+      ...current,
+      { role: "user", content: trimmed },
+      { role: "assistant", content: answer },
+    ]);
+    setQuestion("");
+  }
+
+  return (
+    <div className="grid h-full min-h-0 grid-cols-[minmax(0,1fr)_minmax(320px,400px)] gap-4 p-4 max-lg:grid-cols-1">
+      <section className="flex min-h-0 flex-col gap-4">
+        <div className="panel flex min-h-0 flex-1 flex-col overflow-hidden p-4">
+          <div className="mb-3 flex items-center justify-between gap-3">
+            <div>
+              <h2 className="text-[16px] font-extrabold">微信窗口实时画面</h2>
+              <p className="mt-1 text-[12px] text-muted">
+                {monitorEvent?.target_label ? `预览来源：${monitorEvent.target_label}` : `来自本地监控流：${MONITOR_ENDPOINT}/stream.mjpg`}
+              </p>
+            </div>
+            <div className="flex shrink-0 items-center gap-2">
+              <span className="tag">{streamFps ? `${streamFps.toFixed(1)} FPS` : "FPS --"}</span>
+              <span className={`tag ${monitorEvent ? "text-primary" : ""}`}>{monitorEvent ? "在线" : "等待监控服务"}</span>
+            </div>
+          </div>
+          <div id="wechat-preview-frame" className="relative min-h-0 flex-1 overflow-hidden rounded-[14px] border border-line bg-black">
+            <img
+              src={streamUrl}
+              alt="微信窗口实时画面"
+              className="h-full w-full object-contain"
+              draggable={false}
+              onError={() => {
+                window.setTimeout(() => {
+                  setStreamUrl(`${MONITOR_ENDPOINT}/frame.jpg?t=${Date.now()}`);
+                }, 1000);
+              }}
+            />
+            <div className="absolute left-3 top-3 rounded-full bg-black/70 px-3 py-1 font-mono text-[12px] font-bold text-[#86efac]">
+              {streamFps ? `${streamFps.toFixed(1)} FPS` : "等待帧率"}
+            </div>
+            {annotation?.regions?.map((region) => {
+              const style = regionStyle(region);
+              if (!style) return null;
+              return (
+                <div
+                  key={region.id}
+                  className="pointer-events-none absolute rounded-[8px] border-2 border-[#22c55e] bg-[#22c55e]/10 shadow-[0_0_0_1px_rgba(0,0,0,.45)]"
+                  style={style}
+                >
+                  <div className="absolute -left-0.5 -top-6 rounded bg-[#22c55e] px-2 py-0.5 text-[11px] font-black text-black shadow">
+                    {region.type || region.name}
+                  </div>
+                </div>
+              );
+            })}
+            {streamError ? (
+              <div className="absolute inset-x-0 bottom-4 mx-auto w-fit max-w-[80%] rounded-full bg-black/75 px-4 py-2 text-center text-[12px] font-bold text-[#fca5a5]">
+                {streamError}
+              </div>
+            ) : null}
+          </div>
+        </div>
+      </section>
+
+      <aside className="panel flex min-h-0 max-w-[400px] flex-col overflow-hidden p-4 max-lg:max-w-none">
+        <div className="mb-3 shrink-0">
+          <h2 className="text-[16px] font-extrabold">客户聊天记录</h2>
+          <p className="mt-1 text-[12px] text-muted">展示最近一次 LLM 读取到的对话和建议回复。</p>
+        </div>
+
+        <div className="scrollbar-hidden min-h-0 flex-1 space-y-3 overflow-y-auto pr-1">
+          <div className="rounded-[14px] border border-line bg-surface2 p-3">
+            <div className="text-[11px] font-bold text-muted">最新客户消息</div>
+            <div className="mt-2 text-[14px] font-bold leading-6">{latestUserMessage}</div>
+          </div>
+
+          <div className="space-y-2">
+            {visibleMessages.length ? (
+              visibleMessages.map((message, index) => (
+                <div
+                  key={`${message.role}-${message.content}-${index}`}
+                  className={`rounded-[14px] p-3 ${message.role === "me" ? "bg-[var(--primary-soft)]" : "bg-surface2"}`}
+                >
+                  <div className="mb-1 flex items-center gap-2 text-[11px] font-black text-muted">
+                    {message.role === "me" ? <UserRound size={13} /> : <Bot size={13} />}
+                    {message.sender || messageLabel(message.role)}
+                  </div>
+                  <div className="text-[13px] leading-5">{message.content}</div>
+                </div>
+              ))
+            ) : (
+              <div className="rounded-[14px] border border-dashed border-line p-4 text-center text-[12px] text-muted">
+                暂无聊天记录，等待监测到新消息后读取。
+              </div>
+            )}
+          </div>
+
+          <div className="rounded-[14px] border border-primary/30 bg-[var(--primary-soft)] p-3">
+            <div className="text-[11px] font-black text-primary">建议回复</div>
+            <div className="mt-2 text-[13px] leading-5">{replyText}</div>
+          </div>
+
+          {qaMessages.length ? (
+            <div className="space-y-2 border-t border-line pt-3">
+              {qaMessages.map((message, index) => (
+                <div key={`${message.role}-${index}`} className="rounded-[12px] bg-surface2 p-3 text-[12px] leading-5">
+                  <div className="mb-1 font-black text-muted">{message.role === "user" ? "我" : "AI"}</div>
+                  <div className="whitespace-pre-wrap">{message.content}</div>
+                </div>
+              ))}
+            </div>
+          ) : null}
+        </div>
+
+        <form onSubmit={submitQuestion} className="mt-3 flex shrink-0 items-center gap-2 border-t border-line pt-3">
+          <input
+            value={question}
+            onChange={(event) => setQuestion(event.target.value)}
+            placeholder="询问客户信息或当前对话..."
+            className="min-w-0 flex-1 rounded-[12px] border border-line bg-surface px-3 py-2 text-[13px] outline-none transition focus:border-primary"
+          />
+          <button className="btn-primary h-10 shrink-0 px-3" type="submit" aria-label="发送问题">
+            <SendHorizontal size={16} strokeWidth={2.6} />
+          </button>
+        </form>
+      </aside>
+    </div>
+  );
+}

src/windows/SecondaryWindow.jsx

@@ -1,13 +1,18 @@
 import { useMemo } from "react";
+import { invoke } from "@tauri-apps/api/core";
+import { emit } from "@tauri-apps/api/event";
 import { logs } from "../data/mockData";
 import Terminal from "../components/Terminal";
 import TitleBar from "../components/TitleBar";
+import EngineWorkbench from "./EngineWorkbench";
 import PlaceholderWindow from "./PlaceholderWindow";
 import SettingsWindow from "./SettingsWindow";
+import { closeWindow } from "../utils/navigation";
 
 export default function SecondaryWindow({ route, theme, setTheme, settingSection, setSettingSection }) {
   const title = useMemo(() => {
     if (route.includes("settings")) return "设置";
+    if (route.includes("engine-workbench")) return "微信引擎工作台";
     if (route.includes("engine-logs")) return "分身引擎日志";
     if (route.includes("knowledge-create")) return "新增知识库";
     if (route.includes("knowledge-detail")) return "知识库详情";
@@ -17,12 +22,27 @@ export default function SecondaryWindow({ route, theme, setTheme, settingSection
     return "二级窗口";
   }, [route]);
 
+  async function closeEngineWorkbench() {
+    await invoke("stop_agent").catch(() => {});
+    await invoke("stop_vision_stream").catch(() => {});
+    await emit("engine-state-changed", { enabled: false });
+    await closeWindow();
+  }
+
   return (
     <div className="app-window flex h-dvh w-full flex-col overflow-hidden rounded-[12px]">
-      <TitleBar title={title} theme={theme} setTheme={setTheme} compact />
+      <TitleBar
-      <main className="content-canvas flex min-h-0 flex-1 flex-col overflow-y-auto">
+        title={title}
+        theme={theme}
+        setTheme={setTheme}
+        compact
+        onClose={route.includes("engine-workbench") ? closeEngineWorkbench : undefined}
+      />
+      <main className="content-canvas flex min-h-0 flex-1 flex-col overflow-hidden">
         {route.includes("settings") ? (
           <SettingsWindow active={settingSection} setActive={setSettingSection} />
+        ) : route.includes("engine-workbench") ? (
+          <EngineWorkbench />
         ) : route.includes("engine-logs") ? (
           <div className="flex h-full min-h-0 p-5"><Terminal title="完整运行日志" lines={[...logs, ...logs, ...logs]} tall /></div>
         ) : (

vision_test/app.py

@@ -0,0 +1,6 @@
+
+
+# 测试python服务
+
+print("hello world")
+

wechat_vision/start_wechat_algorithm_live.sh

@@ -27,7 +27,6 @@ nohup ./venv/bin/python wechat_algorithm_live.py \
   --send-reply \
   --typing-chunk-size 2 \
   --typing-delay 0.08 \
-  --open-browser \
   > "$LOG_FILE" 2>&1 &
 
 PID="$!"