From 53d48c450a25d94d703de97168f21d21584d3634 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=A3=AE?= Date: Thu, 25 Jun 2026 09:07:24 +0800 Subject: [PATCH] Add WeChat vision streaming workbench --- .codegraph/.gitignore | 21 +- .codegraph/daemon.pid | 6 - .gitignore | 3 + agent/main.go | 8 + agent/vision_stream.go | 668 +++++++++++++++++++ demos/go-ffmpeg-wechat-stream/README.md | 38 ++ demos/go-ffmpeg-wechat-stream/go.mod | 3 + demos/go-ffmpeg-wechat-stream/main.go | 470 +++++++++++++ src-tauri/src/lib.rs | 121 ++++ src/components/Terminal.jsx | 2 +- src/components/TitleBar.jsx | 4 +- src/pages/ClonePage.jsx | 77 +-- src/windows/EngineWorkbench.jsx | 274 ++++++++ src/windows/SecondaryWindow.jsx | 24 +- vision_test/app.py | 6 + wechat_vision/start_wechat_algorithm_live.sh | 1 - 16 files changed, 1658 insertions(+), 68 deletions(-) delete mode 100644 .codegraph/daemon.pid create mode 100644 agent/vision_stream.go create mode 100644 demos/go-ffmpeg-wechat-stream/README.md create mode 100644 demos/go-ffmpeg-wechat-stream/go.mod create mode 100644 demos/go-ffmpeg-wechat-stream/main.go create mode 100644 src/windows/EngineWorkbench.jsx create mode 100644 vision_test/app.py diff --git a/.codegraph/.gitignore b/.codegraph/.gitignore index 9de0f16..d20c0fe 100644 --- a/.codegraph/.gitignore +++ b/.codegraph/.gitignore @@ -1,16 +1,5 @@ -# CodeGraph data files -# These are local to each machine and should not be committed - -# Database -*.db -*.db-wal -*.db-shm - -# Cache -cache/ - -# Logs -*.log - -# Hook markers -.dirty +# CodeGraph data files — local to each machine, not for committing. +# Ignore everything in .codegraph/ except this file itself, so transient +# files (the database, daemon.pid, sockets, logs) never show up in git. +* +!.gitignore diff --git a/.codegraph/daemon.pid b/.codegraph/daemon.pid deleted file mode 100644 index 736b169..0000000 --- a/.codegraph/daemon.pid +++ /dev/null @@ -1,6 +0,0 @@ -{ - "pid": 30162, - "version": "0.9.7", - "socketPath": "/Users/wang/agent_dev/wechat_ai/.codegraph/daemon.sock", - "startedAt": 1781241939180 -} diff --git a/.gitignore b/.gitignore index 0a77644..7dde481 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,9 @@ wechat_vision/__pycache__/ wechat_vision/ouptsw/ wechat_vision/test/ +vision_test/.venv/ + +demos/go-ffmpeg-wechat-stream/go-ffmpeg-wechat-stream .codegraph/daemon.pid .vscode/ diff --git a/agent/main.go b/agent/main.go index 3c06aa7..5cbf117 100644 --- a/agent/main.go +++ b/agent/main.go @@ -6,6 +6,14 @@ import ( ) func main() { + if len(os.Args) > 1 && os.Args[1] == "vision-stream" { + if err := runVisionStream(os.Args[2:]); err != nil { + logError(err.Error()) + os.Exit(1) + } + return + } + if len(os.Args) > 1 && os.Args[1] == "ax-probe" { if err := runAXProbe(); err != nil { logError(err.Error()) diff --git a/agent/vision_stream.go b/agent/vision_stream.go new file mode 100644 index 0000000..17d1534 --- /dev/null +++ b/agent/vision_stream.go @@ -0,0 +1,668 @@ +package main + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "image" + _ "image/jpeg" + "io" + "math" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" +) + +type visionFrameState struct { + condition *sync.Cond + jpeg []byte + frameWidth int + frameHeight int + sequence int64 + frameIndex int64 + updatedAt time.Time + target previewTarget + annotation annotationMeta + event map[string]any +} + +type annotationMeta struct { + Width int `json:"width"` + Height int `json:"height"` + ScaleFactor float64 `json:"scaleFactor"` +} + +type captureSource struct { + ID string `json:"id"` + Kind string `json:"kind"` + Label string `json:"label"` + AppName string `json:"appName"` + Title string `json:"title"` + PID int `json:"pid"` + X float64 `json:"x"` + Y float64 `json:"y"` + Width float64 `json:"width"` + Height float64 `json:"height"` + ScaleFactor float64 `json:"scaleFactor"` +} + +type previewTarget struct { + Kind string + Label string + WindowID int + Rect WindowBounds +} + +func newVisionFrameState(target previewTarget) *visionFrameState { + return &visionFrameState{ + condition: sync.NewCond(&sync.Mutex{}), + target: target, + event: map[string]any{"type": "waiting"}, + } +} + +func (state *visionFrameState) update(jpeg []byte) { + frameWidth, frameHeight := jpegSize(jpeg) + state.condition.L.Lock() + state.jpeg = append(state.jpeg[:0], jpeg...) + if frameWidth > 0 && frameHeight > 0 { + state.frameWidth = frameWidth + state.frameHeight = frameHeight + } + state.sequence++ + state.frameIndex++ + state.updatedAt = time.Now().UTC() + state.event = map[string]any{ + "type": "preview_frame", + "timestamp": state.updatedAt.Format(time.RFC3339Nano), + "frame_index": state.frameIndex, + "target_kind": state.target.Kind, + "target_label": state.target.Label, + } + state.condition.Broadcast() + state.condition.L.Unlock() +} + +func (state *visionFrameState) updateAnnotation(annotation annotationMeta) { + state.condition.L.Lock() + state.annotation = annotation + state.condition.L.Unlock() +} + +func (state *visionFrameState) updateStatus(event map[string]any, target previewTarget) { + state.condition.L.Lock() + state.updatedAt = time.Now().UTC() + state.event = event + state.target = target + state.condition.Broadcast() + state.condition.L.Unlock() +} + +func (state *visionFrameState) snapshot() ([]byte, map[string]any, int64, time.Time, int, int, annotationMeta) { + state.condition.L.Lock() + defer state.condition.L.Unlock() + jpeg := append([]byte(nil), state.jpeg...) + event := make(map[string]any, len(state.event)) + for key, value := range state.event { + event[key] = value + } + return jpeg, event, state.sequence, state.updatedAt, state.frameWidth, state.frameHeight, state.annotation +} + +func (state *visionFrameState) waitForNext(lastSequence int64) ([]byte, int64) { + state.condition.L.Lock() + defer state.condition.L.Unlock() + for state.sequence == lastSequence { + state.condition.Wait() + } + return append([]byte(nil), state.jpeg...), state.sequence +} + +func runVisionStream(args []string) error { + flags := flag.NewFlagSet("vision-stream", flag.ContinueOnError) + host := flags.String("host", "127.0.0.1", "HTTP server host") + port := flags.Int("port", 8765, "HTTP server port") + input := flags.String("input", "3:none", "FFmpeg avfoundation input for experiments") + fps := flags.Float64("fps", 30, "capture FPS") + cropWechat := flags.Bool("crop-wechat", true, "capture only the WeChat window") + regionsPath := flags.String("regions", defaultRegionsPath(), "annotation regions JSON path") + if err := flags.Parse(args); err != nil { + return err + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + state := newVisionFrameState(previewTarget{Kind: "waiting", Label: "等待标注来源"}) + server := &http.Server{ + Addr: fmt.Sprintf("%s:%d", *host, *port), + Handler: visionHTTPHandler(state), + ReadHeaderTimeout: 5 * time.Second, + } + + go func() { + <-ctx.Done() + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 2*time.Second) + defer shutdownCancel() + _ = server.Shutdown(shutdownCtx) + }() + + errCh := make(chan error, 2) + go func() { + errCh <- server.ListenAndServe() + }() + go func() { + errCh <- runVisionCapture(ctx, state, *input, *fps, *cropWechat, *regionsPath) + }() + + logInfo(fmt.Sprintf("Go annotation source stream started http://%s:%d/stream.mjpg regions=%s fps=%.1f", *host, *port, *regionsPath, *fps)) + + for err := range errCh { + if err == nil || err == http.ErrServerClosed || ctx.Err() != nil { + return nil + } + cancel() + return err + } + + return nil +} + +func visionHTTPHandler(state *visionFrameState) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) { + writer.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = io.WriteString(writer, ``) + }) + mux.HandleFunc("/latest.json", func(writer http.ResponseWriter, request *http.Request) { + _, event, sequence, updatedAt, frameWidth, frameHeight, annotation := state.snapshot() + payload := map[string]any{ + "source": "go-window-capture", + "sequence": sequence, + "updated_at": updatedAt.Format(time.RFC3339Nano), + "event": event, + "frame": map[string]any{ + "width": frameWidth, + "height": frameHeight, + }, + "annotation": annotation, + "target": map[string]any{ + "kind": state.target.Kind, + "label": state.target.Label, + "window_id": state.target.WindowID, + "rect": state.target.Rect, + }, + } + if updatedAt.IsZero() { + payload["updated_at"] = time.Now().UTC().Format(time.RFC3339Nano) + } + writer.Header().Set("Content-Type", "application/json; charset=utf-8") + writer.Header().Set("Cache-Control", "no-store") + _ = json.NewEncoder(writer).Encode(payload) + }) + mux.HandleFunc("/frame.jpg", func(writer http.ResponseWriter, request *http.Request) { + jpeg, _, sequence, _, _, _, _ := state.snapshot() + if len(jpeg) == 0 { + http.Error(writer, "frame not ready", http.StatusServiceUnavailable) + return + } + writer.Header().Set("Content-Type", "image/jpeg") + writer.Header().Set("Cache-Control", "no-store") + writer.Header().Set("X-Frame-Sequence", fmt.Sprintf("%d", sequence)) + _, _ = writer.Write(jpeg) + }) + mux.HandleFunc("/stream.mjpg", func(writer http.ResponseWriter, request *http.Request) { + flusher, ok := writer.(http.Flusher) + if !ok { + http.Error(writer, "streaming unsupported", http.StatusInternalServerError) + return + } + + writer.Header().Set("Content-Type", "multipart/x-mixed-replace; boundary=frame") + writer.Header().Set("Cache-Control", "no-store") + lastSequence := int64(-1) + for request.Context().Err() == nil { + jpeg, sequence := state.waitForNext(lastSequence) + lastSequence = sequence + if len(jpeg) == 0 { + continue + } + _, err := fmt.Fprintf(writer, "--frame\r\nContent-Type: image/jpeg\r\nContent-Length: %d\r\n\r\n", len(jpeg)) + if err != nil { + return + } + if _, err := writer.Write(jpeg); err != nil { + return + } + if _, err := writer.Write([]byte("\r\n")); err != nil { + return + } + flusher.Flush() + } + }) + return mux +} + +func runVisionCapture(ctx context.Context, state *visionFrameState, input string, fps float64, cropWechat bool, regionsPath string) error { + for ctx.Err() == nil { + target, annotation, err := previewTargetFromAnnotation(regionsPath) + if err != nil { + message := "等待标注来源窗口出现：" + err.Error() + logWarning(message) + state.updateStatus(map[string]any{ + "type": "source_waiting", + "timestamp": time.Now().UTC().Format(time.RFC3339Nano), + "message": message, + }, previewTarget{Kind: "waiting", Label: "等待标注来源"}) + select { + case <-ctx.Done(): + return nil + case <-time.After(2 * time.Second): + continue + } + } + + state.updateAnnotation(annotation) + return runResolvedVisionCapture(ctx, state, input, fps, cropWechat, target) + } + return nil +} + +func runResolvedVisionCapture(ctx context.Context, state *visionFrameState, input string, fps float64, cropWechat bool, target previewTarget) error { + if target.Kind == "window" { + logInfo(fmt.Sprintf("using macOS window-only capture window_id=%d", target.WindowID)) + return runScreencaptureMJPEG(ctx, state, fps, &target.Rect, target.WindowID) + } + + if target.Kind == "display" { + logInfo(fmt.Sprintf("using macOS display rectangle capture rect=[%.0f,%.0f,%.0f,%.0f]", target.Rect.X, target.Rect.Y, target.Rect.Width, target.Rect.Height)) + return runScreencaptureMJPEG(ctx, state, fps, &target.Rect, 0) + } + + window, err := maybeFindWeChatWindow(cropWechat) + if err != nil { + return err + } + logInfo("using macOS window-only capture for fallback WeChat preview") + return runScreencaptureMJPEG(ctx, state, fps, window, window.ID) + + // Keep the FFmpeg path below as an experiment. It captures the screen and crops + // by rectangle, so covered windows can leak into the preview. Tauri preview uses + // screencapture -l instead, which captures the WeChat window itself. + ffmpegErr := make(chan error, 1) + ffmpegCtx, cancelFFmpeg := context.WithCancel(ctx) + go func() { + ffmpegErr <- runFFmpegMJPEG(ffmpegCtx, state, input, fps, window) + }() + + timer := time.NewTimer(3 * time.Second) + defer timer.Stop() + select { + case <-ctx.Done(): + cancelFFmpeg() + return nil + case err := <-ffmpegErr: + cancelFFmpeg() + if err != nil { + logWarning("FFmpeg exited before first frame, fallback to screencapture: " + err.Error()) + } + return runScreencaptureMJPEG(ctx, state, fps, window, window.ID) + case <-timer.C: + _, _, sequence, _, _, _, _ := state.snapshot() + if sequence > 0 { + logInfo("FFmpeg produced frames, keep using FFmpeg capture") + return <-ffmpegErr + } + cancelFFmpeg() + logWarning("FFmpeg produced no frames in 3s, fallback to screencapture") + return runScreencaptureMJPEG(ctx, state, fps, window, window.ID) + } +} + +func maybeFindWeChatWindow(cropWechat bool) (*WindowBounds, error) { + if !cropWechat { + return nil, nil + } + bounds, err := findWeChatWindow() + if err != nil { + return nil, fmt.Errorf("find WeChat window failed, fallback to full screen capture: %w", err) + } + logInfo(fmt.Sprintf( + "WeChat window found owner=%s title=%s id=%d rect=[%.0f,%.0f,%.0f,%.0f]", + bounds.Owner, + bounds.Title, + bounds.ID, + bounds.X, + bounds.Y, + bounds.Width, + bounds.Height, + )) + return &bounds, nil +} + +func defaultRegionsPath() string { + home, err := os.UserHomeDir() + if err != nil { + return "data/regions/wechat.json" + } + return filepath.Join(home, "Library", "Application Support", "com.tauri.dev", "data", "regions", "wechat.json") +} + +func previewTargetFromAnnotation(path string) (previewTarget, annotationMeta, error) { + annotation, err := loadAnnotation(path) + if err != nil { + return previewTarget{}, annotationMeta{}, fmt.Errorf("load annotation source failed: %w", err) + } + meta := annotationMeta{ + Width: annotation.ScreenshotWidth, + Height: annotation.ScreenshotHeight, + ScaleFactor: annotation.ScaleFactor, + } + + source := captureSourceFromAnnotation(annotation.Source) + if source.Kind == "" { + target, err := fallbackWeChatPreviewTarget() + return target, meta, err + } + + if source.Kind == "window" { + window, err := findWindowForSource(source) + if err != nil { + return previewTarget{}, meta, err + } + return previewTarget{ + Kind: "window", + Label: source.Label, + WindowID: window.ID, + Rect: window, + }, meta, nil + } + + if source.Kind == "display" { + return previewTarget{ + Kind: "display", + Label: source.Label, + Rect: WindowBounds{ + X: source.X, + Y: source.Y, + Width: source.Width, + Height: source.Height, + Owner: "display", + Title: source.Label, + }, + }, meta, nil + } + + return previewTarget{}, meta, fmt.Errorf("unsupported annotation source kind: %s", source.Kind) +} + +func fallbackWeChatPreviewTarget() (previewTarget, error) { + window, err := findWeChatWindow() + if err != nil { + return previewTarget{}, err + } + return previewTarget{ + Kind: "window", + Label: "微信", + WindowID: window.ID, + Rect: window, + }, nil +} + +func captureSourceFromAnnotation(raw jsonRawOptional) captureSource { + data, err := json.Marshal(raw) + if err != nil { + return captureSource{} + } + var source captureSource + if err := json.Unmarshal(data, &source); err != nil { + return captureSource{} + } + return source +} + +func findWindowForSource(source captureSource) (WindowBounds, error) { + appName := strings.ReplaceAll(source.AppName, "\\", "\\\\") + appName = strings.ReplaceAll(appName, `"`, `\"`) + title := strings.ReplaceAll(source.Title, "\\", "\\\\") + title = strings.ReplaceAll(title, `"`, `\"`) + + script := fmt.Sprintf(` +import CoreGraphics +import Foundation + +let targetOwner = "%s" +let targetTitle = "%s" +let options = CGWindowListOption(arrayLiteral: [.optionOnScreenOnly, .excludeDesktopElements]) +guard let infos = CGWindowListCopyWindowInfo(options, kCGNullWindowID) as? [[String: Any]] else { exit(1) } + +func printWindow(_ info: [String: Any]) { + let owner = info[kCGWindowOwnerName as String] as? String ?? "" + let title = info[kCGWindowName as String] as? String ?? "" + let bounds = info[kCGWindowBounds as String] as? [String: Any] ?? [:] + let x = bounds["X"] as? Double ?? 0 + let y = bounds["Y"] as? Double ?? 0 + let w = bounds["Width"] as? Double ?? 0 + let h = bounds["Height"] as? Double ?? 0 + let id = info[kCGWindowNumber as String] as? Int ?? 0 + let result: [String: Any] = ["x": x, "y": y, "width": w, "height": h, "owner": owner, "title": title, "id": id] + let data = try! JSONSerialization.data(withJSONObject: result) + print(String(data: data, encoding: .utf8)!) +} + +for info in infos { + let owner = info[kCGWindowOwnerName as String] as? String ?? "" + let title = info[kCGWindowName as String] as? String ?? "" + let layer = info[kCGWindowLayer as String] as? Int ?? -1 + if layer == 0 && owner == targetOwner && title == targetTitle { printWindow(info); exit(0) } +} + +for info in infos { + let owner = info[kCGWindowOwnerName as String] as? String ?? "" + let title = info[kCGWindowName as String] as? String ?? "" + let layer = info[kCGWindowLayer as String] as? Int ?? -1 + if layer == 0 && targetOwner != "" && owner.contains(targetOwner) { printWindow(info); exit(0) } +} + +exit(2) +`, appName, title) + + output, err := runSwift(script) + if err != nil { + return WindowBounds{}, fmt.Errorf("find annotation window source failed app=%q title=%q: %w", source.AppName, source.Title, err) + } + var window WindowBounds + if err := json.Unmarshal([]byte(output), &window); err != nil { + return WindowBounds{}, err + } + return window, nil +} + +func runFFmpegMJPEG(ctx context.Context, state *visionFrameState, input string, fps float64, window *WindowBounds) error { + args := []string{ + "-hide_banner", + "-loglevel", "error", + "-f", "avfoundation", + "-framerate", fmt.Sprintf("%.3f", fps), + "-pixel_format", "bgr0", + "-i", input, + } + if window != nil { + args = append(args, "-vf", ffmpegCropFilter(*window)) + } + args = append(args, + "-f", "image2pipe", + "-vcodec", "mjpeg", + "-", + ) + logInfo("FFmpeg command: ffmpeg " + strings.Join(args, " ")) + command := exec.CommandContext(ctx, "ffmpeg", args...) + stdout, err := command.StdoutPipe() + if err != nil { + return err + } + stderr, err := command.StderrPipe() + if err != nil { + return err + } + + if err := command.Start(); err != nil { + return fmt.Errorf("启动 FFmpeg 失败：%w", err) + } + + go drainFFmpegStderr(stderr) + + readErr := readJPEGFrames(stdout, state.update) + waitErr := command.Wait() + if ctx.Err() != nil { + return nil + } + if readErr != nil { + return readErr + } + if waitErr != nil { + return fmt.Errorf("FFmpeg 退出：%w", waitErr) + } + return nil +} + +func runScreencaptureMJPEG(ctx context.Context, state *visionFrameState, fps float64, window *WindowBounds, windowID int) error { + if window == nil { + return errors.New("screencapture requires a capture target") + } + interval := time.Duration(float64(time.Second) / math.Max(fps, 1)) + if interval < 80*time.Millisecond { + interval = 80 * time.Millisecond + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + tempDir, err := os.MkdirTemp("", "wechat-go-vision-*") + if err != nil { + return err + } + defer os.RemoveAll(tempDir) + imagePath := filepath.Join(tempDir, "frame.jpg") + rect := captureRect(*window) + if windowID > 0 { + logInfo(fmt.Sprintf("window-only screencapture started window_id=%d", windowID)) + } else { + logInfo("rectangle screencapture started rect=" + rect) + } + for ctx.Err() == nil { + args := []string{"-x"} + if windowID > 0 { + args = append(args, "-l", fmt.Sprintf("%d", windowID)) + } else { + args = append(args, "-R", rect) + } + args = append(args, imagePath) + command := exec.CommandContext(ctx, "screencapture", args...) + if output, err := command.CombinedOutput(); err != nil { + if ctx.Err() != nil { + return nil + } + logWarning(fmt.Sprintf("screencapture failed: %v output=%s", err, string(output))) + } else if jpeg, err := os.ReadFile(imagePath); err != nil { + logWarning("read screencapture frame failed: " + err.Error()) + } else if len(jpeg) > 0 { + state.update(jpeg) + } + + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + } + return nil +} + +func ffmpegCropFilter(window WindowBounds) string { + width := int(math.Max(1, math.Round(window.Width))) + height := int(math.Max(1, math.Round(window.Height))) + x := int(math.Max(0, math.Round(window.X))) + y := int(math.Max(0, math.Round(window.Y))) + return fmt.Sprintf("crop=%d:%d:%d:%d", width, height, x, y) +} + +func captureRect(window WindowBounds) string { + x := int(math.Round(window.X)) + y := int(math.Round(window.Y)) + width := int(math.Round(window.Width)) + height := int(math.Round(window.Height)) + if x < 0 { + width += x + x = 0 + } + if y < 0 { + height += y + y = 0 + } + if width < 1 { + width = 1 + } + if height < 1 { + height = 1 + } + return fmt.Sprintf("%d,%d,%d,%d", x, y, width, height) +} + +func readJPEGFrames(reader io.Reader, onFrame func([]byte)) error { + buffer := make([]byte, 0, 1024*1024) + chunk := make([]byte, 64*1024) + for { + readCount, err := reader.Read(chunk) + if readCount > 0 { + buffer = append(buffer, chunk[:readCount]...) + for { + start := bytes.Index(buffer, []byte{0xff, 0xd8}) + if start < 0 { + if len(buffer) > 1024*1024 { + buffer = buffer[len(buffer)-2:] + } + break + } + endOffset := bytes.Index(buffer[start+2:], []byte{0xff, 0xd9}) + if endOffset < 0 { + if start > 0 { + buffer = buffer[start:] + } + break + } + end := start + 2 + endOffset + 2 + frame := append([]byte(nil), buffer[start:end]...) + buffer = buffer[end:] + onFrame(frame) + } + } + if err != nil { + if err == io.EOF { + return nil + } + return err + } + } +} + +func jpegSize(jpeg []byte) (int, int) { + config, _, err := image.DecodeConfig(bytes.NewReader(jpeg)) + if err != nil { + return 0, 0 + } + return config.Width, config.Height +} + +func drainFFmpegStderr(reader io.Reader) { + scanner := bufio.NewScanner(reader) + for scanner.Scan() { + logWarning("ffmpeg: " + scanner.Text()) + } +} diff --git a/demos/go-ffmpeg-wechat-stream/README.md b/demos/go-ffmpeg-wechat-stream/README.md new file mode 100644 index 0000000..d7dc702 --- /dev/null +++ b/demos/go-ffmpeg-wechat-stream/README.md @@ -0,0 +1,38 @@ +# Go FFmpeg WeChat Stream Demo + +Standalone demo: Go finds the visible WeChat window, starts FFmpeg `avfoundation` screen capture, crops to the WeChat window rectangle, and serves an MJPEG stream locally. + +If FFmpeg screen capture does not produce frames, the demo automatically falls back to macOS window-only capture with `screencapture -l `. This captures the WeChat window itself, so other windows covering it should not appear in the stream. + +Run: + +```bash +cd demos/go-ffmpeg-wechat-stream +go run . +``` + +Open: + +```text +http://127.0.0.1:8765/ +``` + +Options: + +```bash +go run . -input 3:none -fps 15 -port 8765 -open=true -wait=true +``` + +Notes: + +- macOS only. +- Requires FFmpeg installed and available in `PATH`. +- Requires Screen Recording permission for the terminal/app running the demo. +- WeChat must be visible on screen. +- By default the demo waits until a visible WeChat window is found. Use `-wait=false` to fail immediately. +- FFmpeg `avfoundation` is screen-based, so crop mode can include windows covering WeChat. The fallback `screencapture -l` path is window-based and is closer to Tencent Meeting style app-window sharing. +- `-input` is the FFmpeg `avfoundation` screen device. List devices with: + +```bash +ffmpeg -hide_banner -f avfoundation -list_devices true -i "" +``` diff --git a/demos/go-ffmpeg-wechat-stream/go.mod b/demos/go-ffmpeg-wechat-stream/go.mod new file mode 100644 index 0000000..03998ae --- /dev/null +++ b/demos/go-ffmpeg-wechat-stream/go.mod @@ -0,0 +1,3 @@ +module go-ffmpeg-wechat-stream + +go 1.22 diff --git a/demos/go-ffmpeg-wechat-stream/main.go b/demos/go-ffmpeg-wechat-stream/main.go new file mode 100644 index 0000000..b53c64a --- /dev/null +++ b/demos/go-ffmpeg-wechat-stream/main.go @@ -0,0 +1,470 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "math" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync" + "time" +) + +type windowBounds struct { + X float64 `json:"x"` + Y float64 `json:"y"` + Width float64 `json:"width"` + Height float64 `json:"height"` + Owner string `json:"owner"` + Title string `json:"title"` + ID int `json:"id"` +} + +type streamState struct { + condition *sync.Cond + jpeg []byte + sequence int64 + window windowBounds + startedAt time.Time + updatedAt time.Time +} + +func newStreamState(window windowBounds) *streamState { + return &streamState{ + condition: sync.NewCond(&sync.Mutex{}), + window: window, + startedAt: time.Now(), + } +} + +func (state *streamState) update(jpeg []byte) { + state.condition.L.Lock() + state.jpeg = append(state.jpeg[:0], jpeg...) + state.sequence++ + state.updatedAt = time.Now() + state.condition.Broadcast() + state.condition.L.Unlock() +} + +func (state *streamState) snapshot() (int64, time.Time, windowBounds) { + state.condition.L.Lock() + defer state.condition.L.Unlock() + return state.sequence, state.updatedAt, state.window +} + +func (state *streamState) waitNext(lastSequence int64) ([]byte, int64) { + state.condition.L.Lock() + defer state.condition.L.Unlock() + for state.sequence == lastSequence { + state.condition.Wait() + } + return append([]byte(nil), state.jpeg...), state.sequence +} + +func main() { + if runtime.GOOS != "darwin" { + log.Fatal("this demo currently supports macOS only because it uses avfoundation + CoreGraphics") + } + + var host string + var port int + var input string + var fps float64 + var openBrowser bool + var waitWindow bool + flag.StringVar(&host, "host", "127.0.0.1", "HTTP server host") + flag.IntVar(&port, "port", 8765, "HTTP server port") + flag.StringVar(&input, "input", "3:none", "FFmpeg avfoundation input, e.g. 3:none") + flag.Float64Var(&fps, "fps", 15, "capture frame rate") + flag.BoolVar(&openBrowser, "open", true, "open browser automatically") + flag.BoolVar(&waitWindow, "wait", true, "wait until a visible WeChat window is found") + flag.Parse() + + window, err := waitForWeChatWindow(waitWindow) + if err != nil { + log.Fatalf("find WeChat window failed: %v", err) + } + log.Printf("WeChat window: owner=%q title=%q id=%d rect=[%.0f,%.0f,%.0f,%.0f]", window.Owner, window.Title, window.ID, window.X, window.Y, window.Width, window.Height) + + state := newStreamState(window) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go runCapture(ctx, state, input, fps, window) + + server := &http.Server{ + Addr: fmt.Sprintf("%s:%d", host, port), + Handler: handler(state), + ReadHeaderTimeout: 5 * time.Second, + } + + url := fmt.Sprintf("http://%s:%d/", host, port) + log.Printf("Serving %s", url) + if openBrowser { + go func() { + time.Sleep(500 * time.Millisecond) + _ = exec.Command("open", url).Start() + }() + } + + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Fatal(err) + } +} + +func runCapture(ctx context.Context, state *streamState, input string, fps float64, window windowBounds) { + ffmpegCtx, cancelFFmpeg := context.WithCancel(ctx) + ffmpegErr := make(chan error, 1) + go func() { + ffmpegErr <- runFFmpeg(ffmpegCtx, state, input, fps, window) + }() + + timer := time.NewTimer(3 * time.Second) + defer timer.Stop() + select { + case <-ctx.Done(): + cancelFFmpeg() + return + case err := <-ffmpegErr: + cancelFFmpeg() + log.Printf("FFmpeg stopped before first frame, fallback to screencapture: %v", err) + case <-timer.C: + sequence, _, _ := state.snapshot() + if sequence > 0 { + log.Printf("FFmpeg is producing frames") + if err := <-ffmpegErr; err != nil && ctx.Err() == nil { + log.Printf("FFmpeg stopped: %v", err) + } + return + } + cancelFFmpeg() + log.Printf("FFmpeg produced no frames in 3s, fallback to window-only screencapture") + } + + if err := runScreencapture(ctx, state, fps, window); err != nil && ctx.Err() == nil { + log.Printf("screencapture stopped: %v", err) + } +} + +func waitForWeChatWindow(wait bool) (windowBounds, error) { + for { + window, err := findWeChatWindow() + if err == nil { + return window, nil + } + if !wait { + return windowBounds{}, err + } + log.Printf("waiting for visible WeChat window: %v", err) + time.Sleep(2 * time.Second) + } +} + +func findWeChatWindow() (windowBounds, error) { + script := ` +import CoreGraphics +import Foundation + +let options = CGWindowListOption(arrayLiteral: [.optionOnScreenOnly, .excludeDesktopElements]) +guard let infos = CGWindowListCopyWindowInfo(options, kCGNullWindowID) as? [[String: Any]] else { + exit(1) +} + +for info in infos { + let owner = info[kCGWindowOwnerName as String] as? String ?? "" + let title = info[kCGWindowName as String] as? String ?? "" + let layer = info[kCGWindowLayer as String] as? Int ?? -1 + if layer == 0 && (owner.contains("微信") || owner.lowercased().contains("wechat") || title == "微信") { + if let bounds = info[kCGWindowBounds as String] as? [String: Any], + let x = bounds["X"] as? Double, + let y = bounds["Y"] as? Double, + let w = bounds["Width"] as? Double, + let h = bounds["Height"] as? Double { + let id = info[kCGWindowNumber as String] as? Int ?? 0 + let result: [String: Any] = ["x": x, "y": y, "width": w, "height": h, "owner": owner, "title": title, "id": id] + let data = try! JSONSerialization.data(withJSONObject: result) + print(String(data: data, encoding: .utf8)!) + exit(0) + } + } +} + +exit(2) +` + command := exec.Command("swift", "-") + command.Stdin = strings.NewReader(script) + output, err := command.Output() + if err != nil { + return windowBounds{}, err + } + + var window windowBounds + if err := json.Unmarshal(output, &window); err != nil { + return windowBounds{}, err + } + return window, nil +} + +func runFFmpeg(ctx context.Context, state *streamState, input string, fps float64, window windowBounds) error { + args := []string{ + "-hide_banner", + "-loglevel", "warning", + "-f", "avfoundation", + "-framerate", fmt.Sprintf("%.3f", fps), + "-pixel_format", "bgr0", + "-i", input, + "-vf", cropFilter(window), + "-f", "image2pipe", + "-vcodec", "mjpeg", + "-q:v", "5", + "-", + } + log.Printf("FFmpeg command: ffmpeg %s", strings.Join(args, " ")) + + command := exec.CommandContext(ctx, "ffmpeg", args...) + stdout, err := command.StdoutPipe() + if err != nil { + return err + } + stderr, err := command.StderrPipe() + if err != nil { + return err + } + + if err := command.Start(); err != nil { + return err + } + go logLines("ffmpeg", stderr) + + readErr := readJPEGFrames(stdout, state.update) + waitErr := command.Wait() + if ctx.Err() != nil { + return nil + } + if readErr != nil { + return readErr + } + return waitErr +} + +func runScreencapture(ctx context.Context, state *streamState, fps float64, window windowBounds) error { + interval := time.Duration(float64(time.Second) / math.Max(fps, 1)) + if interval < 80*time.Millisecond { + interval = 80 * time.Millisecond + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + + tempDir, err := os.MkdirTemp("", "go-ffmpeg-wechat-stream-*") + if err != nil { + return err + } + defer os.RemoveAll(tempDir) + + framePath := filepath.Join(tempDir, "frame.jpg") + rect := captureRect(window) + log.Printf("window-only screencapture fallback started window_id=%d rect=%s", window.ID, rect) + for ctx.Err() == nil { + command := exec.CommandContext(ctx, "screencapture", "-x", "-l", fmt.Sprintf("%d", window.ID), framePath) + if output, err := command.CombinedOutput(); err != nil { + if ctx.Err() != nil { + return nil + } + log.Printf("screencapture failed: %v output=%s", err, string(output)) + } else if jpeg, err := os.ReadFile(framePath); err != nil { + log.Printf("read frame failed: %v", err) + } else if len(jpeg) > 0 { + state.update(jpeg) + } + + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + } + return nil +} + +func captureRect(window windowBounds) string { + x := int(math.Round(window.X)) + y := int(math.Round(window.Y)) + width := int(math.Round(window.Width)) + height := int(math.Round(window.Height)) + if x < 0 { + width += x + x = 0 + } + if y < 0 { + height += y + y = 0 + } + if width < 1 { + width = 1 + } + if height < 1 { + height = 1 + } + return fmt.Sprintf("%d,%d,%d,%d", x, y, width, height) +} + +func cropFilter(window windowBounds) string { + width := int(math.Max(1, math.Round(window.Width))) + height := int(math.Max(1, math.Round(window.Height))) + x := int(math.Max(0, math.Round(window.X))) + y := int(math.Max(0, math.Round(window.Y))) + return fmt.Sprintf("crop=%d:%d:%d:%d", width, height, x, y) +} + +func readJPEGFrames(reader io.Reader, onFrame func([]byte)) error { + buffer := make([]byte, 0, 1024*1024) + chunk := make([]byte, 64*1024) + for { + readCount, err := reader.Read(chunk) + if readCount > 0 { + buffer = append(buffer, chunk[:readCount]...) + for { + start := bytes.Index(buffer, []byte{0xff, 0xd8}) + if start < 0 { + if len(buffer) > 1024*1024 { + buffer = buffer[len(buffer)-2:] + } + break + } + endOffset := bytes.Index(buffer[start+2:], []byte{0xff, 0xd9}) + if endOffset < 0 { + if start > 0 { + buffer = buffer[start:] + } + break + } + end := start + 2 + endOffset + 2 + onFrame(append([]byte(nil), buffer[start:end]...)) + buffer = buffer[end:] + } + } + if err != nil { + if err == io.EOF { + return nil + } + return err + } + } +} + +func handler(state *streamState) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) { + writer.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = io.WriteString(writer, ` + + + + Go FFmpeg WeChat Stream + + + +
+
Go FFmpeg WeChat Streamconnecting...
+ +
+ + +`) + }) + mux.HandleFunc("/latest.json", func(writer http.ResponseWriter, request *http.Request) { + sequence, updatedAt, window := state.snapshot() + writer.Header().Set("Content-Type", "application/json; charset=utf-8") + writer.Header().Set("Cache-Control", "no-store") + _ = json.NewEncoder(writer).Encode(map[string]any{ + "source": "go-ffmpeg-demo", + "sequence": sequence, + "updated_at": updatedAt.Format(time.RFC3339Nano), + "window": window, + }) + }) + mux.HandleFunc("/stream.mjpg", func(writer http.ResponseWriter, request *http.Request) { + flusher, ok := writer.(http.Flusher) + if !ok { + http.Error(writer, "streaming unsupported", http.StatusInternalServerError) + return + } + writer.Header().Set("Content-Type", "multipart/x-mixed-replace; boundary=frame") + writer.Header().Set("Cache-Control", "no-store") + lastSequence := int64(-1) + for request.Context().Err() == nil { + jpeg, sequence := state.waitNext(lastSequence) + lastSequence = sequence + if len(jpeg) == 0 { + continue + } + if _, err := fmt.Fprintf(writer, "--frame\r\nContent-Type: image/jpeg\r\nContent-Length: %d\r\n\r\n", len(jpeg)); err != nil { + return + } + if _, err := writer.Write(jpeg); err != nil { + return + } + if _, err := writer.Write([]byte("\r\n")); err != nil { + return + } + flusher.Flush() + } + }) + return mux +} + +func logLines(prefix string, reader io.Reader) { + buffer := make([]byte, 4096) + var line []byte + for { + count, err := reader.Read(buffer) + if count > 0 { + line = append(line, buffer[:count]...) + for { + index := bytes.IndexByte(line, '\n') + if index < 0 { + break + } + log.Printf("%s: %s", prefix, strings.TrimSpace(string(line[:index]))) + line = line[index+1:] + } + } + if err != nil { + if len(line) > 0 { + log.Printf("%s: %s", prefix, strings.TrimSpace(string(line))) + } + return + } + } +} diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 5ef0295..a1ab9f6 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -2,6 +2,7 @@ use screenshots::Screen; use serde::{Deserialize, Serialize}; use std::fs; use std::path::PathBuf; +use std::process::Command; use std::sync::Mutex; use std::time::Instant; use tauri::{Emitter, Manager, WebviewUrl, WebviewWindowBuilder}; @@ -12,6 +13,7 @@ use tauri_plugin_shell::{ use xcap::{Monitor, Window}; struct AgentProcess(Mutex>); +struct VisionStreamProcess(Mutex>); #[derive(Clone, Deserialize, Serialize)] struct AgentLog { @@ -240,6 +242,110 @@ fn stop_agent(app: tauri::AppHandle, process: tauri::State) -> Res Ok(()) } +#[tauri::command] +fn start_vision_stream( + app: tauri::AppHandle, + process: tauri::State, +) -> Result<(), String> { + let mut process_guard = process.0.lock().map_err(|error| error.to_string())?; + if process_guard.is_some() { + return Ok(()); + } + + stop_legacy_python_vision_stream(); + + let (mut rx, child) = app + .shell() + .sidecar("agent") + .map_err(|error| error.to_string())? + .args(["vision-stream"]) + .spawn() + .map_err(|error| error.to_string())?; + + *process_guard = Some(child); + + let app_handle = app.clone(); + tauri::async_runtime::spawn(async move { + while let Some(event) = rx.recv().await { + match event { + CommandEvent::Stdout(line) => { + emit_agent_line( + &app_handle, + "info", + String::from_utf8_lossy(&line).trim().to_string(), + ); + } + CommandEvent::Stderr(line) => { + emit_agent_line( + &app_handle, + "error", + String::from_utf8_lossy(&line).trim().to_string(), + ); + } + CommandEvent::Error(error) => { + emit_agent_log( + &app_handle, + "error", + format!("vision stream error: {error}"), + ); + } + CommandEvent::Terminated(payload) => { + emit_agent_log( + &app_handle, + "warning", + format!("vision stream exited with code {:?}", payload.code), + ); + if let Ok(mut process_guard) = + app_handle.state::().0.lock() + { + *process_guard = None; + } + break; + } + _ => {} + } + } + }); + + Ok(()) +} + +fn stop_legacy_python_vision_stream() { + let Ok(cwd) = std::env::current_dir() else { + return; + }; + let candidates = [ + cwd.join("wechat_vision"), + cwd.join("..").join("wechat_vision"), + cwd.join("..").join("..").join("wechat_vision"), + ]; + + for dir in candidates { + let pid_file = dir.join("ouptsw").join("wechat_algorithm_live.pid"); + let stop_script = dir.join("stop_wechat_algorithm_live.sh"); + if pid_file.exists() && stop_script.exists() { + let _ = Command::new("bash") + .arg(stop_script) + .current_dir(dir) + .output(); + return; + } + } +} + +#[tauri::command] +fn stop_vision_stream( + _app: tauri::AppHandle, + process: tauri::State, +) -> Result<(), String> { + let mut process_guard = process.0.lock().map_err(|error| error.to_string())?; + if let Some(child) = process_guard.take() { + child.kill().map_err(|error| error.to_string())?; + } + + Ok(()) +} + #[tauri::command] async fn capture_screen(app: tauri::AppHandle) -> Result { let screenshot_path = screenshot_path(&app)?; @@ -490,6 +596,12 @@ fn open_popup_window(app: tauri::AppHandle, route: String) -> Result<(), String> let label = format!("popup-{}", route.trim_start_matches('/').replace('/', "-")); if let Some(window) = app.get_webview_window(&label) { + if route == "/window/engine-workbench" { + window + .set_fullscreen(false) + .map_err(|error| error.to_string())?; + window.maximize().map_err(|error| error.to_string())?; + } window.set_focus().map_err(|error| error.to_string())?; return Ok(()); } @@ -497,6 +609,7 @@ fn open_popup_window(app: tauri::AppHandle, route: String) -> Result<(), String> let title = match route.as_str() { "/annotate" => "屏幕区域标注", "/window/settings" => "设置", + "/window/engine-workbench" => "微信引擎工作台", "/window/engine-logs" => "分身引擎日志", "/window/knowledge-create" => "新增知识库", "/window/knowledge-detail" => "知识库详情", @@ -519,6 +632,11 @@ fn open_popup_window(app: tauri::AppHandle, route: String) -> Result<(), String> if route == "/annotate" { builder = builder.maximized(true).fullscreen(true); + } else if route == "/window/engine-workbench" { + builder = builder + .inner_size(1440.0, 920.0) + .maximized(true) + .fullscreen(false); } builder.build().map_err(|error| error.to_string())?; @@ -530,6 +648,7 @@ fn open_popup_window(app: tauri::AppHandle, route: String) -> Result<(), String> pub fn run() { tauri::Builder::default() .manage(AgentProcess(Mutex::new(None))) + .manage(VisionStreamProcess(Mutex::new(None))) .plugin(tauri_plugin_shell::init()) .setup(|app| { if cfg!(debug_assertions) { @@ -549,6 +668,8 @@ pub fn run() { open_popup_window, save_regions, start_agent, + start_vision_stream, + stop_vision_stream, stop_agent ]) .run(tauri::generate_context!()) diff --git a/src/components/Terminal.jsx b/src/components/Terminal.jsx index 8a883e7..b162d0c 100644 --- a/src/components/Terminal.jsx +++ b/src/components/Terminal.jsx @@ -27,7 +27,7 @@ export default function Terminal({ title, lines, tall = false, onClear }) { }; return ( -
+

{title}

diff --git a/src/components/TitleBar.jsx b/src/components/TitleBar.jsx index 0d593ae..b6f0c56 100644 --- a/src/components/TitleBar.jsx +++ b/src/components/TitleBar.jsx @@ -2,7 +2,7 @@ import { LogOut, RotateCcw } from "lucide-react"; import ThemeSwitch from "./ThemeSwitch"; import { closeWindow, startWindowDrag } from "../utils/navigation"; -export default function TitleBar({ title, theme, setTheme, compact = false }) { +export default function TitleBar({ title, theme, setTheme, compact = false, onClose }) { return (
) : ( - + )}
diff --git a/src/pages/ClonePage.jsx b/src/pages/ClonePage.jsx index 4329ded..d06dad0 100644 --- a/src/pages/ClonePage.jsx +++ b/src/pages/ClonePage.jsx @@ -8,8 +8,8 @@ import { openWindow } from "../utils/navigation"; export default function ClonePage() { const [engineEnabled, setEngineEnabled] = useState(false); + const [engineBusy, setEngineBusy] = useState(false); const [agentLogs, setAgentLogs] = useState([]); - const [regionsFile, setRegionsFile] = useState(null); const [annotating, setAnnotating] = useState(false); const [sourcePickerOpen, setSourcePickerOpen] = useState(false); const [captureSources, setCaptureSources] = useState([]); @@ -20,15 +20,18 @@ export default function ClonePage() { let unlisten; let cancelled = false; - - listen("agent-log", (event) => { - setAgentLogs((currentLogs) => [...currentLogs.slice(-300), event.payload]); + listen("engine-state-changed", (event) => { + setEngineEnabled(Boolean(event.payload?.enabled)); + setEngineBusy(false); + setAgentLogs((currentLogs) => [ + ...currentLogs.slice(-300), + { level: "warning", message: "工作台已关闭，引擎状态已同步" }, + ]); }).then((cleanup) => { if (cancelled) { cleanup(); return; } - unlisten = cleanup; }); @@ -38,24 +41,6 @@ export default function ClonePage() { }; }, []); - useEffect(() => { - loadSavedRegions(); - }, []); - - async function loadSavedRegions() { - if (!window.__TAURI_INTERNALS__) return; - - try { - const saved = await invoke("load_regions"); - setRegionsFile(saved); - } catch (error) { - setAgentLogs((currentLogs) => [ - ...currentLogs, - { level: "error", message: `读取标注失败：${String(error)}` }, - ]); - } - } - async function startAnnotation() { if (!window.__TAURI_INTERNALS__) { window.location.hash = "/annotate"; @@ -103,24 +88,50 @@ export default function ClonePage() { } async function toggleAgent() { + if (engineBusy) return; + if (!window.__TAURI_INTERNALS__) { setEngineEnabled((enabled) => !enabled); return; } + setEngineBusy(true); try { if (engineEnabled) { await invoke("stop_agent"); + await invoke("stop_vision_stream").catch((error) => { + setAgentLogs((currentLogs) => [ + ...currentLogs, + { level: "warning", message: `停止视觉流失败：${String(error)}` }, + ]); + }); setEngineEnabled(false); + setAgentLogs((currentLogs) => [ + ...currentLogs.slice(-300), + { level: "warning", message: "引擎已停用，暂未读取 Rust 日志" }, + ]); } else { - await invoke("start_agent"); + await invoke("start_vision_stream"); + await invoke("start_agent").catch((error) => { + setAgentLogs((currentLogs) => [ + ...currentLogs.slice(-300), + { level: "warning", message: `agent 启动失败，预览已继续启动：${String(error)}` }, + ]); + }); setEngineEnabled(true); + setAgentLogs((currentLogs) => [ + ...currentLogs.slice(-300), + { level: "info", message: "引擎已启动，暂未读取 Rust 日志" }, + ]); + await openWindow("/window/engine-workbench"); } } catch (error) { setAgentLogs((currentLogs) => [ ...currentLogs, { level: "error", message: String(error) }, ]); + } finally { + setEngineBusy(false); } } @@ -139,9 +150,10 @@ export default function ClonePage() {
@@ -157,21 +169,6 @@ export default function ClonePage() { {annotating ? "截屏中" : "开始标注"}
-
- {regionsFile?.regions?.length ? ( - regionsFile.regions.map((region) => ( -
-
-
{region.name}
-
screen [{region.bbox_screen.join(", ")}]
-
- {region.type} -
- )) - ) : ( -
暂无已保存区域
- )} -
{sourcePickerOpen ? (
diff --git a/src/windows/EngineWorkbench.jsx b/src/windows/EngineWorkbench.jsx new file mode 100644 index 0000000..227e292 --- /dev/null +++ b/src/windows/EngineWorkbench.jsx @@ -0,0 +1,274 @@ +import { useEffect, useMemo, useRef, useState } from "react"; +import { invoke } from "@tauri-apps/api/core"; +import { Bot, SendHorizontal, UserRound } from "lucide-react"; + +const MONITOR_ENDPOINT = "http://127.0.0.1:8765"; + +function messageLabel(role) { + if (role === "me") return "我"; + if (role === "user") return "客户"; + return "未知"; +} + +export default function EngineWorkbench() { + const [monitorEvent, setMonitorEvent] = useState(null); + const [streamUrl, setStreamUrl] = useState(`${MONITOR_ENDPOINT}/frame.jpg?t=${Date.now()}`); + const [streamFps, setStreamFps] = useState(0); + const [streamError, setStreamError] = useState(""); + const [annotation, setAnnotation] = useState(null); + const [frameMeta, setFrameMeta] = useState(null); + const [videoBox, setVideoBox] = useState({ width: 0, height: 0 }); + const [question, setQuestion] = useState(""); + const [qaMessages, setQaMessages] = useState([]); + const sequenceRef = useRef({ sequence: null, time: 0 }); + const streamRefreshedRef = useRef(false); + + useEffect(() => { + if (!window.__TAURI_INTERNALS__) return; + invoke("start_vision_stream").catch((error) => { + setStreamError(`预览服务启动失败：${String(error)}`); + }); + invoke("load_regions") + .then((saved) => setAnnotation(saved || null)) + .catch(() => setAnnotation(null)); + }, []); + + useEffect(() => { + const updateBox = () => { + const element = document.getElementById("wechat-preview-frame"); + if (!element) return; + const rect = element.getBoundingClientRect(); + setVideoBox({ width: rect.width, height: rect.height }); + }; + updateBox(); + window.addEventListener("resize", updateBox); + return () => window.removeEventListener("resize", updateBox); + }, []); + + useEffect(() => { + let cancelled = false; + + async function pollMonitor() { + try { + const response = await fetch(`${MONITOR_ENDPOINT}/latest.json?t=${Date.now()}`, { cache: "no-store" }); + const data = await response.json(); + if (!cancelled) { + if (data.source !== "go-window-capture") { + setMonitorEvent(null); + setStreamFps(0); + setStreamError(`预览服务来源异常：${data.source || "unknown"}`); + return; + } + if (data.event?.type === "source_waiting") { + setMonitorEvent(null); + setStreamFps(0); + setStreamError(data.event.message || "等待标注来源窗口出现..."); + setFrameMeta(data.frame || null); + return; + } + setStreamError(""); + setFrameMeta(data.frame || null); + const now = performance.now(); + const previous = sequenceRef.current; + if (typeof data.sequence === "number" && typeof previous.sequence === "number" && previous.time) { + const elapsedSeconds = Math.max((now - previous.time) / 1000, 0.001); + setStreamFps(Math.max(0, (data.sequence - previous.sequence) / elapsedSeconds)); + } + sequenceRef.current = { sequence: data.sequence, time: now }; + setMonitorEvent(data.event?.type === "preview_frame" ? data.event : null); + if (typeof data.sequence === "number" && data.sequence > 0) { + streamRefreshedRef.current = true; + setStreamUrl(`${MONITOR_ENDPOINT}/frame.jpg?seq=${data.sequence}&t=${Date.now()}`); + } + } + } catch { + if (!cancelled) { + setMonitorEvent(null); + setStreamError("等待预览服务启动..."); + } + } + } + + pollMonitor(); + const timer = window.setInterval(pollMonitor, 500); + return () => { + cancelled = true; + window.clearInterval(timer); + }; + }, []); + + const latestLlm = monitorEvent?.latest_llm || monitorEvent?.llm_chat_reading || null; + const visibleMessages = useMemo(() => latestLlm?.visible_messages || [], [latestLlm]); + const latestUserMessage = latestLlm?.latest_user_message || "等待识别当前聊天记录"; + const replyText = latestLlm?.reply_text || "暂无建议回复"; + const overlay = useMemo(() => { + const frameWidth = frameMeta?.width || annotation?.screenshotWidth; + const frameHeight = frameMeta?.height || annotation?.screenshotHeight; + if (!frameWidth || !frameHeight || !videoBox.width || !videoBox.height) return null; + const scale = Math.min(videoBox.width / frameWidth, videoBox.height / frameHeight); + const width = frameWidth * scale; + const height = frameHeight * scale; + return { + left: (videoBox.width - width) / 2, + top: (videoBox.height - height) / 2, + scale, + frameWidth, + frameHeight, + }; + }, [annotation, frameMeta, videoBox]); + + function regionStyle(region) { + if (!overlay || !region.bbox_image) return null; + const sourceWidth = annotation?.screenshotWidth || overlay.frameWidth; + const sourceHeight = annotation?.screenshotHeight || overlay.frameHeight; + const [ix1, iy1, ix2, iy2] = region.bbox_image; + const x1 = (ix1 * overlay.frameWidth) / sourceWidth; + const y1 = (iy1 * overlay.frameHeight) / sourceHeight; + const x2 = (ix2 * overlay.frameWidth) / sourceWidth; + const y2 = (iy2 * overlay.frameHeight) / sourceHeight; + return { + left: overlay.left + x1 * overlay.scale, + top: overlay.top + y1 * overlay.scale, + width: Math.max(1, (x2 - x1) * overlay.scale), + height: Math.max(1, (y2 - y1) * overlay.scale), + }; + } + + function submitQuestion(event) { + event.preventDefault(); + const trimmed = question.trim(); + if (!trimmed) return; + + const contextLines = visibleMessages.map((message) => `${messageLabel(message.role)}：${message.content}`).join("\n"); + const answer = contextLines + ? `基于当前已识别聊天记录：\n${contextLines}\n\n建议回复：${replyText}` + : "当前还没有读取到聊天记录。请先确认监控服务已启动，并等待检测到新消息后再询问。"; + + setQaMessages((current) => [ + ...current, + { role: "user", content: trimmed }, + { role: "assistant", content: answer }, + ]); + setQuestion(""); + } + + return ( +
+
+
+
+
+

微信窗口实时画面

+

+ {monitorEvent?.target_label ? `预览来源：${monitorEvent.target_label}` : `来自本地监控流：${MONITOR_ENDPOINT}/stream.mjpg`} +

+
+
+ {streamFps ? `${streamFps.toFixed(1)} FPS` : "FPS --"} + {monitorEvent ? "在线" : "等待监控服务"} +
+
+
+ 微信窗口实时画面 { + window.setTimeout(() => { + setStreamUrl(`${MONITOR_ENDPOINT}/frame.jpg?t=${Date.now()}`); + }, 1000); + }} + /> +
+ {streamFps ? `${streamFps.toFixed(1)} FPS` : "等待帧率"} +
+ {annotation?.regions?.map((region) => { + const style = regionStyle(region); + if (!style) return null; + return ( +
+
+ {region.type || region.name} +
+
+ ); + })} + {streamError ? ( +
+ {streamError} +
+ ) : null} +
+
+
+ + +
+ ); +} diff --git a/src/windows/SecondaryWindow.jsx b/src/windows/SecondaryWindow.jsx index ab5f40b..1a563ac 100644 --- a/src/windows/SecondaryWindow.jsx +++ b/src/windows/SecondaryWindow.jsx @@ -1,13 +1,18 @@ import { useMemo } from "react"; +import { invoke } from "@tauri-apps/api/core"; +import { emit } from "@tauri-apps/api/event"; import { logs } from "../data/mockData"; import Terminal from "../components/Terminal"; import TitleBar from "../components/TitleBar"; +import EngineWorkbench from "./EngineWorkbench"; import PlaceholderWindow from "./PlaceholderWindow"; import SettingsWindow from "./SettingsWindow"; +import { closeWindow } from "../utils/navigation"; export default function SecondaryWindow({ route, theme, setTheme, settingSection, setSettingSection }) { const title = useMemo(() => { if (route.includes("settings")) return "设置"; + if (route.includes("engine-workbench")) return "微信引擎工作台"; if (route.includes("engine-logs")) return "分身引擎日志"; if (route.includes("knowledge-create")) return "新增知识库"; if (route.includes("knowledge-detail")) return "知识库详情"; @@ -17,12 +22,27 @@ export default function SecondaryWindow({ route, theme, setTheme, settingSection return "二级窗口"; }, [route]); + async function closeEngineWorkbench() { + await invoke("stop_agent").catch(() => {}); + await invoke("stop_vision_stream").catch(() => {}); + await emit("engine-state-changed", { enabled: false }); + await closeWindow(); + } + return (
- -
+ +
{route.includes("settings") ? ( + ) : route.includes("engine-workbench") ? ( + ) : route.includes("engine-logs") ? (
) : ( diff --git a/vision_test/app.py b/vision_test/app.py new file mode 100644 index 0000000..aac2268 --- /dev/null +++ b/vision_test/app.py @@ -0,0 +1,6 @@ + + +# 测试python服务 + +print("hello world") + diff --git a/wechat_vision/start_wechat_algorithm_live.sh b/wechat_vision/start_wechat_algorithm_live.sh index 344ecc1..a8dc3dd 100755 --- a/wechat_vision/start_wechat_algorithm_live.sh +++ b/wechat_vision/start_wechat_algorithm_live.sh @@ -27,7 +27,6 @@ nohup ./venv/bin/python wechat_algorithm_live.py \ --send-reply \ --typing-chunk-size 2 \ --typing-delay 0.08 \ - --open-browser \ > "$LOG_FILE" 2>&1 & PID="$!"