Add WeChat vision streaming workbench
This commit is contained in:
王森
parent
f9661f5109
commit
53d48c450a
21
.codegraph/.gitignore
vendored
21
.codegraph/.gitignore
vendored
@ -1,16 +1,5 @@
|
||||
# CodeGraph data files
|
||||
# These are local to each machine and should not be committed
|
||||
|
||||
# Database
|
||||
*.db
|
||||
*.db-wal
|
||||
*.db-shm
|
||||
|
||||
# Cache
|
||||
cache/
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
|
||||
# Hook markers
|
||||
.dirty
|
||||
# CodeGraph data files — local to each machine, not for committing.
|
||||
# Ignore everything in .codegraph/ except this file itself, so transient
|
||||
# files (the database, daemon.pid, sockets, logs) never show up in git.
|
||||
*
|
||||
!.gitignore
|
||||
|
||||
@ -1,6 +0,0 @@
|
||||
{
|
||||
"pid": 30162,
|
||||
"version": "0.9.7",
|
||||
"socketPath": "/Users/wang/agent_dev/wechat_ai/.codegraph/daemon.sock",
|
||||
"startedAt": 1781241939180
|
||||
}
|
||||
3
.gitignore
vendored
3
.gitignore
vendored
@ -14,6 +14,9 @@ wechat_vision/__pycache__/
|
||||
wechat_vision/ouptsw/
|
||||
wechat_vision/test/
|
||||
|
||||
vision_test/.venv/
|
||||
|
||||
demos/go-ffmpeg-wechat-stream/go-ffmpeg-wechat-stream
|
||||
.codegraph/daemon.pid
|
||||
.vscode/
|
||||
|
||||
|
||||
@ -6,6 +6,14 @@ import (
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) > 1 && os.Args[1] == "vision-stream" {
|
||||
if err := runVisionStream(os.Args[2:]); err != nil {
|
||||
logError(err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if len(os.Args) > 1 && os.Args[1] == "ax-probe" {
|
||||
if err := runAXProbe(); err != nil {
|
||||
logError(err.Error())
|
||||
|
||||
668
agent/vision_stream.go
Normal file
668
agent/vision_stream.go
Normal file
@ -0,0 +1,668 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"image"
|
||||
_ "image/jpeg"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type visionFrameState struct {
|
||||
condition *sync.Cond
|
||||
jpeg []byte
|
||||
frameWidth int
|
||||
frameHeight int
|
||||
sequence int64
|
||||
frameIndex int64
|
||||
updatedAt time.Time
|
||||
target previewTarget
|
||||
annotation annotationMeta
|
||||
event map[string]any
|
||||
}
|
||||
|
||||
type annotationMeta struct {
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
ScaleFactor float64 `json:"scaleFactor"`
|
||||
}
|
||||
|
||||
type captureSource struct {
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
Label string `json:"label"`
|
||||
AppName string `json:"appName"`
|
||||
Title string `json:"title"`
|
||||
PID int `json:"pid"`
|
||||
X float64 `json:"x"`
|
||||
Y float64 `json:"y"`
|
||||
Width float64 `json:"width"`
|
||||
Height float64 `json:"height"`
|
||||
ScaleFactor float64 `json:"scaleFactor"`
|
||||
}
|
||||
|
||||
type previewTarget struct {
|
||||
Kind string
|
||||
Label string
|
||||
WindowID int
|
||||
Rect WindowBounds
|
||||
}
|
||||
|
||||
func newVisionFrameState(target previewTarget) *visionFrameState {
|
||||
return &visionFrameState{
|
||||
condition: sync.NewCond(&sync.Mutex{}),
|
||||
target: target,
|
||||
event: map[string]any{"type": "waiting"},
|
||||
}
|
||||
}
|
||||
|
||||
func (state *visionFrameState) update(jpeg []byte) {
|
||||
frameWidth, frameHeight := jpegSize(jpeg)
|
||||
state.condition.L.Lock()
|
||||
state.jpeg = append(state.jpeg[:0], jpeg...)
|
||||
if frameWidth > 0 && frameHeight > 0 {
|
||||
state.frameWidth = frameWidth
|
||||
state.frameHeight = frameHeight
|
||||
}
|
||||
state.sequence++
|
||||
state.frameIndex++
|
||||
state.updatedAt = time.Now().UTC()
|
||||
state.event = map[string]any{
|
||||
"type": "preview_frame",
|
||||
"timestamp": state.updatedAt.Format(time.RFC3339Nano),
|
||||
"frame_index": state.frameIndex,
|
||||
"target_kind": state.target.Kind,
|
||||
"target_label": state.target.Label,
|
||||
}
|
||||
state.condition.Broadcast()
|
||||
state.condition.L.Unlock()
|
||||
}
|
||||
|
||||
func (state *visionFrameState) updateAnnotation(annotation annotationMeta) {
|
||||
state.condition.L.Lock()
|
||||
state.annotation = annotation
|
||||
state.condition.L.Unlock()
|
||||
}
|
||||
|
||||
func (state *visionFrameState) updateStatus(event map[string]any, target previewTarget) {
|
||||
state.condition.L.Lock()
|
||||
state.updatedAt = time.Now().UTC()
|
||||
state.event = event
|
||||
state.target = target
|
||||
state.condition.Broadcast()
|
||||
state.condition.L.Unlock()
|
||||
}
|
||||
|
||||
func (state *visionFrameState) snapshot() ([]byte, map[string]any, int64, time.Time, int, int, annotationMeta) {
|
||||
state.condition.L.Lock()
|
||||
defer state.condition.L.Unlock()
|
||||
jpeg := append([]byte(nil), state.jpeg...)
|
||||
event := make(map[string]any, len(state.event))
|
||||
for key, value := range state.event {
|
||||
event[key] = value
|
||||
}
|
||||
return jpeg, event, state.sequence, state.updatedAt, state.frameWidth, state.frameHeight, state.annotation
|
||||
}
|
||||
|
||||
func (state *visionFrameState) waitForNext(lastSequence int64) ([]byte, int64) {
|
||||
state.condition.L.Lock()
|
||||
defer state.condition.L.Unlock()
|
||||
for state.sequence == lastSequence {
|
||||
state.condition.Wait()
|
||||
}
|
||||
return append([]byte(nil), state.jpeg...), state.sequence
|
||||
}
|
||||
|
||||
func runVisionStream(args []string) error {
|
||||
flags := flag.NewFlagSet("vision-stream", flag.ContinueOnError)
|
||||
host := flags.String("host", "127.0.0.1", "HTTP server host")
|
||||
port := flags.Int("port", 8765, "HTTP server port")
|
||||
input := flags.String("input", "3:none", "FFmpeg avfoundation input for experiments")
|
||||
fps := flags.Float64("fps", 30, "capture FPS")
|
||||
cropWechat := flags.Bool("crop-wechat", true, "capture only the WeChat window")
|
||||
regionsPath := flags.String("regions", defaultRegionsPath(), "annotation regions JSON path")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
state := newVisionFrameState(previewTarget{Kind: "waiting", Label: "等待标注来源"})
|
||||
server := &http.Server{
|
||||
Addr: fmt.Sprintf("%s:%d", *host, *port),
|
||||
Handler: visionHTTPHandler(state),
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
}
|
||||
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer shutdownCancel()
|
||||
_ = server.Shutdown(shutdownCtx)
|
||||
}()
|
||||
|
||||
errCh := make(chan error, 2)
|
||||
go func() {
|
||||
errCh <- server.ListenAndServe()
|
||||
}()
|
||||
go func() {
|
||||
errCh <- runVisionCapture(ctx, state, *input, *fps, *cropWechat, *regionsPath)
|
||||
}()
|
||||
|
||||
logInfo(fmt.Sprintf("Go annotation source stream started http://%s:%d/stream.mjpg regions=%s fps=%.1f", *host, *port, *regionsPath, *fps))
|
||||
|
||||
for err := range errCh {
|
||||
if err == nil || err == http.ErrServerClosed || ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
cancel()
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func visionHTTPHandler(state *visionFrameState) http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) {
|
||||
writer.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
_, _ = io.WriteString(writer, `<html><body style="margin:0;background:#050505;color:#d7ffe4;font-family:monospace"><img src="/stream.mjpg" style="width:100vw;height:100vh;object-fit:contain" /></body></html>`)
|
||||
})
|
||||
mux.HandleFunc("/latest.json", func(writer http.ResponseWriter, request *http.Request) {
|
||||
_, event, sequence, updatedAt, frameWidth, frameHeight, annotation := state.snapshot()
|
||||
payload := map[string]any{
|
||||
"source": "go-window-capture",
|
||||
"sequence": sequence,
|
||||
"updated_at": updatedAt.Format(time.RFC3339Nano),
|
||||
"event": event,
|
||||
"frame": map[string]any{
|
||||
"width": frameWidth,
|
||||
"height": frameHeight,
|
||||
},
|
||||
"annotation": annotation,
|
||||
"target": map[string]any{
|
||||
"kind": state.target.Kind,
|
||||
"label": state.target.Label,
|
||||
"window_id": state.target.WindowID,
|
||||
"rect": state.target.Rect,
|
||||
},
|
||||
}
|
||||
if updatedAt.IsZero() {
|
||||
payload["updated_at"] = time.Now().UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
writer.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
writer.Header().Set("Cache-Control", "no-store")
|
||||
_ = json.NewEncoder(writer).Encode(payload)
|
||||
})
|
||||
mux.HandleFunc("/frame.jpg", func(writer http.ResponseWriter, request *http.Request) {
|
||||
jpeg, _, sequence, _, _, _, _ := state.snapshot()
|
||||
if len(jpeg) == 0 {
|
||||
http.Error(writer, "frame not ready", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
writer.Header().Set("Content-Type", "image/jpeg")
|
||||
writer.Header().Set("Cache-Control", "no-store")
|
||||
writer.Header().Set("X-Frame-Sequence", fmt.Sprintf("%d", sequence))
|
||||
_, _ = writer.Write(jpeg)
|
||||
})
|
||||
mux.HandleFunc("/stream.mjpg", func(writer http.ResponseWriter, request *http.Request) {
|
||||
flusher, ok := writer.(http.Flusher)
|
||||
if !ok {
|
||||
http.Error(writer, "streaming unsupported", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
writer.Header().Set("Content-Type", "multipart/x-mixed-replace; boundary=frame")
|
||||
writer.Header().Set("Cache-Control", "no-store")
|
||||
lastSequence := int64(-1)
|
||||
for request.Context().Err() == nil {
|
||||
jpeg, sequence := state.waitForNext(lastSequence)
|
||||
lastSequence = sequence
|
||||
if len(jpeg) == 0 {
|
||||
continue
|
||||
}
|
||||
_, err := fmt.Fprintf(writer, "--frame\r\nContent-Type: image/jpeg\r\nContent-Length: %d\r\n\r\n", len(jpeg))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := writer.Write(jpeg); err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := writer.Write([]byte("\r\n")); err != nil {
|
||||
return
|
||||
}
|
||||
flusher.Flush()
|
||||
}
|
||||
})
|
||||
return mux
|
||||
}
|
||||
|
||||
func runVisionCapture(ctx context.Context, state *visionFrameState, input string, fps float64, cropWechat bool, regionsPath string) error {
|
||||
for ctx.Err() == nil {
|
||||
target, annotation, err := previewTargetFromAnnotation(regionsPath)
|
||||
if err != nil {
|
||||
message := "等待标注来源窗口出现：" + err.Error()
|
||||
logWarning(message)
|
||||
state.updateStatus(map[string]any{
|
||||
"type": "source_waiting",
|
||||
"timestamp": time.Now().UTC().Format(time.RFC3339Nano),
|
||||
"message": message,
|
||||
}, previewTarget{Kind: "waiting", Label: "等待标注来源"})
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-time.After(2 * time.Second):
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
state.updateAnnotation(annotation)
|
||||
return runResolvedVisionCapture(ctx, state, input, fps, cropWechat, target)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runResolvedVisionCapture(ctx context.Context, state *visionFrameState, input string, fps float64, cropWechat bool, target previewTarget) error {
|
||||
if target.Kind == "window" {
|
||||
logInfo(fmt.Sprintf("using macOS window-only capture window_id=%d", target.WindowID))
|
||||
return runScreencaptureMJPEG(ctx, state, fps, &target.Rect, target.WindowID)
|
||||
}
|
||||
|
||||
if target.Kind == "display" {
|
||||
logInfo(fmt.Sprintf("using macOS display rectangle capture rect=[%.0f,%.0f,%.0f,%.0f]", target.Rect.X, target.Rect.Y, target.Rect.Width, target.Rect.Height))
|
||||
return runScreencaptureMJPEG(ctx, state, fps, &target.Rect, 0)
|
||||
}
|
||||
|
||||
window, err := maybeFindWeChatWindow(cropWechat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logInfo("using macOS window-only capture for fallback WeChat preview")
|
||||
return runScreencaptureMJPEG(ctx, state, fps, window, window.ID)
|
||||
|
||||
// Keep the FFmpeg path below as an experiment. It captures the screen and crops
|
||||
// by rectangle, so covered windows can leak into the preview. Tauri preview uses
|
||||
// screencapture -l instead, which captures the WeChat window itself.
|
||||
ffmpegErr := make(chan error, 1)
|
||||
ffmpegCtx, cancelFFmpeg := context.WithCancel(ctx)
|
||||
go func() {
|
||||
ffmpegErr <- runFFmpegMJPEG(ffmpegCtx, state, input, fps, window)
|
||||
}()
|
||||
|
||||
timer := time.NewTimer(3 * time.Second)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
cancelFFmpeg()
|
||||
return nil
|
||||
case err := <-ffmpegErr:
|
||||
cancelFFmpeg()
|
||||
if err != nil {
|
||||
logWarning("FFmpeg exited before first frame, fallback to screencapture: " + err.Error())
|
||||
}
|
||||
return runScreencaptureMJPEG(ctx, state, fps, window, window.ID)
|
||||
case <-timer.C:
|
||||
_, _, sequence, _, _, _, _ := state.snapshot()
|
||||
if sequence > 0 {
|
||||
logInfo("FFmpeg produced frames, keep using FFmpeg capture")
|
||||
return <-ffmpegErr
|
||||
}
|
||||
cancelFFmpeg()
|
||||
logWarning("FFmpeg produced no frames in 3s, fallback to screencapture")
|
||||
return runScreencaptureMJPEG(ctx, state, fps, window, window.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func maybeFindWeChatWindow(cropWechat bool) (*WindowBounds, error) {
|
||||
if !cropWechat {
|
||||
return nil, nil
|
||||
}
|
||||
bounds, err := findWeChatWindow()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("find WeChat window failed, fallback to full screen capture: %w", err)
|
||||
}
|
||||
logInfo(fmt.Sprintf(
|
||||
"WeChat window found owner=%s title=%s id=%d rect=[%.0f,%.0f,%.0f,%.0f]",
|
||||
bounds.Owner,
|
||||
bounds.Title,
|
||||
bounds.ID,
|
||||
bounds.X,
|
||||
bounds.Y,
|
||||
bounds.Width,
|
||||
bounds.Height,
|
||||
))
|
||||
return &bounds, nil
|
||||
}
|
||||
|
||||
func defaultRegionsPath() string {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return "data/regions/wechat.json"
|
||||
}
|
||||
return filepath.Join(home, "Library", "Application Support", "com.tauri.dev", "data", "regions", "wechat.json")
|
||||
}
|
||||
|
||||
func previewTargetFromAnnotation(path string) (previewTarget, annotationMeta, error) {
|
||||
annotation, err := loadAnnotation(path)
|
||||
if err != nil {
|
||||
return previewTarget{}, annotationMeta{}, fmt.Errorf("load annotation source failed: %w", err)
|
||||
}
|
||||
meta := annotationMeta{
|
||||
Width: annotation.ScreenshotWidth,
|
||||
Height: annotation.ScreenshotHeight,
|
||||
ScaleFactor: annotation.ScaleFactor,
|
||||
}
|
||||
|
||||
source := captureSourceFromAnnotation(annotation.Source)
|
||||
if source.Kind == "" {
|
||||
target, err := fallbackWeChatPreviewTarget()
|
||||
return target, meta, err
|
||||
}
|
||||
|
||||
if source.Kind == "window" {
|
||||
window, err := findWindowForSource(source)
|
||||
if err != nil {
|
||||
return previewTarget{}, meta, err
|
||||
}
|
||||
return previewTarget{
|
||||
Kind: "window",
|
||||
Label: source.Label,
|
||||
WindowID: window.ID,
|
||||
Rect: window,
|
||||
}, meta, nil
|
||||
}
|
||||
|
||||
if source.Kind == "display" {
|
||||
return previewTarget{
|
||||
Kind: "display",
|
||||
Label: source.Label,
|
||||
Rect: WindowBounds{
|
||||
X: source.X,
|
||||
Y: source.Y,
|
||||
Width: source.Width,
|
||||
Height: source.Height,
|
||||
Owner: "display",
|
||||
Title: source.Label,
|
||||
},
|
||||
}, meta, nil
|
||||
}
|
||||
|
||||
return previewTarget{}, meta, fmt.Errorf("unsupported annotation source kind: %s", source.Kind)
|
||||
}
|
||||
|
||||
func fallbackWeChatPreviewTarget() (previewTarget, error) {
|
||||
window, err := findWeChatWindow()
|
||||
if err != nil {
|
||||
return previewTarget{}, err
|
||||
}
|
||||
return previewTarget{
|
||||
Kind: "window",
|
||||
Label: "微信",
|
||||
WindowID: window.ID,
|
||||
Rect: window,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func captureSourceFromAnnotation(raw jsonRawOptional) captureSource {
|
||||
data, err := json.Marshal(raw)
|
||||
if err != nil {
|
||||
return captureSource{}
|
||||
}
|
||||
var source captureSource
|
||||
if err := json.Unmarshal(data, &source); err != nil {
|
||||
return captureSource{}
|
||||
}
|
||||
return source
|
||||
}
|
||||
|
||||
func findWindowForSource(source captureSource) (WindowBounds, error) {
|
||||
appName := strings.ReplaceAll(source.AppName, "\\", "\\\\")
|
||||
appName = strings.ReplaceAll(appName, `"`, `\"`)
|
||||
title := strings.ReplaceAll(source.Title, "\\", "\\\\")
|
||||
title = strings.ReplaceAll(title, `"`, `\"`)
|
||||
|
||||
script := fmt.Sprintf(`
|
||||
import CoreGraphics
|
||||
import Foundation
|
||||
|
||||
let targetOwner = "%s"
|
||||
let targetTitle = "%s"
|
||||
let options = CGWindowListOption(arrayLiteral: [.optionOnScreenOnly, .excludeDesktopElements])
|
||||
guard let infos = CGWindowListCopyWindowInfo(options, kCGNullWindowID) as? [[String: Any]] else { exit(1) }
|
||||
|
||||
func printWindow(_ info: [String: Any]) {
|
||||
let owner = info[kCGWindowOwnerName as String] as? String ?? ""
|
||||
let title = info[kCGWindowName as String] as? String ?? ""
|
||||
let bounds = info[kCGWindowBounds as String] as? [String: Any] ?? [:]
|
||||
let x = bounds["X"] as? Double ?? 0
|
||||
let y = bounds["Y"] as? Double ?? 0
|
||||
let w = bounds["Width"] as? Double ?? 0
|
||||
let h = bounds["Height"] as? Double ?? 0
|
||||
let id = info[kCGWindowNumber as String] as? Int ?? 0
|
||||
let result: [String: Any] = ["x": x, "y": y, "width": w, "height": h, "owner": owner, "title": title, "id": id]
|
||||
let data = try! JSONSerialization.data(withJSONObject: result)
|
||||
print(String(data: data, encoding: .utf8)!)
|
||||
}
|
||||
|
||||
for info in infos {
|
||||
let owner = info[kCGWindowOwnerName as String] as? String ?? ""
|
||||
let title = info[kCGWindowName as String] as? String ?? ""
|
||||
let layer = info[kCGWindowLayer as String] as? Int ?? -1
|
||||
if layer == 0 && owner == targetOwner && title == targetTitle { printWindow(info); exit(0) }
|
||||
}
|
||||
|
||||
for info in infos {
|
||||
let owner = info[kCGWindowOwnerName as String] as? String ?? ""
|
||||
let title = info[kCGWindowName as String] as? String ?? ""
|
||||
let layer = info[kCGWindowLayer as String] as? Int ?? -1
|
||||
if layer == 0 && targetOwner != "" && owner.contains(targetOwner) { printWindow(info); exit(0) }
|
||||
}
|
||||
|
||||
exit(2)
|
||||
`, appName, title)
|
||||
|
||||
output, err := runSwift(script)
|
||||
if err != nil {
|
||||
return WindowBounds{}, fmt.Errorf("find annotation window source failed app=%q title=%q: %w", source.AppName, source.Title, err)
|
||||
}
|
||||
var window WindowBounds
|
||||
if err := json.Unmarshal([]byte(output), &window); err != nil {
|
||||
return WindowBounds{}, err
|
||||
}
|
||||
return window, nil
|
||||
}
|
||||
|
||||
func runFFmpegMJPEG(ctx context.Context, state *visionFrameState, input string, fps float64, window *WindowBounds) error {
|
||||
args := []string{
|
||||
"-hide_banner",
|
||||
"-loglevel", "error",
|
||||
"-f", "avfoundation",
|
||||
"-framerate", fmt.Sprintf("%.3f", fps),
|
||||
"-pixel_format", "bgr0",
|
||||
"-i", input,
|
||||
}
|
||||
if window != nil {
|
||||
args = append(args, "-vf", ffmpegCropFilter(*window))
|
||||
}
|
||||
args = append(args,
|
||||
"-f", "image2pipe",
|
||||
"-vcodec", "mjpeg",
|
||||
"-",
|
||||
)
|
||||
logInfo("FFmpeg command: ffmpeg " + strings.Join(args, " "))
|
||||
command := exec.CommandContext(ctx, "ffmpeg", args...)
|
||||
stdout, err := command.StdoutPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stderr, err := command.StderrPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := command.Start(); err != nil {
|
||||
return fmt.Errorf("启动 FFmpeg 失败：%w", err)
|
||||
}
|
||||
|
||||
go drainFFmpegStderr(stderr)
|
||||
|
||||
readErr := readJPEGFrames(stdout, state.update)
|
||||
waitErr := command.Wait()
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
if readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
if waitErr != nil {
|
||||
return fmt.Errorf("FFmpeg 退出：%w", waitErr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runScreencaptureMJPEG(ctx context.Context, state *visionFrameState, fps float64, window *WindowBounds, windowID int) error {
|
||||
if window == nil {
|
||||
return errors.New("screencapture requires a capture target")
|
||||
}
|
||||
interval := time.Duration(float64(time.Second) / math.Max(fps, 1))
|
||||
if interval < 80*time.Millisecond {
|
||||
interval = 80 * time.Millisecond
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
tempDir, err := os.MkdirTemp("", "wechat-go-vision-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
imagePath := filepath.Join(tempDir, "frame.jpg")
|
||||
rect := captureRect(*window)
|
||||
if windowID > 0 {
|
||||
logInfo(fmt.Sprintf("window-only screencapture started window_id=%d", windowID))
|
||||
} else {
|
||||
logInfo("rectangle screencapture started rect=" + rect)
|
||||
}
|
||||
for ctx.Err() == nil {
|
||||
args := []string{"-x"}
|
||||
if windowID > 0 {
|
||||
args = append(args, "-l", fmt.Sprintf("%d", windowID))
|
||||
} else {
|
||||
args = append(args, "-R", rect)
|
||||
}
|
||||
args = append(args, imagePath)
|
||||
command := exec.CommandContext(ctx, "screencapture", args...)
|
||||
if output, err := command.CombinedOutput(); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
logWarning(fmt.Sprintf("screencapture failed: %v output=%s", err, string(output)))
|
||||
} else if jpeg, err := os.ReadFile(imagePath); err != nil {
|
||||
logWarning("read screencapture frame failed: " + err.Error())
|
||||
} else if len(jpeg) > 0 {
|
||||
state.update(jpeg)
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ffmpegCropFilter(window WindowBounds) string {
|
||||
width := int(math.Max(1, math.Round(window.Width)))
|
||||
height := int(math.Max(1, math.Round(window.Height)))
|
||||
x := int(math.Max(0, math.Round(window.X)))
|
||||
y := int(math.Max(0, math.Round(window.Y)))
|
||||
return fmt.Sprintf("crop=%d:%d:%d:%d", width, height, x, y)
|
||||
}
|
||||
|
||||
func captureRect(window WindowBounds) string {
|
||||
x := int(math.Round(window.X))
|
||||
y := int(math.Round(window.Y))
|
||||
width := int(math.Round(window.Width))
|
||||
height := int(math.Round(window.Height))
|
||||
if x < 0 {
|
||||
width += x
|
||||
x = 0
|
||||
}
|
||||
if y < 0 {
|
||||
height += y
|
||||
y = 0
|
||||
}
|
||||
if width < 1 {
|
||||
width = 1
|
||||
}
|
||||
if height < 1 {
|
||||
height = 1
|
||||
}
|
||||
return fmt.Sprintf("%d,%d,%d,%d", x, y, width, height)
|
||||
}
|
||||
|
||||
func readJPEGFrames(reader io.Reader, onFrame func([]byte)) error {
|
||||
buffer := make([]byte, 0, 1024*1024)
|
||||
chunk := make([]byte, 64*1024)
|
||||
for {
|
||||
readCount, err := reader.Read(chunk)
|
||||
if readCount > 0 {
|
||||
buffer = append(buffer, chunk[:readCount]...)
|
||||
for {
|
||||
start := bytes.Index(buffer, []byte{0xff, 0xd8})
|
||||
if start < 0 {
|
||||
if len(buffer) > 1024*1024 {
|
||||
buffer = buffer[len(buffer)-2:]
|
||||
}
|
||||
break
|
||||
}
|
||||
endOffset := bytes.Index(buffer[start+2:], []byte{0xff, 0xd9})
|
||||
if endOffset < 0 {
|
||||
if start > 0 {
|
||||
buffer = buffer[start:]
|
||||
}
|
||||
break
|
||||
}
|
||||
end := start + 2 + endOffset + 2
|
||||
frame := append([]byte(nil), buffer[start:end]...)
|
||||
buffer = buffer[end:]
|
||||
onFrame(frame)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func jpegSize(jpeg []byte) (int, int) {
|
||||
config, _, err := image.DecodeConfig(bytes.NewReader(jpeg))
|
||||
if err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
return config.Width, config.Height
|
||||
}
|
||||
|
||||
func drainFFmpegStderr(reader io.Reader) {
|
||||
scanner := bufio.NewScanner(reader)
|
||||
for scanner.Scan() {
|
||||
logWarning("ffmpeg: " + scanner.Text())
|
||||
}
|
||||
}
|
||||
38
demos/go-ffmpeg-wechat-stream/README.md
Normal file
38
demos/go-ffmpeg-wechat-stream/README.md
Normal file
@ -0,0 +1,38 @@
|
||||
# Go FFmpeg WeChat Stream Demo
|
||||
|
||||
Standalone demo: Go finds the visible WeChat window, starts FFmpeg `avfoundation` screen capture, crops to the WeChat window rectangle, and serves an MJPEG stream locally.
|
||||
|
||||
If FFmpeg screen capture does not produce frames, the demo automatically falls back to macOS window-only capture with `screencapture -l <window_id>`. This captures the WeChat window itself, so other windows covering it should not appear in the stream.
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd demos/go-ffmpeg-wechat-stream
|
||||
go run .
|
||||
```
|
||||
|
||||
Open:
|
||||
|
||||
```text
|
||||
http://127.0.0.1:8765/
|
||||
```
|
||||
|
||||
Options:
|
||||
|
||||
```bash
|
||||
go run . -input 3:none -fps 15 -port 8765 -open=true -wait=true
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- macOS only.
|
||||
- Requires FFmpeg installed and available in `PATH`.
|
||||
- Requires Screen Recording permission for the terminal/app running the demo.
|
||||
- WeChat must be visible on screen.
|
||||
- By default the demo waits until a visible WeChat window is found. Use `-wait=false` to fail immediately.
|
||||
- FFmpeg `avfoundation` is screen-based, so crop mode can include windows covering WeChat. The fallback `screencapture -l` path is window-based and is closer to Tencent Meeting style app-window sharing.
|
||||
- `-input` is the FFmpeg `avfoundation` screen device. List devices with:
|
||||
|
||||
```bash
|
||||
ffmpeg -hide_banner -f avfoundation -list_devices true -i ""
|
||||
```
|
||||
3
demos/go-ffmpeg-wechat-stream/go.mod
Normal file
3
demos/go-ffmpeg-wechat-stream/go.mod
Normal file
@ -0,0 +1,3 @@
|
||||
module go-ffmpeg-wechat-stream
|
||||
|
||||
go 1.22
|
||||
470
demos/go-ffmpeg-wechat-stream/main.go
Normal file
470
demos/go-ffmpeg-wechat-stream/main.go
Normal file
@ -0,0 +1,470 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type windowBounds struct {
|
||||
X float64 `json:"x"`
|
||||
Y float64 `json:"y"`
|
||||
Width float64 `json:"width"`
|
||||
Height float64 `json:"height"`
|
||||
Owner string `json:"owner"`
|
||||
Title string `json:"title"`
|
||||
ID int `json:"id"`
|
||||
}
|
||||
|
||||
type streamState struct {
|
||||
condition *sync.Cond
|
||||
jpeg []byte
|
||||
sequence int64
|
||||
window windowBounds
|
||||
startedAt time.Time
|
||||
updatedAt time.Time
|
||||
}
|
||||
|
||||
func newStreamState(window windowBounds) *streamState {
|
||||
return &streamState{
|
||||
condition: sync.NewCond(&sync.Mutex{}),
|
||||
window: window,
|
||||
startedAt: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
func (state *streamState) update(jpeg []byte) {
|
||||
state.condition.L.Lock()
|
||||
state.jpeg = append(state.jpeg[:0], jpeg...)
|
||||
state.sequence++
|
||||
state.updatedAt = time.Now()
|
||||
state.condition.Broadcast()
|
||||
state.condition.L.Unlock()
|
||||
}
|
||||
|
||||
func (state *streamState) snapshot() (int64, time.Time, windowBounds) {
|
||||
state.condition.L.Lock()
|
||||
defer state.condition.L.Unlock()
|
||||
return state.sequence, state.updatedAt, state.window
|
||||
}
|
||||
|
||||
func (state *streamState) waitNext(lastSequence int64) ([]byte, int64) {
|
||||
state.condition.L.Lock()
|
||||
defer state.condition.L.Unlock()
|
||||
for state.sequence == lastSequence {
|
||||
state.condition.Wait()
|
||||
}
|
||||
return append([]byte(nil), state.jpeg...), state.sequence
|
||||
}
|
||||
|
||||
func main() {
|
||||
if runtime.GOOS != "darwin" {
|
||||
log.Fatal("this demo currently supports macOS only because it uses avfoundation + CoreGraphics")
|
||||
}
|
||||
|
||||
var host string
|
||||
var port int
|
||||
var input string
|
||||
var fps float64
|
||||
var openBrowser bool
|
||||
var waitWindow bool
|
||||
flag.StringVar(&host, "host", "127.0.0.1", "HTTP server host")
|
||||
flag.IntVar(&port, "port", 8765, "HTTP server port")
|
||||
flag.StringVar(&input, "input", "3:none", "FFmpeg avfoundation input, e.g. 3:none")
|
||||
flag.Float64Var(&fps, "fps", 15, "capture frame rate")
|
||||
flag.BoolVar(&openBrowser, "open", true, "open browser automatically")
|
||||
flag.BoolVar(&waitWindow, "wait", true, "wait until a visible WeChat window is found")
|
||||
flag.Parse()
|
||||
|
||||
window, err := waitForWeChatWindow(waitWindow)
|
||||
if err != nil {
|
||||
log.Fatalf("find WeChat window failed: %v", err)
|
||||
}
|
||||
log.Printf("WeChat window: owner=%q title=%q id=%d rect=[%.0f,%.0f,%.0f,%.0f]", window.Owner, window.Title, window.ID, window.X, window.Y, window.Width, window.Height)
|
||||
|
||||
state := newStreamState(window)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
go runCapture(ctx, state, input, fps, window)
|
||||
|
||||
server := &http.Server{
|
||||
Addr: fmt.Sprintf("%s:%d", host, port),
|
||||
Handler: handler(state),
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("http://%s:%d/", host, port)
|
||||
log.Printf("Serving %s", url)
|
||||
if openBrowser {
|
||||
go func() {
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
_ = exec.Command("open", url).Start()
|
||||
}()
|
||||
}
|
||||
|
||||
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func runCapture(ctx context.Context, state *streamState, input string, fps float64, window windowBounds) {
|
||||
ffmpegCtx, cancelFFmpeg := context.WithCancel(ctx)
|
||||
ffmpegErr := make(chan error, 1)
|
||||
go func() {
|
||||
ffmpegErr <- runFFmpeg(ffmpegCtx, state, input, fps, window)
|
||||
}()
|
||||
|
||||
timer := time.NewTimer(3 * time.Second)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
cancelFFmpeg()
|
||||
return
|
||||
case err := <-ffmpegErr:
|
||||
cancelFFmpeg()
|
||||
log.Printf("FFmpeg stopped before first frame, fallback to screencapture: %v", err)
|
||||
case <-timer.C:
|
||||
sequence, _, _ := state.snapshot()
|
||||
if sequence > 0 {
|
||||
log.Printf("FFmpeg is producing frames")
|
||||
if err := <-ffmpegErr; err != nil && ctx.Err() == nil {
|
||||
log.Printf("FFmpeg stopped: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
cancelFFmpeg()
|
||||
log.Printf("FFmpeg produced no frames in 3s, fallback to window-only screencapture")
|
||||
}
|
||||
|
||||
if err := runScreencapture(ctx, state, fps, window); err != nil && ctx.Err() == nil {
|
||||
log.Printf("screencapture stopped: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func waitForWeChatWindow(wait bool) (windowBounds, error) {
|
||||
for {
|
||||
window, err := findWeChatWindow()
|
||||
if err == nil {
|
||||
return window, nil
|
||||
}
|
||||
if !wait {
|
||||
return windowBounds{}, err
|
||||
}
|
||||
log.Printf("waiting for visible WeChat window: %v", err)
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func findWeChatWindow() (windowBounds, error) {
|
||||
script := `
|
||||
import CoreGraphics
|
||||
import Foundation
|
||||
|
||||
let options = CGWindowListOption(arrayLiteral: [.optionOnScreenOnly, .excludeDesktopElements])
|
||||
guard let infos = CGWindowListCopyWindowInfo(options, kCGNullWindowID) as? [[String: Any]] else {
|
||||
exit(1)
|
||||
}
|
||||
|
||||
for info in infos {
|
||||
let owner = info[kCGWindowOwnerName as String] as? String ?? ""
|
||||
let title = info[kCGWindowName as String] as? String ?? ""
|
||||
let layer = info[kCGWindowLayer as String] as? Int ?? -1
|
||||
if layer == 0 && (owner.contains("微信") || owner.lowercased().contains("wechat") || title == "微信") {
|
||||
if let bounds = info[kCGWindowBounds as String] as? [String: Any],
|
||||
let x = bounds["X"] as? Double,
|
||||
let y = bounds["Y"] as? Double,
|
||||
let w = bounds["Width"] as? Double,
|
||||
let h = bounds["Height"] as? Double {
|
||||
let id = info[kCGWindowNumber as String] as? Int ?? 0
|
||||
let result: [String: Any] = ["x": x, "y": y, "width": w, "height": h, "owner": owner, "title": title, "id": id]
|
||||
let data = try! JSONSerialization.data(withJSONObject: result)
|
||||
print(String(data: data, encoding: .utf8)!)
|
||||
exit(0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
exit(2)
|
||||
`
|
||||
command := exec.Command("swift", "-")
|
||||
command.Stdin = strings.NewReader(script)
|
||||
output, err := command.Output()
|
||||
if err != nil {
|
||||
return windowBounds{}, err
|
||||
}
|
||||
|
||||
var window windowBounds
|
||||
if err := json.Unmarshal(output, &window); err != nil {
|
||||
return windowBounds{}, err
|
||||
}
|
||||
return window, nil
|
||||
}
|
||||
|
||||
func runFFmpeg(ctx context.Context, state *streamState, input string, fps float64, window windowBounds) error {
|
||||
args := []string{
|
||||
"-hide_banner",
|
||||
"-loglevel", "warning",
|
||||
"-f", "avfoundation",
|
||||
"-framerate", fmt.Sprintf("%.3f", fps),
|
||||
"-pixel_format", "bgr0",
|
||||
"-i", input,
|
||||
"-vf", cropFilter(window),
|
||||
"-f", "image2pipe",
|
||||
"-vcodec", "mjpeg",
|
||||
"-q:v", "5",
|
||||
"-",
|
||||
}
|
||||
log.Printf("FFmpeg command: ffmpeg %s", strings.Join(args, " "))
|
||||
|
||||
command := exec.CommandContext(ctx, "ffmpeg", args...)
|
||||
stdout, err := command.StdoutPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stderr, err := command.StderrPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := command.Start(); err != nil {
|
||||
return err
|
||||
}
|
||||
go logLines("ffmpeg", stderr)
|
||||
|
||||
readErr := readJPEGFrames(stdout, state.update)
|
||||
waitErr := command.Wait()
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
if readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
return waitErr
|
||||
}
|
||||
|
||||
func runScreencapture(ctx context.Context, state *streamState, fps float64, window windowBounds) error {
|
||||
interval := time.Duration(float64(time.Second) / math.Max(fps, 1))
|
||||
if interval < 80*time.Millisecond {
|
||||
interval = 80 * time.Millisecond
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
tempDir, err := os.MkdirTemp("", "go-ffmpeg-wechat-stream-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
framePath := filepath.Join(tempDir, "frame.jpg")
|
||||
rect := captureRect(window)
|
||||
log.Printf("window-only screencapture fallback started window_id=%d rect=%s", window.ID, rect)
|
||||
for ctx.Err() == nil {
|
||||
command := exec.CommandContext(ctx, "screencapture", "-x", "-l", fmt.Sprintf("%d", window.ID), framePath)
|
||||
if output, err := command.CombinedOutput(); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
log.Printf("screencapture failed: %v output=%s", err, string(output))
|
||||
} else if jpeg, err := os.ReadFile(framePath); err != nil {
|
||||
log.Printf("read frame failed: %v", err)
|
||||
} else if len(jpeg) > 0 {
|
||||
state.update(jpeg)
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func captureRect(window windowBounds) string {
|
||||
x := int(math.Round(window.X))
|
||||
y := int(math.Round(window.Y))
|
||||
width := int(math.Round(window.Width))
|
||||
height := int(math.Round(window.Height))
|
||||
if x < 0 {
|
||||
width += x
|
||||
x = 0
|
||||
}
|
||||
if y < 0 {
|
||||
height += y
|
||||
y = 0
|
||||
}
|
||||
if width < 1 {
|
||||
width = 1
|
||||
}
|
||||
if height < 1 {
|
||||
height = 1
|
||||
}
|
||||
return fmt.Sprintf("%d,%d,%d,%d", x, y, width, height)
|
||||
}
|
||||
|
||||
func cropFilter(window windowBounds) string {
|
||||
width := int(math.Max(1, math.Round(window.Width)))
|
||||
height := int(math.Max(1, math.Round(window.Height)))
|
||||
x := int(math.Max(0, math.Round(window.X)))
|
||||
y := int(math.Max(0, math.Round(window.Y)))
|
||||
return fmt.Sprintf("crop=%d:%d:%d:%d", width, height, x, y)
|
||||
}
|
||||
|
||||
func readJPEGFrames(reader io.Reader, onFrame func([]byte)) error {
|
||||
buffer := make([]byte, 0, 1024*1024)
|
||||
chunk := make([]byte, 64*1024)
|
||||
for {
|
||||
readCount, err := reader.Read(chunk)
|
||||
if readCount > 0 {
|
||||
buffer = append(buffer, chunk[:readCount]...)
|
||||
for {
|
||||
start := bytes.Index(buffer, []byte{0xff, 0xd8})
|
||||
if start < 0 {
|
||||
if len(buffer) > 1024*1024 {
|
||||
buffer = buffer[len(buffer)-2:]
|
||||
}
|
||||
break
|
||||
}
|
||||
endOffset := bytes.Index(buffer[start+2:], []byte{0xff, 0xd9})
|
||||
if endOffset < 0 {
|
||||
if start > 0 {
|
||||
buffer = buffer[start:]
|
||||
}
|
||||
break
|
||||
}
|
||||
end := start + 2 + endOffset + 2
|
||||
onFrame(append([]byte(nil), buffer[start:end]...))
|
||||
buffer = buffer[end:]
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func handler(state *streamState) http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) {
|
||||
writer.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
_, _ = io.WriteString(writer, `<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>Go FFmpeg WeChat Stream</title>
|
||||
<style>
|
||||
html, body { margin: 0; height: 100%; background: #050505; color: #d7ffe4; font-family: ui-monospace, SFMono-Regular, Menlo, monospace; }
|
||||
.wrap { height: 100%; display: grid; grid-template-rows: auto 1fr; }
|
||||
.bar { display: flex; gap: 16px; align-items: center; padding: 10px 14px; background: #101510; border-bottom: 1px solid #263526; font-size: 13px; }
|
||||
img { width: 100%; height: 100%; object-fit: contain; background: #000; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
<div class="bar"><strong>Go FFmpeg WeChat Stream</strong><span id="status">connecting...</span></div>
|
||||
<img src="/stream.mjpg" />
|
||||
</div>
|
||||
<script>
|
||||
let prevSeq = null, prevTime = performance.now();
|
||||
async function tick() {
|
||||
try {
|
||||
const res = await fetch('/latest.json?t=' + Date.now(), {cache: 'no-store'});
|
||||
const data = await res.json();
|
||||
const now = performance.now();
|
||||
let fps = '--';
|
||||
if (prevSeq !== null && data.sequence >= prevSeq) fps = ((data.sequence - prevSeq) / ((now - prevTime) / 1000)).toFixed(1);
|
||||
prevSeq = data.sequence;
|
||||
prevTime = now;
|
||||
document.getElementById('status').textContent = 'seq=' + data.sequence + ' fps=' + fps + ' window=' + data.window.width + 'x' + data.window.height;
|
||||
} catch (err) {
|
||||
document.getElementById('status').textContent = 'waiting for stream';
|
||||
}
|
||||
}
|
||||
tick(); setInterval(tick, 1000);
|
||||
</script>
|
||||
</body>
|
||||
</html>`)
|
||||
})
|
||||
mux.HandleFunc("/latest.json", func(writer http.ResponseWriter, request *http.Request) {
|
||||
sequence, updatedAt, window := state.snapshot()
|
||||
writer.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
writer.Header().Set("Cache-Control", "no-store")
|
||||
_ = json.NewEncoder(writer).Encode(map[string]any{
|
||||
"source": "go-ffmpeg-demo",
|
||||
"sequence": sequence,
|
||||
"updated_at": updatedAt.Format(time.RFC3339Nano),
|
||||
"window": window,
|
||||
})
|
||||
})
|
||||
mux.HandleFunc("/stream.mjpg", func(writer http.ResponseWriter, request *http.Request) {
|
||||
flusher, ok := writer.(http.Flusher)
|
||||
if !ok {
|
||||
http.Error(writer, "streaming unsupported", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writer.Header().Set("Content-Type", "multipart/x-mixed-replace; boundary=frame")
|
||||
writer.Header().Set("Cache-Control", "no-store")
|
||||
lastSequence := int64(-1)
|
||||
for request.Context().Err() == nil {
|
||||
jpeg, sequence := state.waitNext(lastSequence)
|
||||
lastSequence = sequence
|
||||
if len(jpeg) == 0 {
|
||||
continue
|
||||
}
|
||||
if _, err := fmt.Fprintf(writer, "--frame\r\nContent-Type: image/jpeg\r\nContent-Length: %d\r\n\r\n", len(jpeg)); err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := writer.Write(jpeg); err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := writer.Write([]byte("\r\n")); err != nil {
|
||||
return
|
||||
}
|
||||
flusher.Flush()
|
||||
}
|
||||
})
|
||||
return mux
|
||||
}
|
||||
|
||||
func logLines(prefix string, reader io.Reader) {
|
||||
buffer := make([]byte, 4096)
|
||||
var line []byte
|
||||
for {
|
||||
count, err := reader.Read(buffer)
|
||||
if count > 0 {
|
||||
line = append(line, buffer[:count]...)
|
||||
for {
|
||||
index := bytes.IndexByte(line, '\n')
|
||||
if index < 0 {
|
||||
break
|
||||
}
|
||||
log.Printf("%s: %s", prefix, strings.TrimSpace(string(line[:index])))
|
||||
line = line[index+1:]
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
if len(line) > 0 {
|
||||
log.Printf("%s: %s", prefix, strings.TrimSpace(string(line)))
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -2,6 +2,7 @@ use screenshots::Screen;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use std::sync::Mutex;
|
||||
use std::time::Instant;
|
||||
use tauri::{Emitter, Manager, WebviewUrl, WebviewWindowBuilder};
|
||||
@ -12,6 +13,7 @@ use tauri_plugin_shell::{
|
||||
use xcap::{Monitor, Window};
|
||||
|
||||
struct AgentProcess(Mutex<Option<CommandChild>>);
|
||||
struct VisionStreamProcess(Mutex<Option<CommandChild>>);
|
||||
|
||||
#[derive(Clone, Deserialize, Serialize)]
|
||||
struct AgentLog {
|
||||
@ -240,6 +242,110 @@ fn stop_agent(app: tauri::AppHandle, process: tauri::State<AgentProcess>) -> Res
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
fn start_vision_stream(
|
||||
app: tauri::AppHandle,
|
||||
process: tauri::State<VisionStreamProcess>,
|
||||
) -> Result<(), String> {
|
||||
let mut process_guard = process.0.lock().map_err(|error| error.to_string())?;
|
||||
if process_guard.is_some() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
stop_legacy_python_vision_stream();
|
||||
|
||||
let (mut rx, child) = app
|
||||
.shell()
|
||||
.sidecar("agent")
|
||||
.map_err(|error| error.to_string())?
|
||||
.args(["vision-stream"])
|
||||
.spawn()
|
||||
.map_err(|error| error.to_string())?;
|
||||
|
||||
*process_guard = Some(child);
|
||||
|
||||
let app_handle = app.clone();
|
||||
tauri::async_runtime::spawn(async move {
|
||||
while let Some(event) = rx.recv().await {
|
||||
match event {
|
||||
CommandEvent::Stdout(line) => {
|
||||
emit_agent_line(
|
||||
&app_handle,
|
||||
"info",
|
||||
String::from_utf8_lossy(&line).trim().to_string(),
|
||||
);
|
||||
}
|
||||
CommandEvent::Stderr(line) => {
|
||||
emit_agent_line(
|
||||
&app_handle,
|
||||
"error",
|
||||
String::from_utf8_lossy(&line).trim().to_string(),
|
||||
);
|
||||
}
|
||||
CommandEvent::Error(error) => {
|
||||
emit_agent_log(
|
||||
&app_handle,
|
||||
"error",
|
||||
format!("vision stream error: {error}"),
|
||||
);
|
||||
}
|
||||
CommandEvent::Terminated(payload) => {
|
||||
emit_agent_log(
|
||||
&app_handle,
|
||||
"warning",
|
||||
format!("vision stream exited with code {:?}", payload.code),
|
||||
);
|
||||
if let Ok(mut process_guard) =
|
||||
app_handle.state::<VisionStreamProcess>().0.lock()
|
||||
{
|
||||
*process_guard = None;
|
||||
}
|
||||
break;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn stop_legacy_python_vision_stream() {
|
||||
let Ok(cwd) = std::env::current_dir() else {
|
||||
return;
|
||||
};
|
||||
let candidates = [
|
||||
cwd.join("wechat_vision"),
|
||||
cwd.join("..").join("wechat_vision"),
|
||||
cwd.join("..").join("..").join("wechat_vision"),
|
||||
];
|
||||
|
||||
for dir in candidates {
|
||||
let pid_file = dir.join("ouptsw").join("wechat_algorithm_live.pid");
|
||||
let stop_script = dir.join("stop_wechat_algorithm_live.sh");
|
||||
if pid_file.exists() && stop_script.exists() {
|
||||
let _ = Command::new("bash")
|
||||
.arg(stop_script)
|
||||
.current_dir(dir)
|
||||
.output();
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
fn stop_vision_stream(
|
||||
_app: tauri::AppHandle,
|
||||
process: tauri::State<VisionStreamProcess>,
|
||||
) -> Result<(), String> {
|
||||
let mut process_guard = process.0.lock().map_err(|error| error.to_string())?;
|
||||
if let Some(child) = process_guard.take() {
|
||||
child.kill().map_err(|error| error.to_string())?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
async fn capture_screen(app: tauri::AppHandle) -> Result<CaptureResult, String> {
|
||||
let screenshot_path = screenshot_path(&app)?;
|
||||
@ -490,6 +596,12 @@ fn open_popup_window(app: tauri::AppHandle, route: String) -> Result<(), String>
|
||||
let label = format!("popup-{}", route.trim_start_matches('/').replace('/', "-"));
|
||||
|
||||
if let Some(window) = app.get_webview_window(&label) {
|
||||
if route == "/window/engine-workbench" {
|
||||
window
|
||||
.set_fullscreen(false)
|
||||
.map_err(|error| error.to_string())?;
|
||||
window.maximize().map_err(|error| error.to_string())?;
|
||||
}
|
||||
window.set_focus().map_err(|error| error.to_string())?;
|
||||
return Ok(());
|
||||
}
|
||||
@ -497,6 +609,7 @@ fn open_popup_window(app: tauri::AppHandle, route: String) -> Result<(), String>
|
||||
let title = match route.as_str() {
|
||||
"/annotate" => "屏幕区域标注",
|
||||
"/window/settings" => "设置",
|
||||
"/window/engine-workbench" => "微信引擎工作台",
|
||||
"/window/engine-logs" => "分身引擎日志",
|
||||
"/window/knowledge-create" => "新增知识库",
|
||||
"/window/knowledge-detail" => "知识库详情",
|
||||
@ -519,6 +632,11 @@ fn open_popup_window(app: tauri::AppHandle, route: String) -> Result<(), String>
|
||||
|
||||
if route == "/annotate" {
|
||||
builder = builder.maximized(true).fullscreen(true);
|
||||
} else if route == "/window/engine-workbench" {
|
||||
builder = builder
|
||||
.inner_size(1440.0, 920.0)
|
||||
.maximized(true)
|
||||
.fullscreen(false);
|
||||
}
|
||||
|
||||
builder.build().map_err(|error| error.to_string())?;
|
||||
@ -530,6 +648,7 @@ fn open_popup_window(app: tauri::AppHandle, route: String) -> Result<(), String>
|
||||
pub fn run() {
|
||||
tauri::Builder::default()
|
||||
.manage(AgentProcess(Mutex::new(None)))
|
||||
.manage(VisionStreamProcess(Mutex::new(None)))
|
||||
.plugin(tauri_plugin_shell::init())
|
||||
.setup(|app| {
|
||||
if cfg!(debug_assertions) {
|
||||
@ -549,6 +668,8 @@ pub fn run() {
|
||||
open_popup_window,
|
||||
save_regions,
|
||||
start_agent,
|
||||
start_vision_stream,
|
||||
stop_vision_stream,
|
||||
stop_agent
|
||||
])
|
||||
.run(tauri::generate_context!())
|
||||
|
||||
@ -27,7 +27,7 @@ export default function Terminal({ title, lines, tall = false, onClear }) {
|
||||
};
|
||||
|
||||
return (
|
||||
<div className={`terminal ${tall ? "basis-[650px]" : "basis-[250px]"} flex min-h-0 flex-1 flex-col`}>
|
||||
<div className={`terminal ${tall ? "min-h-[520px] basis-[650px]" : "min-h-[220px] basis-[250px]"} flex flex-1 flex-col`}>
|
||||
<div className="mb-3 flex shrink-0 items-center justify-between">
|
||||
<h3 className="text-[13px] font-black text-[#d7ffe4]">{title}</h3>
|
||||
<button className="cursor-pointer border-0 bg-transparent p-0 text-[11px] font-bold text-[#7ea58b]" onClick={onClear}>清理日志</button>
|
||||
|
||||
@ -2,7 +2,7 @@ import { LogOut, RotateCcw } from "lucide-react";
|
||||
import ThemeSwitch from "./ThemeSwitch";
|
||||
import { closeWindow, startWindowDrag } from "../utils/navigation";
|
||||
|
||||
export default function TitleBar({ title, theme, setTheme, compact = false }) {
|
||||
export default function TitleBar({ title, theme, setTheme, compact = false, onClose }) {
|
||||
return (
|
||||
<header className="glass-bar relative z-10 flex items-center justify-between gap-4 border-b border-line px-5 py-2">
|
||||
<div
|
||||
@ -24,7 +24,7 @@ export default function TitleBar({ title, theme, setTheme, compact = false }) {
|
||||
<button className="icon-btn" aria-label="退出"><LogOut size={16} strokeWidth={2.5} /></button>
|
||||
</>
|
||||
) : (
|
||||
<button className="btn-secondary h-9 px-3" onClick={closeWindow}>关闭</button>
|
||||
<button className="btn-secondary h-9 px-3" onClick={onClose || closeWindow}>关闭</button>
|
||||
)}
|
||||
</div>
|
||||
</header>
|
||||
|
||||
@ -8,8 +8,8 @@ import { openWindow } from "../utils/navigation";
|
||||
|
||||
export default function ClonePage() {
|
||||
const [engineEnabled, setEngineEnabled] = useState(false);
|
||||
const [engineBusy, setEngineBusy] = useState(false);
|
||||
const [agentLogs, setAgentLogs] = useState([]);
|
||||
const [regionsFile, setRegionsFile] = useState(null);
|
||||
const [annotating, setAnnotating] = useState(false);
|
||||
const [sourcePickerOpen, setSourcePickerOpen] = useState(false);
|
||||
const [captureSources, setCaptureSources] = useState([]);
|
||||
@ -20,15 +20,18 @@ export default function ClonePage() {
|
||||
|
||||
let unlisten;
|
||||
let cancelled = false;
|
||||
|
||||
listen("agent-log", (event) => {
|
||||
setAgentLogs((currentLogs) => [...currentLogs.slice(-300), event.payload]);
|
||||
listen("engine-state-changed", (event) => {
|
||||
setEngineEnabled(Boolean(event.payload?.enabled));
|
||||
setEngineBusy(false);
|
||||
setAgentLogs((currentLogs) => [
|
||||
...currentLogs.slice(-300),
|
||||
{ level: "warning", message: "工作台已关闭，引擎状态已同步" },
|
||||
]);
|
||||
}).then((cleanup) => {
|
||||
if (cancelled) {
|
||||
cleanup();
|
||||
return;
|
||||
}
|
||||
|
||||
unlisten = cleanup;
|
||||
});
|
||||
|
||||
@ -38,24 +41,6 @@ export default function ClonePage() {
|
||||
};
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
loadSavedRegions();
|
||||
}, []);
|
||||
|
||||
async function loadSavedRegions() {
|
||||
if (!window.__TAURI_INTERNALS__) return;
|
||||
|
||||
try {
|
||||
const saved = await invoke("load_regions");
|
||||
setRegionsFile(saved);
|
||||
} catch (error) {
|
||||
setAgentLogs((currentLogs) => [
|
||||
...currentLogs,
|
||||
{ level: "error", message: `读取标注失败：${String(error)}` },
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
async function startAnnotation() {
|
||||
if (!window.__TAURI_INTERNALS__) {
|
||||
window.location.hash = "/annotate";
|
||||
@ -103,24 +88,50 @@ export default function ClonePage() {
|
||||
}
|
||||
|
||||
async function toggleAgent() {
|
||||
if (engineBusy) return;
|
||||
|
||||
if (!window.__TAURI_INTERNALS__) {
|
||||
setEngineEnabled((enabled) => !enabled);
|
||||
return;
|
||||
}
|
||||
|
||||
setEngineBusy(true);
|
||||
try {
|
||||
if (engineEnabled) {
|
||||
await invoke("stop_agent");
|
||||
await invoke("stop_vision_stream").catch((error) => {
|
||||
setAgentLogs((currentLogs) => [
|
||||
...currentLogs,
|
||||
{ level: "warning", message: `停止视觉流失败：${String(error)}` },
|
||||
]);
|
||||
});
|
||||
setEngineEnabled(false);
|
||||
setAgentLogs((currentLogs) => [
|
||||
...currentLogs.slice(-300),
|
||||
{ level: "warning", message: "引擎已停用，暂未读取 Rust 日志" },
|
||||
]);
|
||||
} else {
|
||||
await invoke("start_agent");
|
||||
await invoke("start_vision_stream");
|
||||
await invoke("start_agent").catch((error) => {
|
||||
setAgentLogs((currentLogs) => [
|
||||
...currentLogs.slice(-300),
|
||||
{ level: "warning", message: `agent 启动失败，预览已继续启动：${String(error)}` },
|
||||
]);
|
||||
});
|
||||
setEngineEnabled(true);
|
||||
setAgentLogs((currentLogs) => [
|
||||
...currentLogs.slice(-300),
|
||||
{ level: "info", message: "引擎已启动，暂未读取 Rust 日志" },
|
||||
]);
|
||||
await openWindow("/window/engine-workbench");
|
||||
}
|
||||
} catch (error) {
|
||||
setAgentLogs((currentLogs) => [
|
||||
...currentLogs,
|
||||
{ level: "error", message: String(error) },
|
||||
]);
|
||||
} finally {
|
||||
setEngineBusy(false);
|
||||
}
|
||||
}
|
||||
|
||||
@ -139,9 +150,10 @@ export default function ClonePage() {
|
||||
<button
|
||||
className={`${engineEnabled ? "btn-danger" : "btn-primary"} w-full gap-2`}
|
||||
onClick={toggleAgent}
|
||||
disabled={engineBusy}
|
||||
>
|
||||
{engineEnabled ? <PowerOff size={16} strokeWidth={2.6} /> : <Power size={16} strokeWidth={2.6} />}
|
||||
{engineEnabled ? "停用引擎" : "启动引擎"}
|
||||
{engineBusy ? "处理中" : engineEnabled ? "停用引擎" : "启动引擎"}
|
||||
</button>
|
||||
<button className="icon-action shrink-0" onClick={() => openWindow("/window/settings")} aria-label="设置"><Settings size={18} strokeWidth={2.5} /></button>
|
||||
</div>
|
||||
@ -157,21 +169,6 @@ export default function ClonePage() {
|
||||
{annotating ? "截屏中" : "开始标注"}
|
||||
</button>
|
||||
</div>
|
||||
<div className="mt-3 space-y-2">
|
||||
{regionsFile?.regions?.length ? (
|
||||
regionsFile.regions.map((region) => (
|
||||
<div key={region.id} className="list-card !rounded-[12px] !p-3">
|
||||
<div className="min-w-0">
|
||||
<div className="truncate text-[13px] font-extrabold">{region.name}</div>
|
||||
<div className="mt-1 font-mono text-[11px] text-muted">screen [{region.bbox_screen.join(", ")}]</div>
|
||||
</div>
|
||||
<span className="tag shrink-0">{region.type}</span>
|
||||
</div>
|
||||
))
|
||||
) : (
|
||||
<div className="rounded-[12px] border border-dashed border-line p-3 text-center text-[12px] text-muted">暂无已保存区域</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
{sourcePickerOpen ? (
|
||||
<div className="fixed inset-0 z-50 grid place-items-center bg-black/35 px-5">
|
||||
|
||||
274
src/windows/EngineWorkbench.jsx
Normal file
274
src/windows/EngineWorkbench.jsx
Normal file
@ -0,0 +1,274 @@
|
||||
import { useEffect, useMemo, useRef, useState } from "react";
|
||||
import { invoke } from "@tauri-apps/api/core";
|
||||
import { Bot, SendHorizontal, UserRound } from "lucide-react";
|
||||
|
||||
const MONITOR_ENDPOINT = "http://127.0.0.1:8765";
|
||||
|
||||
function messageLabel(role) {
|
||||
if (role === "me") return "我";
|
||||
if (role === "user") return "客户";
|
||||
return "未知";
|
||||
}
|
||||
|
||||
export default function EngineWorkbench() {
|
||||
const [monitorEvent, setMonitorEvent] = useState(null);
|
||||
const [streamUrl, setStreamUrl] = useState(`${MONITOR_ENDPOINT}/frame.jpg?t=${Date.now()}`);
|
||||
const [streamFps, setStreamFps] = useState(0);
|
||||
const [streamError, setStreamError] = useState("");
|
||||
const [annotation, setAnnotation] = useState(null);
|
||||
const [frameMeta, setFrameMeta] = useState(null);
|
||||
const [videoBox, setVideoBox] = useState({ width: 0, height: 0 });
|
||||
const [question, setQuestion] = useState("");
|
||||
const [qaMessages, setQaMessages] = useState([]);
|
||||
const sequenceRef = useRef({ sequence: null, time: 0 });
|
||||
const streamRefreshedRef = useRef(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (!window.__TAURI_INTERNALS__) return;
|
||||
invoke("start_vision_stream").catch((error) => {
|
||||
setStreamError(`预览服务启动失败：${String(error)}`);
|
||||
});
|
||||
invoke("load_regions")
|
||||
.then((saved) => setAnnotation(saved || null))
|
||||
.catch(() => setAnnotation(null));
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
const updateBox = () => {
|
||||
const element = document.getElementById("wechat-preview-frame");
|
||||
if (!element) return;
|
||||
const rect = element.getBoundingClientRect();
|
||||
setVideoBox({ width: rect.width, height: rect.height });
|
||||
};
|
||||
updateBox();
|
||||
window.addEventListener("resize", updateBox);
|
||||
return () => window.removeEventListener("resize", updateBox);
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
|
||||
async function pollMonitor() {
|
||||
try {
|
||||
const response = await fetch(`${MONITOR_ENDPOINT}/latest.json?t=${Date.now()}`, { cache: "no-store" });
|
||||
const data = await response.json();
|
||||
if (!cancelled) {
|
||||
if (data.source !== "go-window-capture") {
|
||||
setMonitorEvent(null);
|
||||
setStreamFps(0);
|
||||
setStreamError(`预览服务来源异常：${data.source || "unknown"}`);
|
||||
return;
|
||||
}
|
||||
if (data.event?.type === "source_waiting") {
|
||||
setMonitorEvent(null);
|
||||
setStreamFps(0);
|
||||
setStreamError(data.event.message || "等待标注来源窗口出现...");
|
||||
setFrameMeta(data.frame || null);
|
||||
return;
|
||||
}
|
||||
setStreamError("");
|
||||
setFrameMeta(data.frame || null);
|
||||
const now = performance.now();
|
||||
const previous = sequenceRef.current;
|
||||
if (typeof data.sequence === "number" && typeof previous.sequence === "number" && previous.time) {
|
||||
const elapsedSeconds = Math.max((now - previous.time) / 1000, 0.001);
|
||||
setStreamFps(Math.max(0, (data.sequence - previous.sequence) / elapsedSeconds));
|
||||
}
|
||||
sequenceRef.current = { sequence: data.sequence, time: now };
|
||||
setMonitorEvent(data.event?.type === "preview_frame" ? data.event : null);
|
||||
if (typeof data.sequence === "number" && data.sequence > 0) {
|
||||
streamRefreshedRef.current = true;
|
||||
setStreamUrl(`${MONITOR_ENDPOINT}/frame.jpg?seq=${data.sequence}&t=${Date.now()}`);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
if (!cancelled) {
|
||||
setMonitorEvent(null);
|
||||
setStreamError("等待预览服务启动...");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pollMonitor();
|
||||
const timer = window.setInterval(pollMonitor, 500);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
window.clearInterval(timer);
|
||||
};
|
||||
}, []);
|
||||
|
||||
const latestLlm = monitorEvent?.latest_llm || monitorEvent?.llm_chat_reading || null;
|
||||
const visibleMessages = useMemo(() => latestLlm?.visible_messages || [], [latestLlm]);
|
||||
const latestUserMessage = latestLlm?.latest_user_message || "等待识别当前聊天记录";
|
||||
const replyText = latestLlm?.reply_text || "暂无建议回复";
|
||||
const overlay = useMemo(() => {
|
||||
const frameWidth = frameMeta?.width || annotation?.screenshotWidth;
|
||||
const frameHeight = frameMeta?.height || annotation?.screenshotHeight;
|
||||
if (!frameWidth || !frameHeight || !videoBox.width || !videoBox.height) return null;
|
||||
const scale = Math.min(videoBox.width / frameWidth, videoBox.height / frameHeight);
|
||||
const width = frameWidth * scale;
|
||||
const height = frameHeight * scale;
|
||||
return {
|
||||
left: (videoBox.width - width) / 2,
|
||||
top: (videoBox.height - height) / 2,
|
||||
scale,
|
||||
frameWidth,
|
||||
frameHeight,
|
||||
};
|
||||
}, [annotation, frameMeta, videoBox]);
|
||||
|
||||
function regionStyle(region) {
|
||||
if (!overlay || !region.bbox_image) return null;
|
||||
const sourceWidth = annotation?.screenshotWidth || overlay.frameWidth;
|
||||
const sourceHeight = annotation?.screenshotHeight || overlay.frameHeight;
|
||||
const [ix1, iy1, ix2, iy2] = region.bbox_image;
|
||||
const x1 = (ix1 * overlay.frameWidth) / sourceWidth;
|
||||
const y1 = (iy1 * overlay.frameHeight) / sourceHeight;
|
||||
const x2 = (ix2 * overlay.frameWidth) / sourceWidth;
|
||||
const y2 = (iy2 * overlay.frameHeight) / sourceHeight;
|
||||
return {
|
||||
left: overlay.left + x1 * overlay.scale,
|
||||
top: overlay.top + y1 * overlay.scale,
|
||||
width: Math.max(1, (x2 - x1) * overlay.scale),
|
||||
height: Math.max(1, (y2 - y1) * overlay.scale),
|
||||
};
|
||||
}
|
||||
|
||||
function submitQuestion(event) {
|
||||
event.preventDefault();
|
||||
const trimmed = question.trim();
|
||||
if (!trimmed) return;
|
||||
|
||||
const contextLines = visibleMessages.map((message) => `${messageLabel(message.role)}：${message.content}`).join("\n");
|
||||
const answer = contextLines
|
||||
? `基于当前已识别聊天记录：\n${contextLines}\n\n建议回复：${replyText}`
|
||||
: "当前还没有读取到聊天记录。请先确认监控服务已启动，并等待检测到新消息后再询问。";
|
||||
|
||||
setQaMessages((current) => [
|
||||
...current,
|
||||
{ role: "user", content: trimmed },
|
||||
{ role: "assistant", content: answer },
|
||||
]);
|
||||
setQuestion("");
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="grid h-full min-h-0 grid-cols-[minmax(0,1fr)_minmax(320px,400px)] gap-4 p-4 max-lg:grid-cols-1">
|
||||
<section className="flex min-h-0 flex-col gap-4">
|
||||
<div className="panel flex min-h-0 flex-1 flex-col overflow-hidden p-4">
|
||||
<div className="mb-3 flex items-center justify-between gap-3">
|
||||
<div>
|
||||
<h2 className="text-[16px] font-extrabold">微信窗口实时画面</h2>
|
||||
<p className="mt-1 text-[12px] text-muted">
|
||||
{monitorEvent?.target_label ? `预览来源：${monitorEvent.target_label}` : `来自本地监控流：${MONITOR_ENDPOINT}/stream.mjpg`}
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex shrink-0 items-center gap-2">
|
||||
<span className="tag">{streamFps ? `${streamFps.toFixed(1)} FPS` : "FPS --"}</span>
|
||||
<span className={`tag ${monitorEvent ? "text-primary" : ""}`}>{monitorEvent ? "在线" : "等待监控服务"}</span>
|
||||
</div>
|
||||
</div>
|
||||
<div id="wechat-preview-frame" className="relative min-h-0 flex-1 overflow-hidden rounded-[14px] border border-line bg-black">
|
||||
<img
|
||||
src={streamUrl}
|
||||
alt="微信窗口实时画面"
|
||||
className="h-full w-full object-contain"
|
||||
draggable={false}
|
||||
onError={() => {
|
||||
window.setTimeout(() => {
|
||||
setStreamUrl(`${MONITOR_ENDPOINT}/frame.jpg?t=${Date.now()}`);
|
||||
}, 1000);
|
||||
}}
|
||||
/>
|
||||
<div className="absolute left-3 top-3 rounded-full bg-black/70 px-3 py-1 font-mono text-[12px] font-bold text-[#86efac]">
|
||||
{streamFps ? `${streamFps.toFixed(1)} FPS` : "等待帧率"}
|
||||
</div>
|
||||
{annotation?.regions?.map((region) => {
|
||||
const style = regionStyle(region);
|
||||
if (!style) return null;
|
||||
return (
|
||||
<div
|
||||
key={region.id}
|
||||
className="pointer-events-none absolute rounded-[8px] border-2 border-[#22c55e] bg-[#22c55e]/10 shadow-[0_0_0_1px_rgba(0,0,0,.45)]"
|
||||
style={style}
|
||||
>
|
||||
<div className="absolute -left-0.5 -top-6 rounded bg-[#22c55e] px-2 py-0.5 text-[11px] font-black text-black shadow">
|
||||
{region.type || region.name}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
{streamError ? (
|
||||
<div className="absolute inset-x-0 bottom-4 mx-auto w-fit max-w-[80%] rounded-full bg-black/75 px-4 py-2 text-center text-[12px] font-bold text-[#fca5a5]">
|
||||
{streamError}
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<aside className="panel flex min-h-0 max-w-[400px] flex-col overflow-hidden p-4 max-lg:max-w-none">
|
||||
<div className="mb-3 shrink-0">
|
||||
<h2 className="text-[16px] font-extrabold">客户聊天记录</h2>
|
||||
<p className="mt-1 text-[12px] text-muted">展示最近一次 LLM 读取到的对话和建议回复。</p>
|
||||
</div>
|
||||
|
||||
<div className="scrollbar-hidden min-h-0 flex-1 space-y-3 overflow-y-auto pr-1">
|
||||
<div className="rounded-[14px] border border-line bg-surface2 p-3">
|
||||
<div className="text-[11px] font-bold text-muted">最新客户消息</div>
|
||||
<div className="mt-2 text-[14px] font-bold leading-6">{latestUserMessage}</div>
|
||||
</div>
|
||||
|
||||
<div className="space-y-2">
|
||||
{visibleMessages.length ? (
|
||||
visibleMessages.map((message, index) => (
|
||||
<div
|
||||
key={`${message.role}-${message.content}-${index}`}
|
||||
className={`rounded-[14px] p-3 ${message.role === "me" ? "bg-[var(--primary-soft)]" : "bg-surface2"}`}
|
||||
>
|
||||
<div className="mb-1 flex items-center gap-2 text-[11px] font-black text-muted">
|
||||
{message.role === "me" ? <UserRound size={13} /> : <Bot size={13} />}
|
||||
{message.sender || messageLabel(message.role)}
|
||||
</div>
|
||||
<div className="text-[13px] leading-5">{message.content}</div>
|
||||
</div>
|
||||
))
|
||||
) : (
|
||||
<div className="rounded-[14px] border border-dashed border-line p-4 text-center text-[12px] text-muted">
|
||||
暂无聊天记录，等待监测到新消息后读取。
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="rounded-[14px] border border-primary/30 bg-[var(--primary-soft)] p-3">
|
||||
<div className="text-[11px] font-black text-primary">建议回复</div>
|
||||
<div className="mt-2 text-[13px] leading-5">{replyText}</div>
|
||||
</div>
|
||||
|
||||
{qaMessages.length ? (
|
||||
<div className="space-y-2 border-t border-line pt-3">
|
||||
{qaMessages.map((message, index) => (
|
||||
<div key={`${message.role}-${index}`} className="rounded-[12px] bg-surface2 p-3 text-[12px] leading-5">
|
||||
<div className="mb-1 font-black text-muted">{message.role === "user" ? "我" : "AI"}</div>
|
||||
<div className="whitespace-pre-wrap">{message.content}</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
<form onSubmit={submitQuestion} className="mt-3 flex shrink-0 items-center gap-2 border-t border-line pt-3">
|
||||
<input
|
||||
value={question}
|
||||
onChange={(event) => setQuestion(event.target.value)}
|
||||
placeholder="询问客户信息或当前对话..."
|
||||
className="min-w-0 flex-1 rounded-[12px] border border-line bg-surface px-3 py-2 text-[13px] outline-none transition focus:border-primary"
|
||||
/>
|
||||
<button className="btn-primary h-10 shrink-0 px-3" type="submit" aria-label="发送问题">
|
||||
<SendHorizontal size={16} strokeWidth={2.6} />
|
||||
</button>
|
||||
</form>
|
||||
</aside>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@ -1,13 +1,18 @@
|
||||
import { useMemo } from "react";
|
||||
import { invoke } from "@tauri-apps/api/core";
|
||||
import { emit } from "@tauri-apps/api/event";
|
||||
import { logs } from "../data/mockData";
|
||||
import Terminal from "../components/Terminal";
|
||||
import TitleBar from "../components/TitleBar";
|
||||
import EngineWorkbench from "./EngineWorkbench";
|
||||
import PlaceholderWindow from "./PlaceholderWindow";
|
||||
import SettingsWindow from "./SettingsWindow";
|
||||
import { closeWindow } from "../utils/navigation";
|
||||
|
||||
export default function SecondaryWindow({ route, theme, setTheme, settingSection, setSettingSection }) {
|
||||
const title = useMemo(() => {
|
||||
if (route.includes("settings")) return "设置";
|
||||
if (route.includes("engine-workbench")) return "微信引擎工作台";
|
||||
if (route.includes("engine-logs")) return "分身引擎日志";
|
||||
if (route.includes("knowledge-create")) return "新增知识库";
|
||||
if (route.includes("knowledge-detail")) return "知识库详情";
|
||||
@ -17,12 +22,27 @@ export default function SecondaryWindow({ route, theme, setTheme, settingSection
|
||||
return "二级窗口";
|
||||
}, [route]);
|
||||
|
||||
async function closeEngineWorkbench() {
|
||||
await invoke("stop_agent").catch(() => {});
|
||||
await invoke("stop_vision_stream").catch(() => {});
|
||||
await emit("engine-state-changed", { enabled: false });
|
||||
await closeWindow();
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="app-window flex h-dvh w-full flex-col overflow-hidden rounded-[12px]">
|
||||
<TitleBar title={title} theme={theme} setTheme={setTheme} compact />
|
||||
<main className="content-canvas flex min-h-0 flex-1 flex-col overflow-y-auto">
|
||||
<TitleBar
|
||||
title={title}
|
||||
theme={theme}
|
||||
setTheme={setTheme}
|
||||
compact
|
||||
onClose={route.includes("engine-workbench") ? closeEngineWorkbench : undefined}
|
||||
/>
|
||||
<main className="content-canvas flex min-h-0 flex-1 flex-col overflow-hidden">
|
||||
{route.includes("settings") ? (
|
||||
<SettingsWindow active={settingSection} setActive={setSettingSection} />
|
||||
) : route.includes("engine-workbench") ? (
|
||||
<EngineWorkbench />
|
||||
) : route.includes("engine-logs") ? (
|
||||
<div className="flex h-full min-h-0 p-5"><Terminal title="完整运行日志" lines={[...logs, ...logs, ...logs]} tall /></div>
|
||||
) : (
|
||||
|
||||
6
vision_test/app.py
Normal file
6
vision_test/app.py
Normal file
@ -0,0 +1,6 @@
|
||||
|
||||
|
||||
# 测试python服务
|
||||
|
||||
print("hello world")
|
||||
|
||||
@ -27,7 +27,6 @@ nohup ./venv/bin/python wechat_algorithm_live.py \
|
||||
--send-reply \
|
||||
--typing-chunk-size 2 \
|
||||
--typing-delay 0.08 \
|
||||
--open-browser \
|
||||
> "$LOG_FILE" 2>&1 &
|
||||
|
||||
PID="$!"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user