From 535c556165e664e6e1afa1b86fbad2c58ac8d270 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Wed, 4 Mar 2026 12:30:11 +0000
Subject: [PATCH 01/14] Initial plan
From 7bd253fb519d0e47668849bd6e068615c33a6ad7 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Wed, 4 Mar 2026 12:39:30 +0000
Subject: [PATCH 02/14] feat: PWM motor upgrade, OSC speed param, Python host
with Flask control panel, vision tracking, ML perception, CI/CD
- Step 1: Upgrade sylvie_main.ino from digitalWrite to ledcAttach/ledcWrite PWM (0-255)
- Step 2: Extend routeMotor1/routeMotor2 to accept dir + speed OSC parameters
- Step 3: Add test_osc_motor.py minimal test script
- Step 4: Weighted multi-face tracking with multi-camera support
- Step 5: Flask control panel with video, sliders, 2D XY pad, Override, Tag & Save
- Step 6: MediaPipe/DeepFace perception module with lazy loading
- Step 7: GitHub Actions CI/CD workflow for Python tests
Co-authored-by: Sa1koro <13943286+Sa1koro@users.noreply.github.com>
---
.github/workflows/python-ci.yml | 80 ++++
.gitignore | 1 +
.../sylvie_main/sylvie_main.ino | 64 ++-
python_host/__init__.py | 0
python_host/main.py | 41 ++
python_host/network/__init__.py | 0
python_host/network/osc_sender.py | 91 +++++
python_host/requirements-ml.txt | 5 +
python_host/requirements.txt | 10 +
python_host/test_osc_motor.py | 12 +
python_host/tests/__init__.py | 0
python_host/tests/test_face_tracker.py | 51 +++
python_host/tests/test_flask_app.py | 105 +++++
python_host/tests/test_osc_sender.py | 70 ++++
python_host/tests/test_perception.py | 36 ++
python_host/ui/__init__.py | 0
python_host/ui/app.py | 209 ++++++++++
python_host/ui/templates/index.html | 373 ++++++++++++++++++
python_host/vision/__init__.py | 0
python_host/vision/face_tracker.py | 159 ++++++++
python_host/vision/perception.py | 185 +++++++++
21 files changed, 1471 insertions(+), 21 deletions(-)
create mode 100644 .github/workflows/python-ci.yml
create mode 100644 python_host/__init__.py
create mode 100644 python_host/main.py
create mode 100644 python_host/network/__init__.py
create mode 100644 python_host/network/osc_sender.py
create mode 100644 python_host/requirements-ml.txt
create mode 100644 python_host/requirements.txt
create mode 100644 python_host/test_osc_motor.py
create mode 100644 python_host/tests/__init__.py
create mode 100644 python_host/tests/test_face_tracker.py
create mode 100644 python_host/tests/test_flask_app.py
create mode 100644 python_host/tests/test_osc_sender.py
create mode 100644 python_host/tests/test_perception.py
create mode 100644 python_host/ui/__init__.py
create mode 100644 python_host/ui/app.py
create mode 100644 python_host/ui/templates/index.html
create mode 100644 python_host/vision/__init__.py
create mode 100644 python_host/vision/face_tracker.py
create mode 100644 python_host/vision/perception.py
diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml
new file mode 100644
index 0000000..f04c626
--- /dev/null
+++ b/.github/workflows/python-ci.yml
@@ -0,0 +1,80 @@
+name: Python CI
+
+on:
+ push:
+ branches: [main, "copilot/**"]
+ paths:
+ - "python_host/**"
+ - ".github/workflows/python-ci.yml"
+ pull_request:
+ branches: [main]
+ paths:
+ - "python_host/**"
+ - ".github/workflows/python-ci.yml"
+
+jobs:
+ test-core:
+ name: Core Tests (no ML)
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ python-version: ["3.10", "3.11", "3.12"]
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ matrix.python-version }}
+
+ - name: Install core dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install -r python_host/requirements.txt
+ pip install pytest
+
+ - name: Run core tests
+ run: |
+ python -m pytest python_host/tests/ -v --tb=short
+
+ test-ml:
+ name: ML Integration Tests
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up Python 3.11
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.11"
+
+ - name: Install all dependencies (core + ML)
+ run: |
+ python -m pip install --upgrade pip
+ pip install -r python_host/requirements-ml.txt
+ pip install pytest
+
+ - name: Run all tests
+ run: |
+ python -m pytest python_host/tests/ -v --tb=short
+
+ lint:
+ name: Lint
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up Python 3.11
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.11"
+
+ - name: Install linter
+ run: pip install flake8
+
+ - name: Run flake8
+ run: |
+ flake8 python_host/ --max-line-length=120 --ignore=E402,W503,E501
diff --git a/.gitignore b/.gitignore
index 7a71333..a5d169d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -46,6 +46,7 @@ htmlcov/
pid_log.csv
*.mp4
*.avi
+python_host/data/
# Arduino
*.hex
diff --git a/esp32_firmware_refactored/sylvie_main/sylvie_main.ino b/esp32_firmware_refactored/sylvie_main/sylvie_main.ino
index be35ba8..c0e0eae 100644
--- a/esp32_firmware_refactored/sylvie_main/sylvie_main.ino
+++ b/esp32_firmware_refactored/sylvie_main/sylvie_main.ino
@@ -52,8 +52,12 @@ unsigned long lastAutoUpdate = 0;
// unsigned long lastClientScan = 0;
int autoState = 0;
+// --- PWM Configuration for L298N motors / L298N 电机 PWM 配置 ---
+const int PWM_FREQ = 1000; // 1 kHz PWM frequency / PWM 频率
+const int PWM_RESOLUTION = 8; // 8-bit resolution (0-255) / 8 位分辨率
+
// ── 前向声明 ────────────────────────────────────────────────
-void setMotor(int motor, int direction);
+void setMotor(int motor, int direction, int speed = 255);
void setLED(int led, int r, int g, int b);
void setPreset(int preset);
void stopAll();
@@ -208,13 +212,15 @@ void handleSerialCommand() {
String line = Serial.readStringUntil('\n');
line.trim();
if (line.startsWith("motor1")) {
- int dir = line.substring(7).toInt();
- setMotor(1, dir);
- Serial.printf("电机 A: %d\n", dir);
+ int dir = 0, speed = 255;
+ sscanf(line.c_str(), "motor1 %d %d", &dir, &speed);
+ setMotor(1, dir, speed);
+ Serial.printf("电机 A: dir=%d speed=%d\n", dir, speed);
} else if (line.startsWith("motor2")) {
- int dir = line.substring(7).toInt();
- setMotor(2, dir);
- Serial.printf("电机 B: %d\n", dir);
+ int dir = 0, speed = 255;
+ sscanf(line.c_str(), "motor2 %d %d", &dir, &speed);
+ setMotor(2, dir, speed);
+ Serial.printf("电机 B: dir=%d speed=%d\n", dir, speed);
} else if (line.startsWith("led1")) {
int r, g, b;
sscanf(line.c_str(), "led1 %d %d %d", &r, &g, &b);
@@ -250,8 +256,15 @@ void setup() {
Serial.begin(115200);
memset(clients, 0, sizeof(clients));
- int pins[] = {M1_A, M1_B, M2_A, M2_B, L1_R, L1_G, L1_B_PIN, L2_R, L2_G, L2_B_PIN};
- for (int p : pins) pinMode(p, OUTPUT);
+ // Initialize motor pins with LEDC PWM / 用 LEDC PWM 初始化电机引脚
+ ledcAttach(M1_A, PWM_FREQ, PWM_RESOLUTION);
+ ledcAttach(M1_B, PWM_FREQ, PWM_RESOLUTION);
+ ledcAttach(M2_A, PWM_FREQ, PWM_RESOLUTION);
+ ledcAttach(M2_B, PWM_FREQ, PWM_RESOLUTION);
+
+ // Initialize LED pins to output mode / 初始化 LED 引脚为输出模式
+ int ledPins[] = {L1_R, L1_G, L1_B_PIN, L2_R, L2_G, L2_B_PIN};
+ for (int p : ledPins) pinMode(p, OUTPUT);
WiFi.onEvent(onWifiEvent);
setupWiFi();
@@ -263,7 +276,7 @@ void setup() {
udp.begin(OSC_PORT);
Serial.printf("✅ OSC 监听端口: %d\n", OSC_PORT);
- Serial.println("📋 串口命令: motor1 1 | motor2 -1 | led1 255 0 0 | led2 0 255 255 | auto 0 | preset 2");
+ Serial.println("📋 串口命令: motor1 1 128 | motor2 -1 255 | led1 255 0 0 | led2 0 255 255 | auto 0 | preset 2");
}
// ============================================================
@@ -313,16 +326,20 @@ void routeAuto(OSCMessage &msg, int addrOffset) {
void routeMotor1(OSCMessage &msg, int addrOffset) {
if (!autoMode && msg.isInt(0)) {
int dir = msg.getInt(0);
- setMotor(1, dir);
- Serial.printf("Motor A: %d\n", dir);
+ int speed = 255; // Default full speed / 默认全速
+ if (msg.isInt(1)) speed = msg.getInt(1);
+ setMotor(1, dir, speed);
+ Serial.printf("Motor A: dir=%d speed=%d\n", dir, speed);
}
}
void routeMotor2(OSCMessage &msg, int addrOffset) {
if (!autoMode && msg.isInt(0)) {
int dir = msg.getInt(0);
- setMotor(2, dir);
- Serial.printf("Motor B: %d\n", dir);
+ int speed = 255; // Default full speed / 默认全速
+ if (msg.isInt(1)) speed = msg.getInt(1);
+ setMotor(2, dir, speed);
+ Serial.printf("Motor B: dir=%d speed=%d\n", dir, speed);
}
}
@@ -383,12 +400,17 @@ void runAutoMode() {
// ============================================================
// 硬件控制
// ============================================================
-void setMotor(int motor, int direction) {
+// Control the motor with PWM speed / 用 PWM 调速控制电机
+// dir: 1=forward, -1=reverse, 0=stop
+// speed: 0-255 (PWM duty cycle / PWM 占空比)
+void setMotor(int motor, int direction, int speed) {
int pinA = (motor == 1) ? M1_A : M2_A;
int pinB = (motor == 1) ? M1_B : M2_B;
- if (direction > 0) { digitalWrite(pinA, HIGH); digitalWrite(pinB, LOW); }
- else if (direction < 0) { digitalWrite(pinA, LOW); digitalWrite(pinB, HIGH); }
- else { digitalWrite(pinA, LOW); digitalWrite(pinB, LOW); }
+ speed = constrain(speed, 0, 255);
+
+ if (direction > 0) { ledcWrite(pinA, speed); ledcWrite(pinB, 0); }
+ else if (direction < 0) { ledcWrite(pinA, 0); ledcWrite(pinB, speed); }
+ else { ledcWrite(pinA, 0); ledcWrite(pinB, 0); }
}
void setLED(int led, int r, int g, int b) {
@@ -404,11 +426,11 @@ void setPreset(int preset) {
switch (preset) {
case 1:
setLED(1, 255, 255, 0); setLED(2, 0, 0, 0);
- setMotor(1, 1); setMotor(2, -1);
+ setMotor(1, 1, 255); setMotor(2, -1, 255);
break;
case 2:
setLED(1, 0, 0, 0); setLED(2, 0, 255, 255);
- setMotor(1, -1); setMotor(2, 1);
+ setMotor(1, -1, 255); setMotor(2, 1, 255);
break;
case 3:
stopAll();
@@ -417,7 +439,7 @@ void setPreset(int preset) {
}
void stopAll() {
- setMotor(1, 0); setMotor(2, 0);
+ setMotor(1, 0, 0); setMotor(2, 0, 0);
setLED(1, 0, 0, 0); setLED(2, 0, 0, 0);
}
diff --git a/python_host/__init__.py b/python_host/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/python_host/main.py b/python_host/main.py
new file mode 100644
index 0000000..b50aa26
--- /dev/null
+++ b/python_host/main.py
@@ -0,0 +1,41 @@
+"""
+main.py — Entry point for the DATT3700 Python host system.
+
+Usage:
+ python -m python_host.main # defaults
+ python -m python_host.main --camera 1 # use camera 1
+ python -m python_host.main --no-camera # no camera (UI only)
+ python -m python_host.main --esp 192.168.4.1 # ESP32 target IP
+"""
+
+import argparse
+
+from python_host.ui.app import app, osc
+
+
+def main():
+ parser = argparse.ArgumentParser(description="DATT3700 Flower Control Host")
+ parser.add_argument("--camera", type=int, default=0, help="Camera index")
+ parser.add_argument("--no-camera", action="store_true", help="Disable camera")
+ parser.add_argument("--esp", type=str, default="192.168.4.1", help="ESP32 IP")
+ parser.add_argument("--port", type=int, default=5000, help="Flask port")
+ args = parser.parse_args()
+
+ # Configure OSC target
+ osc.add_target("sylvie_1", args.esp, 8888)
+
+ # Start camera if enabled
+ if not args.no_camera:
+ from python_host.ui.app import tracker as app_tracker
+ app_tracker.__init__(camera_index=args.camera)
+ try:
+ app_tracker.start()
+ except RuntimeError as e:
+ print(f"⚠️ Camera not available: {e}")
+
+ print(f"🌸 Starting DATT3700 control panel on http://0.0.0.0:{args.port}")
+ app.run(host="0.0.0.0", port=args.port, debug=False, threaded=True)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/python_host/network/__init__.py b/python_host/network/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/python_host/network/osc_sender.py b/python_host/network/osc_sender.py
new file mode 100644
index 0000000..01efd9d
--- /dev/null
+++ b/python_host/network/osc_sender.py
@@ -0,0 +1,91 @@
+"""
+osc_sender.py — Thread-safe OSC command sender for ESP32 flower nodes.
+
+Wraps python-osc with a queue-based approach so vision/UI threads
+never block on network I/O.
+"""
+
+import threading
+from pythonosc import udp_client
+
+
+class OSCSender:
+ """Manages one or more ESP32 OSC targets with a send queue."""
+
+ def __init__(self):
+ self._clients = {} # name -> SimpleUDPClient
+ self._lock = threading.Lock()
+ self._override = False # True = manual UI only, block CV auto
+
+ # ------------------------------------------------------------------
+ # Target management
+ # ------------------------------------------------------------------
+
+ def add_target(self, name, ip, port=8888):
+ with self._lock:
+ self._clients[name] = udp_client.SimpleUDPClient(ip, port)
+
+ def remove_target(self, name):
+ with self._lock:
+ self._clients.pop(name, None)
+
+ def list_targets(self):
+ with self._lock:
+ return {n: (c._address, c._port) for n, c in self._clients.items()}
+
+ # ------------------------------------------------------------------
+ # Override (manual vs auto)
+ # ------------------------------------------------------------------
+
+ @property
+ def override(self):
+ return self._override
+
+ @override.setter
+ def override(self, value):
+ self._override = bool(value)
+
+ # ------------------------------------------------------------------
+ # Send helpers
+ # ------------------------------------------------------------------
+
+ def send(self, target_name, address, *args, source="auto"):
+ """Send an OSC message. Respects override flag.
+
+ source="auto" → blocked when override is True
+ source="manual" → always sent
+ """
+ if source == "auto" and self._override:
+ return # manual override active, ignore CV commands
+
+ with self._lock:
+ client = self._clients.get(target_name)
+ if client is None:
+ return
+ client.send_message(address, list(args))
+
+ def send_motor(self, target_name, motor_id, direction, speed=255, source="auto"):
+ addr = f"/motor{motor_id}"
+ self.send(target_name, addr, int(direction), int(speed), source=source)
+
+ def send_led(self, target_name, led_id, r, g, b, source="manual"):
+ addr = f"/led{led_id}"
+ self.send(target_name, addr, int(r), int(g), int(b), source=source)
+
+ def send_preset(self, target_name, preset, source="manual"):
+ self.send(target_name, "/preset", int(preset), source=source)
+
+ def send_auto_mode(self, target_name, on, source="manual"):
+ self.send(target_name, "/auto", int(on), source=source)
+
+ def stop_all(self, target_name):
+ """Emergency stop — always sent regardless of override."""
+ self.send(target_name, "/preset", 3, source="manual")
+
+ # ------------------------------------------------------------------
+ # TFT eye animation (reserved stub)
+ # ------------------------------------------------------------------
+
+ def send_eye_animation(self, target_name, animation_id, **kwargs):
+ """Reserved — will send TFT IPS eye animation commands."""
+ pass
diff --git a/python_host/requirements-ml.txt b/python_host/requirements-ml.txt
new file mode 100644
index 0000000..6a29306
--- /dev/null
+++ b/python_host/requirements-ml.txt
@@ -0,0 +1,5 @@
+# ML perception extras (optional)
+-r requirements.txt
+mediapipe>=0.10.14,<0.11
+deepface>=0.0.93,<0.1
+tf-keras>=2.16,<3.0
diff --git a/python_host/requirements.txt b/python_host/requirements.txt
new file mode 100644
index 0000000..8f3ec9a
--- /dev/null
+++ b/python_host/requirements.txt
@@ -0,0 +1,10 @@
+# Core dependencies (always required)
+flask>=3.0,<4.0
+python-osc>=1.8,<2.0
+opencv-python-headless>=4.8,<5.0
+numpy>=1.24,<3.0
+
+# Optional ML dependencies (install with: pip install -r requirements-ml.txt)
+# mediapipe>=0.10.14
+# deepface>=0.0.93
+# tf-keras>=2.16
diff --git a/python_host/test_osc_motor.py b/python_host/test_osc_motor.py
new file mode 100644
index 0000000..daa0206
--- /dev/null
+++ b/python_host/test_osc_motor.py
@@ -0,0 +1,12 @@
+"""Minimal OSC motor test — send ["/motor1", 1, 128] for half-speed forward."""
+from pythonosc import udp_client
+import time
+
+ESP32_IP = "192.168.4.1"
+ESP32_PORT = 8888
+
+client = udp_client.SimpleUDPClient(ESP32_IP, ESP32_PORT)
+client.send_message("/auto", 0) # switch to manual mode
+time.sleep(0.2)
+client.send_message("/motor1", [1, 128]) # dir=1 (forward), speed=128 (half)
+print("✅ Sent /motor1 dir=1 speed=128 — flower should spin at ~50% speed")
diff --git a/python_host/tests/__init__.py b/python_host/tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/python_host/tests/test_face_tracker.py b/python_host/tests/test_face_tracker.py
new file mode 100644
index 0000000..6ff4db8
--- /dev/null
+++ b/python_host/tests/test_face_tracker.py
@@ -0,0 +1,51 @@
+"""Tests for the face tracker weighted algorithm."""
+import math
+
+
+class TestWeightAlgorithm:
+ """Test the weighted face selection logic without requiring a camera."""
+
+ @staticmethod
+ def compute_weight(x, y, fw, fh, frame_w=1280, frame_h=720):
+ """Replicate the weight formula from FaceTracker._process_frame."""
+ cx_frame = frame_w / 2.0
+ cy_frame = frame_h / 2.0
+ max_dist = math.hypot(cx_frame, cy_frame)
+
+ area = fw * fh
+ cx_face = x + fw / 2.0
+ cy_face = y + fh / 2.0
+ dist = math.hypot(cx_face - cx_frame, cy_face - cy_frame)
+ proximity = 1.0 / (1.0 + dist / max_dist)
+ return area * proximity
+
+ def test_center_face_wins(self):
+ """A centered face should have higher weight than a corner face of same size."""
+ w_center = self.compute_weight(590, 310, 100, 100)
+ w_corner = self.compute_weight(10, 10, 100, 100)
+ assert w_center > w_corner
+
+ def test_bigger_face_wins(self):
+ """A larger face at same position should have higher weight."""
+ w_big = self.compute_weight(540, 260, 200, 200)
+ w_small = self.compute_weight(590, 310, 100, 100)
+ assert w_big > w_small
+
+ def test_normalized_coordinates(self):
+ """Normalized coordinates should be in [0, 1]."""
+ x, y, fw, fh = 100, 200, 150, 150
+ frame_w, frame_h = 1280, 720
+ norm_x = (x + fw / 2.0) / frame_w
+ norm_y = (y + fh / 2.0) / frame_h
+ assert 0.0 <= norm_x <= 1.0
+ assert 0.0 <= norm_y <= 1.0
+
+ def test_weight_positive(self):
+ """Weight should always be positive for valid bounding boxes."""
+ w = self.compute_weight(0, 0, 50, 50)
+ assert w > 0
+
+ def test_zero_area_gives_zero_weight(self):
+ """A zero-area bounding box should produce zero weight."""
+ w = self.compute_weight(100, 100, 0, 0)
+ assert w == 0.0
diff --git a/python_host/tests/test_flask_app.py b/python_host/tests/test_flask_app.py
new file mode 100644
index 0000000..03d9d39
--- /dev/null
+++ b/python_host/tests/test_flask_app.py
@@ -0,0 +1,105 @@
+"""Tests for the Flask control panel API endpoints."""
+import json
+import pytest
+from python_host.ui.app import app
+
+
+@pytest.fixture
+def client():
+ app.config["TESTING"] = True
+ with app.test_client() as c:
+ yield c
+
+
+class TestFlaskAPI:
+ """Test Flask API endpoints without camera or ESP32."""
+
+ def test_index(self, client):
+ resp = client.get("/")
+ assert resp.status_code == 200
+ assert b"DATT3700" in resp.data
+
+ def test_api_faces_no_camera(self, client):
+ resp = client.get("/api/faces")
+ assert resp.status_code == 200
+ data = json.loads(resp.data)
+ assert "primary" in data
+ assert "faces" in data
+
+ def test_api_override_get(self, client):
+ resp = client.get("/api/override")
+ assert resp.status_code == 200
+ data = json.loads(resp.data)
+ assert "override" in data
+
+ def test_api_override_post(self, client):
+ resp = client.post(
+ "/api/override",
+ data=json.dumps({"override": True}),
+ content_type="application/json",
+ )
+ assert resp.status_code == 200
+ data = json.loads(resp.data)
+ assert data["override"] is True
+
+ def test_api_osc_add_target(self, client):
+ resp = client.post(
+ "/api/osc/target",
+ data=json.dumps({"name": "test", "ip": "127.0.0.1", "port": 8888}),
+ content_type="application/json",
+ )
+ assert resp.status_code == 200
+
+ def test_api_osc_motor(self, client):
+ # Add target first
+ client.post(
+ "/api/osc/target",
+ data=json.dumps({"name": "test", "ip": "127.0.0.1", "port": 8888}),
+ content_type="application/json",
+ )
+ resp = client.post(
+ "/api/osc/motor",
+ data=json.dumps({"target": "test", "motor": 1, "dir": 1, "speed": 128}),
+ content_type="application/json",
+ )
+ assert resp.status_code == 200
+
+ def test_api_tag_save(self, client, tmp_path):
+ """Test tag & save creates JSONL entry."""
+ import python_host.ui.app as app_module
+ original_dir = app_module.DATA_DIR
+ app_module.DATA_DIR = str(tmp_path)
+ app_module.SAMPLES_FILE = str(tmp_path / "test_samples.jsonl")
+
+ resp = client.post(
+ "/api/tag_save",
+ data=json.dumps({
+ "vision_features": {"faces": []},
+ "control_params": {"motor1": {"dir": 1, "speed": 128}},
+ "emotion_label": "happy",
+ }),
+ content_type="application/json",
+ )
+ assert resp.status_code == 200
+ data = json.loads(resp.data)
+ assert data["status"] == "saved"
+
+ # Restore
+ app_module.DATA_DIR = original_dir
+
+ def test_api_perception_status(self, client):
+ resp = client.get("/api/perception/status")
+ assert resp.status_code == 200
+ data = json.loads(resp.data)
+ assert "mediapipe" in data
+ assert "deepface" in data
+
+ def test_api_eye_animation_stub(self, client):
+ resp = client.post(
+ "/api/eye_animation",
+ data=json.dumps({"target": "test", "animation_id": 1}),
+ content_type="application/json",
+ )
+ assert resp.status_code == 200
+ data = json.loads(resp.data)
+ assert data["status"] == "stub_ok"
diff --git a/python_host/tests/test_osc_sender.py b/python_host/tests/test_osc_sender.py
new file mode 100644
index 0000000..44a99a3
--- /dev/null
+++ b/python_host/tests/test_osc_sender.py
@@ -0,0 +1,70 @@
+"""Tests for the OSC sender module."""
+from unittest.mock import MagicMock
+from python_host.network.osc_sender import OSCSender
+
+
+class TestOSCSender:
+ """Test OSC sender logic without network access."""
+
+ def test_add_and_list_targets(self):
+ sender = OSCSender()
+ sender.add_target("test", "127.0.0.1", 8888)
+ targets = sender.list_targets()
+ assert "test" in targets
+
+ def test_remove_target(self):
+ sender = OSCSender()
+ sender.add_target("test", "127.0.0.1", 8888)
+ sender.remove_target("test")
+ targets = sender.list_targets()
+ assert "test" not in targets
+
+ def test_override_blocks_auto(self):
+ sender = OSCSender()
+ sender.add_target("test", "127.0.0.1", 8888)
+ sender.override = True
+
+ # Mock the internal client to track calls
+ mock_client = MagicMock()
+ sender._clients["test"] = mock_client
+
+ sender.send("test", "/motor1", 1, 128, source="auto")
+ mock_client.send_message.assert_not_called()
+
+ def test_override_allows_manual(self):
+ sender = OSCSender()
+ sender.add_target("test", "127.0.0.1", 8888)
+ sender.override = True
+
+ mock_client = MagicMock()
+ sender._clients["test"] = mock_client
+
+ sender.send("test", "/motor1", 1, 128, source="manual")
+ mock_client.send_message.assert_called_once()
+
+ def test_send_motor_formats_address(self):
+ sender = OSCSender()
+ mock_client = MagicMock()
+ sender._clients["test"] = mock_client
+
+ sender.send_motor("test", 1, 1, 128, source="manual")
+ mock_client.send_message.assert_called_once_with("/motor1", [1, 128])
+
+ def test_send_to_nonexistent_target_silent(self):
+ sender = OSCSender()
+ # Should not raise
+ sender.send("nonexistent", "/motor1", 1, 128, source="manual")
+
+ def test_stop_all_ignores_override(self):
+ sender = OSCSender()
+ sender.override = True
+ mock_client = MagicMock()
+ sender._clients["test"] = mock_client
+
+ sender.stop_all("test")
+ mock_client.send_message.assert_called_once_with("/preset", [3])
+
+ def test_eye_animation_stub(self):
+ sender = OSCSender()
+ # Should not raise
+ sender.send_eye_animation("test", 0)
diff --git a/python_host/tests/test_perception.py b/python_host/tests/test_perception.py
new file mode 100644
index 0000000..c72470d
--- /dev/null
+++ b/python_host/tests/test_perception.py
@@ -0,0 +1,36 @@
+"""Tests for the perception module (lazy-loading, no hardware required)."""
+from python_host.vision.perception import PerceptionModule
+
+
+class TestPerceptionModule:
+ """Test perception module initialization and graceful degradation."""
+
+ def test_init_no_crash(self):
+ pm = PerceptionModule()
+ assert pm._running is False
+ assert pm._results["emotion"] is None
+
+ def test_get_results_empty(self):
+ pm = PerceptionModule()
+ results = pm.get_results()
+ assert results["emotion"] is None
+ assert results["pose"] is None
+ assert results["face_analysis"] is None
+
+ def test_lazy_load_mediapipe(self):
+ """MediaPipe loading should not crash even if not installed."""
+ pm = PerceptionModule()
+ # This should return True or False without crashing
+ result = pm._try_load_mediapipe()
+ assert isinstance(result, bool)
+
+ def test_lazy_load_deepface(self):
+ """DeepFace loading should not crash even if not installed."""
+ pm = PerceptionModule()
+ result = pm._try_load_deepface()
+ assert isinstance(result, bool)
+
+ def test_stop_before_start(self):
+ """Stopping before starting should not crash."""
+ pm = PerceptionModule()
+ pm.stop() # Should not raise
diff --git a/python_host/ui/__init__.py b/python_host/ui/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/python_host/ui/app.py b/python_host/ui/app.py
new file mode 100644
index 0000000..619d249
--- /dev/null
+++ b/python_host/ui/app.py
@@ -0,0 +1,209 @@
+"""
+app.py — Flask control panel for DATT3700 interactive flower installation.
+
+Layout:
+ Left: Live video stream preview with face detection overlay
+ Right: Motor/LED sliders, 2D XY pad, Override switch, Tag & Save
+"""
+
+import json
+import os
+import time
+
+from flask import Flask, render_template, Response, request, jsonify
+
+from python_host.vision.face_tracker import FaceTracker
+from python_host.network.osc_sender import OSCSender
+
+# ── Globals ──────────────────────────────────────────────────
+
+app = Flask(
+ __name__,
+ template_folder=os.path.join(os.path.dirname(__file__), "templates"),
+ static_folder=os.path.join(os.path.dirname(__file__), "static"),
+)
+
+tracker = FaceTracker(camera_index=0)
+osc = OSCSender()
+
+DATA_DIR = os.path.join(os.path.dirname(__file__), "..", "data")
+SAMPLES_FILE = os.path.join(DATA_DIR, "training_samples.jsonl")
+
+# ── Routes ───────────────────────────────────────────────────
+
+
+@app.route("/")
+def index():
+ return render_template("index.html")
+
+
+# ── Video streaming ──────────────────────────────────────────
+
+
+def _generate_frames():
+ while True:
+ jpeg = tracker.get_frame_jpeg()
+ if jpeg is None:
+ time.sleep(0.03)
+ continue
+ yield (
+ b"--frame\r\n"
+ b"Content-Type: image/jpeg\r\n\r\n" + jpeg + b"\r\n"
+ )
+
+
+@app.route("/video_feed")
+def video_feed():
+ return Response(
+ _generate_frames(),
+ mimetype="multipart/x-mixed-replace; boundary=frame",
+ )
+
+
+# ── Face data API ────────────────────────────────────────────
+
+
+@app.route("/api/faces")
+def api_faces():
+ target = tracker.get_primary_target()
+ faces = tracker.get_all_faces()
+ return jsonify({"primary": target, "faces": faces})
+
+
+# ── Camera switching ─────────────────────────────────────────
+
+
+@app.route("/api/cameras")
+def api_cameras():
+ return jsonify({"cameras": FaceTracker.list_cameras()})
+
+
+@app.route("/api/camera/switch", methods=["POST"])
+def api_camera_switch():
+ idx = request.json.get("index", 0)
+ tracker.switch_camera(int(idx))
+ return jsonify({"status": "ok", "camera": idx})
+
+
+# ── OSC control endpoints ────────────────────────────────────
+
+
+@app.route("/api/osc/targets")
+def api_osc_targets():
+ return jsonify(osc.list_targets())
+
+
+@app.route("/api/osc/target", methods=["POST"])
+def api_osc_add_target():
+ data = request.json
+ osc.add_target(data["name"], data["ip"], data.get("port", 8888))
+ return jsonify({"status": "ok"})
+
+
+@app.route("/api/osc/motor", methods=["POST"])
+def api_osc_motor():
+ d = request.json
+ osc.send_motor(
+ d["target"], d["motor"], d["dir"], d.get("speed", 255), source="manual"
+ )
+ return jsonify({"status": "ok"})
+
+
+@app.route("/api/osc/led", methods=["POST"])
+def api_osc_led():
+ d = request.json
+ osc.send_led(d["target"], d["led"], d["r"], d["g"], d["b"])
+ return jsonify({"status": "ok"})
+
+
+@app.route("/api/osc/preset", methods=["POST"])
+def api_osc_preset():
+ d = request.json
+ osc.send_preset(d["target"], d["preset"])
+ return jsonify({"status": "ok"})
+
+
+@app.route("/api/osc/stop", methods=["POST"])
+def api_osc_stop():
+ d = request.json
+ osc.stop_all(d["target"])
+ return jsonify({"status": "ok"})
+
+
+# ── Override toggle ──────────────────────────────────────────
+
+
+@app.route("/api/override", methods=["GET", "POST"])
+def api_override():
+ if request.method == "POST":
+ osc.override = request.json.get("override", False)
+ return jsonify({"override": osc.override})
+
+
+# ── Tag & Save (data labeling) ───────────────────────────────
+
+
+@app.route("/api/tag_save", methods=["POST"])
+def api_tag_save():
+ """Save current vision features + manual control params as a training sample."""
+ d = request.json
+ sample = {
+ "timestamp": time.time(),
+ "vision_features": d.get("vision_features", {}),
+ "control_params": d.get("control_params", {}),
+ "emotion_label": d.get("emotion_label", ""),
+ }
+ os.makedirs(DATA_DIR, exist_ok=True)
+ with open(SAMPLES_FILE, "a") as f:
+ f.write(json.dumps(sample) + "\n")
+ return jsonify({"status": "saved", "sample": sample})
+
+
+# ── TFT eye animation stub ──────────────────────────────────
+
+
+@app.route("/api/eye_animation", methods=["POST"])
+def api_eye_animation():
+ """Reserved endpoint for TFT IPS eye animation commands."""
+ d = request.json
+ osc.send_eye_animation(d.get("target"), d.get("animation_id", 0))
+ return jsonify({"status": "stub_ok"})
+
+
+# ── ML perception endpoints ─────────────────────────────────
+
+
+@app.route("/api/perception/status")
+def api_perception_status():
+ """Check which perception modules are available."""
+ modules = {"mediapipe": False, "deepface": False}
+ try:
+ import mediapipe # noqa: F401
+ modules["mediapipe"] = True
+ except ImportError:
+ pass
+ try:
+ from deepface import DeepFace # noqa: F401
+ modules["deepface"] = True
+ except ImportError:
+ pass
+ return jsonify(modules)
+
+
+# ── Entry point ──────────────────────────────────────────────
+
+
+def create_app(camera_index=0, esp32_targets=None):
+ """Factory for external callers / testing."""
+ global tracker
+ tracker = FaceTracker(camera_index=camera_index)
+ if esp32_targets:
+ for name, (ip, port) in esp32_targets.items():
+ osc.add_target(name, ip, port)
+ return app
+
+
+if __name__ == "__main__":
+ tracker.start()
+ osc.add_target("sylvie_1", "192.168.4.1", 8888)
+ app.run(host="0.0.0.0", port=5000, debug=False, threaded=True)
diff --git a/python_host/ui/templates/index.html b/python_host/ui/templates/index.html
new file mode 100644
index 0000000..9cfba13
--- /dev/null
+++ b/python_host/ui/templates/index.html
@@ -0,0 +1,373 @@
+
+
+
+
+
+ DATT3700 Flower Control Panel
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+
+
+
+
Primary Target
+
No face detected
+
+
+ ML modules: checking…
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Override (Manual Takeover)
+
When ON, blocks auto-tracking commands
+
+
+
+
+
+
+
Motor Control
+
+
+
+
+ Motor 1
+ Dir: 0 | Speed: 0
+
+
+
+
+
+
+
+
+
+ Motor 2
+ Dir: 0 | Speed: 0
+
+
+
+
+
+
+
+
+
+
+
+
Flower Pad (X: Speed/Open · Y: Jitter)
+
+
+
+
+
+
LED Color
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Tag & Save (Data Labeling)
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/python_host/vision/__init__.py b/python_host/vision/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/python_host/vision/face_tracker.py b/python_host/vision/face_tracker.py
new file mode 100644
index 0000000..905bb60
--- /dev/null
+++ b/python_host/vision/face_tracker.py
@@ -0,0 +1,159 @@
+"""
+face_tracker.py — Weighted multi-face tracking with multi-camera support.
+
+Selects the primary target using:
+ weight = bbox_area × (1 / (1 + center_distance))
+
+Outputs only the primary target's normalized coordinates (0.0-1.0).
+No heavy ML dependencies — uses only OpenCV Haar Cascade.
+"""
+
+import cv2
+import math
+import threading
+import time
+
+
+class FaceTracker:
+ """Lightweight face tracker with weighted target selection."""
+
+ def __init__(self, camera_index=0, frame_width=1280, frame_height=720):
+ self._camera_index = camera_index
+ self._frame_width = frame_width
+ self._frame_height = frame_height
+
+ self._cap = None
+ self._cascade = cv2.CascadeClassifier(
+ cv2.data.haarcascades + "haarcascade_frontalface_default.xml"
+ )
+
+ self._lock = threading.Lock()
+ self._latest_frame = None
+ self._primary_target = None # (norm_x, norm_y, weight)
+ self._all_faces = []
+ self._running = False
+
+ # ------------------------------------------------------------------
+ # Public API
+ # ------------------------------------------------------------------
+
+ def start(self):
+ """Open camera and begin capture thread."""
+ self._cap = cv2.VideoCapture(self._camera_index)
+ self._cap.set(cv2.CAP_PROP_FRAME_WIDTH, self._frame_width)
+ self._cap.set(cv2.CAP_PROP_FRAME_HEIGHT, self._frame_height)
+ if not self._cap.isOpened():
+ raise RuntimeError(f"Cannot open camera {self._camera_index}")
+ self._running = True
+ self._thread = threading.Thread(target=self._capture_loop, daemon=True)
+ self._thread.start()
+
+ def stop(self):
+ """Release camera resources."""
+ self._running = False
+ if self._cap:
+ self._cap.release()
+ self._cap = None
+
+ def switch_camera(self, camera_index):
+ """Hot-switch to another camera (e.g. iPhone Continuity Camera)."""
+ self.stop()
+ self._camera_index = camera_index
+ self.start()
+
+ def get_primary_target(self):
+ """Return (norm_x, norm_y, weight) of highest-weight face or None."""
+ with self._lock:
+ return self._primary_target
+
+ def get_all_faces(self):
+ """Return list of face dicts for overlay rendering."""
+ with self._lock:
+ return list(self._all_faces)
+
+ def get_frame_jpeg(self):
+ """Return the latest frame as JPEG bytes (for Flask streaming)."""
+ with self._lock:
+ frame = self._latest_frame
+ if frame is None:
+ return None
+ _, buf = cv2.imencode(".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, 70])
+ return buf.tobytes()
+
+ # ------------------------------------------------------------------
+ # Internals
+ # ------------------------------------------------------------------
+
+ def _capture_loop(self):
+ while self._running:
+ ok, frame = self._cap.read()
+ if not ok:
+ time.sleep(0.01)
+ continue
+ self._process_frame(frame)
+
+ def _process_frame(self, frame):
+ h, w = frame.shape[:2]
+ cx_frame, cy_frame = w / 2.0, h / 2.0
+ max_dist = math.hypot(cx_frame, cy_frame)
+
+ gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+ rects = self._cascade.detectMultiScale(
+ gray, scaleFactor=1.1, minNeighbors=5, minSize=(60, 60)
+ )
+
+ faces = []
+ best_weight = -1.0
+ best_target = None
+
+ for x, y, fw, fh in rects:
+ area = fw * fh
+ cx_face = x + fw / 2.0
+ cy_face = y + fh / 2.0
+ dist = math.hypot(cx_face - cx_frame, cy_face - cy_frame)
+ proximity = 1.0 / (1.0 + dist / max_dist)
+ weight = area * proximity
+
+ norm_x = cx_face / w
+ norm_y = cy_face / h
+
+ face_info = {
+ "x": int(x), "y": int(y), "w": int(fw), "h": int(fh),
+ "norm_x": round(norm_x, 4),
+ "norm_y": round(norm_y, 4),
+ "weight": round(weight, 2),
+ }
+ faces.append(face_info)
+
+ if weight > best_weight:
+ best_weight = weight
+ best_target = (round(norm_x, 4), round(norm_y, 4), round(weight, 2))
+
+ # Draw bounding box on frame for preview
+ cv2.rectangle(frame, (x, y), (x + fw, y + fh), (0, 255, 0), 2)
+
+ # Highlight primary target
+ if best_target and faces:
+ primary = max(faces, key=lambda f: f["weight"])
+ cv2.rectangle(
+ frame,
+ (primary["x"], primary["y"]),
+ (primary["x"] + primary["w"], primary["y"] + primary["h"]),
+ (0, 0, 255), 3,
+ )
+
+ with self._lock:
+ self._latest_frame = frame
+ self._all_faces = faces
+ self._primary_target = best_target
+
+ @staticmethod
+ def list_cameras(max_check=5):
+ """Probe available camera indices."""
+ available = []
+ for i in range(max_check):
+ cap = cv2.VideoCapture(i)
+ if cap.isOpened():
+ available.append(i)
+ cap.release()
+ return available
diff --git a/python_host/vision/perception.py b/python_host/vision/perception.py
new file mode 100644
index 0000000..c071361
--- /dev/null
+++ b/python_host/vision/perception.py
@@ -0,0 +1,185 @@
+"""
+perception.py — Optional ML perception modules (MediaPipe + DeepFace).
+
+Uses lazy imports so the system works without ML dependencies installed.
+Thread-safe: runs inference in a background thread, exposes results via
+a locked dict.
+
+Best practices followed:
+ - mediapipe >= 0.10.14 uses the new Tasks API (not legacy mp.solutions)
+ - deepface uses lightweight backends by default
+ - No pyav / ffmpeg dependency (pure OpenCV capture)
+"""
+
+import threading
+import time
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class PerceptionModule:
+ """Runs optional emotion + pose detection on frames from FaceTracker."""
+
+ def __init__(self):
+ self._lock = threading.Lock()
+ self._results = {
+ "emotion": None, # e.g. {"dominant": "happy", "scores": {...}}
+ "pose": None, # e.g. {"landmarks": [...], "gesture": "..."}
+ "face_analysis": None, # e.g. {"age": 25, "gender": "Man", ...}
+ }
+ self._running = False
+ self._tracker = None
+
+ # Lazy-loaded modules
+ self._mp = None
+ self._deepface = None
+ self._mp_face_mesh = None
+ self._mp_pose = None
+
+ # ------------------------------------------------------------------
+ # Init
+ # ------------------------------------------------------------------
+
+ def _try_load_mediapipe(self):
+ try:
+ import mediapipe as mp
+ self._mp = mp
+ # Use new Tasks API (mediapipe >= 0.10.14)
+ self._mp_face_mesh = mp.solutions.face_mesh.FaceMesh(
+ static_image_mode=False,
+ max_num_faces=1,
+ refine_landmarks=True,
+ min_detection_confidence=0.5,
+ min_tracking_confidence=0.5,
+ )
+ self._mp_pose = mp.solutions.pose.Pose(
+ static_image_mode=False,
+ model_complexity=0,
+ min_detection_confidence=0.5,
+ min_tracking_confidence=0.5,
+ )
+ logger.info("MediaPipe loaded successfully")
+ return True
+ except ImportError:
+ logger.warning("MediaPipe not installed — pose/mesh disabled")
+ return False
+
+ def _try_load_deepface(self):
+ try:
+ from deepface import DeepFace
+ self._deepface = DeepFace
+ logger.info("DeepFace loaded successfully")
+ return True
+ except ImportError:
+ logger.warning("DeepFace not installed — emotion analysis disabled")
+ return False
+
+ # ------------------------------------------------------------------
+ # Lifecycle
+ # ------------------------------------------------------------------
+
+ def start(self, tracker):
+ """Begin perception loop reading frames from a FaceTracker."""
+ self._tracker = tracker
+ self._try_load_mediapipe()
+ self._try_load_deepface()
+ self._running = True
+ self._thread = threading.Thread(target=self._loop, daemon=True)
+ self._thread.start()
+
+ def stop(self):
+ self._running = False
+ if self._mp_face_mesh:
+ self._mp_face_mesh.close()
+ if self._mp_pose:
+ self._mp_pose.close()
+
+ def get_results(self):
+ with self._lock:
+ return dict(self._results)
+
+ # ------------------------------------------------------------------
+ # Main loop
+ # ------------------------------------------------------------------
+
+ def _loop(self):
+ while self._running:
+ if self._tracker is None:
+ time.sleep(0.1)
+ continue
+
+ # Borrow the latest frame
+ frame_jpeg = self._tracker.get_frame_jpeg()
+ if frame_jpeg is None:
+ time.sleep(0.05)
+ continue
+
+ # Decode JPEG back to numpy (avoids holding tracker lock)
+ import cv2
+ import numpy as np
+ arr = np.frombuffer(frame_jpeg, dtype=np.uint8)
+ frame = cv2.imdecode(arr, cv2.IMREAD_COLOR)
+ if frame is None:
+ time.sleep(0.05)
+ continue
+
+ results = {}
+
+ # ── MediaPipe Face Mesh ──
+ if self._mp_face_mesh:
+ try:
+ import cv2 as cv
+ rgb = cv.cvtColor(frame, cv.COLOR_BGR2RGB)
+ mesh_result = self._mp_face_mesh.process(rgb)
+ if mesh_result.multi_face_landmarks:
+ landmarks = []
+ for lm in mesh_result.multi_face_landmarks[0].landmark:
+ landmarks.append({
+ "x": round(lm.x, 4),
+ "y": round(lm.y, 4),
+ "z": round(lm.z, 4),
+ })
+ results["pose"] = {"landmarks_count": len(landmarks)}
+ except Exception as e:
+ logger.debug(f"MediaPipe mesh error: {e}")
+
+ # ── MediaPipe Pose ──
+ if self._mp_pose:
+ try:
+ import cv2 as cv
+ rgb = cv.cvtColor(frame, cv.COLOR_BGR2RGB)
+ pose_result = self._mp_pose.process(rgb)
+ if pose_result.pose_landmarks:
+ results["pose_body"] = {
+ "landmarks_count": len(pose_result.pose_landmarks.landmark)
+ }
+ except Exception as e:
+ logger.debug(f"MediaPipe pose error: {e}")
+
+ # ── DeepFace emotion analysis (throttled) ──
+ if self._deepface:
+ try:
+ analysis = self._deepface.analyze(
+ frame,
+ actions=["emotion"],
+ enforce_detection=False,
+ silent=True,
+ )
+ if analysis and len(analysis) > 0:
+ a = analysis[0]
+ results["emotion"] = {
+ "dominant": a.get("dominant_emotion", "unknown"),
+ "scores": a.get("emotion", {}),
+ }
+ results["face_analysis"] = {
+ "region": a.get("region", {}),
+ }
+ except Exception as e:
+ logger.debug(f"DeepFace error: {e}")
+
+ with self._lock:
+ self._results.update(results)
+
+ # Throttle to ~5 FPS for ML inference
+ time.sleep(0.2)
From f9283b5f83cf9b644ff325fa0e643d7a2c06403d Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Wed, 4 Mar 2026 12:43:06 +0000
Subject: [PATCH 03/14] fix: address code review feedback and security scan
findings
- Replace deprecated substr() with substring() in HTML template
- Replace alert() with inline toast notification for better UX
- Remove duplicate cv2 imports in perception.py
- Store target info separately instead of accessing private _address/_port
- Use proper FaceTracker re-instantiation instead of __init__ call
- Add permissions: contents: read to CI workflow
Co-authored-by: Sa1koro <13943286+Sa1koro@users.noreply.github.com>
---
.github/workflows/python-ci.yml | 3 +++
python_host/main.py | 7 ++++---
python_host/network/osc_sender.py | 5 ++++-
python_host/ui/templates/index.html | 13 +++++++++----
python_host/vision/perception.py | 6 ++----
5 files changed, 22 insertions(+), 12 deletions(-)
diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml
index f04c626..a3786b0 100644
--- a/.github/workflows/python-ci.yml
+++ b/.github/workflows/python-ci.yml
@@ -12,6 +12,9 @@ on:
- "python_host/**"
- ".github/workflows/python-ci.yml"
+permissions:
+ contents: read
+
jobs:
test-core:
name: Core Tests (no ML)
diff --git a/python_host/main.py b/python_host/main.py
index b50aa26..a25382c 100644
--- a/python_host/main.py
+++ b/python_host/main.py
@@ -10,6 +10,7 @@
import argparse
+from python_host.vision.face_tracker import FaceTracker
from python_host.ui.app import app, osc
@@ -26,10 +27,10 @@ def main():
# Start camera if enabled
if not args.no_camera:
- from python_host.ui.app import tracker as app_tracker
- app_tracker.__init__(camera_index=args.camera)
+ import python_host.ui.app as app_module
+ app_module.tracker = FaceTracker(camera_index=args.camera)
try:
- app_tracker.start()
+ app_module.tracker.start()
except RuntimeError as e:
print(f"⚠️ Camera not available: {e}")
diff --git a/python_host/network/osc_sender.py b/python_host/network/osc_sender.py
index 01efd9d..7c5cfd5 100644
--- a/python_host/network/osc_sender.py
+++ b/python_host/network/osc_sender.py
@@ -14,6 +14,7 @@ class OSCSender:
def __init__(self):
self._clients = {} # name -> SimpleUDPClient
+ self._target_info = {} # name -> (ip, port)
self._lock = threading.Lock()
self._override = False # True = manual UI only, block CV auto
@@ -24,14 +25,16 @@ def __init__(self):
def add_target(self, name, ip, port=8888):
with self._lock:
self._clients[name] = udp_client.SimpleUDPClient(ip, port)
+ self._target_info[name] = (ip, port)
def remove_target(self, name):
with self._lock:
self._clients.pop(name, None)
+ self._target_info.pop(name, None)
def list_targets(self):
with self._lock:
- return {n: (c._address, c._port) for n, c in self._clients.items()}
+ return dict(self._target_info)
# ------------------------------------------------------------------
# Override (manual vs auto)
diff --git a/python_host/ui/templates/index.html b/python_host/ui/templates/index.html
index 9cfba13..4554792 100644
--- a/python_host/ui/templates/index.html
+++ b/python_host/ui/templates/index.html
@@ -272,9 +272,9 @@ Tag & Save (Data Labeling)
// ── LED ──
async function sendLED() {
const hex = document.getElementById('ledColor').value;
- const r = parseInt(hex.substr(1,2), 16);
- const g = parseInt(hex.substr(3,2), 16);
- const b = parseInt(hex.substr(5,2), 16);
+ const r = parseInt(hex.substring(1,3), 16);
+ const g = parseInt(hex.substring(3,5), 16);
+ const b = parseInt(hex.substring(5,7), 16);
const led = parseInt(document.getElementById('ledGroup').value);
await postJSON('/api/osc/led', {target: TARGET(), led, r, g, b});
}
@@ -298,7 +298,12 @@ Tag & Save (Data Labeling)
emotion_label: label
});
document.getElementById('emotionLabel').value = '';
- alert('Sample saved!');
+ // Show inline toast instead of alert
+ const btn = document.querySelector('[onclick="tagAndSave()"]');
+ const orig = btn.textContent;
+ btn.textContent = '✅ Saved!';
+ btn.classList.replace('bg-emerald-600', 'bg-emerald-400');
+ setTimeout(() => { btn.textContent = orig; btn.classList.replace('bg-emerald-400', 'bg-emerald-600'); }, 1500);
}
// ── 2D XY Pad ──
diff --git a/python_host/vision/perception.py b/python_host/vision/perception.py
index c071361..ea56718 100644
--- a/python_host/vision/perception.py
+++ b/python_host/vision/perception.py
@@ -129,8 +129,7 @@ def _loop(self):
# ── MediaPipe Face Mesh ──
if self._mp_face_mesh:
try:
- import cv2 as cv
- rgb = cv.cvtColor(frame, cv.COLOR_BGR2RGB)
+ rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
mesh_result = self._mp_face_mesh.process(rgb)
if mesh_result.multi_face_landmarks:
landmarks = []
@@ -147,8 +146,7 @@ def _loop(self):
# ── MediaPipe Pose ──
if self._mp_pose:
try:
- import cv2 as cv
- rgb = cv.cvtColor(frame, cv.COLOR_BGR2RGB)
+ rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
pose_result = self._mp_pose.process(rgb)
if pose_result.pose_landmarks:
results["pose_body"] = {
From 6ba80f3493d172a6ad3b7fb8f5c532a5e0ed7c47 Mon Sep 17 00:00:00 2001
From: Sakura Tsuki
Date: Wed, 4 Mar 2026 12:51:40 -0500
Subject: [PATCH 04/14] refactor(project): restructure ESP32 firmware and
update project configuration
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
- Move ESP32 firmware files to new refactored directory structure
- Update Flask server port from 5000 to 15000 in main.py
- Rename Chinese component label from "舵机" to English "Servo" in hardware guide
- Add Kait node section to hardware documentation
- Create pyproject.toml with project metadata and pyserial dependency
- Generate uv.lock file with package dependencies
- Maintain all Python test scripts and control utilities in new structure
---
esp32_firmware/esp32_sylvie/HARDWARE_GUIDE.md | 4 +++-
.../esp32_wifi_relay.py | 0
.../esp32_wifi_test.py | 0
.../osc_controll.py | 0
.../test_motor_pwm.py | 0
pyproject.toml | 8 +++++++
python_host/main.py | 2 +-
uv.lock | 23 +++++++++++++++++++
8 files changed, 35 insertions(+), 2 deletions(-)
rename {esp32_firmware/esp32_sylvie => esp32_firmware_refactored}/esp32_wifi_relay.py (100%)
rename {esp32_firmware/esp32_sylvie => esp32_firmware_refactored}/esp32_wifi_test.py (100%)
rename {esp32_firmware/esp32_sylvie => esp32_firmware_refactored}/osc_controll.py (100%)
rename {esp32_firmware/esp32_sylvie => esp32_firmware_refactored}/test_motor_pwm.py (100%)
create mode 100644 pyproject.toml
create mode 100644 uv.lock
diff --git a/esp32_firmware/esp32_sylvie/HARDWARE_GUIDE.md b/esp32_firmware/esp32_sylvie/HARDWARE_GUIDE.md
index 05f6534..16ddf94 100644
--- a/esp32_firmware/esp32_sylvie/HARDWARE_GUIDE.md
+++ b/esp32_firmware/esp32_sylvie/HARDWARE_GUIDE.md
@@ -40,6 +40,8 @@
| 组件 | 功能 | ESP32 引脚 | 备注说明 |
| --- | --- | --- | --- |
-| **舵机** | 开合控制 | GPIO 18 | 闭合: 60°, 张开: 120° |
+| **Servo** | 开合控制 | GPIO 18 | 闭合: 60°, 张开: 120° |
| **LED 1** | 红色指示灯 (危险) | GPIO 22 | |
| **LED 2** | 绿色指示灯 (放松) | GPIO 23 | |
+
+## 3. Kait 节点 (1x Servo, 待提交)
\ No newline at end of file
diff --git a/esp32_firmware/esp32_sylvie/esp32_wifi_relay.py b/esp32_firmware_refactored/esp32_wifi_relay.py
similarity index 100%
rename from esp32_firmware/esp32_sylvie/esp32_wifi_relay.py
rename to esp32_firmware_refactored/esp32_wifi_relay.py
diff --git a/esp32_firmware/esp32_sylvie/esp32_wifi_test.py b/esp32_firmware_refactored/esp32_wifi_test.py
similarity index 100%
rename from esp32_firmware/esp32_sylvie/esp32_wifi_test.py
rename to esp32_firmware_refactored/esp32_wifi_test.py
diff --git a/esp32_firmware/esp32_sylvie/osc_controll.py b/esp32_firmware_refactored/osc_controll.py
similarity index 100%
rename from esp32_firmware/esp32_sylvie/osc_controll.py
rename to esp32_firmware_refactored/osc_controll.py
diff --git a/esp32_firmware/esp32_sylvie/test_motor_pwm.py b/esp32_firmware_refactored/test_motor_pwm.py
similarity index 100%
rename from esp32_firmware/esp32_sylvie/test_motor_pwm.py
rename to esp32_firmware_refactored/test_motor_pwm.py
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..bd821c3
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,8 @@
+[project]
+name = "datt3700"
+version = "0.1.0"
+description = "Add your description here"
+requires-python = ">=3.11"
+dependencies = [
+ "pyserial>=3.5",
+]
diff --git a/python_host/main.py b/python_host/main.py
index a25382c..daaa9f2 100644
--- a/python_host/main.py
+++ b/python_host/main.py
@@ -19,7 +19,7 @@ def main():
parser.add_argument("--camera", type=int, default=0, help="Camera index")
parser.add_argument("--no-camera", action="store_true", help="Disable camera")
parser.add_argument("--esp", type=str, default="192.168.4.1", help="ESP32 IP")
- parser.add_argument("--port", type=int, default=5000, help="Flask port")
+ parser.add_argument("--port", type=int, default=15000, help="Flask port")
args = parser.parse_args()
# Configure OSC target
diff --git a/uv.lock b/uv.lock
new file mode 100644
index 0000000..929cf5c
--- /dev/null
+++ b/uv.lock
@@ -0,0 +1,23 @@
+version = 1
+revision = 3
+requires-python = ">=3.11"
+
+[[package]]
+name = "datt3700"
+version = "0.1.0"
+source = { virtual = "." }
+dependencies = [
+ { name = "pyserial" },
+]
+
+[package.metadata]
+requires-dist = [{ name = "pyserial", specifier = ">=3.5" }]
+
+[[package]]
+name = "pyserial"
+version = "3.5"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/1e/7d/ae3f0a63f41e4d2f6cb66a5b57197850f919f59e558159a4dd3a818f5082/pyserial-3.5.tar.gz", hash = "sha256:3c77e014170dfffbd816e6ffc205e9842efb10be9f58ec16d3e8675b4925cddb", size = 159125, upload-time = "2020-11-23T03:59:15.045Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/07/bc/587a445451b253b285629263eb51c2d8e9bcea4fc97826266d186f96f558/pyserial-3.5-py2.py3-none-any.whl", hash = "sha256:c4451db6ba391ca6ca299fb3ec7bae67a5c55dde170964c7a14ceefec02f2cf0", size = 90585, upload-time = "2020-11-23T03:59:13.41Z" },
+]
From 53bc56cd6fa94633d0358d0e670bb14b4552d8af Mon Sep 17 00:00:00 2001
From: Sakura Tsuki
Date: Wed, 4 Mar 2026 13:44:16 -0500
Subject: [PATCH 05/14] feat(network): add device discovery and multi-node
control system
- Implement mDNS and gateway-based device discovery mechanisms
- Add device registry with node type inference and labeling
- Create device selection and management API endpoints
- Integrate threading support for concurrent device operations
- Add raw OSC console with history logging capabilities
- Implement dynamic UI controls based on detected node types
- Replace static target configuration with dynamic device management
- Add UTF-8 encoding for training sample file operations
- Create device registry JSON configuration file
- Update frontend UI with device scanning and selection controls
---
python_host/README.md | 28 ++
python_host/main.py | 23 +-
python_host/network/node_discovery.py | 206 +++++++++
python_host/network/osc_sender.py | 151 ++++++-
python_host/requirements.txt | 1 +
python_host/tests/test_flask_app.py | 72 +++-
python_host/tests/test_osc_sender.py | 56 ++-
python_host/ui/app.py | 186 +++++++-
python_host/ui/device_registry.json | 40 ++
python_host/ui/templates/index.html | 585 ++++++++++++--------------
10 files changed, 998 insertions(+), 350 deletions(-)
create mode 100644 python_host/README.md
create mode 100644 python_host/network/node_discovery.py
create mode 100644 python_host/ui/device_registry.json
diff --git a/python_host/README.md b/python_host/README.md
new file mode 100644
index 0000000..a21f9b0
--- /dev/null
+++ b/python_host/README.md
@@ -0,0 +1,28 @@
+# python_host
+
+Flask control panel for DATT3700 multi-node ESP32 setup.
+
+## Features
+
+- mDNS scan for ESP32 nodes (`_datt_flower._tcp`, `_osc._udp`)
+- Gateway fallback scan via OSC (`/info/clients`, `/info/self`)
+- Node-type-aware control rendering for `sylvie`, `sue`, `face_track`
+- Universal raw OSC console with send/receive history
+
+## Quick start
+
+```bash
+python -m pip install -r python_host/requirements.txt
+python -m python_host.main --port 5000
+```
+
+Open `http://127.0.0.1:5000`.
+
+## Key API endpoints
+
+- `POST /api/devices/scan` with `{"mode":"mdns|gateway|auto"}`
+- `GET /api/devices`
+- `POST /api/devices/select`
+- `POST /api/osc/raw`
+- `GET /api/osc/history`
+
diff --git a/python_host/main.py b/python_host/main.py
index daaa9f2..ad8da2b 100644
--- a/python_host/main.py
+++ b/python_host/main.py
@@ -11,7 +11,7 @@
import argparse
from python_host.vision.face_tracker import FaceTracker
-from python_host.ui.app import app, osc
+from python_host.ui.app import app
def main():
@@ -22,19 +22,30 @@ def main():
parser.add_argument("--port", type=int, default=15000, help="Flask port")
args = parser.parse_args()
- # Configure OSC target
- osc.add_target("sylvie_1", args.esp, 8888)
+ import python_host.ui.app as app_module
+
+ # Configure default OSC target in the shared device registry.
+ app_module._register_device(
+ {
+ "name": "sylvie_1",
+ "ip": args.esp,
+ "port": 8888,
+ "node_type": "sylvie",
+ "source": "startup",
+ "metadata": {},
+ }
+ )
+ app_module._selected_device = "sylvie_1"
# Start camera if enabled
if not args.no_camera:
- import python_host.ui.app as app_module
app_module.tracker = FaceTracker(camera_index=args.camera)
try:
app_module.tracker.start()
except RuntimeError as e:
- print(f"⚠️ Camera not available: {e}")
+ print(f"Camera not available: {e}")
- print(f"🌸 Starting DATT3700 control panel on http://0.0.0.0:{args.port}")
+ print(f"Starting DATT3700 control panel on http://0.0.0.0:{args.port}")
app.run(host="0.0.0.0", port=args.port, debug=False, threaded=True)
diff --git a/python_host/network/node_discovery.py b/python_host/network/node_discovery.py
new file mode 100644
index 0000000..c0bb5f8
--- /dev/null
+++ b/python_host/network/node_discovery.py
@@ -0,0 +1,206 @@
+"""Device discovery helpers for ESP32 flower nodes."""
+
+from __future__ import annotations
+
+import json
+import os
+import time as time_module
+
+try:
+ from zeroconf import ServiceBrowser, ServiceListener, Zeroconf
+except ImportError: # pragma: no cover - optional dependency in tests
+ Zeroconf = None
+ ServiceBrowser = None
+ ServiceListener = object
+
+
+REGISTRY_PATH = os.path.join(
+ os.path.dirname(__file__), "..", "ui", "device_registry.json"
+)
+
+
+class _MDNSListener(ServiceListener):
+ """Collect mDNS services while the browser runs."""
+
+ def __init__(self, zeroconf, service_type):
+ self._zeroconf = zeroconf
+ self._service_type = service_type
+ self.records = []
+
+ def remove_service(self, zeroconf, service_type, name):
+ pass
+
+ def update_service(self, zeroconf, service_type, name):
+ pass
+
+ def add_service(self, zeroconf, service_type, name):
+ info = self._zeroconf.get_service_info(self._service_type, name, timeout=300)
+ if info is None:
+ return
+
+ try:
+ addresses = info.parsed_addresses()
+ except Exception:
+ addresses = []
+ if not addresses:
+ return
+
+ txt = {}
+ for key, value in info.properties.items():
+ k = key.decode("utf-8", errors="ignore")
+ if isinstance(value, bytes):
+ txt[k] = value.decode("utf-8", errors="ignore")
+ else:
+ txt[k] = str(value)
+
+ self.records.append(
+ {
+ "service_name": name,
+ "hostname": info.server.rstrip(".") if info.server else "",
+ "ip": addresses[0],
+ "port": int(info.port),
+ "txt": txt,
+ }
+ )
+
+
+def load_registry(registry_path: str = REGISTRY_PATH) -> dict:
+ if not os.path.exists(registry_path):
+ return {
+ "default_osc_port": 8888,
+ "known_devices": {},
+ "name_rules": [],
+ "node_types": {},
+ }
+ with open(registry_path, "r", encoding="utf-8") as f:
+ return json.load(f)
+
+
+def infer_node_type(name: str, txt_node_type: str | None = None, registry: dict | None = None) -> str:
+ if txt_node_type:
+ return txt_node_type
+
+ registry = registry or load_registry()
+ known = registry.get("known_devices", {})
+ if name in known and known[name].get("node_type"):
+ return known[name]["node_type"]
+
+ lowered = (name or "").lower()
+ for rule in registry.get("name_rules", []):
+ token = (rule.get("contains") or "").lower()
+ if token and token in lowered:
+ return rule.get("node_type", "unknown")
+
+ return "unknown"
+
+
+def discover_mdns_nodes(timeout_sec: float = 1.2, registry: dict | None = None) -> list[dict]:
+ """Discover devices from mDNS service advertisements."""
+ if Zeroconf is None or ServiceBrowser is None:
+ return []
+
+ browser_cls = ServiceBrowser
+ zeroconf_cls = Zeroconf
+ if browser_cls is None or zeroconf_cls is None:
+ return []
+
+ registry = registry or load_registry()
+ services = ["_datt_flower._tcp.local.", "_osc._udp.local."]
+ seen = {}
+
+ zc = zeroconf_cls()
+ try:
+ listeners = []
+ browsers = []
+ for service in services:
+ listener = _MDNSListener(zc, service)
+ listeners.append(listener)
+ browsers.append(browser_cls(zc, service, listener))
+
+ # Let the browser collect announcements for a short window.
+ end_at = time_module.time() + timeout_sec
+ while time_module.time() < end_at:
+ time_module.sleep(0.05)
+
+ for listener in listeners:
+ for item in listener.records:
+ name = item["hostname"].split(".")[0] if item["hostname"] else item["service_name"]
+ txt_node_type = item["txt"].get("node_type")
+ node_type = infer_node_type(name, txt_node_type=txt_node_type, registry=registry)
+ key = f"{name}@{item['ip']}"
+ seen[key] = {
+ "name": name,
+ "ip": item["ip"],
+ "port": item["port"] or registry.get("default_osc_port", 8888),
+ "node_type": node_type,
+ "source": "mdns",
+ "metadata": {
+ "service_name": item["service_name"],
+ "txt": item["txt"],
+ },
+ }
+ finally:
+ zc.close()
+
+ return list(seen.values())
+
+
+def discover_nodes_via_gateway(
+ osc_sender,
+ gateway_ip: str,
+ gateway_port: int = 8888,
+ timeout_sec: float = 0.8,
+ registry: dict | None = None,
+) -> list[dict]:
+ """Query a gateway ESP32 for AP client list and probe each client via /info/self."""
+ registry = registry or load_registry()
+
+ results = []
+ seen = set()
+
+ gateway_self = osc_sender.query_info_self_ip(gateway_ip, gateway_port, timeout=timeout_sec)
+ gateway_name = gateway_self.get("name") if gateway_self else "gateway"
+ gateway_type = infer_node_type(gateway_name, registry=registry)
+ results.append(
+ {
+ "name": gateway_name,
+ "ip": gateway_ip,
+ "port": gateway_port,
+ "node_type": gateway_type,
+ "source": "gateway_self",
+ "metadata": gateway_self or {},
+ }
+ )
+ seen.add(gateway_ip)
+
+ payload = osc_sender.query_info_clients_ip(gateway_ip, gateway_port, timeout=timeout_sec)
+ if not payload:
+ return results
+
+ clients = payload.get("clients", [])
+ for idx, client in enumerate(clients, start=1):
+ ip = client.get("ip")
+ if not ip or ip in seen:
+ continue
+ seen.add(ip)
+
+ info = osc_sender.query_info_self_ip(ip, gateway_port, timeout=timeout_sec) or {}
+ name = info.get("name") or f"client_{idx}"
+ node_type = infer_node_type(name, registry=registry)
+
+ results.append(
+ {
+ "name": name,
+ "ip": ip,
+ "port": gateway_port,
+ "node_type": node_type,
+ "source": "gateway_clients",
+ "metadata": {
+ "mac": client.get("mac", ""),
+ "self": info,
+ },
+ }
+ )
+
+ return results
+
diff --git a/python_host/network/osc_sender.py b/python_host/network/osc_sender.py
index 7c5cfd5..730e495 100644
--- a/python_host/network/osc_sender.py
+++ b/python_host/network/osc_sender.py
@@ -5,10 +5,67 @@
never block on network I/O.
"""
+import socket
+import struct
import threading
+import time
+from collections import deque
+
from pythonosc import udp_client
+def _pad4(data: bytes) -> bytes:
+ pad = (4 - (len(data) % 4)) % 4
+ return data + (b"\x00" * pad)
+
+
+def _build_osc_message(address, args=None):
+ args = list(args or [])
+ address_bin = _pad4(address.encode("utf-8") + b"\x00")
+
+ type_tags = [","]
+ payload = b""
+ for arg in args:
+ if isinstance(arg, int):
+ type_tags.append("i")
+ payload += struct.pack(">i", int(arg))
+ else:
+ type_tags.append("s")
+ payload += _pad4(str(arg).encode("utf-8") + b"\x00")
+
+ tag_bin = _pad4("".join(type_tags).encode("utf-8") + b"\x00")
+ return address_bin + tag_bin + payload
+
+
+def _read_osc_string(data, offset):
+ end = data.find(b"\x00", offset)
+ if end < 0:
+ return "", len(data)
+ value = data[offset:end].decode("utf-8", errors="ignore")
+ next_offset = (end + 4) & ~0x03
+ return value, next_offset
+
+
+def _parse_osc_message(data):
+ address, offset = _read_osc_string(data, 0)
+ type_tags, offset = _read_osc_string(data, offset)
+
+ args = []
+ for tag in type_tags[1:]: # skip leading comma
+ if tag == "i":
+ if offset + 4 > len(data):
+ break
+ args.append(struct.unpack(">i", data[offset:offset + 4])[0])
+ offset += 4
+ elif tag == "s":
+ value, offset = _read_osc_string(data, offset)
+ args.append(value)
+ else:
+ break
+
+ return address, args
+
+
class OSCSender:
"""Manages one or more ESP32 OSC targets with a send queue."""
@@ -17,6 +74,7 @@ def __init__(self):
self._target_info = {} # name -> (ip, port)
self._lock = threading.Lock()
self._override = False # True = manual UI only, block CV auto
+ self._history = deque(maxlen=200)
# ------------------------------------------------------------------
# Target management
@@ -48,6 +106,22 @@ def override(self):
def override(self, value):
self._override = bool(value)
+ def _push_history(self, direction, address, args, target_name=None, ip=None, port=None):
+ self._history.append(
+ {
+ "ts": time.time(),
+ "direction": direction,
+ "target": target_name,
+ "ip": ip,
+ "port": port,
+ "address": address,
+ "args": list(args or []),
+ }
+ )
+
+ def get_history(self, limit=80):
+ return list(self._history)[-int(limit):]
+
# ------------------------------------------------------------------
# Send helpers
# ------------------------------------------------------------------
@@ -59,13 +133,20 @@ def send(self, target_name, address, *args, source="auto"):
source="manual" → always sent
"""
if source == "auto" and self._override:
- return # manual override active, ignore CV commands
+ return False # manual override active, ignore CV commands
with self._lock:
client = self._clients.get(target_name)
+ target = self._target_info.get(target_name)
if client is None:
- return
+ return False
client.send_message(address, list(args))
+ ip, port = target if target else (None, None)
+ self._push_history("tx", address, args, target_name=target_name, ip=ip, port=port)
+ return True
+
+ def send_raw(self, target_name, address, args=None, source="manual"):
+ return self.send(target_name, address, *(args or []), source=source)
def send_motor(self, target_name, motor_id, direction, speed=255, source="auto"):
addr = f"/motor{motor_id}"
@@ -92,3 +173,69 @@ def stop_all(self, target_name):
def send_eye_animation(self, target_name, animation_id, **kwargs):
"""Reserved — will send TFT IPS eye animation commands."""
pass
+
+ # ------------------------------------------------------------------
+ # Lightweight request/reply helpers for discovery endpoints
+ # ------------------------------------------------------------------
+
+ def _request_reply(self, ip, port, address, args=None, timeout=0.8):
+ packet = _build_osc_message(address, args=args)
+ sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+ try:
+ sock.settimeout(timeout)
+ sock.bind(("0.0.0.0", 0))
+ sock.sendto(packet, (ip, int(port)))
+ self._push_history("tx", address, args or [], ip=ip, port=port)
+ data, src = sock.recvfrom(2048)
+ reply_addr, reply_args = _parse_osc_message(data)
+ self._push_history("rx", reply_addr, reply_args, ip=src[0], port=src[1])
+ return {"address": reply_addr, "args": reply_args, "ip": src[0], "port": src[1]}
+ except OSError:
+ return None
+ finally:
+ sock.close()
+
+ def query_info_self_ip(self, ip, port=8888, timeout=0.8):
+ reply = self._request_reply(ip, port, "/info/self", timeout=timeout)
+ if not reply or reply.get("address") != "/info/self":
+ return None
+ args = reply.get("args", [])
+ if len(args) < 4:
+ return None
+ return {
+ "name": str(args[0]),
+ "mac": str(args[1]),
+ "mode": str(args[2]),
+ "ip": str(args[3]),
+ }
+
+ def query_info_clients_ip(self, ip, port=8888, timeout=0.8):
+ reply = self._request_reply(ip, port, "/info/clients", timeout=timeout)
+ if not reply or reply.get("address") != "/info/clients":
+ return None
+
+ args = reply.get("args", [])
+ count = int(args[0]) if args and isinstance(args[0], int) else 0
+ clients = []
+ idx = 1
+ while idx + 1 < len(args):
+ clients.append({"mac": str(args[idx]), "ip": str(args[idx + 1])})
+ idx += 2
+
+ return {"count": count, "clients": clients}
+
+ def query_info_self(self, target_name, timeout=0.8):
+ with self._lock:
+ target = self._target_info.get(target_name)
+ if not target:
+ return None
+ ip, port = target
+ return self.query_info_self_ip(ip, port, timeout=timeout)
+
+ def query_info_clients(self, target_name, timeout=0.8):
+ with self._lock:
+ target = self._target_info.get(target_name)
+ if not target:
+ return None
+ ip, port = target
+ return self.query_info_clients_ip(ip, port, timeout=timeout)
diff --git a/python_host/requirements.txt b/python_host/requirements.txt
index 8f3ec9a..c85e6c3 100644
--- a/python_host/requirements.txt
+++ b/python_host/requirements.txt
@@ -3,6 +3,7 @@ flask>=3.0,<4.0
python-osc>=1.8,<2.0
opencv-python-headless>=4.8,<5.0
numpy>=1.24,<3.0
+zeroconf>=0.132,<1.0
# Optional ML dependencies (install with: pip install -r requirements-ml.txt)
# mediapipe>=0.10.14
diff --git a/python_host/tests/test_flask_app.py b/python_host/tests/test_flask_app.py
index 03d9d39..928b26a 100644
--- a/python_host/tests/test_flask_app.py
+++ b/python_host/tests/test_flask_app.py
@@ -1,6 +1,9 @@
"""Tests for the Flask control panel API endpoints."""
import json
+
import pytest
+
+import python_host.ui.app as app_module
from python_host.ui.app import app
@@ -66,8 +69,8 @@ def test_api_osc_motor(self, client):
def test_api_tag_save(self, client, tmp_path):
"""Test tag & save creates JSONL entry."""
- import python_host.ui.app as app_module
original_dir = app_module.DATA_DIR
+ original_samples = app_module.SAMPLES_FILE
app_module.DATA_DIR = str(tmp_path)
app_module.SAMPLES_FILE = str(tmp_path / "test_samples.jsonl")
@@ -86,6 +89,7 @@ def test_api_tag_save(self, client, tmp_path):
# Restore
app_module.DATA_DIR = original_dir
+ app_module.SAMPLES_FILE = original_samples
def test_api_perception_status(self, client):
resp = client.get("/api/perception/status")
@@ -103,3 +107,69 @@ def test_api_eye_animation_stub(self, client):
assert resp.status_code == 200
data = json.loads(resp.data)
assert data["status"] == "stub_ok"
+
+ def test_api_registry_and_devices(self, client):
+ reg = client.get("/api/devices/registry")
+ assert reg.status_code == 200
+ reg_data = json.loads(reg.data)
+ assert "node_types" in reg_data
+
+ devices = client.get("/api/devices")
+ assert devices.status_code == 200
+ data = json.loads(devices.data)
+ assert "devices" in data
+
+ def test_api_scan_mdns_with_mock(self, client, monkeypatch):
+ monkeypatch.setattr(
+ app_module,
+ "discover_mdns_nodes",
+ lambda timeout_sec, registry: [
+ {
+ "name": "F7OWER_00",
+ "ip": "192.168.4.1",
+ "port": 8888,
+ "node_type": "sylvie",
+ "source": "mdns",
+ "metadata": {},
+ }
+ ],
+ )
+ monkeypatch.setattr(app_module, "discover_nodes_via_gateway", lambda **kwargs: [])
+
+ resp = client.post(
+ "/api/devices/scan",
+ data=json.dumps({"mode": "mdns"}),
+ content_type="application/json",
+ )
+ assert resp.status_code == 200
+ payload = json.loads(resp.data)
+ assert payload["count"] >= 1
+ assert any(d["name"] == "F7OWER_00" for d in payload["devices"])
+
+ def test_api_select_and_raw(self, client):
+ client.post(
+ "/api/osc/target",
+ data=json.dumps({"name": "raw_test", "ip": "127.0.0.1", "port": 8888}),
+ content_type="application/json",
+ )
+ sel = client.post(
+ "/api/devices/select",
+ data=json.dumps({"name": "raw_test"}),
+ content_type="application/json",
+ )
+ assert sel.status_code == 200
+
+ raw = client.post(
+ "/api/osc/raw",
+ data=json.dumps({"address": "/state", "args": ["relax"]}),
+ content_type="application/json",
+ )
+ assert raw.status_code == 200
+ raw_data = json.loads(raw.data)
+ assert raw_data["target"] == "raw_test"
+
+ history = client.get("/api/osc/history")
+ assert history.status_code == 200
+ hist_data = json.loads(history.data)
+ assert "items" in hist_data
+
diff --git a/python_host/tests/test_osc_sender.py b/python_host/tests/test_osc_sender.py
index 44a99a3..2a66b64 100644
--- a/python_host/tests/test_osc_sender.py
+++ b/python_host/tests/test_osc_sender.py
@@ -1,5 +1,6 @@
"""Tests for the OSC sender module."""
from unittest.mock import MagicMock
+
from python_host.network.osc_sender import OSCSender
@@ -28,8 +29,9 @@ def test_override_blocks_auto(self):
mock_client = MagicMock()
sender._clients["test"] = mock_client
- sender.send("test", "/motor1", 1, 128, source="auto")
+ sent = sender.send("test", "/motor1", 1, 128, source="auto")
mock_client.send_message.assert_not_called()
+ assert sent is False
def test_override_allows_manual(self):
sender = OSCSender()
@@ -39,13 +41,15 @@ def test_override_allows_manual(self):
mock_client = MagicMock()
sender._clients["test"] = mock_client
- sender.send("test", "/motor1", 1, 128, source="manual")
+ sent = sender.send("test", "/motor1", 1, 128, source="manual")
mock_client.send_message.assert_called_once()
+ assert sent is True
def test_send_motor_formats_address(self):
sender = OSCSender()
mock_client = MagicMock()
sender._clients["test"] = mock_client
+ sender._target_info["test"] = ("127.0.0.1", 8888)
sender.send_motor("test", 1, 1, 128, source="manual")
mock_client.send_message.assert_called_once_with("/motor1", [1, 128])
@@ -53,17 +57,63 @@ def test_send_motor_formats_address(self):
def test_send_to_nonexistent_target_silent(self):
sender = OSCSender()
# Should not raise
- sender.send("nonexistent", "/motor1", 1, 128, source="manual")
+ sent = sender.send("nonexistent", "/motor1", 1, 128, source="manual")
+ assert sent is False
def test_stop_all_ignores_override(self):
sender = OSCSender()
sender.override = True
mock_client = MagicMock()
sender._clients["test"] = mock_client
+ sender._target_info["test"] = ("127.0.0.1", 8888)
sender.stop_all("test")
mock_client.send_message.assert_called_once_with("/preset", [3])
+ def test_send_raw_and_history(self):
+ sender = OSCSender()
+ mock_client = MagicMock()
+ sender._clients["test"] = mock_client
+ sender._target_info["test"] = ("127.0.0.1", 8888)
+
+ sender.send_raw("test", "/state", ["relax"], source="manual")
+ history = sender.get_history(limit=5)
+
+ assert history
+ assert history[-1]["address"] == "/state"
+ assert history[-1]["args"] == ["relax"]
+
+ def test_query_info_self_parsing(self):
+ sender = OSCSender()
+ sender._request_reply = MagicMock(
+ return_value={
+ "address": "/info/self",
+ "args": ["F7OWER_00", "AA:BB", "AP", "192.168.4.1"],
+ "ip": "192.168.4.1",
+ "port": 8888,
+ }
+ )
+
+ info = sender.query_info_self_ip("192.168.4.1", 8888)
+ assert info["name"] == "F7OWER_00"
+ assert info["mode"] == "AP"
+
+ def test_query_info_clients_parsing(self):
+ sender = OSCSender()
+ sender._request_reply = MagicMock(
+ return_value={
+ "address": "/info/clients",
+ "args": [2, "AA:BB", "192.168.4.2", "CC:DD", "192.168.4.3"],
+ "ip": "192.168.4.1",
+ "port": 8888,
+ }
+ )
+
+ info = sender.query_info_clients_ip("192.168.4.1", 8888)
+ assert info["count"] == 2
+ assert len(info["clients"]) == 2
+ assert info["clients"][0]["ip"] == "192.168.4.2"
+
def test_eye_animation_stub(self):
sender = OSCSender()
# Should not raise
diff --git a/python_host/ui/app.py b/python_host/ui/app.py
index 619d249..167beb8 100644
--- a/python_host/ui/app.py
+++ b/python_host/ui/app.py
@@ -8,12 +8,19 @@
import json
import os
+import threading
import time
from flask import Flask, render_template, Response, request, jsonify
-from python_host.vision.face_tracker import FaceTracker
+from python_host.network.node_discovery import (
+ discover_mdns_nodes,
+ discover_nodes_via_gateway,
+ infer_node_type,
+ load_registry,
+)
from python_host.network.osc_sender import OSCSender
+from python_host.vision.face_tracker import FaceTracker
# ── Globals ──────────────────────────────────────────────────
@@ -25,10 +32,54 @@
tracker = FaceTracker(camera_index=0)
osc = OSCSender()
+registry = load_registry()
DATA_DIR = os.path.join(os.path.dirname(__file__), "..", "data")
SAMPLES_FILE = os.path.join(DATA_DIR, "training_samples.jsonl")
+_devices_lock = threading.Lock()
+_devices = {}
+_selected_device = None
+
+
+def _device_label(device):
+ known = registry.get("known_devices", {}).get(device["name"], {})
+ if known.get("label"):
+ return known["label"]
+ node_meta = registry.get("node_types", {}).get(device.get("node_type", "unknown"), {})
+ return node_meta.get("label", device["name"])
+
+
+def _register_device(device):
+ node_type = infer_node_type(device.get("name", ""), txt_node_type=device.get("node_type"), registry=registry)
+ entry = {
+ "name": device["name"],
+ "ip": device["ip"],
+ "port": int(device.get("port") or registry.get("default_osc_port", 8888)),
+ "node_type": node_type,
+ "source": device.get("source", "manual"),
+ "metadata": device.get("metadata", {}),
+ }
+ entry["label"] = _device_label(entry)
+ osc.add_target(entry["name"], entry["ip"], entry["port"])
+
+ with _devices_lock:
+ _devices[entry["name"]] = entry
+ return entry
+
+
+def _list_devices():
+ with _devices_lock:
+ return list(_devices.values())
+
+
+def _selected_target(fallback=None):
+ if fallback:
+ return fallback
+ with _devices_lock:
+ return _selected_device
+
+
# ── Routes ───────────────────────────────────────────────────
@@ -85,6 +136,77 @@ def api_camera_switch():
return jsonify({"status": "ok", "camera": idx})
+# ── Device discovery & selection ─────────────────────────────
+
+
+@app.route("/api/devices/registry")
+def api_device_registry():
+ return jsonify(registry)
+
+
+@app.route("/api/devices")
+def api_devices():
+ return jsonify({"devices": _list_devices(), "selected": _selected_target()})
+
+
+@app.route("/api/devices/select", methods=["POST"])
+def api_devices_select():
+ global _selected_device
+ name = request.json.get("name")
+ with _devices_lock:
+ if name not in _devices:
+ return jsonify({"status": "error", "message": "device not found"}), 404
+ _selected_device = name
+ return jsonify({"status": "ok", "selected": _selected_device})
+
+
+@app.route("/api/devices/scan", methods=["POST"])
+def api_devices_scan():
+ global _selected_device
+
+ data = request.json or {}
+ mode = data.get("mode", "auto")
+ timeout_sec = float(data.get("timeout", 1.2))
+ gateway_ip = data.get("gateway_ip", "192.168.4.1")
+ gateway_port = int(data.get("gateway_port", 8888))
+
+ discovered = []
+ if mode in ("mdns", "auto"):
+ discovered.extend(discover_mdns_nodes(timeout_sec=timeout_sec, registry=registry))
+ if mode in ("gateway", "auto"):
+ discovered.extend(
+ discover_nodes_via_gateway(
+ osc_sender=osc,
+ gateway_ip=gateway_ip,
+ gateway_port=gateway_port,
+ timeout_sec=min(timeout_sec, 0.8),
+ registry=registry,
+ )
+ )
+
+ merged = []
+ seen = set()
+ for item in discovered:
+ key = (item.get("name"), item.get("ip"))
+ if key in seen:
+ continue
+ seen.add(key)
+ merged.append(_register_device(item))
+
+ if merged and _selected_device is None:
+ _selected_device = merged[0]["name"]
+
+ return jsonify(
+ {
+ "status": "ok",
+ "mode": mode,
+ "count": len(merged),
+ "selected": _selected_target(),
+ "devices": _list_devices(),
+ }
+ )
+
+
# ── OSC control endpoints ────────────────────────────────────
@@ -96,37 +218,68 @@ def api_osc_targets():
@app.route("/api/osc/target", methods=["POST"])
def api_osc_add_target():
data = request.json
- osc.add_target(data["name"], data["ip"], data.get("port", 8888))
- return jsonify({"status": "ok"})
+ entry = _register_device(
+ {
+ "name": data["name"],
+ "ip": data["ip"],
+ "port": data.get("port", 8888),
+ "node_type": data.get("node_type"),
+ "source": "manual",
+ "metadata": {},
+ }
+ )
+ return jsonify({"status": "ok", "device": entry})
+
+
+@app.route("/api/osc/raw", methods=["POST"])
+def api_osc_raw():
+ d = request.json or {}
+ target = _selected_target(d.get("target"))
+ address = d.get("address", "").strip()
+ if not target:
+ return jsonify({"status": "error", "message": "no selected target"}), 400
+ if not address.startswith("/"):
+ return jsonify({"status": "error", "message": "invalid OSC address"}), 400
+
+ sent = osc.send_raw(target, address, d.get("args", []), source=d.get("source", "manual"))
+ return jsonify({"status": "ok" if sent else "error", "target": target, "sent": bool(sent)})
+
+
+@app.route("/api/osc/history")
+def api_osc_history():
+ limit = int(request.args.get("limit", 80))
+ return jsonify({"items": osc.get_history(limit=limit)})
@app.route("/api/osc/motor", methods=["POST"])
def api_osc_motor():
d = request.json
- osc.send_motor(
- d["target"], d["motor"], d["dir"], d.get("speed", 255), source="manual"
- )
+ target = _selected_target(d.get("target"))
+ osc.send_motor(target, d["motor"], d["dir"], d.get("speed", 255), source="manual")
return jsonify({"status": "ok"})
@app.route("/api/osc/led", methods=["POST"])
def api_osc_led():
d = request.json
- osc.send_led(d["target"], d["led"], d["r"], d["g"], d["b"])
+ target = _selected_target(d.get("target"))
+ osc.send_led(target, d["led"], d["r"], d["g"], d["b"])
return jsonify({"status": "ok"})
@app.route("/api/osc/preset", methods=["POST"])
def api_osc_preset():
d = request.json
- osc.send_preset(d["target"], d["preset"])
+ target = _selected_target(d.get("target"))
+ osc.send_preset(target, d["preset"])
return jsonify({"status": "ok"})
@app.route("/api/osc/stop", methods=["POST"])
def api_osc_stop():
- d = request.json
- osc.stop_all(d["target"])
+ d = request.json or {}
+ target = _selected_target(d.get("target"))
+ osc.stop_all(target)
return jsonify({"status": "ok"})
@@ -154,7 +307,7 @@ def api_tag_save():
"emotion_label": d.get("emotion_label", ""),
}
os.makedirs(DATA_DIR, exist_ok=True)
- with open(SAMPLES_FILE, "a") as f:
+ with open(SAMPLES_FILE, "a", encoding="utf-8") as f:
f.write(json.dumps(sample) + "\n")
return jsonify({"status": "saved", "sample": sample})
@@ -166,7 +319,8 @@ def api_tag_save():
def api_eye_animation():
"""Reserved endpoint for TFT IPS eye animation commands."""
d = request.json
- osc.send_eye_animation(d.get("target"), d.get("animation_id", 0))
+ target = _selected_target(d.get("target"))
+ osc.send_eye_animation(target, d.get("animation_id", 0))
return jsonify({"status": "stub_ok"})
@@ -195,15 +349,17 @@ def api_perception_status():
def create_app(camera_index=0, esp32_targets=None):
"""Factory for external callers / testing."""
- global tracker
+ global tracker, _selected_device
tracker = FaceTracker(camera_index=camera_index)
if esp32_targets:
for name, (ip, port) in esp32_targets.items():
- osc.add_target(name, ip, port)
+ _register_device({"name": name, "ip": ip, "port": port, "source": "bootstrap"})
+ _selected_device = next(iter(esp32_targets.keys()))
return app
if __name__ == "__main__":
tracker.start()
- osc.add_target("sylvie_1", "192.168.4.1", 8888)
+ _register_device({"name": "sylvie_1", "ip": "192.168.4.1", "port": 8888, "source": "default"})
+ _selected_device = "sylvie_1"
app.run(host="0.0.0.0", port=5000, debug=False, threaded=True)
diff --git a/python_host/ui/device_registry.json b/python_host/ui/device_registry.json
new file mode 100644
index 0000000..0f9a4bd
--- /dev/null
+++ b/python_host/ui/device_registry.json
@@ -0,0 +1,40 @@
+{
+ "default_osc_port": 8888,
+ "known_devices": {
+ "F7OWER_00": {"node_type": "sylvie", "label": "Sylvie Gateway"},
+ "F7OWER_01": {"node_type": "sylvie", "label": "Sylvie Client"},
+ "sue_1": {"node_type": "sue", "label": "Sue Node"},
+ "face_track_1": {"node_type": "face_track", "label": "Face Tracking Node"}
+ },
+ "name_rules": [
+ {"contains": "f7ower", "node_type": "sylvie"},
+ {"contains": "sylvie", "node_type": "sylvie"},
+ {"contains": "sue", "node_type": "sue"},
+ {"contains": "face", "node_type": "face_track"},
+ {"contains": "track", "node_type": "face_track"},
+ {"contains": "kait", "node_type": "kait"}
+ ],
+ "node_types": {
+ "sylvie": {
+ "label": "Sylvie",
+ "description": "2x DC motors, 2x RGB LED, presets"
+ },
+ "sue": {
+ "label": "Sue",
+ "description": "1x Servo, 2x mono LED channels"
+ },
+ "face_track": {
+ "label": "Face Tracking",
+ "description": "8x Servo (4 pan/tilt pairs)"
+ },
+ "kait": {
+ "label": "Kait",
+ "description": "1x rotation servo"
+ },
+ "unknown": {
+ "label": "Unknown",
+ "description": "Raw OSC only"
+ }
+ }
+}
+
diff --git a/python_host/ui/templates/index.html b/python_host/ui/templates/index.html
index 4554792..aa67aac 100644
--- a/python_host/ui/templates/index.html
+++ b/python_host/ui/templates/index.html
@@ -3,376 +3,315 @@
- DATT3700 Flower Control Panel
+ DATT3700 Multi-Node Control
-
-
-
-
-
-
-
-
+
+
-

+
-
-
Primary Target
-
No face detected
-
-
- ML modules: checking…
-
+
Primary Face
+
No face detected
-
-
-
-
-
-
ESP32 Target
-
-
-
-
-
-
-
Motor Control
-
-
-
-
- Motor 1
- Dir: 0 | Speed: 0
-
-
-
-
-
-
-
-
-
- Motor 2
- Dir: 0 | Speed: 0
-
-
-
-
-
-
+
Connected Devices
+
+
+
+
+
-
+
-
Flower Pad (X: Speed/Open · Y: Jitter)
-
+
Node Controls
+
Scan and select a device to load controls.
-
-
LED Color
-
-
-
-
+
Raw OSC Console
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
+
+ await refreshDevices();
+ await refreshHistory();
+ }
+
+ init();
+