PoC #130 & #208: uinput virtual devices + WebRTC streaming

src/components/Trackpad/ScreenMirror.tsx

@@ -4,6 +4,7 @@ import type React from "react"
 import { useRef } from "react"
 import { useConnection } from "../../contexts/ConnectionProvider"
 import { useMirrorStream } from "../../hooks/useMirrorStream"
+import { useWebRTCStream } from "../../hooks/useWebRTCStream"
 
 interface ScreenMirrorProps {
 	scrollMode: boolean
@@ -23,7 +24,13 @@ export const ScreenMirror = ({
 }: ScreenMirrorProps) => {
 	const { wsRef, status } = useConnection()
 	const canvasRef = useRef<HTMLCanvasElement>(null)
-	const { hasFrame } = useMirrorStream(wsRef, canvasRef, status)
+
+	// Try WebRTC first (PoC #208), fall back to old method
+	const { hasFrame: hasWebRTCFrame } = useWebRTCStream(wsRef, canvasRef, status)
+	const { hasFrame: hasLegacyFrame } = useMirrorStream(wsRef, canvasRef, status)
+
+	// Prefer WebRTC if connected
+	const hasFrame = status === "connected" ? hasWebRTCFrame : hasLegacyFrame
 
 	return (
 		<div className="absolute inset-0 flex items-center justify-center bg-black overflow-hidden select-none touch-none">

src/hooks/useClipboardSync.ts

@@ -0,0 +1,125 @@
+/**
+ * useClipboardSync — PoC hook for bidirectional clipboard sync (Issue #97)
+ *
+ * Problem: navigator.clipboard.writeText() requires HTTPS, which TanStack
+ * Router doesn't support yet (https://github.com/TanStack/router/issues/4287).
+ *
+ * Solution: Use the WebSocket channel to relay clipboard text between the
+ * phone client and the desktop server without requiring HTTPS on the client.
+ *
+ * Flow:
+ *   COPY  → client reads its own clipboard → sends {type:"clipboard-push", text}
+ *           → server calls keyboard.pressKey(Ctrl+C) then writes text to host clipboard
+ *   PASTE → client sends {type:"clipboard-pull"}
+ *           → server reads host clipboard → responds {type:"clipboard-text", text}
+ *           → client writes text into active input via document.execCommand('insertText')
+ *             (works over plain HTTP, no HTTPS required)
+ */
+
+import { useCallback, useEffect, useRef } from "react"
+
+export interface ClipboardSyncOptions {
+	/** Live WebSocket connection to the Rein server */
+	socket: WebSocket | null
+	/** Called when a paste payload arrives from the server */
+	onPasteReceived?: (text: string) => void
+}
+
+export function useClipboardSync({
+	socket,
+	onPasteReceived,
+}: ClipboardSyncOptions) {
+	const onPasteRef = useRef(onPasteReceived)
+	useEffect(() => {
+		onPasteRef.current = onPasteReceived
+	}, [onPasteReceived])
+
+	// Listen for clipboard-text messages arriving from the server
+	useEffect(() => {
+		if (!socket) return
+
+		const handler = (event: MessageEvent) => {
+			try {
+				const msg = JSON.parse(event.data as string) as {
+					type: string
+					text?: string
+				}
+				if (msg.type === "clipboard-text" && typeof msg.text === "string") {
+					// Try modern clipboard API first (only works over HTTPS / localhost)
+					if (navigator.clipboard && window.isSecureContext) {
+						navigator.clipboard.writeText(msg.text).catch(() => {
+							insertTextFallback(msg.text)
+						})
+					} else {
+						// Fallback: insert into focused element via execCommand (HTTP-safe)
+						insertTextFallback(msg.text)
+					}
+					onPasteRef.current?.(msg.text)
+				}
+			} catch {
+				// Not a JSON message — ignore
+			}
+		}
+
+		socket.addEventListener("message", handler)
+		return () => socket.removeEventListener("message", handler)
+	}, [socket])
+
+	/**
+	 * pushCopy — reads the client clipboard and sends it to the server so the
+	 * host clipboard is updated and Ctrl+C is emitted on the focused element.
+	 */
+	const pushCopy = useCallback(async () => {
+		if (!socket || socket.readyState !== WebSocket.OPEN) return
+
+		let text = ""
+		try {
+			if (navigator.clipboard && window.isSecureContext) {
+				text = await navigator.clipboard.readText()
+			} else {
+				// Fallback: try to read selected text from DOM
+				text = window.getSelection()?.toString() ?? ""
+			}
+		} catch {
+			text = window.getSelection()?.toString() ?? ""
+		}
+
+		socket.send(JSON.stringify({ type: "clipboard-push", text }))
+	}, [socket])
+
+	/**
+	 * requestPaste — asks the server for the current host clipboard content.
+	 * The server will respond with {type:"clipboard-text", text}.
+	 */
+	const requestPaste = useCallback(() => {
+		if (!socket || socket.readyState !== WebSocket.OPEN) return
+		socket.send(JSON.stringify({ type: "clipboard-pull" }))
+	}, [socket])
+
+	return { pushCopy, requestPaste }
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * HTTP-safe text insertion — uses the deprecated but universally supported
+ * execCommand('insertText') which works without HTTPS.
+ */
+function insertTextFallback(text: string): void {
+	const active = document.activeElement as
+		| HTMLInputElement
+		| HTMLTextAreaElement
+		| null
+	if (active && "value" in active) {
+		const start = active.selectionStart ?? active.value.length
+		const end = active.selectionEnd ?? active.value.length
+		active.value = active.value.slice(0, start) + text + active.value.slice(end)
+		active.selectionStart = active.selectionEnd = start + text.length
+		active.dispatchEvent(new Event("input", { bubbles: true }))
+	} else {
+		// Last resort: execCommand (works in most browsers even on HTTP)
+		document.execCommand("insertText", false, text)
+	}
+}

src/hooks/useWebRTCCaptureProvider.ts

@@ -0,0 +1,149 @@
+"use client"
+
+import { useCallback, useEffect, useRef, useState } from "react"
+
+interface RTCMessage {
+	type: "offer" | "answer" | "ice-candidate"
+	payload: RTCSessionDescriptionInit | RTCIceCandidateInit
+}
+
+export function useWebRTCCaptureProvider(
+	wsRef: React.RefObject<WebSocket | null>,
+	onConnectedChange?: (connected: boolean) => void,
+) {
+	const [isSharing, setIsSharing] = useState(false)
+	const peerConnectionRef = useRef<RTCPeerConnection | null>(null)
+	const localStreamRef = useRef<MediaStream | null>(null)
+
+	const stopSharing = useCallback(() => {
+		if (localStreamRef.current) {
+			for (const track of localStreamRef.current.getTracks()) {
+				track.stop()
+			}
+			localStreamRef.current = null
+		}
+
+		if (peerConnectionRef.current) {
+			peerConnectionRef.current.close()
+			peerConnectionRef.current = null
+		}
+
+		if (wsRef.current && wsRef.current.readyState === WebSocket.OPEN) {
+			wsRef.current.send(JSON.stringify({ type: "stop-webrtc-provider" }))
+		}
+
+		setIsSharing(false)
+		onConnectedChange?.(false)
+	}, [wsRef, onConnectedChange])
+
+	const startSharing = useCallback(async () => {
+		try {
+			const stream = await navigator.mediaDevices.getDisplayMedia({
+				video: {
+					displaySurface: "monitor",
+					width: { ideal: 1920 },
+					height: { ideal: 1080 },
+					frameRate: { ideal: 60 },
+				},
+				audio: false,
+			})
+
+			localStreamRef.current = stream
+
+			const config: RTCConfiguration = {
+				iceServers: [],
+			}
+
+			const pc = new RTCPeerConnection(config)
+			peerConnectionRef.current = pc
+
+			for (const track of stream.getVideoTracks()) {
+				pc.addTrack(track, stream)
+			}
+
+			pc.onicecandidate = (event) => {
+				if (event.candidate && wsRef.current?.readyState === WebSocket.OPEN) {
+					wsRef.current.send(
+						JSON.stringify({
+							type: "webrtc-ice",
+							payload: event.candidate.toJSON(),
+						}),
+					)
+				}
+			}
+
+			pc.onconnectionstatechange = () => {
+				const state = pc.connectionState
+				console.log("[WebRTC] Connection state:", state)
+
+				if (state === "connected") {
+					onConnectedChange?.(true)
+				} else if (
+					state === "disconnected" ||
+					state === "failed" ||
+					state === "closed"
+				) {
+					onConnectedChange?.(false)
+				}
+			}
+
+			const offer = await pc.createOffer()
+			await pc.setLocalDescription(offer)
+
+			if (wsRef.current && wsRef.current.readyState === WebSocket.OPEN) {
+				wsRef.current.send(
+					JSON.stringify({
+						type: "webrtc-offer",
+						payload: offer,
+					}),
+				)
+			}
+
+			stream.getVideoTracks()[0].onended = () => {
+				stopSharing()
+			}
+
+			setIsSharing(true)
+		} catch (err) {
+			console.error("Failed to start WebRTC screen capture:", err)
+			setIsSharing(false)
+		}
+	}, [wsRef, stopSharing, onConnectedChange])
+
+	useEffect(() => {
+		return () => {
+			stopSharing()
+		}
+	}, [stopSharing])
+
+	useEffect(() => {
+		if (!wsRef.current) return
+
+		const handleMessage = (event: MessageEvent) => {
+			if (peerConnectionRef.current?.remoteDescription) return
+
+			try {
+				const data = JSON.parse(event.data)
+
+				if (data.type === "webrtc-answer") {
+					const answer = new RTCSessionDescription(data.payload)
+					peerConnectionRef.current?.setRemoteDescription(answer)
+				} else if (data.type === "webrtc-ice") {
+					const candidate = new RTCIceCandidate(data.payload)
+					peerConnectionRef.current?.addIceCandidate(candidate)
+				}
+			} catch (err) {
+				console.error("Error handling WebRTC message:", err)
+			}
+		}
+
+		wsRef.current.addEventListener("message", handleMessage)
+		return () => wsRef.current?.removeEventListener("message", handleMessage)
+	}, [wsRef])
+
+	return {
+		isSharing,
+		startSharing,
+		stopSharing,
+	}
+}