diff --git a/src/components/Trackpad/ScreenMirror.tsx b/src/components/Trackpad/ScreenMirror.tsx index 4121e635..804da7bd 100644 --- a/src/components/Trackpad/ScreenMirror.tsx +++ b/src/components/Trackpad/ScreenMirror.tsx @@ -4,6 +4,7 @@ import type React from "react" import { useRef } from "react" import { useConnection } from "../../contexts/ConnectionProvider" import { useMirrorStream } from "../../hooks/useMirrorStream" +import { useWebRTCStream } from "../../hooks/useWebRTCStream" interface ScreenMirrorProps { scrollMode: boolean @@ -23,7 +24,13 @@ export const ScreenMirror = ({ }: ScreenMirrorProps) => { const { wsRef, status } = useConnection() const canvasRef = useRef(null) - const { hasFrame } = useMirrorStream(wsRef, canvasRef, status) + + // Try WebRTC first (PoC #208), fall back to old method + const { hasFrame: hasWebRTCFrame } = useWebRTCStream(wsRef, canvasRef, status) + const { hasFrame: hasLegacyFrame } = useMirrorStream(wsRef, canvasRef, status) + + // Prefer WebRTC if connected + const hasFrame = status === "connected" ? hasWebRTCFrame : hasLegacyFrame return (
diff --git a/src/hooks/useClipboardSync.ts b/src/hooks/useClipboardSync.ts new file mode 100644 index 00000000..631da005 --- /dev/null +++ b/src/hooks/useClipboardSync.ts @@ -0,0 +1,125 @@ +/** + * useClipboardSync — PoC hook for bidirectional clipboard sync (Issue #97) + * + * Problem: navigator.clipboard.writeText() requires HTTPS, which TanStack + * Router doesn't support yet (https://github.com/TanStack/router/issues/4287). + * + * Solution: Use the WebSocket channel to relay clipboard text between the + * phone client and the desktop server without requiring HTTPS on the client. + * + * Flow: + * COPY → client reads its own clipboard → sends {type:"clipboard-push", text} + * → server calls keyboard.pressKey(Ctrl+C) then writes text to host clipboard + * PASTE → client sends {type:"clipboard-pull"} + * → server reads host clipboard → responds {type:"clipboard-text", text} + * → client writes text into active input via document.execCommand('insertText') + * (works over plain HTTP, no HTTPS required) + */ + +import { useCallback, useEffect, useRef } from "react" + +export interface ClipboardSyncOptions { + /** Live WebSocket connection to the Rein server */ + socket: WebSocket | null + /** Called when a paste payload arrives from the server */ + onPasteReceived?: (text: string) => void +} + +export function useClipboardSync({ + socket, + onPasteReceived, +}: ClipboardSyncOptions) { + const onPasteRef = useRef(onPasteReceived) + useEffect(() => { + onPasteRef.current = onPasteReceived + }, [onPasteReceived]) + + // Listen for clipboard-text messages arriving from the server + useEffect(() => { + if (!socket) return + + const handler = (event: MessageEvent) => { + try { + const msg = JSON.parse(event.data as string) as { + type: string + text?: string + } + if (msg.type === "clipboard-text" && typeof msg.text === "string") { + // Try modern clipboard API first (only works over HTTPS / localhost) + if (navigator.clipboard && window.isSecureContext) { + navigator.clipboard.writeText(msg.text).catch(() => { + insertTextFallback(msg.text) + }) + } else { + // Fallback: insert into focused element via execCommand (HTTP-safe) + insertTextFallback(msg.text) + } + onPasteRef.current?.(msg.text) + } + } catch { + // Not a JSON message — ignore + } + } + + socket.addEventListener("message", handler) + return () => socket.removeEventListener("message", handler) + }, [socket]) + + /** + * pushCopy — reads the client clipboard and sends it to the server so the + * host clipboard is updated and Ctrl+C is emitted on the focused element. + */ + const pushCopy = useCallback(async () => { + if (!socket || socket.readyState !== WebSocket.OPEN) return + + let text = "" + try { + if (navigator.clipboard && window.isSecureContext) { + text = await navigator.clipboard.readText() + } else { + // Fallback: try to read selected text from DOM + text = window.getSelection()?.toString() ?? "" + } + } catch { + text = window.getSelection()?.toString() ?? "" + } + + socket.send(JSON.stringify({ type: "clipboard-push", text })) + }, [socket]) + + /** + * requestPaste — asks the server for the current host clipboard content. + * The server will respond with {type:"clipboard-text", text}. + */ + const requestPaste = useCallback(() => { + if (!socket || socket.readyState !== WebSocket.OPEN) return + socket.send(JSON.stringify({ type: "clipboard-pull" })) + }, [socket]) + + return { pushCopy, requestPaste } +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** + * HTTP-safe text insertion — uses the deprecated but universally supported + * execCommand('insertText') which works without HTTPS. + */ +function insertTextFallback(text: string): void { + const active = document.activeElement as + | HTMLInputElement + | HTMLTextAreaElement + | null + if (active && "value" in active) { + const start = active.selectionStart ?? active.value.length + const end = active.selectionEnd ?? active.value.length + active.value = active.value.slice(0, start) + text + active.value.slice(end) + active.selectionStart = active.selectionEnd = start + text.length + active.dispatchEvent(new Event("input", { bubbles: true })) + } else { + // Last resort: execCommand (works in most browsers even on HTTP) + document.execCommand("insertText", false, text) + } +} diff --git a/src/hooks/useWebRTCCaptureProvider.ts b/src/hooks/useWebRTCCaptureProvider.ts new file mode 100644 index 00000000..1824d912 --- /dev/null +++ b/src/hooks/useWebRTCCaptureProvider.ts @@ -0,0 +1,149 @@ +"use client" + +import { useCallback, useEffect, useRef, useState } from "react" + +interface RTCMessage { + type: "offer" | "answer" | "ice-candidate" + payload: RTCSessionDescriptionInit | RTCIceCandidateInit +} + +export function useWebRTCCaptureProvider( + wsRef: React.RefObject, + onConnectedChange?: (connected: boolean) => void, +) { + const [isSharing, setIsSharing] = useState(false) + const peerConnectionRef = useRef(null) + const localStreamRef = useRef(null) + + const stopSharing = useCallback(() => { + if (localStreamRef.current) { + for (const track of localStreamRef.current.getTracks()) { + track.stop() + } + localStreamRef.current = null + } + + if (peerConnectionRef.current) { + peerConnectionRef.current.close() + peerConnectionRef.current = null + } + + if (wsRef.current && wsRef.current.readyState === WebSocket.OPEN) { + wsRef.current.send(JSON.stringify({ type: "stop-webrtc-provider" })) + } + + setIsSharing(false) + onConnectedChange?.(false) + }, [wsRef, onConnectedChange]) + + const startSharing = useCallback(async () => { + try { + const stream = await navigator.mediaDevices.getDisplayMedia({ + video: { + displaySurface: "monitor", + width: { ideal: 1920 }, + height: { ideal: 1080 }, + frameRate: { ideal: 60 }, + }, + audio: false, + }) + + localStreamRef.current = stream + + const config: RTCConfiguration = { + iceServers: [], + } + + const pc = new RTCPeerConnection(config) + peerConnectionRef.current = pc + + for (const track of stream.getVideoTracks()) { + pc.addTrack(track, stream) + } + + pc.onicecandidate = (event) => { + if (event.candidate && wsRef.current?.readyState === WebSocket.OPEN) { + wsRef.current.send( + JSON.stringify({ + type: "webrtc-ice", + payload: event.candidate.toJSON(), + }), + ) + } + } + + pc.onconnectionstatechange = () => { + const state = pc.connectionState + console.log("[WebRTC] Connection state:", state) + + if (state === "connected") { + onConnectedChange?.(true) + } else if ( + state === "disconnected" || + state === "failed" || + state === "closed" + ) { + onConnectedChange?.(false) + } + } + + const offer = await pc.createOffer() + await pc.setLocalDescription(offer) + + if (wsRef.current && wsRef.current.readyState === WebSocket.OPEN) { + wsRef.current.send( + JSON.stringify({ + type: "webrtc-offer", + payload: offer, + }), + ) + } + + stream.getVideoTracks()[0].onended = () => { + stopSharing() + } + + setIsSharing(true) + } catch (err) { + console.error("Failed to start WebRTC screen capture:", err) + setIsSharing(false) + } + }, [wsRef, stopSharing, onConnectedChange]) + + useEffect(() => { + return () => { + stopSharing() + } + }, [stopSharing]) + + useEffect(() => { + if (!wsRef.current) return + + const handleMessage = (event: MessageEvent) => { + if (peerConnectionRef.current?.remoteDescription) return + + try { + const data = JSON.parse(event.data) + + if (data.type === "webrtc-answer") { + const answer = new RTCSessionDescription(data.payload) + peerConnectionRef.current?.setRemoteDescription(answer) + } else if (data.type === "webrtc-ice") { + const candidate = new RTCIceCandidate(data.payload) + peerConnectionRef.current?.addIceCandidate(candidate) + } + } catch (err) { + console.error("Error handling WebRTC message:", err) + } + } + + wsRef.current.addEventListener("message", handleMessage) + return () => wsRef.current?.removeEventListener("message", handleMessage) + }, [wsRef]) + + return { + isSharing, + startSharing, + stopSharing, + } +} diff --git a/src/hooks/useWebRTCStream.ts b/src/hooks/useWebRTCStream.ts new file mode 100644 index 00000000..9639e6e7 --- /dev/null +++ b/src/hooks/useWebRTCStream.ts @@ -0,0 +1,182 @@ +"use client" + +import { useCallback, useEffect, useRef, useState } from "react" + +export function useWebRTCStream( + wsRef: React.RefObject, + canvasRef: React.RefObject, + status: "connecting" | "connected" | "disconnected", +) { + const [hasFrame, setHasFrame] = useState(false) + const peerConnectionRef = useRef(null) + const remoteStreamRef = useRef(null) + const videoRef = useRef(null) + const rAFRef = useRef(null) + const frameRef = useRef(null) + + const renderFrame = useCallback(() => { + if (!canvasRef.current || !frameRef.current) return + + const canvas = canvasRef.current + const ctx = canvas.getContext("2d", { + alpha: false, + desynchronized: true, + }) + if (!ctx) return + + if ( + canvas.width !== frameRef.current.width || + canvas.height !== frameRef.current.height + ) { + canvas.width = frameRef.current.width + canvas.height = frameRef.current.height + } + + ctx.drawImage(frameRef.current, 0, 0) + rAFRef.current = null + }, [canvasRef]) + + const processVideoFrame = useCallback( + async (stream: MediaStream) => { + const video = document.createElement("video") + video.srcObject = stream + video.muted = true + video.playsInline = true + await video.play() + + const processFrame = () => { + const canvas = document.createElement("canvas") + canvas.width = video.videoWidth + canvas.height = video.videoHeight + const ctx = canvas.getContext("2d", { alpha: false }) + if (!ctx) return + + ctx.drawImage(video, 0, 0) + + createImageBitmap(canvas).then((bitmap) => { + if (frameRef.current) { + frameRef.current.close() + } + frameRef.current = bitmap + setHasFrame(true) + + if (!rAFRef.current) { + rAFRef.current = requestAnimationFrame(renderFrame) + } + }) + + if (remoteStreamRef.current) { + rAFRef.current = requestAnimationFrame(processFrame) + } + } + + processFrame() + }, + [renderFrame], + ) + + const startStream = useCallback(() => { + const pc = new RTCPeerConnection({ iceServers: [] }) + peerConnectionRef.current = pc + + pc.ontrack = (event) => { + if (event.streams[0]) { + remoteStreamRef.current = event.streams[0] + processVideoFrame(event.streams[0]) + } + } + + pc.onicecandidate = (event) => { + if (event.candidate && wsRef.current?.readyState === WebSocket.OPEN) { + wsRef.current.send( + JSON.stringify({ + type: "webrtc-ice", + payload: event.candidate.toJSON(), + }), + ) + } + } + + wsRef.current.send(JSON.stringify({ type: "start-webrtc-consumer" })) + }, [wsRef, processVideoFrame]) + + const stopStream = useCallback(() => { + if (rAFRef.current) { + cancelAnimationFrame(rAFRef.current) + rAFRef.current = null + } + + if (frameRef.current) { + frameRef.current.close() + frameRef.current = null + } + + if (remoteStreamRef.current) { + for (const t of remoteStreamRef.current.getTracks()) { + t.stop() + } + remoteStreamRef.current = null + } + + if (peerConnectionRef.current) { + peerConnectionRef.current.close() + peerConnectionRef.current = null + } + + setHasFrame(false) + }, []) + + useEffect(() => { + if ( + status === "connected" && + wsRef.current?.readyState === WebSocket.OPEN + ) { + startStream() + } else if (status === "disconnected") { + stopStream() + } + + return () => { + stopStream() + } + }, [status, startStream, stopStream, wsRef.current?.readyState]) + + useEffect(() => { + if (!wsRef.current || status !== "connected") return + + const handleMessage = async (event: MessageEvent) => { + try { + const data = JSON.parse(event.data) + + if (data.type === "webrtc-offer") { + if (!peerConnectionRef.current) return + + const offer = new RTCSessionDescription(data.payload) + await peerConnectionRef.current.setRemoteDescription(offer) + + const answer = await peerConnectionRef.current.createAnswer() + await peerConnectionRef.current.setLocalDescription(answer) + + wsRef.current?.send( + JSON.stringify({ + type: "webrtc-answer", + payload: answer, + }), + ) + } else if (data.type === "webrtc-ice") { + if (!peerConnectionRef.current) return + + const candidate = new RTCIceCandidate(data.payload) + await peerConnectionRef.current.addIceCandidate(candidate) + } + } catch (err) { + console.error("Error handling WebRTC message:", err) + } + } + + wsRef.current.addEventListener("message", handleMessage) + return () => wsRef.current?.removeEventListener("message", handleMessage) + }, [wsRef, status]) + + return { hasFrame } +} diff --git a/src/routes/__root.tsx b/src/routes/__root.tsx index c9542781..30665411 100644 --- a/src/routes/__root.tsx +++ b/src/routes/__root.tsx @@ -13,6 +13,7 @@ import { useConnection, } from "../contexts/ConnectionProvider" import { useCaptureProvider } from "../hooks/useCaptureProvider" +import { useWebRTCCaptureProvider } from "../hooks/useWebRTCCaptureProvider" export const Route = createRootRoute({ component: AppWithConnection, @@ -46,7 +47,9 @@ function RootComponent() { function DesktopCaptureProvider() { const { wsRef, status } = useConnection() - const { startSharing } = useCaptureProvider(wsRef) + const { startSharing: startCaptureProvider } = useCaptureProvider(wsRef) + const { startSharing: startWebRTC, isSharing: isWebRTCSharing } = + useWebRTCCaptureProvider(wsRef) const hasStartedRef = useRef(false) useEffect(() => { @@ -58,9 +61,12 @@ function DesktopCaptureProvider() { if (!isMobile && canShare) { hasStartedRef.current = true - startSharing() + // Try WebRTC first (PoC #208), fall back to capture provider + startWebRTC().catch(() => { + startCaptureProvider() + }) } - }, [status, startSharing]) + }, [status, startWebRTC, startCaptureProvider]) return null } diff --git a/src/server/InputHandler.ts b/src/server/InputHandler.ts index 63f9b26e..6cfbe3ef 100644 --- a/src/server/InputHandler.ts +++ b/src/server/InputHandler.ts @@ -1,6 +1,14 @@ import { Button, Key, Point, keyboard, mouse } from "@nut-tree-fork/nut-js" import { KEY_MAP } from "./KeyMap" -import { moveRelative } from "./ydotool" +import { moveRelative as moveRelativeYdotool } from "./ydotool" +import { + moveRelative as moveRelativeUinput, + pressButton as pressButtonUinput, + scroll as scrollUinput, + pressKey as pressKeyUinput, + releaseKey as releaseKeyUinput, + typeText as typeTextUinput, +} from "./uinput" import os from "node:os" export interface InputMessage { @@ -124,19 +132,20 @@ export class InputHandler { Number.isFinite(msg.dy) ) { try { - // Attempt ydotool relative movement first - const success = await moveRelative(msg.dx, msg.dy) + const success = await moveRelativeUinput(msg.dx, msg.dy) - // Fallback to absolute positioning if ydotool is unavailable or fails if (!success) { - const currentPos = await mouse.getPosition() - - await mouse.setPosition( - new Point( - Math.round(currentPos.x + msg.dx), - Math.round(currentPos.y + msg.dy), - ), - ) + const fallbackSuccess = await moveRelativeYdotool(msg.dx, msg.dy) + if (!fallbackSuccess) { + const currentPos = await mouse.getPosition() + + await mouse.setPosition( + new Point( + Math.round(currentPos.x + msg.dx), + Math.round(currentPos.y + msg.dy), + ), + ) + } } } catch (err) { console.error("Move event failed:", err) @@ -147,22 +156,32 @@ export class InputHandler { case "click": { const VALID_BUTTONS = ["left", "right", "middle"] if (msg.button && VALID_BUTTONS.includes(msg.button)) { - const btn = - msg.button === "left" - ? Button.LEFT - : msg.button === "right" - ? Button.RIGHT - : Button.MIDDLE - try { - if (msg.press) { - await mouse.pressButton(btn) - } else { - await mouse.releaseButton(btn) + const uinputSuccess = await pressButtonUinput(msg.button, msg.press) + + if (!uinputSuccess) { + const btn = + msg.button === "left" + ? Button.LEFT + : msg.button === "right" + ? Button.RIGHT + : Button.MIDDLE + + if (msg.press) { + await mouse.pressButton(btn) + } else { + await mouse.releaseButton(btn) + } } } catch (err) { console.error("Click event failed:", err) - // ensure release just in case + const btn = + msg.button === "left" + ? Button.LEFT + : msg.button === "right" + ? Button.RIGHT + : Button.MIDDLE + await mouse.releaseButton(btn).catch(() => {}) } } @@ -198,35 +217,42 @@ export class InputHandler { case "scroll": { const MAX_SCROLL = 100 - const promises: Promise[] = [] - - // Vertical scroll - if (this.isFiniteNumber(msg.dy) && Math.round(msg.dy) !== 0) { - const amount = this.clamp(Math.round(msg.dy), -MAX_SCROLL, MAX_SCROLL) - if (amount > 0) { - promises.push(mouse.scrollDown(amount)) - } else if (amount < 0) { - promises.push(mouse.scrollUp(-amount)) - } - } - // Horizontal scroll - if (this.isFiniteNumber(msg.dx) && Math.round(msg.dx) !== 0) { - const amount = this.clamp(Math.round(msg.dx), -MAX_SCROLL, MAX_SCROLL) - if (amount > 0) { - promises.push(mouse.scrollRight(amount)) - } else if (amount < 0) { - promises.push(mouse.scrollLeft(-amount)) - } - } + const clampedDx = this.isFiniteNumber(msg.dx) + ? this.clamp(Math.round(msg.dx), -MAX_SCROLL, MAX_SCROLL) + : 0 + const clampedDy = this.isFiniteNumber(msg.dy) + ? this.clamp(Math.round(msg.dy), -MAX_SCROLL, MAX_SCROLL) + : 0 - if (promises.length) { - const results = await Promise.allSettled(promises) - for (const result of results) { - if (result.status === "rejected") { - console.error("Scroll event failed:", result.reason) + try { + const uinputSuccess = await scrollUinput(clampedDx, clampedDy) + + if (!uinputSuccess) { + const promises: Promise[] = [] + + if (clampedDy !== 0) { + if (clampedDy > 0) { + promises.push(mouse.scrollDown(clampedDy)) + } else { + promises.push(mouse.scrollUp(-clampedDy)) + } + } + + if (clampedDx !== 0) { + if (clampedDx > 0) { + promises.push(mouse.scrollRight(clampedDx)) + } else { + promises.push(mouse.scrollLeft(-clampedDx)) + } + } + + if (promises.length) { + await Promise.allSettled(promises) } } + } catch (err) { + console.error("Scroll event failed:", err) } break } @@ -260,24 +286,33 @@ export class InputHandler { case "key": if (msg.key && typeof msg.key === "string" && msg.key.length <= 50) { console.log(`Processing key: ${msg.key}`) - const nutKey = KEY_MAP[msg.key.toLowerCase()] try { - if (nutKey !== undefined) { - await keyboard.pressKey(nutKey) - await keyboard.releaseKey(nutKey) - } else if (msg.key === " " || msg.key?.toLowerCase() === "space") { - const spaceKey = KEY_MAP.space - await keyboard.pressKey(spaceKey) - await keyboard.releaseKey(spaceKey) - } else if (msg.key.length === 1) { - await keyboard.type(msg.key) + const uinputSuccess = await pressKeyUinput(msg.key) + if (uinputSuccess) { + await releaseKeyUinput(msg.key) } else { - console.log(`Unmapped key: ${msg.key}`) + const nutKey = KEY_MAP[msg.key.toLowerCase()] + + if (nutKey !== undefined) { + await keyboard.pressKey(nutKey) + await keyboard.releaseKey(nutKey) + } else if ( + msg.key === " " || + msg.key?.toLowerCase() === "space" + ) { + const spaceKey = KEY_MAP.space + await keyboard.pressKey(spaceKey) + await keyboard.releaseKey(spaceKey) + } else if (msg.key.length === 1) { + await keyboard.type(msg.key) + } else { + console.log(`Unmapped key: ${msg.key}`) + } } } catch (err) { console.warn("Key press failed:", err) - // ensure release just in case + const nutKey = KEY_MAP[msg.key.toLowerCase()] if (nutKey !== undefined) await keyboard.releaseKey(nutKey).catch(() => {}) if (msg.key === " " || msg.key?.toLowerCase() === "space") @@ -343,7 +378,11 @@ export class InputHandler { case "text": if (msg.text && typeof msg.text === "string") { try { - await keyboard.type(msg.text) + const uinputSuccess = await typeTextUinput(msg.text) + + if (!uinputSuccess) { + await keyboard.type(msg.text) + } } catch (err) { console.error("Failed to type text:", err) } diff --git a/src/server/clipboard-server.ts b/src/server/clipboard-server.ts new file mode 100644 index 00000000..53229d67 --- /dev/null +++ b/src/server/clipboard-server.ts @@ -0,0 +1,111 @@ +/** + * clipboard-server.ts — Server-side clipboard relay for Issue #97 + * + * Handles two new WebSocket message types: + * + * clipboard-push { type: "clipboard-push", text: string } + * ← Client sends its clipboard text. Server writes it to the host + * clipboard via clipboardy, then triggers Ctrl+C on the focused element + * so the text is "copied" at the OS level too. + * + * clipboard-pull { type: "clipboard-pull" } + * ← Client wants the host clipboard. Server reads it via clipboardy and + * responds with { type: "clipboard-text", text: string }. + * + * Why clipboardy? + * - Works over plain HTTP (no HTTPS requirement). + * - Cross-platform: macOS pbpaste/pbcopy, Linux xclip/xsel/wl-paste, + * Windows PowerShell Get-Clipboard / Set-Clipboard. + * - No native module compilation required. + * + * Integration point: import handleClipboardMessage and call it from the + * existing ws.on("message") handler in websocket.ts, after the input + * message block. + */ + +import { Key, keyboard } from "@nut-tree-fork/nut-js" +import os from "node:os" +import type { WebSocket } from "ws" + +// --------------------------------------------------------------------------- +// Clipboard backend — thin wrapper so it's easy to swap in a native FFI +// driver later (ties naturally into Issue #130 / PR #290). +// --------------------------------------------------------------------------- + +/** Reads the host clipboard. Returns "" on failure. */ +async function readHostClipboard(): Promise { + try { + // Dynamic import keeps this optional — the dependency is only required + // at runtime, so the rest of the server still boots if it's absent. + const { default: clipboardy } = await import("clipboardy") + return await clipboardy.read() + } catch { + return "" + } +} + +/** Writes text to the host clipboard. No-op on failure. */ +async function writeHostClipboard(text: string): Promise { + try { + const { default: clipboardy } = await import("clipboardy") + await clipboardy.write(text) + } catch { + // Silently degrade — the paste text was already sent to the client + } +} + +// --------------------------------------------------------------------------- +// Message handler +// --------------------------------------------------------------------------- + +const modifier = os.platform() === "darwin" ? Key.LeftSuper : Key.LeftControl + +export async function handleClipboardMessage( + msg: { type: string; text?: string }, + ws: WebSocket, +): Promise { + // Returns true if the message was handled (caller should skip further handling) + + if (msg.type === "clipboard-push") { + const text = typeof msg.text === "string" ? msg.text.slice(0, 4096) : "" + + // 1. Write to host clipboard so Ctrl+V works normally afterwards + await writeHostClipboard(text) + + // 2. Emit Ctrl+C to copy the selected text on the host side as well + // (matches the issue description: "send Ctrl+C and client's clipboard content") + try { + await keyboard.pressKey(modifier, Key.C) + } finally { + await Promise.allSettled([ + keyboard.releaseKey(Key.C), + keyboard.releaseKey(modifier), + ]) + } + + return true + } + + if (msg.type === "clipboard-pull") { + const text = await readHostClipboard() + ws.send(JSON.stringify({ type: "clipboard-text", text })) + return true + } + + return false +} + +// --------------------------------------------------------------------------- +// Patch for websocket.ts — insert into VALID_INPUT_TYPES and message handler +// --------------------------------------------------------------------------- +// +// In websocket.ts, add to VALID_INPUT_TYPES: +// "clipboard-push", "clipboard-pull" +// +// And before `await inputHandler.handleMessage(...)`, add: +// +// import { handleClipboardMessage } from "./clipboard-server" +// ... +// if (await handleClipboardMessage(msg, ws)) return +// +// --------------------------------------------------------------------------- diff --git a/src/server/uinput.ts b/src/server/uinput.ts new file mode 100644 index 00000000..0019ba98 --- /dev/null +++ b/src/server/uinput.ts @@ -0,0 +1,607 @@ +import * as fs from "node:fs/promises" +import * as fsSync from "node:fs" + +const UINPUT_PATH = "/dev/uinput" + +interface VirtualInputConfig { + name: string + vendorId?: number + productId?: number + version?: number +} + +const EV_KEY = 0x01 +const EV_REL = 0x02 +const EV_ABS = 0x03 +const EV_SYN = 0x00 + +const REL_X = 0x00 +const REL_Y = 0x01 +const REL_WHEEL = 0x08 +const REL_HWHEEL = 0x06 + +const BTN_LEFT = 0x110 +const BTN_RIGHT = 0x111 +const BTN_MIDDLE = 0x112 + +const KEY_UNKNOWN = 0x00 +const KEY_SPACE = 0x39 +const KEY_ENTER = 0x1c +const KEY_TAB = 0x0f +const KEY_BACKSPACE = 0x0e +const KEY_ESC = 0x01 +const KEY_DELETE = 0x4f +const KEY_INSERT = 0x4b +const KEY_HOME = 0x4e +const KEY_END = 0x4d +const KEY_PAGEUP = 0x49 +const KEY_PAGEDOWN = 0x4a +const KEY_UP = 0x48 +const KEY_DOWN = 0x50 +const KEY_LEFT = 0x4b +const KEY_RIGHT = 0x4d +const KEY_F1 = 0x3b +const KEY_F2 = 0x3c +const KEY_F3 = 0x3d +const KEY_F4 = 0x3e +const KEY_F5 = 0x3f +const KEY_F6 = 0x40 +const KEY_F7 = 0x41 +const KEY_F8 = 0x42 +const KEY_F9 = 0x43 +const KEY_F10 = 0x44 +const KEY_F11 = 0x57 +const KEY_F12 = 0x58 +const KEY_LEFTSHIFT = 0x2a +const KEY_RIGHTSHIFT = 0x36 +const KEY_LEFTCTRL = 0x1d +const KEY_RIGHTCTRL = 0x1d +const KEY_LEFTALT = 0x38 +const KEY_RIGHTALT = 0x38 +const KEY_LEFTMETA = 0x5b +const KEY_RIGHTMETA = 0x5c + +const KEY_A = 0x1e +const KEY_B = 0x32 +const KEY_C = 0x33 +const KEY_D = 0x20 +const KEY_E = 0x12 +const KEY_F = 0x21 +const KEY_G = 0x22 +const KEY_H = 0x23 +const KEY_I = 0x17 +const KEY_J = 0x24 +const KEY_K = 0x25 +const KEY_L = 0x26 +const KEY_M = 0x34 +const KEY_N = 0x31 +const KEY_O = 0x18 +const KEY_P = 0x19 +const KEY_Q = 0x10 +const KEY_R = 0x13 +const KEY_S = 0x1f +const KEY_T = 0x14 +const KEY_U = 0x16 +const KEY_V = 0x2f +const KEY_W = 0x11 +const KEY_X = 0x2d +const KEY_Y = 0x15 +const KEY_Z = 0x2c + +const KEY_0 = 0x0b +const KEY_1 = 0x02 +const KEY_2 = 0x03 +const KEY_3 = 0x04 +const KEY_4 = 0x05 +const KEY_5 = 0x06 +const KEY_6 = 0x07 +const KEY_7 = 0x08 +const KEY_8 = 0x09 +const KEY_9 = 0x0a + +const KEY_MINUS = 0x0c +const KEY_EQUAL = 0x0d +const KEY_LEFTBRACE = 0x1a +const KEY_RIGHTBRACE = 0x1b +const KEY_BACKSLASH = 0x2b +const KEY_SEMICOLON = 0x27 +const KEY_QUOTE = 0x28 +const KEY_COMMA = 0x32 +const KEY_DOT = 0x34 +const KEY_SLASH = 0x35 +const KEY_GRAVE = 0x29 + +const KEY_CAPSLOCK = 0x3a +const KEY_NUMLOCK = 0x45 +const KEY_SCROLLLOCK = 0x46 + +const KEY_KP0 = 0x52 +const KEY_KP1 = 0x4f +const KEY_KP2 = 0x50 +const KEY_KP3 = 0x51 +const KEY_KP4 = 0x4b +const KEY_KP5 = 0x4c +const KEY_KP6 = 0x4d +const KEY_KP7 = 0x47 +const KEY_KP8 = 0x48 +const KEY_KP9 = 0x49 +const KEY_KPMINUS = 0x4a +const KEY_KPPLUS = 0x4e +const KEY_KPDOT = 0x53 +const KEY_KPENTER = 0x5c + +const ui_user_dev = (name: string) => { + const nameBytes = Buffer.from(name.padEnd(80, "\0").slice(0, 80)) + return Buffer.alloc(4 + 4 + 4 + 4 + 80) +} + +const createUinputDevice = async ( + fd: number, + config: VirtualInputConfig, +): Promise => { + const buffer = Buffer.alloc(4 + 4 + 4 + 4 + 80) + buffer.writeUInt16LE(config.vendorId ?? 0x1234, 0) + buffer.writeUInt16LE(config.productId ?? 0x5678, 2) + buffer.writeUInt16LE(config.version ?? 1, 4) + const nameBuffer = Buffer.from(`${config.name.slice(0, 79)}\0`) + nameBuffer.copy(buffer, 8) + fsSync.writeSync(fd, buffer) +} + +const setupMouseEvents = async (fd: number): Promise => { + const events = [ + [EV_KEY, BTN_LEFT], + [EV_KEY, BTN_RIGHT], + [EV_KEY, BTN_MIDDLE], + [EV_REL, REL_X], + [EV_REL, REL_Y], + [EV_REL, REL_WHEEL], + [EV_REL, REL_HWHEEL], + [EV_SYN, 0], + ] + for (const [type, code] of events) { + const buffer = Buffer.alloc(8) + buffer.writeUInt16LE(type, 0) + buffer.writeUInt16LE(code, 2) + fsSync.writeSync(fd, buffer) + } +} + +const setupKeyboardEvents = async (fd: number): Promise => { + const keys = [ + KEY_UNKNOWN, + KEY_ESC, + KEY_1, + KEY_2, + KEY_3, + KEY_4, + KEY_5, + KEY_6, + KEY_7, + KEY_8, + KEY_9, + KEY_0, + KEY_MINUS, + KEY_EQUAL, + KEY_BACKSPACE, + KEY_TAB, + KEY_Q, + KEY_W, + KEY_E, + KEY_R, + KEY_T, + KEY_Y, + KEY_U, + KEY_I, + KEY_O, + KEY_P, + KEY_LEFTBRACE, + KEY_RIGHTBRACE, + KEY_ENTER, + KEY_LEFTCTRL, + KEY_A, + KEY_S, + KEY_D, + KEY_F, + KEY_G, + KEY_H, + KEY_J, + KEY_K, + KEY_L, + KEY_SEMICOLON, + KEY_QUOTE, + KEY_GRAVE, + KEY_LEFTSHIFT, + KEY_BACKSLASH, + KEY_Z, + KEY_X, + KEY_C, + KEY_V, + KEY_B, + KEY_N, + KEY_M, + KEY_COMMA, + KEY_DOT, + KEY_SLASH, + KEY_RIGHTSHIFT, + KEY_KPMULT, + KEY_LEFTALT, + KEY_SPACE, + KEY_CAPSLOCK, + KEY_F1, + KEY_F2, + KEY_F3, + KEY_F4, + KEY_F5, + KEY_F6, + KEY_F7, + KEY_F8, + KEY_F9, + KEY_F10, + KEY_F11, + KEY_F12, + KEY_NUMLOCK, + KEY_SCROLLLOCK, + KEY_KP7, + KEY_KP8, + KEY_KP9, + KEY_KPMINUS, + KEY_KP4, + KEY_KP5, + KEY_KP6, + KEY_KPPLUS, + KEY_KP1, + KEY_KP2, + KEY_KP3, + KEY_KP0, + KEY_KPDOT, + KEY_KPENTER, + KEY_RIGHTCTRL, + KEY_KPDIV, + KEY_KP0, + KEY_KP0, + KEY_RIGHTALT, + KEY_HOME, + KEY_UP, + KEY_PAGEUP, + KEY_LEFT, + KEY_RIGHT, + KEY_END, + KEY_DOWN, + KEY_PAGEDOWN, + KEY_INSERT, + KEY_DELETE, + KEY_LEFTMETA, + KEY_RIGHTMETA, + ] + for (const code of keys) { + const buffer = Buffer.alloc(8) + buffer.writeUInt16LE(EV_KEY, 0) + buffer.writeUInt16LE(code, 2) + fsSync.writeSync(fd, buffer) + } + const syncBuffer = Buffer.alloc(8) + syncBuffer.writeUInt16LE(EV_SYN, 0) + fsSync.writeSync(fd, syncBuffer) +} + +const KEY_MAP: Record = { + escape: KEY_ESC, + "`": KEY_GRAVE, + "1": KEY_1, + "2": KEY_2, + "3": KEY_3, + "4": KEY_4, + "5": KEY_5, + "6": KEY_6, + "7": KEY_7, + "8": KEY_8, + "9": KEY_9, + "0": KEY_0, + "-": KEY_MINUS, + "=": KEY_EQUAL, + backspace: KEY_BACKSPACE, + tab: KEY_TAB, + q: KEY_Q, + w: KEY_W, + e: KEY_E, + r: KEY_R, + t: KEY_T, + y: KEY_Y, + u: KEY_U, + i: KEY_I, + o: KEY_O, + p: KEY_P, + "[": KEY_LEFTBRACE, + "]": KEY_RIGHTBRACE, + enter: KEY_ENTER, + control: KEY_LEFTCTRL, + a: KEY_A, + s: KEY_S, + d: KEY_D, + f: KEY_F, + g: KEY_G, + h: KEY_H, + j: KEY_J, + k: KEY_K, + l: KEY_L, + ";": KEY_SEMICOLON, + "'": KEY_QUOTE, + "\\": KEY_BACKSLASH, + shift: KEY_LEFTSHIFT, + z: KEY_Z, + x: KEY_X, + c: KEY_C, + v: KEY_V, + b: KEY_B, + n: KEY_N, + m: KEY_M, + ",": KEY_COMMA, + ".": KEY_DOT, + "/": KEY_SLASH, + alt: KEY_LEFTALT, + " ": KEY_SPACE, + capslock: KEY_CAPSLOCK, + f1: KEY_F1, + f2: KEY_F2, + f3: KEY_F3, + f4: KEY_F4, + f5: KEY_F5, + f6: KEY_F6, + f7: KEY_F7, + f8: KEY_F8, + f9: KEY_F9, + f10: KEY_F10, + f11: KEY_F11, + f12: KEY_F12, + numlock: KEY_NUMLOCK, + scrolllock: KEY_SCROLLLOCK, + home: KEY_HOME, + end: KEY_END, + pageup: KEY_PAGEUP, + pagedown: KEY_PAGEDOWN, + insert: KEY_INSERT, + delete: KEY_DELETE, + arrowup: KEY_UP, + arrowdown: KEY_DOWN, + arrowleft: KEY_LEFT, + arrowright: KEY_RIGHT, + meta: KEY_LEFTMETA, +} + +let isUinputAvailable: boolean | null = null +let deviceFd: number | null = null +let lastFailureTime = 0 +const COOLDOWN_MS = 5000 + +const UINPUT_DEVICE_NAME = "rein-virtual-input" + +export async function checkUinput(): Promise { + if (isUinputAvailable !== null) { + return isUinputAvailable + } + + const now = Date.now() + if (now - lastFailureTime < COOLDOWN_MS) { + return false + } + + try { + await fs.access(UINPUT_PATH) + isUinputAvailable = true + console.log("[uinput] /dev/uinput is available") + return true + } catch { + isUinputAvailable = false + lastFailureTime = now + console.warn("[uinput] /dev/uinput is not available") + return false + } +} + +async function initUinputDevice(): Promise { + if (deviceFd !== null) { + return true + } + + if (!(await checkUinput())) { + return false + } + + try { + const fd = fsSync.open(UINPUT_PATH, fsSync.O_WRONLY).fd + if (fd === undefined) { + return false + } + deviceFd = fd + + await createUinputDevice(deviceFd, { + name: UINPUT_DEVICE_NAME, + }) + + await setupMouseEvents(deviceFd) + await setupKeyboardEvents(deviceFd) + + const ioctlEnable = Buffer.alloc(4) + ioctlEnable.writeUInt16LE(0x40, 0) + ioctlEnable.writeUInt8(0x03, 2) + ioctlEnable.writeUInt8(0x00, 3) + + try { + fsSync.writeSync(deviceFd, Buffer.from([0x40, 0x03, 0x00, 0x00])) + } catch { + console.log("[uinput] ioctl skipped (may not be needed)") + } + + console.log("[uinput] Virtual input device created") + return true + } catch (err) { + console.error("[uinput] Failed to initialize device:", err) + if (deviceFd !== null) { + try { + fsSync.close(deviceFd) + } catch { + // Ignore close errors + } + deviceFd = null + } + lastFailureTime = Date.now() + return false + } +} + +const sendEvent = (type: number, code: number, value: number): void => { + if (deviceFd === null) return + const buffer = Buffer.alloc(16) + buffer.writeUInt16LE(type, 0) + buffer.writeUInt16LE(code, 2) + buffer.writeInt32LE(value, 4) + fsSync.writeSync(deviceFd, buffer) +} + +const sendSync = (): void => { + sendEvent(EV_SYN, 0, 0) +} + +export async function moveRelative(dx: number, dy: number): Promise { + if (!(await initUinputDevice())) { + return false + } + + try { + if (dx !== 0) { + sendEvent(EV_REL, REL_X, Math.round(dx)) + } + if (dy !== 0) { + sendEvent(EV_REL, REL_Y, Math.round(dy)) + } + sendSync() + return true + } catch (err) { + console.error("[uinput] Error in moveRelative:", err) + isUinputAvailable = null + lastFailureTime = Date.now() + return false + } +} + +export async function pressButton( + button: "left" | "right" | "middle", + press: boolean, +): Promise { + if (!(await initUinputDevice())) { + return false + } + + const btnCode = + button === "left" ? BTN_LEFT : button === "right" ? BTN_RIGHT : BTN_MIDDLE + + try { + sendEvent(EV_KEY, btnCode, press ? 1 : 0) + sendSync() + return true + } catch (err) { + console.error("[uinput] Error in pressButton:", err) + return false + } +} + +export async function scroll(dx: number, dy: number): Promise { + if (!(await initUinputDevice())) { + return false + } + + try { + if (dy !== 0) { + sendEvent(EV_REL, REL_WHEEL, Math.round(dy)) + } + if (dx !== 0) { + sendEvent(EV_REL, REL_HWHEEL, Math.round(dx)) + } + sendSync() + return true + } catch (err) { + console.error("[uinput] Error in scroll:", err) + return false + } +} + +export async function pressKey(key: string): Promise { + if (!(await initUinputDevice())) { + return false + } + + const keyCode = KEY_MAP[key.toLowerCase()] + + if (keyCode === undefined) { + console.warn(`[uinput] Unknown key: ${key}`) + return false + } + + try { + sendEvent(EV_KEY, keyCode, 1) + sendSync() + return true + } catch (err) { + console.error("[uinput] Error in pressKey:", err) + return false + } +} + +export async function releaseKey(key: string): Promise { + if (!(await initUinputDevice())) { + return false + } + + const keyCode = KEY_MAP[key.toLowerCase()] + + if (keyCode === undefined) { + return false + } + + try { + sendEvent(EV_KEY, keyCode, 0) + sendSync() + return true + } catch (err) { + console.error("[uinput] Error in releaseKey:", err) + return false + } +} + +export async function typeText(text: string): Promise { + if (!(await initUinputDevice())) { + return false + } + + try { + for (const char of text) { + const code = char.charCodeAt(0) + + if (code >= 32 && code <= 126) { + sendEvent(EV_KEY, code, 1) + sendSync() + await new Promise((r) => setTimeout(r, 5)) + sendEvent(EV_KEY, code, 0) + sendSync() + await new Promise((r) => setTimeout(r, 5)) + } + } + return true + } catch (err) { + console.error("[uinput] Error in typeText:", err) + return false + } +} + +export async function cleanupUinput(): Promise { + if (deviceFd !== null) { + try { + fsSync.close(deviceFd) + deviceFd = null + console.log("[uinput] Device cleaned up") + } catch (err) { + console.error("[uinput] Error during cleanup:", err) + } + } +} diff --git a/src/server/websocket.ts b/src/server/websocket.ts index 0388a12a..9dedf09f 100644 --- a/src/server/websocket.ts +++ b/src/server/websocket.ts @@ -23,6 +23,7 @@ function isLocalhost(request: IncomingMessage): boolean { interface ExtWebSocket extends WebSocket { isConsumer?: boolean isProvider?: boolean + isWebRTCConsumer?: boolean } export async function createWsServer( @@ -230,6 +231,75 @@ export async function createWsServer( return } + if (msg.type === "start-webrtc-consumer") { + ;(ws as ExtWebSocket).isWebRTCConsumer = true + logger.info("Client registered as WebRTC Consumer") + return + } + + if (msg.type === "stop-webrtc-provider") { + ;(ws as ExtWebSocket).isProvider = false + logger.info("Client unregistered as WebRTC Provider") + return + } + + if (msg.type === "webrtc-offer") { + logger.info("Received WebRTC offer, relaying to consumer") + for (const client of wss.clients) { + if ( + (client as ExtWebSocket).isWebRTCConsumer && + client.readyState === WebSocket.OPEN + ) { + client.send( + JSON.stringify({ + type: "webrtc-offer", + payload: msg.payload, + }), + ) + } + } + return + } + + if (msg.type === "webrtc-answer") { + logger.info("Received WebRTC answer, relaying to provider") + for (const client of wss.clients) { + if ( + (client as ExtWebSocket).isProvider && + client.readyState === WebSocket.OPEN + ) { + client.send( + JSON.stringify({ + type: "webrtc-answer", + payload: msg.payload, + }), + ) + } + } + return + } + + if (msg.type === "webrtc-ice") { + const isFromProvider = (ws as ExtWebSocket).isProvider + const targetRole = isFromProvider ? "consumer" : "provider" + + for (const client of wss.clients) { + const isTarget = isFromProvider + ? (client as ExtWebSocket).isWebRTCConsumer + : (client as ExtWebSocket).isProvider + + if (isTarget && client.readyState === WebSocket.OPEN) { + client.send( + JSON.stringify({ + type: "webrtc-ice", + payload: msg.payload, + }), + ) + } + } + return + } + if (msg.type === "update-config") { try { if (