From f52b19a52315adde9f80aa81ba3b55f83c85d14c Mon Sep 17 00:00:00 2001 From: Josh de Leeuw Date: Fri, 10 Jun 2022 15:48:30 -0400 Subject: [PATCH 1/3] add web worker config --- .vscode/settings.json | 3 + examples/test-web-worker.html | 15 ++++ package-lock.json | 17 +++++ package.json | 1 + rollup.config.js | 3 +- src/Eyetracker.ts | 132 +++++++++++++++++++--------------- src/Worker.js | 17 +++++ tests/index.test.ts | 5 +- workers.d.ts | 4 ++ 9 files changed, 135 insertions(+), 62 deletions(-) create mode 100644 .vscode/settings.json create mode 100644 examples/test-web-worker.html create mode 100644 src/Worker.js create mode 100644 workers.d.ts diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..6b665aa --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "liveServer.settings.port": 5501 +} diff --git a/examples/test-web-worker.html b/examples/test-web-worker.html new file mode 100644 index 0000000..dff2559 --- /dev/null +++ b/examples/test-web-worker.html @@ -0,0 +1,15 @@ + + + + + + + + + diff --git a/package-lock.json b/package-lock.json index 5ba4821..b20862e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30,6 +30,7 @@ "pretty-quick": "^3.1.3", "process": "^0.11.10", "rollup": "^2.74.1", + "rollup-plugin-web-worker-loader": "^1.6.1", "ts-jest": "^28.0.3", "tslib": "^2.4.0", "typescript": "^4.6.4" @@ -4247,6 +4248,15 @@ "fsevents": "~2.3.2" } }, + "node_modules/rollup-plugin-web-worker-loader": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/rollup-plugin-web-worker-loader/-/rollup-plugin-web-worker-loader-1.6.1.tgz", + "integrity": "sha512-4QywQSz1NXFHKdyiou16mH3ijpcfLtLGOrAqvAqu1Gx+P8+zj+3gwC2BSL/VW1d+LW4nIHC8F7d7OXhs9UdR2A==", + "dev": true, + "peerDependencies": { + "rollup": "^1.9.2 || ^2.0.0" + } + }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -8110,6 +8120,13 @@ "fsevents": "~2.3.2" } }, + "rollup-plugin-web-worker-loader": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/rollup-plugin-web-worker-loader/-/rollup-plugin-web-worker-loader-1.6.1.tgz", + "integrity": "sha512-4QywQSz1NXFHKdyiou16mH3ijpcfLtLGOrAqvAqu1Gx+P8+zj+3gwC2BSL/VW1d+LW4nIHC8F7d7OXhs9UdR2A==", + "dev": true, + "requires": {} + }, "safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", diff --git a/package.json b/package.json index 76f7b92..d3e4927 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "pretty-quick": "^3.1.3", "process": "^0.11.10", "rollup": "^2.74.1", + "rollup-plugin-web-worker-loader": "^1.6.1", "ts-jest": "^28.0.3", "tslib": "^2.4.0", "typescript": "^4.6.4" diff --git a/rollup.config.js b/rollup.config.js index 054cf6f..def0833 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -1,6 +1,7 @@ import commonjs from "@rollup/plugin-commonjs"; import nodeResolve from "@rollup/plugin-node-resolve"; import typescript from "@rollup/plugin-typescript"; +import webWorkerLoader from "rollup-plugin-web-worker-loader"; export default { input: "src/index.ts", @@ -9,5 +10,5 @@ export default { format: "iife", name: "eyetrack", }, - plugins: [nodeResolve(), typescript(), commonjs()], + plugins: [nodeResolve(), webWorkerLoader(), commonjs(), typescript()], }; diff --git a/src/Eyetracker.ts b/src/Eyetracker.ts index 2a8e209..7b1b342 100644 --- a/src/Eyetracker.ts +++ b/src/Eyetracker.ts @@ -7,51 +7,60 @@ import { import "@tensorflow/tfjs-core"; import "@tensorflow/tfjs-backend-webgl"; import "@mediapipe/face_mesh"; - +import DemoWorker from "web-worker:./Worker.js"; export class Eyetracker { - private stream: MediaStream | undefined; private video: HTMLVideoElement | undefined; private canvas: HTMLCanvasElement | undefined; private detector: FaceLandmarksDetector | undefined; private ctx: CanvasRenderingContext2D | undefined; + private worker: Worker | undefined; + + constructor() { + this.worker = new DemoWorker(); + this.worker.addEventListener("message", (e) => { + console.log(e.data); + }); + } + + foo() { + this.worker?.postMessage({ cmd: "foo" }); + } - /** - * This is a function to add two numbers together. - * - * @param a A number to add - * @param b A number to add - * @returns The sum of both numbers - */ - add(a: number, b: number): number { - return a + b; + bar() { + this.worker?.postMessage({ cmd: "bar" }); } async getCameraPermission() { - this.stream = await navigator.mediaDevices.getUserMedia({ video: true, audio: false }); + this.stream = await navigator.mediaDevices.getUserMedia({ + video: true, + audio: false, + }); } async getListOfCameras(): Promise> { const devices = await navigator.mediaDevices.enumerateDevices(); const videoDevices = devices.filter((d) => { - d.kind === 'videoinput'; - return d.kind === 'videoinput'; - }) + d.kind === "videoinput"; + return d.kind === "videoinput"; + }); return videoDevices; } async setCamera(device: MediaDeviceInfo) { - this.stream = await navigator.mediaDevices.getUserMedia({ video: { deviceId: device.deviceId } }); + this.stream = await navigator.mediaDevices.getUserMedia({ + video: { deviceId: device.deviceId }, + }); } async createVideo(Id: string) { - let video = document.createElement('video'); - video.setAttribute('id', Id); + let video = document.createElement("video"); + video.setAttribute("id", Id); document.body.appendChild(video); - video.style.transform = 'scaleX(-1)'; - (video.srcObject as (undefined | MediaProvider | null)) = this.stream; + video.style.transform = "scaleX(-1)"; + (video.srcObject as undefined | MediaProvider | null) = this.stream; video.autoplay = true; return new Promise((resolve) => { video.onloadedmetadata = () => { @@ -64,87 +73,90 @@ export class Eyetracker { } createDisplay(Id: string) { - let canvas = document.createElement("canvas") - canvas.setAttribute('id', Id); + let canvas = document.createElement("canvas"); + canvas.setAttribute("id", Id); document.body.appendChild(canvas); - this.canvas = canvas + this.canvas = canvas; if (this.video != null) { canvas.height = this.video.height; canvas.width = this.video.width; this.canvas = canvas; - var ctx = canvas.getContext('2d'); + var ctx = canvas.getContext("2d"); if (ctx != null) { ctx.translate(canvas.width, 0); ctx.scale(-1, 1); ctx.fillStyle = "green"; - (this.ctx as (undefined | CanvasRenderingContext2D | null)) = ctx; - return canvas - } - else { - console.log('canvas.getContext(\'2d\') return null'); - return canvas + (this.ctx as undefined | CanvasRenderingContext2D | null) = ctx; + return canvas; + } else { + console.log("canvas.getContext('2d') return null"); + return canvas; } + } else { + console.log('Undefined Property "this.video"'); } - else { console.log('Undefined Property \"this.video\"'); } } // Need to test this out setDisplay(canvas: HTMLCanvasElement) { let video = this.video; this.canvas = canvas; - if ((canvas != undefined) && (video != undefined)) { - canvas.height = video.height - canvas.width = video.width - var ctx = canvas.getContext('2d'); + if (canvas != undefined && video != undefined) { + canvas.height = video.height; + canvas.width = video.width; + var ctx = canvas.getContext("2d"); if (ctx != null) { ctx.translate(canvas.width, 0); ctx.scale(-1, 1); ctx.fillStyle = "green"; } - (this.ctx as (undefined | CanvasRenderingContext2D | null)) = ctx; + (this.ctx as undefined | CanvasRenderingContext2D | null) = ctx; + } else { + console.log('/"this.canvas/", /"this.video/" Undefined'); } - else { console.log('/"this.canvas/", /"this.video/" Undefined'); } } showDisplay() { let ctx = this.ctx; let video = this.video; - if ((ctx != undefined) && (video != undefined)) { - ctx.drawImage(video, 0, 0) + if (ctx != undefined && video != undefined) { + ctx.drawImage(video, 0, 0); + } else { + console.log('"this.ctx", "this.video" Undefined'); } - else { console.log('\"this.ctx\", \"this.video\" Undefined') } } hideDisplay(canvas: HTMLCanvasElement) { - canvas.style.visibility = 'hidden' + canvas.style.visibility = "hidden"; } async createOverlay(): Promise { - try { let ctx = this.ctx; let video = this.video; let detector = this.detector; - if ((detector != undefined) && (video != undefined) && (ctx != undefined)) { - + if (detector != undefined && video != undefined && ctx != undefined) { const coordinates = (await detector.estimateFaces(video))[0]; const boxCoords = coordinates.box; const keypoints = coordinates.keypoints; - ctx.drawImage(video, 0, 0) + ctx.drawImage(video, 0, 0); for (let keypoint = 468; keypoint < keypoints.length; keypoint++) { - const x = keypoints[keypoint]['x'] - const y = keypoints[keypoint]['y'] - + const x = keypoints[keypoint]["x"]; + const y = keypoints[keypoint]["y"]; + ctx.beginPath(); ctx.rect(x, y, 2, 2); ctx.stroke(); } window.requestAnimationFrame(await this.createOverlay()); + } else { + console.log('"this.detector", "this.video", "this.ctx" Undefined'); } - else { console.log('\"this.detector\", \"this.video\", \"this.ctx\" Undefined'); } + } catch (err) { + this.showDisplay(); + window.requestAnimationFrame(await this.createOverlay()); } - catch (err) { this.showDisplay(); window.requestAnimationFrame(await this.createOverlay()); } } async init() { @@ -155,22 +167,26 @@ export class Eyetracker { refineLandmarks: true, }; const detector = await createDetector(model, detectorConfig); - this.detector = detector - return detector + this.detector = detector; + return detector; } async isFaceValid(): Promise { let detector = this.detector; let video = this.video; - if ((detector != undefined) && (video != undefined)) { - const faces = await detector.estimateFaces(video, { flipHorizontal: true }) + if (detector != undefined && video != undefined) { + const faces = await detector.estimateFaces(video, { + flipHorizontal: true, + }); if (faces.length > 0) { - console.log('FACE DETECTED'); - console.log(faces) + console.log("FACE DETECTED"); + console.log(faces); + } else { + console.log("Waiting for faces..."); } - else { console.log('Waiting for faces...') } window.requestAnimationFrame(await this.isFaceValid()); + } else { + console.log('"this.detector", "this.video" Undefined'); } - else { console.log('\"this.detector\", \"this.video\" Undefined') } } } diff --git a/src/Worker.js b/src/Worker.js new file mode 100644 index 0000000..888882a --- /dev/null +++ b/src/Worker.js @@ -0,0 +1,17 @@ +self.addEventListener("message", (e) => { + const data = e.data; + switch (data.cmd) { + case "foo": + self.postMessage(foo()); + case "bar": + self.postMessage(bar()); + } +}); + +const foo = () => { + return Math.random(); +}; + +const bar = () => { + return Math.floor(Math.random() * 100); +}; diff --git a/tests/index.test.ts b/tests/index.test.ts index 9973466..b56187f 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -1,6 +1,5 @@ import { initEyetracker } from "../src/index"; -test("test adds two numbers", () => { - const eye = initEyetracker(); - expect(eye.add(2, 2)).toEqual(4); +test("nothing", () => { + expect(true).toBe(true); }); diff --git a/workers.d.ts b/workers.d.ts new file mode 100644 index 0000000..940ec77 --- /dev/null +++ b/workers.d.ts @@ -0,0 +1,4 @@ +declare module "web-worker:*" { + const WorkerFactory: new () => Worker; + export default WorkerFactory; +} From 43369f39049f97951445b7a37eb0d46841c75204 Mon Sep 17 00:00:00 2001 From: Josh de Leeuw Date: Fri, 10 Jun 2022 15:49:47 -0400 Subject: [PATCH 2/3] ignore vscode --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 6704566..a7dc02a 100644 --- a/.gitignore +++ b/.gitignore @@ -102,3 +102,6 @@ dist # TernJS port file .tern-port + +# VS Code configuration +.vscode \ No newline at end of file From b40b0eae7b3b6d8355e13d8724772cda27073f0f Mon Sep 17 00:00:00 2001 From: Josh de Leeuw Date: Fri, 10 Jun 2022 15:57:14 -0400 Subject: [PATCH 3/3] Delete settings.json --- .vscode/settings.json | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 6b665aa..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "liveServer.settings.port": 5501 -}