script.js

// Copyright 2023 The MediaPipe Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//      http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
import vision from "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.3";
import * as THREE from "https://cdn.skypack.dev/three@0.150.1";
import { OrbitControls } from "https://cdn.skypack.dev/three@0.150.1/examples/jsm/controls/OrbitControls";
import { GLTFLoader } from "https://cdn.skypack.dev/three@0.150.1/examples/jsm/loaders/GLTFLoader";




// Functions to deal with the Avatar


/**
 * Returns the world-space dimensions of the viewport at `depth` units away from
 * the camera.
 */
function getViewportSizeAtDepth(camera, depth) {
    const viewportHeightAtDepth = 2 * depth * Math.tan(THREE.MathUtils.degToRad(0.5 * camera.fov));
    const viewportWidthAtDepth = viewportHeightAtDepth * camera.aspect;
    return new THREE.Vector2(viewportWidthAtDepth, viewportHeightAtDepth);
}

/**
 * Creates a `THREE.Mesh` which fully covers the `camera` viewport, is `depth`
 * units away from the camera and uses `material`.
 */
function createCameraPlaneMesh(camera, depth, material) {
    if (camera.near > depth || depth > camera.far) {
        console.warn("Camera plane geometry will be clipped by the `camera`!");
    }
    const viewportSize = getViewportSizeAtDepth(camera, depth);
    const cameraPlaneGeometry = new THREE.PlaneGeometry(viewportSize.width, viewportSize.height);
    cameraPlaneGeometry.translate(0, 0, -depth);
    return new THREE.Mesh(cameraPlaneGeometry, material);
}

class BasicScene {
    constructor() {
        this.lastTime = 0;
        this.callbacks = [];
        // Initialize the canvas with the same aspect ratio as the video input
        const mycontainer = document.getElementById("liveView");
        const mycanvas = document.getElementById("avatar_canvas");
        const myvideo = document.getElementById("webcam");
        // this.width = mycontainer.clientWidth;
        // this.height = mycontainer.clientHeight;
        this.width = 700;
        this.height = 525;
        // this.height = window.innerHeight;
        // this.width = (this.height * 1280) / 720;
        // Set up the Three.js scene, camera, and renderer
        this.scene = new THREE.Scene();
        this.camera = new THREE.PerspectiveCamera(60, this.width / this.height, 0.01, 5000);
        this.renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true});
        this.renderer.setSize(this.width, this.height);
        THREE.ColorManagement.legacy = false;
        this.renderer.outputEncoding = THREE.sRGBEncoding;
        this.renderer.domElement.setAttribute("id", "avatar_canvas");
        mycontainer.appendChild(this.renderer.domElement);
        // Set up the basic lighting for the scene
        const ambientLight = new THREE.AmbientLight(0xffffff, 0.5);
        this.scene.add(ambientLight);
        const directionalLight = new THREE.DirectionalLight(0xffffff, 0.5);
        directionalLight.position.set(0, 1, 0);
        this.scene.add(directionalLight);
        // Set up the camera position and controls
        this.camera.position.z = 0;
        this.controls = new OrbitControls(this.camera, this.renderer.domElement);
        let orbitTarget = this.camera.position.clone();
        orbitTarget.z -= 5;
        this.controls.target = orbitTarget;
        this.controls.update();
        // Add a video background
        // const inputFrameTexture = new THREE.VideoTexture(myvideo);
        // if (!inputFrameTexture) {
        //     throw new Error("Failed to get the 'input_frame' texture!");
        // }
        // inputFrameTexture.encoding = THREE.sRGBEncoding;
        // const inputFramesDepth = 500;
        // const inputFramesPlane = createCameraPlaneMesh(this.camera, inputFramesDepth, new THREE.MeshBasicMaterial({ map: inputFrameTexture }));
        // this.scene.add(inputFramesPlane);
        // Render the scene
        this.render();
        window.addEventListener("resize", this.resize.bind(this));
    }
    resize() {
        const myvideo = document.getElementById("webcam");
        // this.width = myvideo.videoWidth;
        // this.height = myvideo.videoHeight;
        this.width = 700;
        this.height = 525;
        this.camera.aspect = this.width / this.height;
        this.camera.updateProjectionMatrix();
        this.renderer.setSize(this.width, this.height);
        this.renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
        this.renderer.render(this.scene, this.camera);
    }
    render(time = this.lastTime) {
        const delta = (time - this.lastTime) / 1000;
        this.lastTime = time;
        // Call all registered callbacks with deltaTime parameter
        for (const callback of this.callbacks) {
            callback(delta);
        }
        // Render the scene
        this.renderer.render(this.scene, this.camera);
        // Request next frame
        requestAnimationFrame((t) => this.render(t));
    }
}

class Avatar {
    constructor(url, scene) {
        this.loader = new GLTFLoader();
        this.morphTargetMeshes = [];
        this.url = url;
        this.scene = scene;
        this.loadModel(this.url);
    }
    loadModel(url) {
        this.url = url;
        this.loader.load(
        // URL of the model you want to load
        url, 
        // Callback when the resource is loaded
        (gltf) => {
            if (this.gltf) {
                // Reset GLTF and morphTargetMeshes if a previous model was loaded.
                this.gltf.scene.remove();
                this.morphTargetMeshes = [];
            }
            this.gltf = gltf;
            console.log();
            this.scene.add(gltf.scene);
            this.init(gltf);
        }, 
        // Called while loading is progressing
        (progress) => console.log("Loading model...", 100.0 * (progress.loaded / progress.total), "%"), 
        // Called when loading has errors
        (error) => console.error(error));
    }
    init(gltf) {
        gltf.scene.traverse((object) => {
            // Register first bone found as the root
            if (object.isBone && !this.root) {
                this.root = object;
                console.log(object);
            }
            // Return early if no mesh is found.
            if (!object.isMesh) {
                console.warn(`No mesh found`);
                return;
            }
            const mesh = object;
            // Reduce clipping when model is close to camera.
            mesh.frustumCulled = false;
            // Return early if mesh doesn't include morphable targets
            if (!mesh.morphTargetDictionary || !mesh.morphTargetInfluences) {
                console.warn(`Mesh ${mesh.name} does not have morphable targets`);
                return;
            }
            this.morphTargetMeshes.push(mesh);
        });
    }
    remove() {
        console.log("Trying to remove")
        this.scene.remove(this.gltf.scene);
        this.morphTargetMeshes = [];
    }
    updateBlendshapes(blendshapes) {
        for (const mesh of this.morphTargetMeshes) {
            if (!mesh.morphTargetDictionary || !mesh.morphTargetInfluences) {
                console.warn(`Mesh ${mesh.name} does not have morphable targets`);
                continue;
            }
            for (const [name, value] of blendshapes) {
                if (!Object.keys(mesh.morphTargetDictionary).includes(name)) {
                    console.warn(`Model morphable target ${name} not found`);
                    continue;
                }
                const idx = mesh.morphTargetDictionary[name];
                mesh.morphTargetInfluences[idx] = value;
            }
        }
    }
    /**
     * Apply a position, rotation, scale matrix to current GLTF.scene
     * @param matrix
     * @param matrixRetargetOptions
     * @returns
     */
    applyMatrix(matrix, matrixRetargetOptions) {
        const { decompose = false, scale = 1 } = matrixRetargetOptions || {};
        if (!this.gltf) {
            return;
        }
        // Three.js will update the object matrix when it render the page
        // according the object position, scale, rotation.
        // To manually set the object matrix, you have to set autoupdate to false.
        matrix.scale(new THREE.Vector3(scale, scale, scale));
        
        this.gltf.scene.matrixAutoUpdate = false;
        // Set new position and rotation from matrix
        this.gltf.scene.matrix.copy(matrix);
    }
    /**
     * Takes the root object in the avatar and offsets its position for retargetting.
     * @param offset
     * @param rotation
     */
    offsetRoot(offset, rotation) {
        if (this.root) {
            this.root.position.copy(offset);

            if (rotation) {
                let offsetQuat = new THREE.Quaternion().setFromEuler(new THREE.Euler(rotation.x, rotation.y, rotation.z));
                this.root.quaternion.copy(offsetQuat);
            }
        }
    }
}

function retarget(blendshapes) {
    const categories = blendshapes.categories;
    let coefsMap = new Map();
    for (let i = 0; i < categories.length; ++i) {
        const blendshape = categories[i];
        // Adjust certain blendshape values to be less prominent.
        switch (blendshape.categoryName) {
            case "browOuterUpLeft":
                blendshape.score *= 1.2;
                break;
            case "browOuterUpRight":
                blendshape.score *= 1.2;
                break;
            case "eyeBlinkLeft":
                blendshape.score *= 1.2;
                break;
            case "eyeBlinkRight":
                blendshape.score *= 1.2;
                break;
            default:
        }
        coefsMap.set(categories[i].categoryName, categories[i].score);
    }
    return coefsMap;
}


// start the facelandmarker

const { FaceLandmarker, FilesetResolver, DrawingUtils } = vision;
const videoBlendShapes1 = document.getElementById("video-blend-shapes-1");
const videoBlendShapes2 = document.getElementById("video-blend-shapes-2");
let faceLandmarker;
const videoWidth = 700;
// Before we can use FaceLandmarker class we must wait for it to finish
// loading. Machine Learning models can be large and take a moment to
// get everything needed to run.
async function createFaceLandmarker() {
    const filesetResolver = await FilesetResolver.forVisionTasks("https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.3/wasm");
    faceLandmarker = await FaceLandmarker.createFromOptions(filesetResolver, {
        baseOptions: {
            modelAssetPath: `https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task`,
            delegate: "GPU"
        },
        outputFaceBlendshapes: true,
        outputFacialTransformationMatrixes: true,
        runningMode: "VIDEO",
        numFaces: 2
    });

}
createFaceLandmarker().then(function() {enableCam()} );

// initialise the avatar
const scene = new BasicScene();
let avatar1 = new Avatar("raccoon_head.glb", scene.scene);
let avatar1_scale = 40;
// const avatar1 = new Avatar("EmojiWithGlasses.glb", scene.scene);
// const avatar1_scale = 1750;
let avatar2 = new Avatar("raccoon_red.glb", scene.scene);
let avatar2_scale = 40;

/********************************************************************
// Demo: Continuously grab image from webcam stream and detect it.
********************************************************************/
const video = document.getElementById("webcam");
const canvasElement = document.getElementById("grid_canvas");
const canvasCtx = canvasElement.getContext("2d");

// Enable the live webcam view and start detection.
function enableCam() {
    if (!faceLandmarker) {
        console.log("Wait! faceLandmarker not loaded yet.");
        return;
    }
    // getUsermedia parameters.
    const constraints = {
        video: true
    };
    // Activate the webcam stream.
    navigator.mediaDevices.getUserMedia(constraints).then((stream) => {
        video.srcObject = stream;
        video.addEventListener("loadeddata", predictWebcam);
    });
}

let lastVideoTime = -1;
let results = undefined;
const drawingUtils = new DrawingUtils(canvasCtx);


async function predictWebcam() {
    const radio = video.videoHeight / video.videoWidth;
    video.style.width = videoWidth + "px";
    video.style.height = videoWidth * radio + "px";
    canvasElement.style.width = videoWidth + "px";
    canvasElement.style.height = videoWidth * radio + "px";
    canvasElement.width = video.videoWidth;
    canvasElement.height = video.videoHeight;
    // scene.resize()

    let startTimeMs = performance.now();
    if (lastVideoTime !== video.currentTime) {
        lastVideoTime = video.currentTime;
        results = faceLandmarker.detectForVideo(video, startTimeMs);
    }
    if (results.faceLandmarks) {
        let outputType = document.querySelector('input[name="output_type"]:checked').value;
        if (outputType == 'grid') {
            for (const landmarks of results.faceLandmarks) {
                drawingUtils.drawConnectors(landmarks, FaceLandmarker.FACE_LANDMARKS_TESSELATION, { color: "#C0C0C070", lineWidth: 1 });
                drawingUtils.drawConnectors(landmarks, FaceLandmarker.FACE_LANDMARKS_RIGHT_EYE, { color: "#FF3030" });
                drawingUtils.drawConnectors(landmarks, FaceLandmarker.FACE_LANDMARKS_RIGHT_EYEBROW, { color: "#FF3030" });
                drawingUtils.drawConnectors(landmarks, FaceLandmarker.FACE_LANDMARKS_LEFT_EYE, { color: "#30FF30" });
                drawingUtils.drawConnectors(landmarks, FaceLandmarker.FACE_LANDMARKS_LEFT_EYEBROW, { color: "#30FF30" });
                drawingUtils.drawConnectors(landmarks, FaceLandmarker.FACE_LANDMARKS_FACE_OVAL, { color: "#E0E0E0" });
                drawingUtils.drawConnectors(landmarks, FaceLandmarker.FACE_LANDMARKS_LIPS, { color: "#E0E0E0" });
                drawingUtils.drawConnectors(landmarks, FaceLandmarker.FACE_LANDMARKS_RIGHT_IRIS, { color: "#FF3030" });
                drawingUtils.drawConnectors(landmarks, FaceLandmarker.FACE_LANDMARKS_LEFT_IRIS, { color: "#30FF30" });
            }
        } else if (outputType == 'text') {
            if(results.faceBlendshapes[0]) {
                drawBlendShapes(videoBlendShapes1, results.faceBlendshapes[0]);
            }
            if(results.faceBlendshapes[1]) {
                drawBlendShapes(videoBlendShapes2, results.faceBlendshapes[1]);
            }
        } else if (outputType == 'avatar') {
            // Apply transformation
            const transformationMatrices = results.facialTransformationMatrixes;
            if (transformationMatrices[0]) {
                let matrix1 = new THREE.Matrix4().fromArray(transformationMatrices[0].data);
                // Example of applying matrix directly to the avatar
                avatar1.applyMatrix(matrix1, { scale: avatar1_scale });
            }
            if(transformationMatrices[1]) {
                let matrix2 = new THREE.Matrix4().fromArray(transformationMatrices[1].data);
                // Example of applying matrix directly to the avatar
                avatar2.applyMatrix(matrix2, { scale: avatar2_scale });
            }


            // Apply Blendshapes to avatars
            const blendshapes = results.faceBlendshapes;
            if (blendshapes[0]) {
                const coefsMap = retarget(blendshapes[0]);
                avatar1.updateBlendshapes(coefsMap);
            }
            if(blendshapes[1]) {
                const coefsMap = retarget(blendshapes[1]);
                avatar2.updateBlendshapes(coefsMap);
            }

        }
    }

    // Call this function again to keep predicting when the browser is ready.
    window.requestAnimationFrame(predictWebcam);
}

function drawBlendShapes(el, blendShapes) {
    // console.log(blendShapes);
    let htmlMaker = "";
    blendShapes.categories.map((shape) => {
        htmlMaker += `
      <li class="blend-shapes-item">
        <span class="blend-shapes-label">${shape.displayName || shape.categoryName}</span>
        <span class="blend-shapes-value" style="width: calc(${+shape.score * 100}% - 120px)">${(+shape.score).toFixed(4)}</span>
      </li>
    `;
    });
    el.innerHTML = htmlMaker;
}

// event listeners for form elements

const optionGrid = document.getElementById("grid");
const optionText = document.getElementById("text");
const optionAvatar = document.getElementById("avatar");

optionGrid.addEventListener("change", updateUI);
optionText.addEventListener("change", updateUI);
optionAvatar.addEventListener("change", updateUI);

function updateUI(event){
    const shapes1 = document.getElementById("blend-shapes-1");
    const shapes2 = document.getElementById("blend-shapes-2");
    const grid = document.getElementById("grid_canvas");
    const avatar = document.getElementById("avatar_canvas");

    if (event.target.value == 'grid') {
        grid.classList.remove("hidden");
        avatar.classList.add("hidden");
        shapes1.classList.add("hidden");
        shapes2.classList.add("hidden");
    } else if (event.target.value == 'text') {
        grid.classList.add("hidden");
        avatar.classList.add("hidden");
        shapes1.classList.remove("hidden");
        shapes2.classList.remove("hidden");
    } else if (event.target.value == 'avatar') {
        grid.classList.add("hidden");
        avatar.classList.remove("hidden");
        shapes1.classList.add("hidden");
        shapes2.classList.add("hidden");
    }
}

const selectAvatar = document.getElementById("avatar_1_file");
selectAvatar.addEventListener("change", changeAvatar);

function changeAvatar(event) {
    if (event.target.value == 'red_raccoon') {
        avatar1.remove()
        avatar1.loadModel("raccoon_red.glb");
        avatar1_scale = 40;
    } else if (event.target.value == 'raccoon') {
        avatar1.remove()
        avatar1.loadModel("raccoon_head.glb");
        avatar1_scale = 40;
    } else if (event.target.value == 'smile') {
        avatar1.remove()
        avatar1.loadModel("EmojiWithGlasses.glb");
        avatar1_scale = 1750;
    }
}