From 388c58021d64c98f81c0ec0603150afebc3baa4c Mon Sep 17 00:00:00 2001 From: Nikita Ostapenko Date: Thu, 26 Oct 2023 20:11:08 -0400 Subject: [PATCH 01/19] enable VR support in aquarium --- aquarium/aquarium-config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aquarium/aquarium-config.js b/aquarium/aquarium-config.js index dfa531f0..33d6e587 100644 --- a/aquarium/aquarium-config.js +++ b/aquarium/aquarium-config.js @@ -2,5 +2,5 @@ var g_aquariumConfig = { aquariumRoot: '', - enableVR: false + enableVR: true, }; From d0c965ce5908c87914921c036592526a40900831 Mon Sep 17 00:00:00 2001 From: Nikita Ostapenko Date: Thu, 26 Oct 2023 20:13:24 -0400 Subject: [PATCH 02/19] add xr compatible to gl context and xr support detection --- aquarium/aquarium.js | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/aquarium/aquarium.js b/aquarium/aquarium.js index e1b7f4b7..778b8d4d 100755 --- a/aquarium/aquarium.js +++ b/aquarium/aquarium.js @@ -883,7 +883,7 @@ function main() { g_fpsTimer = new tdl.fps.FPSTimer(); if (isMultiviewSupportEnabled()) { - gl = tdl.webgl.setupWebGL(canvas, {antialias: false}, undefined, 'webgl2'); + gl = tdl.webgl.setupWebGL(canvas, {antialias: false, xrCompatible: true}, undefined, 'webgl2'); multiview = gl.getExtension('OVR_multiview2'); } else { gl = tdl.webgl.setupWebGL(canvas); @@ -2202,6 +2202,16 @@ $(function(){ function initPostDOMLoaded() { if (g_aquariumConfig.enableVR) { + if (navigator.xr) { + // Checks to ensure that 'immersive-ar' mode is available, and only + // enables the button if so. + navigator.xr.isSessionSupported('immersive-vr').then((supported) => { + if (supported) { + vrButton = addButton("Enter VR", "E", getCurrentUrl() + "/vr_assets/button.png", onRequestPresent); + } + }); + } + /* if(navigator.getVRDisplays) { g_frameData = new VRFrameData(); @@ -2232,6 +2242,7 @@ $(function(){ console.log("Your browser does not support WebVR. See webvr.info for assistance"); } } +*/ // Regardless of if we have WebVR support, we can demonstrate stereo rendering inside the window. stereoDemoButton = addButton("Toggle Stereo Demo", "", getCurrentUrl() + "/vr_assets/button.png", toggleStereoDemo); } From 7eceecdf14dfc9521e5401197ea5c7c067281322 Mon Sep 17 00:00:00 2001 From: Nikita Ostapenko Date: Thu, 26 Oct 2023 20:28:23 -0400 Subject: [PATCH 03/19] enter and exit xr session work --- aquarium/aquarium.js | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/aquarium/aquarium.js b/aquarium/aquarium.js index 778b8d4d..7097b94f 100755 --- a/aquarium/aquarium.js +++ b/aquarium/aquarium.js @@ -2156,21 +2156,44 @@ $(function(){ } } + var g_session = null; + function onRequestPresent() { + return navigator.xr.requestSession('immersive-vr').then((session) => { + removeButton(vrButton); + vrButton = addButton("Exit VR", "E", getCurrentUrl() + "/vr_assets/button.png", onExitPresent); + session.isImmersive = true; + g_session = session; + session.addEventListener('end', onSessionEnded); + }); + /* g_vrDisplay.requestPresent([{ source: canvas }]).then(function() {}, function() { console.error("request present failed."); }); + */ } function onExitPresent() { + session.end(); + + + /* if (!g_vrDisplay.isPresenting) return; g_vrDisplay.exitPresent().then(function() {}, function() { console.error("exit present failed."); }); + */ } + function onSessionEnded(event) { + if (event.session.isImmersive) { + removeButton(vrButton); + vrButton = addButton("Enter VR", "E", getCurrentUrl() + "/vr_assets/button.png", onRequestPresent); + } + } + function toggleStereoDemo() { g_stereoDemoActive = !g_stereoDemoActive; g_shadersNeedUpdate = true; From 71276c6ceb48dc571029f09a70475e89234a9b52 Mon Sep 17 00:00:00 2001 From: Nikita Ostapenko Date: Wed, 27 Dec 2023 22:39:20 -0500 Subject: [PATCH 04/19] basic XR rendering works --- aquarium/aquarium.js | 113 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 108 insertions(+), 5 deletions(-) diff --git a/aquarium/aquarium.js b/aquarium/aquarium.js index 7097b94f..1bd91000 100755 --- a/aquarium/aquarium.js +++ b/aquarium/aquarium.js @@ -19,7 +19,7 @@ tdl.require('tdl.webgl'); const g_query = parseQueryString(window.location.search); function isMultiviewSupportEnabled() { - return g_aquariumConfig.enableVR && g_query.enableMultiview && g_query.enableMultiview == 'true'; + return g_aquariumConfig.enableVR || (g_query.enableMultiview && g_query.enableMultiview == 'true'); } if (isMultiviewSupportEnabled()) { @@ -88,6 +88,9 @@ var g_lightRayRotRange = 1.0; var g_lightRayRotLerp = 0.2; var g_lightRayOffset = Math.PI * 2 / g_numLightRays; var g_lightRayInfo = []; +var g_session = null; +var g_xrImmersiveRefSpace = null; +var g_onXRFrame = ()=>{}; var g_ui = [ { obj: 'globals', name: 'speed', value: 1, max: 4 }, @@ -1259,7 +1262,7 @@ function initialize() { gl.clearColor(0,0.8,1,0); gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT | gl.STENCIL_BUFFER_BIT); - var presentingVR = g_vrDisplay && g_vrDisplay.isPresenting; + var presentingVR = g_session && g_session.isImmersive; var uiMatrix = new Float32Array(16); @@ -1272,7 +1275,7 @@ function initialize() { } fast.matrix4.copy(viewInverse, viewInverseMatrix); // TODO: Support VRUI when using multiview? Would require adding multiview shaders to UI and changing UI matrices when multiview is on. - if (!useMultiview && presentingVR && pose) { + if (!useMultiview && presentingVR && pose && g_vrUi) { // Hard coded FPS translation vector and pin the whole UI in front of the user in VR mode. This hard coded position // vector used only once here. calculateViewMatrix(uiMatrix, pose.orientation, [0, 0, 10]); @@ -1701,6 +1704,96 @@ function initialize() { render(monoProjection, viewInverseTemp); } + function onXRFrame(now, frame) { + let session = frame.session; + let refSpace = g_xrImmersiveRefSpace; + let pose = frame.getViewerPose(refSpace); + + var elapsedTime; + if(then == 0.0) { + elapsedTime = 0.0; + } else { + elapsedTime = now - then; + } + then = now; + + frameCount++; + g_fpsTimer.update(elapsedTime); + fpsElem.innerHTML = g_fpsTimer.averageFPS; + + if (g_shadersNeedUpdate) { + setShaders(true); + g_shadersNeedUpdate = false; + } + + if (g_vrUi) { + // Set fps and prepare rendering it. + g_vrUi.setFps(g_fpsTimer.averageFPS); + + // Query gamepad button clicked event. + g_vrUi.queryGamepadStatus(); + + // TODO: Support VRUI when doing multiview rendering. + if (!useMultiviewForStereo() && g_vrUi.isMenuMode) { + + // When VR UI in menu mode, UI need a cursor to help user do select operation. Currently, cursor uses + // head-neck model which means a point in front of user and user could move the point by rotating their head(with HMD). + // A click event will be triggered when user stare at a label 2 seconds. + // TODO : add gamepad support to control cursor and trigger select event with VR controllers. + + // Jquery selector description. + var selectorDescription; + + // VR UI return whether there is an option been selected in VR mode. + var clickedLabel = g_vrUi.queryClickedLabel([0, 0, 0], g_frameData.pose.orientation); + if (clickedLabel != null) { + if (clickedLabel.isAdvancedSettings) { + selectorDescription = "#optionsContainer > div:contains(" + clickedLabel.name + ")"; + $(selectorDescription).click(); + } else if (clickedLabel.name == "options") { + $("#options").click(); + } else { + selectorDescription = "#setSetting" + clickedLabel.name; + $(selectorDescription).click(); + } + } + } + } + + // Using head-neck model in VR mode because of unclear distance measurement(vr return position using meters), + // user could see around but couldn't move around. + eyePosition[0] = g.globals.eyeRadius; + eyePosition[1] = g.globals.eyeHeight; + eyePosition[2] = g.globals.eyeRadius; + + let vrPose = [pose.transform.orientation.x, pose.transform.orientation.y, pose.transform.orientation.z, 1] + calculateViewMatrix(viewInverseTemp, vrPose, eyePosition); + + let glLayer = session.renderState.baseLayer; + + gl.bindFramebuffer(gl.FRAMEBUFFER, glLayer.framebuffer); + gl.enable(gl.SCISSOR_TEST); + + let xrViewport = glLayer.getViewport(pose.views[0]); + gl.viewport(xrViewport.x, xrViewport.y, xrViewport.width, xrViewport.height); + gl.scissor(xrViewport.x, xrViewport.y, xrViewport.width, xrViewport.height); + render(pose.views[0].projectionMatrix, viewInverseTemp, false, pose); + + xrViewport = glLayer.getViewport(pose.views[1]); + gl.viewport(xrViewport.x, xrViewport.y, xrViewport.width, xrViewport.height); + gl.scissor(xrViewport.x, xrViewport.y, xrViewport.width, xrViewport.height); + render(pose.views[1].projectionMatrix, viewInverseTemp, false, pose); + //renderStereo(pose.views[0].projectionMatrix, pose.views[1].projectionMatrix, viewInverseTemp, vrPose); + // gl.clearColor(Math.cos(now / 2000), + // Math.cos(now / 4000), + // Math.cos(now / 6000), 1.0); + + // gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT); + session.requestAnimationFrame(onXRFrame); + } + + g_onXRFrame = onXRFrame; + function onAnimationFrame() { var now = theClock.getTime(); var elapsedTime; @@ -2156,14 +2249,21 @@ $(function(){ } } - var g_session = null; - function onRequestPresent() { return navigator.xr.requestSession('immersive-vr').then((session) => { removeButton(vrButton); vrButton = addButton("Exit VR", "E", getCurrentUrl() + "/vr_assets/button.png", onExitPresent); session.isImmersive = true; g_session = session; + + session.updateRenderState({ baseLayer: new XRWebGLLayer(session, gl) }); + + let refSpaceType = 'local'; + session.requestReferenceSpace(refSpaceType).then((refSpace) => { + g_xrImmersiveRefSpace = refSpace + g_shadersNeedUpdate = true; + session.requestAnimationFrame(g_onXRFrame); + }); session.addEventListener('end', onSessionEnded); }); /* @@ -2191,6 +2291,7 @@ $(function(){ if (event.session.isImmersive) { removeButton(vrButton); vrButton = addButton("Enter VR", "E", getCurrentUrl() + "/vr_assets/button.png", onRequestPresent); + g_shadersNeedUpdate = true; } } @@ -2231,6 +2332,8 @@ $(function(){ navigator.xr.isSessionSupported('immersive-vr').then((supported) => { if (supported) { vrButton = addButton("Enter VR", "E", getCurrentUrl() + "/vr_assets/button.png", onRequestPresent); + g_vrUi = new Ui(gl, g_numFish); + g_vrUi.load("./vr_assets/ui/config.js"); } }); } From c84257319ac752e6076fe53c922f734ea8219160 Mon Sep 17 00:00:00 2001 From: Nikita Ostapenko Date: Wed, 27 Dec 2023 22:44:55 -0500 Subject: [PATCH 05/19] correct clock calculation --- aquarium/aquarium.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/aquarium/aquarium.js b/aquarium/aquarium.js index 1bd91000..9ca7271c 100755 --- a/aquarium/aquarium.js +++ b/aquarium/aquarium.js @@ -1709,6 +1709,7 @@ function initialize() { let refSpace = g_xrImmersiveRefSpace; let pose = frame.getViewerPose(refSpace); + var now = theClock.getTime(); var elapsedTime; if(then == 0.0) { elapsedTime = 0.0; @@ -1717,6 +1718,15 @@ function initialize() { } then = now; + if (g.net.sync) { + clock = now * g.globals.speed; + eyeClock = now * g.globals.eyeSpeed; + } else { + // we have our own clock. + clock += elapsedTime * g.globals.speed; + eyeClock += elapsedTime * g.globals.eyeSpeed; + } + frameCount++; g_fpsTimer.update(elapsedTime); fpsElem.innerHTML = g_fpsTimer.averageFPS; From 28fd1a441d07e178ac31f26db53934766d916536 Mon Sep 17 00:00:00 2001 From: Nikita Ostapenko Date: Wed, 27 Dec 2023 23:06:58 -0500 Subject: [PATCH 06/19] change eye height and radius for xr session --- aquarium/aquarium.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/aquarium/aquarium.js b/aquarium/aquarium.js index 9ca7271c..2b4693ad 100755 --- a/aquarium/aquarium.js +++ b/aquarium/aquarium.js @@ -96,7 +96,7 @@ var g_ui = [ { obj: 'globals', name: 'speed', value: 1, max: 4 }, { obj: 'globals', name: 'targetHeight', value: 0, max: 150 }, { obj: 'globals', name: 'targetRadius', value: 88, max: 200 }, - { obj: 'globals', name: 'eyeHeight', value: 19, max: 150 }, + { obj: 'globals', name: 'eyeHeight', value: 150, max: 150 }, { obj: 'globals', name: 'eyeRadius', value: 60, max: 200 }, { obj: 'globals', name: 'eyeSpeed', value: 0.06, max: 1 }, { obj: 'globals', name: 'fieldOfView', value: 85, max: 179, min: 1}, @@ -2272,6 +2272,8 @@ $(function(){ session.requestReferenceSpace(refSpaceType).then((refSpace) => { g_xrImmersiveRefSpace = refSpace g_shadersNeedUpdate = true; + g.globals.eyeHeight = 15; + g.globals.eyeRadius = 1; session.requestAnimationFrame(g_onXRFrame); }); session.addEventListener('end', onSessionEnded); @@ -2285,7 +2287,8 @@ $(function(){ function onExitPresent() { session.end(); - + g.globals.eyeHeight = g_viewSettings[0].eyeHeight; + g.globals.eyeRadius = g_viewSettings[0].eyeRadius; /* if (!g_vrDisplay.isPresenting) From eb106367258e69992777fd30d7ee70dc251c4a8e Mon Sep 17 00:00:00 2001 From: Nikita Ostapenko Date: Wed, 27 Dec 2023 23:28:02 -0500 Subject: [PATCH 07/19] fixed vr exit --- aquarium/aquarium.js | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/aquarium/aquarium.js b/aquarium/aquarium.js index 2b4693ad..57906f9d 100755 --- a/aquarium/aquarium.js +++ b/aquarium/aquarium.js @@ -91,6 +91,7 @@ var g_lightRayInfo = []; var g_session = null; var g_xrImmersiveRefSpace = null; var g_onXRFrame = ()=>{}; +var g_onAnimationFrame = () => {}; var g_ui = [ { obj: 'globals', name: 'speed', value: 1, max: 4 }, @@ -1965,6 +1966,7 @@ function initialize() { } } + g_onAnimationFrame = onAnimationFrame; onAnimationFrame(); return true; } @@ -2286,10 +2288,9 @@ $(function(){ } function onExitPresent() { - session.end(); - g.globals.eyeHeight = g_viewSettings[0].eyeHeight; - g.globals.eyeRadius = g_viewSettings[0].eyeRadius; - + g_session.end(); + location.reload(); + /* if (!g_vrDisplay.isPresenting) return; @@ -2302,9 +2303,7 @@ $(function(){ function onSessionEnded(event) { if (event.session.isImmersive) { - removeButton(vrButton); - vrButton = addButton("Enter VR", "E", getCurrentUrl() + "/vr_assets/button.png", onRequestPresent); - g_shadersNeedUpdate = true; + onExitPresent(); } } From 588cb647f59cbd16d1008e039f3c4c473b7bf138 Mon Sep 17 00:00:00 2001 From: Nikita Ostapenko Date: Wed, 27 Dec 2023 23:32:43 -0500 Subject: [PATCH 08/19] remove obsolete g_vrDisplay and related --- aquarium/aquarium.js | 144 ++----------------------------------------- 1 file changed, 6 insertions(+), 138 deletions(-) diff --git a/aquarium/aquarium.js b/aquarium/aquarium.js index 57906f9d..6934be50 100755 --- a/aquarium/aquarium.js +++ b/aquarium/aquarium.js @@ -57,7 +57,6 @@ var g_getCount = 0; var g_putCount = 0; var g_frameData; -var g_vrDisplay; var g_vrUi; var g_multiviewFb; // multiview framebuffer. @@ -1863,70 +1862,15 @@ function initialize() { if (!g.options.reflection.enabled) { g.options.reflection.toggle(); } } - if (g_vrDisplay) { - g_requestId = g_vrDisplay.requestAnimationFrame(onAnimationFrame); - g_vrDisplay.getFrameData(g_frameData); - } else { - g_requestId = requestAnimationFrame(onAnimationFrame); - } + g_requestId = requestAnimationFrame(onAnimationFrame); if (g_shadersNeedUpdate) { - var isInStereoMode = (g_vrDisplay && g_vrDisplay.isPresenting) || g_stereoDemoActive; + var isInStereoMode = g_stereoDemoActive; setShaders(isInStereoMode && useMultiviewForStereo()); g_shadersNeedUpdate = false; } - if (g_vrDisplay && g_vrDisplay.isPresenting) { - /* VR UI is enabled in VR Mode. VR UI has two mode, menu mode is the mirror of control panel of - * aquarium and non-menu mode may presents fps(could be turn off) in front of user. These two - * mode is controlled by isMenuMode flag and this flag is set by any keyboard event or gamepad - * button click. - */ - - // Set fps and prepare rendering it. - g_vrUi.setFps(g_fpsTimer.averageFPS); - - // Query gamepad button clicked event. - g_vrUi.queryGamepadStatus(); - - // TODO: Support VRUI when doing multiview rendering. - if (!useMultiviewForStereo() && g_vrUi.isMenuMode) { - - // When VR UI in menu mode, UI need a cursor to help user do select operation. Currently, cursor uses - // head-neck model which means a point in front of user and user could move the point by rotating their head(with HMD). - // A click event will be triggered when user stare at a label 2 seconds. - // TODO : add gamepad support to control cursor and trigger select event with VR controllers. - - // Jquery selector description. - var selectorDescription; - - // VR UI return whether there is an option been selected in VR mode. - var clickedLabel = g_vrUi.queryClickedLabel([0, 0, 0], g_frameData.pose.orientation); - if (clickedLabel != null) { - if (clickedLabel.isAdvancedSettings) { - selectorDescription = "#optionsContainer > div:contains(" + clickedLabel.name + ")"; - $(selectorDescription).click(); - } else if (clickedLabel.name == "options") { - $("#options").click(); - } else { - selectorDescription = "#setSetting" + clickedLabel.name; - $(selectorDescription).click(); - } - } - } - - // Using head-neck model in VR mode because of unclear distance measurement(vr return position using meters), - // user could see around but couldn't move around. - eyePosition[0] = g.globals.eyeRadius; - eyePosition[1] = g.globals.eyeHeight; - eyePosition[2] = g.globals.eyeRadius; - - calculateViewMatrix(viewInverseTemp, g_frameData.pose.orientation, eyePosition); - - renderStereo(g_frameData.leftProjectionMatrix, g_frameData.rightProjectionMatrix, viewInverseTemp, g_frameData.pose); - - g_vrDisplay.submitFrame(); - } else if (g_stereoDemoActive) { + if (g_stereoDemoActive) { var near = 1; var far = 25000; var aspect = (canvas.clientWidth * 0.5) / canvas.clientHeight; @@ -2241,26 +2185,6 @@ $(function(){ return path.substring(0, path.lastIndexOf('/')); } - function onPresentChange() { - // When we begin or end presenting, the canvas should be resized - // to the recommended dimensions for the display. - resize(); - - g_shadersNeedUpdate = true; - - if (g_vrDisplay.isPresenting) { - if (g_vrDisplay.capabilities.hasExternalDisplay) { - removeButton(vrButton); - vrButton = addButton("Exit VR", "E", getCurrentUrl() + "/vr_assets/button.png", onExitPresent); - } - } else { - if (g_vrDisplay.capabilities.hasExternalDisplay) { - removeButton(vrButton); - vrButton = addButton("Enter VR", "E", getCurrentUrl() + "/vr_assets/button.png", onRequestPresent); - } - } - } - function onRequestPresent() { return navigator.xr.requestSession('immersive-vr').then((session) => { removeButton(vrButton); @@ -2280,25 +2204,11 @@ $(function(){ }); session.addEventListener('end', onSessionEnded); }); - /* - g_vrDisplay.requestPresent([{ source: canvas }]).then(function() {}, function() { - console.error("request present failed."); - }); - */ } function onExitPresent() { g_session.end(); location.reload(); - - /* - if (!g_vrDisplay.isPresenting) - return; - - g_vrDisplay.exitPresent().then(function() {}, function() { - console.error("exit present failed."); - }); - */ } function onSessionEnded(event) { @@ -2313,19 +2223,9 @@ $(function(){ } function resize() { - if (g_vrDisplay && g_vrDisplay.isPresenting) { - // If we're presenting we want to use the drawing buffer size - // recommended by the VRDisplay, since that will ensure the best - // results post-distortion. - var leftEye = g_vrDisplay.getEyeParameters("left"); - var rightEye = g_vrDisplay.getEyeParameters("right"); - - setCanvasSize(canvas, Math.max(leftEye.renderWidth, rightEye.renderWidth) * 2, Math.max(leftEye.renderHeight, rightEye.renderHeight)); - } else { - // When we're not presenting, we want to change the size of the canvas - // to match the window dimensions. - setCanvasSize(canvas, g.globals.width, g.globals.height); - } + // When we're not presenting, we want to change the size of the canvas + // to match the window dimensions. + setCanvasSize(canvas, g.globals.width, g.globals.height); } function onResize() { @@ -2349,38 +2249,6 @@ $(function(){ } }); } - /* - if(navigator.getVRDisplays) { - g_frameData = new VRFrameData(); - - navigator.getVRDisplays().then(function(displays) { - if (displays.length > 0) { - g_vrDisplay = displays[0]; - g_vrDisplay.depthNear = 0.1; - g_vrDisplay.depthFar = 1024.0; - - if (g_vrDisplay.capabilities.canPresent) { - vrButton = addButton("Enter VR", "E", getCurrentUrl() + "/vr_assets/button.png", onRequestPresent); - } - g_vrUi = new Ui(gl, g_numFish); - g_vrUi.load("./vr_assets/ui/config.js"); - - window.addEventListener('vrdisplaypresentchange', onPresentChange, false); - window.addEventListener('vrdisplayactivate', onRequestPresent, false); - window.addEventListener('vrdisplaydeactivate', onExitPresent, false); - window.addEventListener('keydown', function() { g_vrUi.isMenuMode = !g_vrUi.isMenuMode; }, false); - } else { - console.log("WebVR supported, but no VRDisplays found.") - } - }); - } else { - if (navigator.getVRDevices) { - console.log("Your browser supports WebVR but not the latest version. See webvr.info for more info."); - } else { - console.log("Your browser does not support WebVR. See webvr.info for assistance"); - } - } -*/ // Regardless of if we have WebVR support, we can demonstrate stereo rendering inside the window. stereoDemoButton = addButton("Toggle Stereo Demo", "", getCurrentUrl() + "/vr_assets/button.png", toggleStereoDemo); } From 2f2b3ebc02a5aee4291f881cf604b4e74f0c2f33 Mon Sep 17 00:00:00 2001 From: Nikita Ostapenko Date: Sat, 30 Dec 2023 20:27:35 -0500 Subject: [PATCH 09/19] Add device change listener and update vr button Untested --- aquarium/aquarium.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/aquarium/aquarium.js b/aquarium/aquarium.js index 6934be50..2f6f8b32 100755 --- a/aquarium/aquarium.js +++ b/aquarium/aquarium.js @@ -2217,6 +2217,19 @@ $(function(){ } } + function onDeviceChange() { + vrButton ?? removeButton(vrButton); + // Checks to ensure that 'immersive-ar' mode is available, and only + // enables the button if so. + navigator.xr.isSessionSupported('immersive-vr').then((supported) => { + if (supported) { + vrButton = addButton("Enter VR", "E", getCurrentUrl() + "/vr_assets/button.png", onRequestPresent); + g_vrUi = new Ui(gl, g_numFish); + g_vrUi.load("./vr_assets/ui/config.js"); + } + }); + } + function toggleStereoDemo() { g_stereoDemoActive = !g_stereoDemoActive; g_shadersNeedUpdate = true; @@ -2248,6 +2261,7 @@ $(function(){ g_vrUi.load("./vr_assets/ui/config.js"); } }); + navigator.xr.addEventListener('devicechange', onDeviceChange); } // Regardless of if we have WebVR support, we can demonstrate stereo rendering inside the window. stereoDemoButton = addButton("Toggle Stereo Demo", "", getCurrentUrl() + "/vr_assets/button.png", toggleStereoDemo); From 4612417e01a5fd7dfea81370dd4fc4a157130e07 Mon Sep 17 00:00:00 2001 From: Nikita Ostapenko Date: Mon, 8 Jan 2024 22:19:40 -0500 Subject: [PATCH 10/19] add w component to orientation instead of fixed 1 --- aquarium/aquarium.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aquarium/aquarium.js b/aquarium/aquarium.js index 2f6f8b32..d936b024 100755 --- a/aquarium/aquarium.js +++ b/aquarium/aquarium.js @@ -1776,7 +1776,7 @@ function initialize() { eyePosition[1] = g.globals.eyeHeight; eyePosition[2] = g.globals.eyeRadius; - let vrPose = [pose.transform.orientation.x, pose.transform.orientation.y, pose.transform.orientation.z, 1] + let vrPose = [pose.transform.orientation.x, pose.transform.orientation.y, pose.transform.orientation.z, pose.transform.orientation.w] calculateViewMatrix(viewInverseTemp, vrPose, eyePosition); let glLayer = session.renderState.baseLayer; From a8c333fe4de3ede3fa0effde0919bcd57ae80b55 Mon Sep 17 00:00:00 2001 From: Nikita Ostapenko Date: Mon, 8 Jan 2024 22:24:45 -0500 Subject: [PATCH 11/19] change generated eye position to XR pose position --- aquarium/aquarium.js | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/aquarium/aquarium.js b/aquarium/aquarium.js index d936b024..f880e49b 100755 --- a/aquarium/aquarium.js +++ b/aquarium/aquarium.js @@ -1770,11 +1770,8 @@ function initialize() { } } - // Using head-neck model in VR mode because of unclear distance measurement(vr return position using meters), - // user could see around but couldn't move around. - eyePosition[0] = g.globals.eyeRadius; - eyePosition[1] = g.globals.eyeHeight; - eyePosition[2] = g.globals.eyeRadius; + const viewPos = pose.views[n].transform.position; + eyePosition = [viewPos.x, viewPos.y, viewPos.z]; let vrPose = [pose.transform.orientation.x, pose.transform.orientation.y, pose.transform.orientation.z, pose.transform.orientation.w] calculateViewMatrix(viewInverseTemp, vrPose, eyePosition); From b35d63d069535de49c550511894240ec34ee7548 Mon Sep 17 00:00:00 2001 From: Nikita Ostapenko Date: Mon, 8 Jan 2024 22:29:52 -0500 Subject: [PATCH 12/19] Iterate through all available views on rendering. Also remove obsolete calculateViewMatrix usage --- aquarium/aquarium.js | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/aquarium/aquarium.js b/aquarium/aquarium.js index f880e49b..8b1440ba 100755 --- a/aquarium/aquarium.js +++ b/aquarium/aquarium.js @@ -1770,26 +1770,17 @@ function initialize() { } } - const viewPos = pose.views[n].transform.position; - eyePosition = [viewPos.x, viewPos.y, viewPos.z]; - - let vrPose = [pose.transform.orientation.x, pose.transform.orientation.y, pose.transform.orientation.z, pose.transform.orientation.w] - calculateViewMatrix(viewInverseTemp, vrPose, eyePosition); - let glLayer = session.renderState.baseLayer; gl.bindFramebuffer(gl.FRAMEBUFFER, glLayer.framebuffer); gl.enable(gl.SCISSOR_TEST); - let xrViewport = glLayer.getViewport(pose.views[0]); - gl.viewport(xrViewport.x, xrViewport.y, xrViewport.width, xrViewport.height); - gl.scissor(xrViewport.x, xrViewport.y, xrViewport.width, xrViewport.height); - render(pose.views[0].projectionMatrix, viewInverseTemp, false, pose); - - xrViewport = glLayer.getViewport(pose.views[1]); - gl.viewport(xrViewport.x, xrViewport.y, xrViewport.width, xrViewport.height); - gl.scissor(xrViewport.x, xrViewport.y, xrViewport.width, xrViewport.height); - render(pose.views[1].projectionMatrix, viewInverseTemp, false, pose); + for (const view of pose.views) { + let xrViewport = glLayer.getViewport(view); + gl.viewport(xrViewport.x, xrViewport.y, xrViewport.width, xrViewport.height); + gl.scissor(xrViewport.x, xrViewport.y, xrViewport.width, xrViewport.height); + render(view.projectionMatrix, view.transform.inverse.matrix, false, view); // Not clear on what the final arg is used for here? Glancing at the code it looks like it should probably be view instead of pose. + } //renderStereo(pose.views[0].projectionMatrix, pose.views[1].projectionMatrix, viewInverseTemp, vrPose); // gl.clearColor(Math.cos(now / 2000), // Math.cos(now / 4000), From 89bd539342c201f0699d298e873cfbb62a657c2c Mon Sep 17 00:00:00 2001 From: Nikita Ostapenko Date: Sun, 17 Mar 2024 19:57:00 -0400 Subject: [PATCH 13/19] Use view.transform.matrix instead of inverse --- aquarium/aquarium.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aquarium/aquarium.js b/aquarium/aquarium.js index 8b1440ba..f5b928bd 100755 --- a/aquarium/aquarium.js +++ b/aquarium/aquarium.js @@ -1779,7 +1779,7 @@ function initialize() { let xrViewport = glLayer.getViewport(view); gl.viewport(xrViewport.x, xrViewport.y, xrViewport.width, xrViewport.height); gl.scissor(xrViewport.x, xrViewport.y, xrViewport.width, xrViewport.height); - render(view.projectionMatrix, view.transform.inverse.matrix, false, view); // Not clear on what the final arg is used for here? Glancing at the code it looks like it should probably be view instead of pose. + render(view.projectionMatrix, view.transform.matrix, false, view); // Not clear on what the final arg is used for here? Glancing at the code it looks like it should probably be view instead of pose. } //renderStereo(pose.views[0].projectionMatrix, pose.views[1].projectionMatrix, viewInverseTemp, vrPose); // gl.clearColor(Math.cos(now / 2000), From 7c86198e2cb409e3f78ad188fb5ec4b918626202 Mon Sep 17 00:00:00 2001 From: Nikita Ostapenko Date: Mon, 18 Mar 2024 23:21:51 -0400 Subject: [PATCH 14/19] Fix VR button icon --- aquarium/aquarium.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/aquarium/aquarium.js b/aquarium/aquarium.js index f5b928bd..2f76740a 100755 --- a/aquarium/aquarium.js +++ b/aquarium/aquarium.js @@ -2079,6 +2079,7 @@ $(function(){ (function() { "use strict"; var vrButton; + var vrButtonURL; var stereoDemoButton; function getButtonContainer () { @@ -2173,10 +2174,12 @@ $(function(){ return path.substring(0, path.lastIndexOf('/')); } + vrButtonURL = getCurrentUrl() + "/../aquarium-vr/vr_assets/button.png"; + function onRequestPresent() { return navigator.xr.requestSession('immersive-vr').then((session) => { removeButton(vrButton); - vrButton = addButton("Exit VR", "E", getCurrentUrl() + "/vr_assets/button.png", onExitPresent); + vrButton = addButton("Exit VR", "E", vrButtonURL, onExitPresent); session.isImmersive = true; g_session = session; @@ -2211,7 +2214,7 @@ $(function(){ // enables the button if so. navigator.xr.isSessionSupported('immersive-vr').then((supported) => { if (supported) { - vrButton = addButton("Enter VR", "E", getCurrentUrl() + "/vr_assets/button.png", onRequestPresent); + vrButton = addButton("Enter VR", "E", vrButtonURL, onRequestPresent); g_vrUi = new Ui(gl, g_numFish); g_vrUi.load("./vr_assets/ui/config.js"); } @@ -2244,7 +2247,7 @@ $(function(){ // enables the button if so. navigator.xr.isSessionSupported('immersive-vr').then((supported) => { if (supported) { - vrButton = addButton("Enter VR", "E", getCurrentUrl() + "/vr_assets/button.png", onRequestPresent); + vrButton = addButton("Enter VR", "E", vrButtonURL, onRequestPresent); g_vrUi = new Ui(gl, g_numFish); g_vrUi.load("./vr_assets/ui/config.js"); } @@ -2252,7 +2255,7 @@ $(function(){ navigator.xr.addEventListener('devicechange', onDeviceChange); } // Regardless of if we have WebVR support, we can demonstrate stereo rendering inside the window. - stereoDemoButton = addButton("Toggle Stereo Demo", "", getCurrentUrl() + "/vr_assets/button.png", toggleStereoDemo); + stereoDemoButton = addButton("Toggle Stereo Demo", "", vrButtonURL, toggleStereoDemo); } window.addEventListener('resize', function() {onResize();}, false); onResize(); From e53df9b1d2e7ffb674f687f9ded3bcac0043d200 Mon Sep 17 00:00:00 2001 From: Nikita Ostapenko Date: Fri, 29 Mar 2024 22:42:24 -0400 Subject: [PATCH 15/19] Remove obsolette and unnecessary comments. --- aquarium/aquarium.js | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/aquarium/aquarium.js b/aquarium/aquarium.js index 2f76740a..d71fc6bc 100755 --- a/aquarium/aquarium.js +++ b/aquarium/aquarium.js @@ -1779,14 +1779,8 @@ function initialize() { let xrViewport = glLayer.getViewport(view); gl.viewport(xrViewport.x, xrViewport.y, xrViewport.width, xrViewport.height); gl.scissor(xrViewport.x, xrViewport.y, xrViewport.width, xrViewport.height); - render(view.projectionMatrix, view.transform.matrix, false, view); // Not clear on what the final arg is used for here? Glancing at the code it looks like it should probably be view instead of pose. + render(view.projectionMatrix, view.transform.matrix, false, view); } - //renderStereo(pose.views[0].projectionMatrix, pose.views[1].projectionMatrix, viewInverseTemp, vrPose); - // gl.clearColor(Math.cos(now / 2000), - // Math.cos(now / 4000), - // Math.cos(now / 6000), 1.0); - - // gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT); session.requestAnimationFrame(onXRFrame); } From a3c7a29788c07460c4daa2a4b384b2f7bb20e1dd Mon Sep 17 00:00:00 2001 From: Nikita Ostapenko Date: Sat, 30 Mar 2024 13:22:29 -0400 Subject: [PATCH 16/19] Add toggleable URL search param stereo-enabled. If set to true will render the stereo demo button Also changed stereo demo button comment to reflect this --- aquarium/aquarium.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/aquarium/aquarium.js b/aquarium/aquarium.js index d71fc6bc..f73729d1 100755 --- a/aquarium/aquarium.js +++ b/aquarium/aquarium.js @@ -46,6 +46,8 @@ var g_sceneGroups = {}; // the placement of the models var g_fog = true; var g_numFish = [1, 100, 500, 1000, 5000, 10000, 15000, 20000, 25000, 30000]; +const searchParams = new URLSearchParams(window.location.search); +var g_stereoDemoAvailable = searchParams.get("stereo-enabled")=="true" ?? false; var g_stereoDemoActive = false; var g_shadersNeedUpdate = false; // Set to true whenever the state has changed so that shaders may need to be changed. @@ -2248,8 +2250,10 @@ $(function(){ }); navigator.xr.addEventListener('devicechange', onDeviceChange); } - // Regardless of if we have WebVR support, we can demonstrate stereo rendering inside the window. - stereoDemoButton = addButton("Toggle Stereo Demo", "", vrButtonURL, toggleStereoDemo); + // Regardless of if we have WebVR support, we can demonstrate stereo rendering inside the window if enable-stereo param is included in url + if (g_stereoDemoAvailable) { + stereoDemoButton = addButton("Toggle Stereo Demo", "", vrButtonURL, toggleStereoDemo); + } } window.addEventListener('resize', function() {onResize();}, false); onResize(); From 7d2c7c794026366a11f42d7fad2d65fe4546a0d9 Mon Sep 17 00:00:00 2001 From: Nikita Ostapenko Date: Mon, 1 Apr 2024 22:54:59 -0400 Subject: [PATCH 17/19] Initialize eye position and adjust globals for VR --- aquarium/aquarium.js | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/aquarium/aquarium.js b/aquarium/aquarium.js index f73729d1..78ed1ffe 100755 --- a/aquarium/aquarium.js +++ b/aquarium/aquarium.js @@ -91,7 +91,7 @@ var g_lightRayOffset = Math.PI * 2 / g_numLightRays; var g_lightRayInfo = []; var g_session = null; var g_xrImmersiveRefSpace = null; -var g_onXRFrame = ()=>{}; +var g_startXRRendering = () => {}; var g_onAnimationFrame = () => {}; var g_ui = [ @@ -1706,6 +1706,15 @@ function initialize() { render(monoProjection, viewInverseTemp); } + function startXRRendering(now, frame) { + var eyeClock = 0; + eyePosition[0] = Math.sin(eyeClock) * g.globals.eyeRadius; + eyePosition[1] = g.globals.eyeHeight; + eyePosition[2] = Math.cos(eyeClock) * g.globals.eyeRadius; + + onXRFrame(now, frame); + } + function onXRFrame(now, frame) { let session = frame.session; let refSpace = g_xrImmersiveRefSpace; @@ -1786,7 +1795,7 @@ function initialize() { session.requestAnimationFrame(onXRFrame); } - g_onXRFrame = onXRFrame; + g_startXRRendering = startXRRendering; function onAnimationFrame() { var now = theClock.getTime(); @@ -2185,9 +2194,9 @@ $(function(){ session.requestReferenceSpace(refSpaceType).then((refSpace) => { g_xrImmersiveRefSpace = refSpace g_shadersNeedUpdate = true; - g.globals.eyeHeight = 15; - g.globals.eyeRadius = 1; - session.requestAnimationFrame(g_onXRFrame); + g.globals.eyeHeight = 150; + g.globals.eyeRadius = 10; + session.requestAnimationFrame(g_startXRRendering); }); session.addEventListener('end', onSessionEnded); }); From 2718d7b097f584f9df9c263f7f471bccb50a3194 Mon Sep 17 00:00:00 2001 From: Nikita Ostapenko Date: Tue, 2 Apr 2024 16:01:28 -0400 Subject: [PATCH 18/19] Add translationScale and floorAdjust to render(). translationScale adjusts vr mode movement to be close to real world. floorAdjust lifts eye position to the fish level. --- aquarium/aquarium.js | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/aquarium/aquarium.js b/aquarium/aquarium.js index 78ed1ffe..d5dc9be8 100755 --- a/aquarium/aquarium.js +++ b/aquarium/aquarium.js @@ -1247,7 +1247,8 @@ function initialize() { lightRay.setProgram(lightRay.programSet.getProgram(shadingSettings)); } - function render(projectionMatrix, viewInverseMatrix, useMultiview, pose) { + function render(projectionMatrix, viewInverseMatrix, useMultiview, pose, + translationScale, floorAdjust) { var genericConstInUse = useMultiview ? genericConstMultiview : genericConst; var outsideConstInUse = useMultiview ? outsideConstMultiview : outsideConst; var innerConstInUse = useMultiview ? innerConstMultiview : innerConst; @@ -1276,6 +1277,19 @@ function initialize() { fast.matrix4.copy(projection, projectionMatrix); } fast.matrix4.copy(viewInverse, viewInverseMatrix); + + if (translationScale != null) { + fast.matrix4.getTranslation(v3t0, viewInverse); + fast.mulScalarVector(v3t0, translationScale, v3t0); + + if (floorAdjust != null) { + v3t0[1] += floorAdjust; + } + + fast.matrix4.setTranslation(viewInverse, v3t0); + } + + // TODO: Support VRUI when using multiview? Would require adding multiview shaders to UI and changing UI matrices when multiview is on. if (!useMultiview && presentingVR && pose && g_vrUi) { // Hard coded FPS translation vector and pin the whole UI in front of the user in VR mode. This hard coded position @@ -1790,7 +1804,7 @@ function initialize() { let xrViewport = glLayer.getViewport(view); gl.viewport(xrViewport.x, xrViewport.y, xrViewport.width, xrViewport.height); gl.scissor(xrViewport.x, xrViewport.y, xrViewport.width, xrViewport.height); - render(view.projectionMatrix, view.transform.matrix, false, view); + render(view.projectionMatrix, view.transform.matrix, false, view, 10, 5); } session.requestAnimationFrame(onXRFrame); } From 4296838e4aa6c76ebfa52d52305979112de8130e Mon Sep 17 00:00:00 2001 From: Nikita Ostapenko Date: Tue, 2 Apr 2024 16:12:35 -0400 Subject: [PATCH 19/19] Reduce fish size a bit for VR mode. --- aquarium/aquarium.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/aquarium/aquarium.js b/aquarium/aquarium.js index d5dc9be8..85965737 100755 --- a/aquarium/aquarium.js +++ b/aquarium/aquarium.js @@ -1248,7 +1248,7 @@ function initialize() { } function render(projectionMatrix, viewInverseMatrix, useMultiview, pose, - translationScale, floorAdjust) { + translationScale, floorAdjust, fishScale) { var genericConstInUse = useMultiview ? genericConstMultiview : genericConst; var outsideConstInUse = useMultiview ? outsideConstMultiview : outsideConst; var innerConstInUse = useMultiview ? innerConstMultiview : innerConst; @@ -1289,6 +1289,9 @@ function initialize() { fast.matrix4.setTranslation(viewInverse, v3t0); } + if (fishScale == null) { + fishScale = 1; + } // TODO: Support VRUI when using multiview? Would require adding multiview shaders to UI and changing UI matrices when multiview is on. if (!useMultiview && presentingVR && pose && g_vrUi) { @@ -1403,7 +1406,7 @@ function initialize() { for (var ii = 0; ii < numFish; ++ii) { var fishClock = fishBaseClock + ii * fishOffset; var speed = fishSpeed + math.pseudoRandom() * fishSpeedRange; - var scale = 1.0 + math.pseudoRandom() * 1; + var scale = (1.0 + math.pseudoRandom() * 1) * fishScale; var xRadius = fishRadius + pseudoRandom() * fishRadiusRange; var yRadius = 2.0 + pseudoRandom() * fishHeightRange; var zRadius = fishRadius + pseudoRandom() * fishRadiusRange; @@ -1804,7 +1807,7 @@ function initialize() { let xrViewport = glLayer.getViewport(view); gl.viewport(xrViewport.x, xrViewport.y, xrViewport.width, xrViewport.height); gl.scissor(xrViewport.x, xrViewport.y, xrViewport.width, xrViewport.height); - render(view.projectionMatrix, view.transform.matrix, false, view, 10, 5); + render(view.projectionMatrix, view.transform.matrix, false, view, 10, 5, 0.75); } session.requestAnimationFrame(onXRFrame); }