Needle Engine 3.32.16-alpha

Files changed (12) hide show

src/engine-components/webxr/Avatar.ts +12 -1
src/engine/engine_element.ts +32 -4
src/engine/engine_time.ts +2 -2
src/engine/xr/NeedleXRController.ts +58 -23
src/engine/xr/NeedleXRSession.ts +48 -19
src/engine-components-experimental/networking/PlayerSync.ts +8 -2
src/engine-components/export/usdz/USDZExporter.ts +13 -0
src/engine-components/webxr/WebARSessionRoot.ts +26 -6
src/engine-components/webxr/WebXR.ts +1 -2
src/engine-components/webxr/WebXRButtons.ts +19 -0
src/engine-components/webxr/controllers/XRControllerModel.ts +56 -23
src/engine-components/webxr/controllers/XRControllerMovement.ts +16 -2

src/engine-components/webxr/Avatar.ts CHANGED Viewed

@@ -42,7 +42,9 @@
             marker.avatar = this.gameObject;
             marker.connectionId = playerstate.owner;
         }
-        else if(this.context.connection.isConnected) console.error("No player state found for avatar", this);
+        else if (this.context.connection.isConnected) console.error("No player state found for avatar", this);
+        // don't destroy the avatar when entering XR and not connected to a networking backend
+        else if (playerstate && !this.context.connection.isConnected) playerstate.dontDestroy = true;
     }
     onLeaveXR(_args: NeedleXREventArgs): void {
@@ -182,6 +184,9 @@
             this.head = new AssetReference("", this.sourceId, head);
             if (debug) console.log("Create head", head);
         }
+        else if (this.head instanceof Object3D) {
+            this.head = new AssetReference("", this.sourceId, this.head);
+        }
         if (!this.rightHand) {
             const rightHand = new Object3D();
@@ -190,6 +195,9 @@
             this.rightHand = new AssetReference("", this.sourceId, rightHand);
             if (debug) console.log("Create right hand", rightHand);
         }
+        else if (this.rightHand instanceof Object3D) {
+            this.rightHand = new AssetReference("", this.sourceId, this.rightHand);
+        }
         if (!this.leftHand) {
             const leftHand = new Object3D();
@@ -198,6 +206,9 @@
             this.leftHand = new AssetReference("", this.sourceId, leftHand);
             if (debug) console.log("Create left hand", leftHand);
         }
+        else if (this.leftHand instanceof Object3D) {
+            this.leftHand = new AssetReference("", this.sourceId, this.leftHand);
+        }
         await this.loadAvatarObjects(this.head, this.leftHand, this.rightHand);

src/engine/engine_element.ts CHANGED Viewed

@@ -346,10 +346,15 @@
             totalProgress01: this._loadingProgress01
         };
         const progressEvent = new CustomEvent("progress", { detail: progressEventDetail });
+        const displayNames = new Array<string>();
         const args: ContextCreateArgs = {
             files: filesToLoad,
             onLoadingProgress: evt => {
-                evt.name = getNameFromUrl(evt.name);
+                const index = evt.index;
+                if (!displayNames[index] && evt.name) {
+                    displayNames[index] = getDisplayName(evt.name);
+                }
+                evt.name = displayNames[index];
                 if (useDefaultLoading) this._loadingView?.onLoadingUpdate(evt);
                 progressEventDetail.name = evt.name;
                 progressEventDetail.progress = evt.progress;
@@ -500,7 +505,6 @@
     onEnterAR(session: XRSession) {
         this.onSetupAR();
         const overlayContainer = this.getAROverlayContainer();
-        console.log("onEnterAR", session, overlayContainer);
         this._overlay_ar.onBegin(this._context!, overlayContainer, session);
         this.dispatchEvent(new CustomEvent("enter-ar", { detail: { session: session, context: this._context, htmlContainer: this._overlay_ar?.ARContainer } }));
     }
@@ -615,12 +619,36 @@
     return hash;
 }
-function getNameFromUrl(str: string) {
+function getDisplayName(str: string) {
     const parts = str.split("/");
     let name = parts[parts.length - 1];
     // Remove params
     const index = name.indexOf("?")
     if (index > 0)
         name = name.substring(0, index);
-    return decodeURIComponent(name);
+    const extension = name.split(".").pop();
+    if (extension === "glb" || extension === "gltf")
+        name = name.substring(0, name.length - 4);
+    name = decodeURIComponent(name);
+    if (name.length > 3) {
+        let displayName = "";
+        for (let i = 0; i < name.length; i++) {
+            let c = name[i];
+            if (c === ' ' && displayName.length <= 0) continue;
+            const isFirstCharacter = displayName.length === 0;
+            if (isFirstCharacter == false && c === c.toUpperCase()) {
+                displayName += " " + c;
+            }
+            else {
+                if (isFirstCharacter) {
+                    c = c.toUpperCase();
+                }
+                displayName += c;
+            }
+        }
+        if (debug) console.log("displayName", name, displayName);
+        return displayName;
+    }
+    if (debug) console.log("displayName", name);
+    return name;
 }

src/engine/engine_time.ts CHANGED Viewed

@@ -46,8 +46,8 @@
         this.frame += 1;
         this.time += this.deltaTime;
-        if (this._fpsSamples.length < 30) this._fpsSamples.push(this.deltaTime);
-        else this._fpsSamples[(this._fpsSampleIndex++) % 30] = this.deltaTime;
+        if (this._fpsSamples.length < 60) this._fpsSamples.push(this.deltaTime);
+        else this._fpsSamples[(this._fpsSampleIndex++) % 60] = this.deltaTime;
         let sum = 0;
         for (let i = 0; i < this._fpsSamples.length; i++)
             sum += this._fpsSamples[i];

src/engine/xr/NeedleXRController.ts CHANGED Viewed

@@ -95,6 +95,8 @@
      * You should usually use the `getButton` and `getStick` methods instead to get access to named buttons and sticks
      */
     get gamepad() { return this.inputSource.gamepad; }
+    /** @returns true if this is a hand (otherwise this is a controller) */
+    get isHand() { return this.inputSource.hand != undefined; }
     /**
      * If this is a hand then this is the hand info (XRHand)
      * @link https://developer.mozilla.org/en-US/docs/Web/API/XRHand
@@ -138,6 +140,18 @@
         return this.xr.getHitTest(this);
     }
+    /** This is cleared at the beginning of each frame */
+    private readonly _handJointPoses: Map<XRJointSpace, XRJointPose> = new Map();
+    /** Get the hand joint pose from the current XRFrame. Results are cached for a frame to avoid calling getJointPose multiple times */
+    getHandJointPose(joint: XRJointSpace) {
+        if (!this.hand || !this.xr.frame?.getJointPose || !this.xr.referenceSpace) return null;
+        let pose = this._handJointPoses?.get(joint);
+        if (pose) return pose;
+        pose = this.xr.frame.getJointPose(joint, this.xr.referenceSpace);
+        if (pose) this._handJointPoses.set(joint, pose);
+        return pose;
+    }
     private readonly _gripPosition = new Vector3();
     private readonly _gripQuaternion = new Quaternion();
     private readonly _linearVelocity: Vector3 = new Vector3();
@@ -274,6 +288,9 @@
     }
     private onUpdateFrame(frame: XRFrame) {
+        // make sure this is cleared every frame
+        this._handJointPoses.clear();
         if (!this.xr.referenceSpace) {
             this._isTracking = false;
             return;
@@ -312,15 +329,13 @@
             // TODO check why types are not correct here
             // @ts-ignore
             const wrist = hand.get("wrist");
-            if (wrist && frame.getJointPose) {
-                const pose = frame.getJointPose(wrist, this.xr.referenceSpace);
-                if (pose) {
-                    gotWrist = true;
-                    const p = pose.transform.position;
-                    const q = pose.transform.orientation;
-                    this._object.position.set(p.x, p.y, p.z);
-                    this._object.quaternion.set(q.x, q.y, q.z, q.w).multiply(flipForwardQuaternion);
-                }
+            const writePose = wrist && this.getHandJointPose(wrist);
+            if (writePose) {
+                gotWrist = true;
+                const p = writePose.transform.position;
+                const q = writePose.transform.orientation;
+                this._object.position.set(p.x, p.y, p.z);
+                this._object.quaternion.set(q.x, q.y, q.z, q.w).multiply(flipForwardQuaternion);
             }
             if (!gotWrist) {
                 this._object.position.copy(this._rayPosition);
@@ -329,16 +344,14 @@
             //@ts-ignore
             const middle = hand.get("middle-finger-metacarpal");
-            if (middle && frame.getJointPose) {
-                const pose = frame.getJointPose(middle, this.xr.referenceSpace);
-                if (pose) {
-                    const p = pose.transform.position;
-                    const q = pose.transform.orientation;
-                    // for some reason the grip rotation is different from the wrist rotation
-                    // but we want to use the wrist rotation for the grip
-                    this._gripPosition.set(p.x, p.y, p.z);
-                    this._gripQuaternion.set(q.x, q.y, q.z, q.w);
-                }
+            const middlePose = middle && this.getHandJointPose(middle);
+            if (middlePose) {
+                const p = middlePose.transform.position;
+                const q = middlePose.transform.orientation;
+                // for some reason the grip rotation is different from the wrist rotation
+                // but we want to use the wrist rotation for the grip
+                this._gripPosition.set(p.x, p.y, p.z);
+                this._gripQuaternion.set(q.x, q.y, q.z, q.w);
             }
         }
         else {
@@ -687,11 +700,33 @@
     }
+    private _didMoveLastFrame = false;
+    private readonly _lastPointerMovePosition = new Vector3();
+    private readonly _lastPointerMoveQuaternion = new Quaternion();
     private onUpdateMove() {
-        let button = this.xr.context.input.getFirstPressedButtonForPointer(this.index);
-        if (button === undefined) button = 0;
-        const pressure = this.gamepad?.buttons[button]?.value;
-        this.emitPointerEvent("pointermove", button, "none", false, null, pressure);
+        let didMove = false;
+        const dist = this._lastPointerMovePosition.distanceTo(this.gripWorldPosition);
+        if (dist > .02) didMove = true;
+        if (!didMove) {
+            const angle = this._lastPointerMoveQuaternion.angleTo(this.gripWorldQuaternion);
+            if (angle > .02) didMove = true;
+        }
+        if (didMove) {
+            this._didMoveLastFrame = true;
+            this._lastPointerMovePosition.copy(this.gripWorldPosition);
+            this._lastPointerMoveQuaternion.copy(this.gripWorldQuaternion);
+            if (debug) Gizmos.DrawLabel(this.rayWorldPosition.add(this.object.worldForward.multiplyScalar(.1)), "move", .01);
+            let button = this.xr.context.input.getFirstPressedButtonForPointer(this.index);
+            if (button === undefined) button = 0;
+            const pressure = this.gamepad?.buttons[button]?.value;
+            this.emitPointerEvent("pointermove", button, "none", false, null, pressure);
+        }
+        else {
+            this._didMoveLastFrame = false;
+        }
     }

src/engine/xr/NeedleXRSession.ts CHANGED Viewed

@@ -1,4 +1,4 @@
-import { Camera, DoubleSide, Mesh, MeshBasicMaterial, Object3D, PerspectiveCamera, PlaneGeometry, PlaneHelper, Quaternion, Vector3, WebXRArrayCamera } from "three";
+import { Camera, Object3D, PerspectiveCamera, Quaternion, Vector3 } from "three";
 import { isDevEnvironment, showBalloonMessage, showBalloonWarning } from "../debug/index.js";
 import { Context, FrameEvent } from "../engine_context.js";
@@ -6,7 +6,6 @@
 import { isDestroyed } from "../engine_gameobject.js";
 import { Gizmos } from "../engine_gizmos.js";
 import { registerFrameEventCallback, unregisterFrameEventCallback } from "../engine_lifecycle_functions_internal.js";
-import { Mathf } from "../engine_math.js";
 import { getTempQuaternion, getTempVector, getWorldPosition, getWorldQuaternion, getWorldScale, setWorldPosition, setWorldQuaternion, setWorldScale } from "../engine_three_utils.js";
 import { ICamera, IComponent, INeedleXRSession } from "../engine_types.js";
 import { delay, getParam, isDesktop, isQuest } from "../engine_utils.js";
@@ -17,19 +16,24 @@
 import { TemporaryXRContext } from "./TempXRContext.js";
 import type { IXRRig } from "./XRRig.js";
+/** @link https://developer.mozilla.org/en-US/docs/Web/API/XRFrame/fillPoses */
+declare type FillPosesFunction = (spaces: IterableIterator<XRJointSpace>, referenceSpace: XRSpace, targetArray: Float32Array) => void;
+declare type NeedleXRFrame = XRFrame & { fillPoses?: FillPosesFunction };
 /** NeedleXRSession event argument.
  * Use `args.xr` to access the NeedleXRSession */
-export type NeedleXREventArgs = { xr: NeedleXRSession }
+export type NeedleXREventArgs = { readonly xr: NeedleXRSession }
 export type SessionChangedEvt = (args: NeedleXREventArgs) => void;
-export type SessionRequestedEvent = (args: { mode: XRSessionMode, init: XRSessionInit }) => void;
-export type SessionRequestedEndEvent = (args: { mode: XRSessionMode, init: XRSessionInit, newSession: XRSession | null }) => void;
+export type SessionRequestedEvent = (args: { readonly mode: XRSessionMode, readonly init: XRSessionInit }) => void;
+export type SessionRequestedEndEvent = (args: { readonly mode: XRSessionMode, readonly init: XRSessionInit, newSession: XRSession | null }) => void;
 /** Result of a XR hit-test
  * @property {XRHitTestResult} hit The original XRHitTestResult
  * @property {Vector3} position The hit position in world space
  * @property {Quaternion} quaternion The hit rotation in world space
  */
-export type NeedleXRHitTestResult = { hit: XRHitTestResult, position: Vector3, quaternion: Quaternion };
+export type NeedleXRHitTestResult = { readonly hit: XRHitTestResult, readonly position: Vector3, readonly quaternion: Quaternion };
 const debug = getParam("debugwebxr");
 const debugFPS = getParam("stats");
@@ -203,17 +207,28 @@
     }
     private static readonly _sessionRequestEndListeners: SessionRequestedEndEvent[] = [];
-    /** Listen to XR session started */
-    static onXRStart(evt: SessionChangedEvt) {
+    /** Listen to XR session started. Unsubscribe with `offXRSessionStart` */
+    static onXRSessionStart(evt: SessionChangedEvt) {
         this._xrStartListeners.push(evt);
     };
     /** Unsubscribe from XRSession started events */
-    static offXRStart(evt: SessionChangedEvt) {
+    static offXRSessionStart(evt: SessionChangedEvt) {
         const index = this._xrStartListeners.indexOf(evt);
         if (index >= 0) this._xrStartListeners.splice(index, 1);
     }
     private static readonly _xrStartListeners: SessionChangedEvt[] = [];
+    /** Listen to XR session ended. Unsubscribe with `offXRSessionEnd` */
+    static onXRSessionEnd(evt: SessionChangedEvt) {
+        this._xrEndListeners.push(evt);
+    };
+    /** Unsubscribe from XRSession started events */
+    static offXRSessionEnd(evt: SessionChangedEvt) {
+        const index = this._xrEndListeners.indexOf(evt);
+        if (index >= 0) this._xrEndListeners.splice(index, 1);
+    }
+    private static readonly _xrEndListeners: SessionChangedEvt[] = [];
     /** Listen to controller added events.
      * Events are cleared when starting a new session
      **/
@@ -452,7 +467,7 @@
      * The current XR frame
      * @link https://developer.mozilla.org/en-US/docs/Web/API/XRFrame
      */
-    get frame(): XRFrame { return this.context.xrFrame!; }
+    get frame(): NeedleXRFrame { return this.context.xrFrame! as NeedleXRFrame; }
     /** The currently active/connected controllers */
     readonly controllers: NeedleXRController[] = [];
@@ -820,6 +835,10 @@
         this.context.renderer.xr.enabled = false;
         this.context.mainCameraComponent?.applyClearFlags();
+        for (const listener of NeedleXRSession._xrEndListeners) {
+            listener({ xr: this });
+        }
         // make sure we disconnect all controllers
         for (let i = 0; i < this.controllers.length; i++) {
             this.disconnectInputSource(this.controllers[i].inputSource);
@@ -898,15 +917,6 @@
             this.updateActiveXRRig();
         }
-        if ((debug || debugFPS) && this.rig) {
-            const pos = this.rig.gameObject.worldPosition;
-            const forward = this.rig.gameObject.worldForward;
-            pos.add(forward.multiplyScalar(1.5));
-            const upwards = this.rig.gameObject.worldUp;
-            pos.add(upwards.multiplyScalar(2.5));
-            Gizmos.DrawLabel(pos, this.context.time.smoothedFps.toFixed(1));
-        }
         // make sure the camera is parented to the active rig
         if (this.rig && this._mainCamera?.gameObject) {
             const currentParent = this._mainCamera?.gameObject?.parent;
@@ -1037,11 +1047,30 @@
         this.sync?.onUpdate(this);
+        this.onRenderDebug();
+    }
+    private onRenderDebug() {
         if (debug) {
             for (const controller of this.controllers) {
                 controller.onRenderDebug();
             }
         }
+        if ((debug || debugFPS) && this.rig) {
+            const pos = this.rig.gameObject.worldPosition;
+            const forward = this.rig.gameObject.worldForward;
+            pos.add(forward.multiplyScalar(1.5));
+            const upwards = this.rig.gameObject.worldUp;
+            pos.add(upwards.multiplyScalar(2.5));
+            let debugLabel = "";
+            debugLabel += this.context.time.smoothedFps.toFixed(1);
+            if (debug) {
+                for (const ctrl of this.controllers) {
+                    debugLabel += `\n${ctrl.hand ? "hand" : "ctrl"} ${ctrl.inputSource.handedness}[${ctrl.index}] con:${ctrl.connected} tr:${ctrl.isTracking}`;
+                }
+            }
+            Gizmos.DrawLabel(pos, debugLabel);
+        }
     }
     private onBeforeRender = () => {

src/engine-components-experimental/networking/PlayerSync.ts CHANGED Viewed

@@ -180,6 +180,9 @@
     @syncField(PlayerState.prototype.onOwnerChange)
     owner?: string;
+    /** when enabled PlayerSync will not destroy itself when not connected anymore */
+    dontDestroy: boolean = false;
     get isLocalPlayer(): boolean {
         return this.owner === this.context.connection.connectionId;
     }
@@ -267,8 +270,11 @@
             // we could also do this in a timeout and check if the owner is still not assigned after a second (but that would be a hack)
             setTimeout(() => {
                 if (!this.destroyed && !this.owner) {
-                    if (debug) console.warn(`PlayerState.start → owner is still undefined: destroying \"${this.name}\" instance now`);
-                    this.doDestroy();
+                    if (!this.dontDestroy) {
+                        if (debug) console.warn(`PlayerState.start → owner is still undefined: destroying \"${this.name}\" instance now`);
+                        this.doDestroy();
+                    }
+                    else if (debug) console.warn("PlayerState.start → owner is still undefined but dontDestroy is set to true", this.name);
                 }
                 else console.log("PlayerState.start → owner is assigned", this.owner);
             }, 2000);

src/engine-components/export/usdz/USDZExporter.ts CHANGED Viewed

@@ -8,6 +8,7 @@
 import { Behaviour, GameObject } from "../../Component.js";
 import { Renderer } from "../../Renderer.js"
 import { WebARSessionRoot } from "../../webxr/WebARSessionRoot.js";
+import { NeedleWebXRHtmlElement } from "../../webxr/WebXRButtons.js";
 import { XRFlag, XRState, XRStateFlag } from "../../webxr/XRFlag.js";
 import type { IUSDExporterExtension } from "./Extension.js";
 import { AnimationExtension } from "./extensions/Animation.js"
@@ -76,6 +77,7 @@
     extensions: IUSDExporterExtension[] = [];
     private link!: HTMLAnchorElement;
+    private button?: HTMLButtonElement;
     start() {
         if (debug) {
@@ -113,6 +115,8 @@
         const ios = isiOS()
         const safari = isSafari();
         if (debug || (ios && safari)) {
+            this.button = this.createQuicklookButton();
             this.lastCallback = this.quicklookCallback.bind(this);
             this.link = ensureQuicklookLinkIsCreated(this.context);
             this.link.addEventListener('message', this.lastCallback);
@@ -124,6 +128,7 @@
     }
     onDisable() {
+        this.button?.remove();
         this.link?.removeEventListener('message', this.lastCallback);
         // const ios = isiOS()
         // const safari = isSafari();
@@ -407,4 +412,12 @@
             target.matrix.multiply(new Matrix4().makeRotationY(Math.PI));
         }
     }
+    private createQuicklookButton() {
+        const buttoncontainer = NeedleWebXRHtmlElement.getOrCreate(this.context);
+        const button = buttoncontainer.createQuicklookButton();
+        if(!button.parentNode) buttoncontainer.appendChild(button);
+        return button;
+    }
 }

src/engine-components/webxr/WebARSessionRoot.ts CHANGED Viewed

@@ -1,9 +1,10 @@
 import { AxesHelper, DoubleSide, Matrix4, Mesh, MeshBasicMaterial, Object3D, Plane, Quaternion, Raycaster, RingGeometry, Scene, Vector3 } from "three";
 import { isDevEnvironment, showBalloonWarning } from "../../engine/debug/index.js";
+import { AssetReference } from "../../engine/engine_addressables.js";
 import { Context } from "../../engine/engine_context.js";
 import { ObjectUtils, PrimitiveType } from "../../engine/engine_create_objects.js";
-import { destroy } from "../../engine/engine_gameobject.js";
+import { destroy, instantiate } from "../../engine/engine_gameobject.js";
 import { NEPointerEvent } from "../../engine/engine_input.js";
 import { serializable } from "../../engine/engine_serialization_decorator.js";
 import { IComponent, IGameObject } from "../../engine/engine_types.js";
@@ -40,6 +41,10 @@
     @serializable()
     invertForward: boolean = false;
+    /** When assigned this asset will be loaded and visualize the placement while in AR */
+    @serializable(AssetReference)
+    customReticle?: AssetReference;
     /** When enabled we will create a XR anchor for the scene placement
      * and make sure the scene is at that anchored point during a XR session */
     @serializable()
@@ -77,6 +82,10 @@
     /** user input is used for ar touch transform */
     private userInput?: WebXRSessionRootUserInput;
+    onEnable(): void {
+        this.customReticle?.preload();
+    }
     supportsXR(mode: XRSessionMode): boolean {
         return mode === "immersive-ar";
     }
@@ -216,17 +225,28 @@
         let reticle = this._reticle[i];
         if (!reticle) {
-            reticle = new Mesh(
-                new RingGeometry(0.07, 0.09, 32).rotateX(- Math.PI / 2),
-                new MeshBasicMaterial({ side: DoubleSide })
-            ) as any as IGameObject;
+            if (this.customReticle) {
+                if (this.customReticle.asset) {
+                    reticle = instantiate(this.customReticle.asset);
+                }
+                else {
+                    this.customReticle.loadAssetAsync();
+                    return;
+                }
+            }
+            else {
+                reticle = new Mesh(
+                    new RingGeometry(0.07, 0.09, 32).rotateX(- Math.PI / 2),
+                    new MeshBasicMaterial({ side: DoubleSide })
+                ) as any as IGameObject;
+                reticle.name = "AR Placement Reticle";
+            }
             if (debug) {
                 const axes = new AxesHelper(1);
                 axes.position.y += .01;
                 reticle.add(axes);
             }
             this._reticle[i] = reticle;
-            reticle.name = "AR Placement Reticle";
             reticle.matrixAutoUpdate = false;
             reticle.visible = false;
         }

src/engine-components/webxr/WebXR.ts CHANGED Viewed

@@ -264,8 +264,7 @@
      * @returns the Needle WebXR button container */
     getButtonsContainer(): NeedleWebXRHtmlElement {
         if (!this._container) {
-            this._container = NeedleWebXRHtmlElement.create();
-            this.context.domElement.shadowRoot?.appendChild(this._container);
+            this._container = NeedleWebXRHtmlElement.getOrCreate(this.context);
         }
         return this._container;
     }

src/engine-components/webxr/WebXRButtons.ts CHANGED Viewed

@@ -1,4 +1,5 @@
 import { isDevEnvironment } from "../../engine/debug/index.js";
+import { Context } from "../../engine/engine_context.js";
 import { generateQRCode } from "../../engine/engine_utils.js";
 import { isMozillaXR } from "../../engine/engine_utils.js";
 import { NeedleXRSession } from "../../engine/engine_xr.js";
@@ -15,6 +16,16 @@
         return document.createElement(webXRElementName) as NeedleWebXRHtmlElement;
     }
+    static getOrCreate(context: Context) {
+        const domElement = context.domElement;
+        let el = domElement.querySelector(webXRElementName);
+        if (!el) {
+            el = NeedleWebXRHtmlElement.create();
+            domElement.appendChild(el);
+        };
+        return el as NeedleWebXRHtmlElement;
+    }
     private readonly root: HTMLElement;
     constructor() {
@@ -310,6 +321,14 @@
             button.disabled = button["was-disabled"];
             // button.innerText = button["original-text"];
         });
+        NeedleXRSession.onXRSessionStart(_ => {
+            button["previous-display"] = button.style.display;
+            button.style.display = "none";
+        });
+        NeedleXRSession.onXRSessionEnd(_ => {
+            if (button["previous-display"] != undefined)
+                button.style.display = button["previous-display"];
+        });
     }
 }

src/engine-components/webxr/controllers/XRControllerModel.ts CHANGED Viewed

@@ -1,4 +1,4 @@
-import { AxesHelper, Group, Material, Mesh, Object3D } from "three";
+import { AxesHelper, Group, Material, Mesh, Object3D, XRHandSpace } from "three";
 import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js";
 import { XRControllerModelFactory } from "three/examples/jsm/webxr/XRControllerModelFactory.js";
 import { XRHandMeshModel } from "three/examples/jsm/webxr/XRHandMeshModel.js";
@@ -16,6 +16,9 @@
 const debug = getParam("debugwebxr");
+const handsJointBuffer = new Float32Array(16 * 25);
+const renderingUpdateTimings = new Array<number>();
 export class XRControllerModel extends Behaviour {
     @serializable()
@@ -75,8 +78,7 @@
                         this.scene.add(model);
                         // The controller mesh should by default inherit layers.
                         model.traverse(child => {
-                            child.layers.disableAll();
-                            child.layers.enable(2);
+                            child.layers.set(2);
                         });
                     }
                     else {
@@ -103,8 +105,31 @@
     onBeforeRender() {
         if (!NeedleXRSession.active) return;
-        const xr = NeedleXRSession.active;
+        if (debug) renderingUpdateTimings[0] = Date.now();
+        // update model
+        this.updateRendering(NeedleXRSession.active);
+        if (debug) {
+            const dt = Date.now() - renderingUpdateTimings[0];
+            renderingUpdateTimings.push(dt);
+            if (renderingUpdateTimings.length >= 30) {
+                renderingUpdateTimings[0] = 0;
+                const avrg = renderingUpdateTimings.reduce((a, b) => a + b, 0) / renderingUpdateTimings.length;
+                renderingUpdateTimings.length = 0;
+                console.log("[XRControllerModel] " + avrg.toFixed(2) + " ms");
+            }
+        }
+    }
+    onLeaveXR(_args: NeedleXREventArgs): void {
+        for (const entry of this._models) {
+            if (!entry) continue;
+            entry.model?.removeFromParent();
+        }
+        this._models = [];
+    }
+    private updateRendering(xr: NeedleXRSession) {
         for (let i = 0; i < this._models.length; i++) {
             const entry = this._models[i];
             if (!entry) continue;
@@ -130,19 +155,35 @@
             else if (ctrl.inputSource.hand && entry.handmesh) {
                 const referenceSpace = xr.referenceSpace;
                 const hand = this.context.renderer.xr.getHand(ctrl.index);
+                // if (referenceSpace && xr.frame.fillPoses) {
+                //     xr.frame.fillPoses(ctrl.inputSource.hand.values(), referenceSpace, handsJointBuffer);
+                //     let j = 0;
+                //     for (const space of ctrl.inputSource.hand.values()) {
+                //         const joint = hand.joints[space.jointName];
+                //         if (joint) {
+                //             joint.matrix.fromArray(handsJointBuffer, j * 16);
+                //             joint.matrix.decompose(joint.position, joint.quaternion, joint.scale);
+                //             joint.visible = true;
+                //         }
+                //         j++;
+                //     }
+                // }
+                // else
                 if (referenceSpace && xr.frame.getJointPose) {
                     for (const inputjoint of ctrl.inputSource.hand.values()) {
-                        // Update the joints groups with the XRJoint poses
-                        const jointPose = xr.frame.getJointPose(inputjoint, referenceSpace);
                         // The transform of this joint will be updated with the joint pose on each frame
                         const joint = hand.joints[inputjoint.jointName];
                         if (joint) {
+                            // Update the joints groups with the XRJoint poses
+                            const jointPose = ctrl.getHandJointPose(inputjoint);
                             if (jointPose) {
+                                // joint.matrixAutoUpdate = false;
+                                // joint.matrix.fromArray(jointPose.transform.matrix);
+                                // joint.matrix.decompose(joint.position, joint.quaternion, joint.scale);
                                 const { position, quaternion } = xr.convertSpace(jointPose.transform);
                                 joint.position.copy(position);
                                 joint.quaternion.copy(quaternion);
-                                joint.matrixWorldNeedsUpdate = true;
-                                // joint.jointRadius = jointPose.radius;
+                                joint.matrixWorldAutoUpdate = false;
                             }
                             joint.visible = jointPose != null;
                         }
@@ -150,24 +191,17 @@
                     // ensure that the hand renders in rig space
                     if (entry.model) {
                         entry.model.visible = ctrl.isTracking;
-                        if (entry.model.parent !== xr.rig?.gameObject) {
-                            entry.model.position.set(0, 0, 0);
+                        if (entry.model.visible && entry.model.parent !== xr.rig?.gameObject) {
                             xr.rig?.gameObject.add(entry.model);
                         }
+                        entry.model.position.set(0, 0, 0);
                     }
-                    entry.handmesh?.updateMesh();
+                    if (entry.model?.visible) entry.handmesh?.updateMesh();
                 }
             }
         }
     }
-    onLeaveXR(_args: NeedleXREventArgs): void {
-        for (const entry of this._models) {
-            if (!entry) continue;
-            entry.model?.removeFromParent();
-        }
-        this._models = [];
-    }
     protected async loadModel(controller: NeedleXRController, url: string): Promise<IGameObject | null> {
         if (!controller.connected) {
@@ -210,7 +244,7 @@
             else {
                 const basePath = customHand.uri.substring(0, customHand.uri.indexOf(expectedHandModelName));
                 loader.setPath(basePath);
-                if(debug) console.log("XRControllerModel: loading custom hand model from " + basePath);
+                if (debug) console.log("XRControllerModel: loading custom hand model from " + basePath);
             }
         }
@@ -221,13 +255,12 @@
         const handmesh = new XRHandMeshModel(handObject, hand, loader.path, controller.inputSource.handedness, loader, (object: Object3D) => {
             // The hand mesh should not receive raycasts
             object.traverseVisible(child => {
-                child.layers.disableAll();
-                child.layers.enable(2);
+                child.layers.set(2);
                 if (NeedleXRSession.active?.isPassThrough)
                     this.makeOccluder(child);
             });
             if (!controller.connected) {
-                if(debug) Gizmos.DrawLabel(controller.rayWorldPosition, "Hand is loaded but not connected anymore", .05, 5);
+                if (debug) Gizmos.DrawLabel(controller.rayWorldPosition, "Hand is loaded but not connected anymore", .05, 5);
                 object.removeFromParent();
             }
         });
@@ -250,7 +283,7 @@
             }
         }
         else {
-            if(debug) {
+            if (debug) {
                 Gizmos.DrawLabel(controller.rayWorldPosition, "No inputSource.hand found for index " + controller.index, .05, 5);
             }
         }

src/engine-components/webxr/controllers/XRControllerMovement.ts CHANGED Viewed

@@ -204,16 +204,29 @@
         for (let i = 0; i < session.controllers.length; i++) {
             const ctrl = session.controllers[i];
             if (!ctrl.connected || !ctrl.isTracking) continue;
+            // save performance by only raycasting every nth frame
+            if (this.context.time.frame % 2 !== 0) {
+                const disc = this._hitDiscs[i];
+                // if the disc had a hit last frame, we can show it again
+                if (disc && disc["hit"]) disc.visible = true;
+                continue;
+            }
             const hit = this.context.physics.raycastFromRay(ctrl.ray, { testObject: this.hitPointRaycastFilter })[0];
             this._hitDistances[i] = hit?.distance;
+            let disc = this._hitDiscs[i];
+            if (disc) // save the hit object on the disc
+                disc["hit"] = hit;
             if (hit) {
                 const rigScale = (session.rigScale ?? 1);
                 if (debug) {
-                    Gizmos.DrawWireSphere(hit.point, .025 * rigScale, 0xff0000, .2);
+                    Gizmos.DrawWireSphere(hit.point, .025 * rigScale, 0xff0000);
                     Gizmos.DrawLabel(getTempVector(0, .2, 0).add(hit.point), hit.object.name, .02, 0);
                 }
-                let disc = this._hitDiscs[i];
                 if (!disc) {
                     disc = this.createHitPointObject();
                     this._hitDiscs[i] = disc;
@@ -223,6 +236,7 @@
                 disc.scale.set(size, size, size);
                 disc.layers.disableAll();
                 disc.layers.enable(2);
+                disc["hit"] = hit;
                 if (hit.normal) {
                     const factor = 0.02 * rigScale;