Needle Engine 3.32.15-alpha

Files changed (11) hide show

src/engine-components/ui/Button.ts +7 -5
src/engine-components/ui/Canvas.ts +9 -2
src/engine-components/DragControls.ts +10 -0
src/engine/engine_context.ts +1 -1
src/engine/engine_gizmos.ts +9 -8
src/engine/engine_types.ts +3 -1
src/engine-components/ui/EventSystem.ts +57 -20
src/engine/xr/NeedleXRController.ts +106 -6
src/engine-components/export/usdz/ThreeUSDZExporter.ts +3 -1
src/engine-components/webxr/controllers/XRControllerModel.ts +16 -4
src/engine-components/webxr/controllers/XRControllerMovement.ts +8 -13

src/engine-components/ui/Button.ts CHANGED Viewed

@@ -65,12 +65,12 @@
     @serializable(EventList)
     onClick?: EventList;
-    private _isHovered: boolean = false;
+    private _isHovered: number = 0;
     onPointerEnter(_) {
+        this._isHovered += 1;
         if (debug)
-            console.log("Button Enter", this.animationTriggers?.highlightedTrigger, this.animator);
-        this._isHovered = true;
+            console.warn("Button Enter", this._isHovered, this.animationTriggers?.highlightedTrigger, this.animator);
         if (!this.interactable) return;
         if (this.transition == Transition.Animation && this.animationTriggers && this.animator) {
             this.animator.setTrigger(this.animationTriggers.highlightedTrigger);
@@ -82,10 +82,12 @@
     }
     onPointerExit() {
+        this._isHovered -= 1;
         if (debug)
-            console.log("Button Exit", this.animationTriggers?.highlightedTrigger, this.animator);
-        this._isHovered = false;
+            console.log("Button Exit", this._isHovered, this.animationTriggers?.highlightedTrigger, this.animator);
         if (!this.interactable) return;
+        if (this._isHovered > 0) return;
+        this._isHovered = 0;
         if (this.transition == Transition.Animation && this.animationTriggers && this.animator) {
             this.animator.setTrigger(this.animationTriggers.normalTrigger);
         }

src/engine-components/ui/Canvas.ts CHANGED Viewed

@@ -4,7 +4,7 @@
 import { Mathf } from "../../engine/engine_math.js";
 import { serializable } from "../../engine/engine_serialization_decorator.js";
 import { FrameEvent } from "../../engine/engine_setup.js";
-import { getParam } from "../../engine/engine_utils.js";
+import { delayForFrames, getParam } from "../../engine/engine_utils.js";
 import { NeedleXREventArgs } from "../../engine/xr/index.js";
 import { Camera } from "../Camera.js";
 import { GameObject } from "../Component.js";
@@ -202,12 +202,19 @@
         }
     }
-    onEnterXR(args: NeedleXREventArgs): void {
+    async onEnterXR(args: NeedleXREventArgs) {
+        // workaround for https://linear.app/needle/issue/NE-4114
         if (this.screenspace) {
             if (args.xr.isVR || args.xr.isPassThrough) {
                 this.gameObject.visible = false;
             }
         }
+        else {
+            this.gameObject.visible = false;
+            await delayForFrames(1).then(()=>{
+                this.gameObject.visible = true;
+            });
+        }
     }
     onLeaveXR(args: NeedleXREventArgs): void {
         if (this.screenspace) {

src/engine-components/DragControls.ts CHANGED Viewed

@@ -451,6 +451,16 @@
     private _initialDistance: number = 0;
     private alignManipulator() {
+        if (!this.handlerA || !this.handlerB) {
+            console.error("alignManipulator called on MultiTouchDragHandler without valid handlers. This is likely a bug.", this);
+            return;
+        }
+        if (!this.handlerA.followObject || !this.handlerB.followObject) {
+            console.error("alignManipulator called on MultiTouchDragHandler without valid follow objects. This is likely a bug.", this.handlerA, this.handlerB);
+            return;
+        }
         this._tempVec1.copy(this._handlerAAttachmentPoint);
         this._tempVec2.copy(this._handlerBAttachmentPoint);
         this.handlerA.followObject.localToWorld(this._tempVec1);

src/engine/engine_context.ts CHANGED Viewed

@@ -1005,7 +1005,7 @@
 				if ((isDevEnvironment() || debug) && (err instanceof Error || err instanceof TypeError))
 					showBalloonMessage(`Caught unhandled exception during render-loop - see console for details.`, LogType.Error);
 				console.error("Frame #" + this.time.frame + "\n", err);
-				if (this._renderlooperrors > 10) {
+				if (this._renderlooperrors >= 3) {
 					console.warn("Stopping render loop due to error")
 					this.renderer.setAnimationLoop(null);
 				}

src/engine/engine_gizmos.ts CHANGED Viewed

@@ -208,12 +208,12 @@
             width: "auto",
             fontSize: size,
             color: color,
-            lineHeight: .75,
+            lineHeight: 1,
             backgroundColor: backgroundColor ?? undefined,
             backgroundOpacity: opacity,
             textContent: text,
-            borderRadius: 1 * size,
-            padding: 1 * size,
+            borderRadius: .5 * size,
+            padding: .8 * size,
             whiteSpace: 'pre',
         };
@@ -232,7 +232,6 @@
             // handle.setText(text);
         }
         this.tmuiNeedsUpdate = true;
-        element.layers.enableAll();
         this.registerTimedObject(Context.Current, element as any, duration, this.textLabelCache as any);
         return element as Text & LabelHandle;
     }
@@ -320,8 +319,10 @@
             context.post_render_callbacks.push(postRender);
         }
-        object.layers.disableAll();
-        object.layers.enable(2);
+        object.traverse(obj => {
+            obj.layers.disableAll();
+            obj.layers.enable(2);
+        });
         object.renderOrder = 999999;
         object[$cacheSymbol] = cache;
@@ -349,7 +350,7 @@
                     continue;
                 }
                 const isInXR = ctx.isInVR;
-                const keepUp = isInXR;
+                const keepUp = false;
                 const copyRotation = !isInXR;
                 lookAtObject(obj as any, ctx.mainCamera, keepUp, copyRotation);
             }
@@ -364,7 +365,7 @@
                 objects.splice(i, 1);
                 times.splice(i, 1);
                 obj.removeFromParent();
-                if (isDestroyed(obj) == false) {
+                if (isDestroyed(obj) != true) {
                     const cache = obj[$cacheSymbol];
                     cache.push(obj);
                 }

src/engine/engine_types.ts CHANGED Viewed

@@ -526,5 +526,7 @@
 export type XRControllerButtonName = "thumbrest" | "xr-standard-trigger" | "xr-standard-squeeze" | "xr-standard-thumbstick" | "xr-standard-touchpad" | "menu" | GamepadButtonName;
+export type XRGestureName = "pinch";
 /** All known (types) button names for various devices and cases combined. You should use the device specific names if you e.g. know you only deal with a mouse use MouseButtonName  */
-export type ButtonName = "unknown" | MouseButtonName | GamepadButtonName | XRControllerButtonName;
+export type ButtonName = "unknown" | MouseButtonName | GamepadButtonName | XRControllerButtonName | XRGestureName;

src/engine-components/ui/EventSystem.ts CHANGED Viewed

@@ -28,6 +28,8 @@
     hasActiveUI: boolean
 }
+declare type IComponentCanMaybeReceiveEvents = IPointerEventHandler & IComponent & { interactable?: boolean };
 export class EventSystem extends Behaviour {
     private static _eventSystemMap = new Map<Context, EventSystem[]>();
@@ -340,7 +342,7 @@
         // thus is not hovering over anything
         const hoveredData = this.hoveredByID.get(args.pointerId);
         if (hoveredData) {
-            this.triggerOnExit(hoveredData.obj, hoveredData.data, null);
+            this.propagatePointerExit(hoveredData.obj, hoveredData.data, null);
         }
         this.hoveredByID.delete(args.pointerId);
@@ -444,7 +446,7 @@
         // trigger onPointerExit
         if (isNewlyHovering && prevHovering) {
-            this.triggerOnExit(prevHovering, hovering.data, object);
+            this.propagatePointerExit(prevHovering, hovering.data, object);
         }
         // save hovered object
@@ -513,35 +515,29 @@
             case "hand":
                 // for hands and controller we assume they are never totally still (except for simulated environments)
                 // we might want to add a threshold here (e.g. if a user holds their hand very still or controller)
-                // so maybe check the angle every frame?
+                // so maybe check the angle everxy frame?
                 break;
         }
         this.propagate(object, (behaviour) => {
-            const comp = behaviour as any;
+            const comp = behaviour as IComponentCanMaybeReceiveEvents;
             if (comp.interactable === false) return;
             if (!comp.activeAndEnabled || !comp.enabled) return;
             if (comp.onPointerEnter) {
                 if (hoveredObjectChanged) {
-                    if (!comp[this.pointerEnterSymbol]) {
-                        comp[this.pointerEnterSymbol] = true;
-                        delete comp[this.pointerExitSymbol];
-                        comp.onPointerEnter(args);
-                    }
+                    this.handlePointerEnter(comp, args);
                 }
             }
             if (args.isDown) {
                 if (comp.onPointerDown) {
                     comp.onPointerDown(args);
                     // Set the handler that we called the down event on
                     // So we can call the up event on the same handler
                     // In a scenario where we Down on one object and Up on another
                     pressedEvent?.handlers.add(comp);
                     this.handlePointerCapture(args, comp);
                 }
             }
@@ -582,11 +578,8 @@
         }
     }
-    /**
-     * Propagate up in hiearchy and call OnExit regardless of the pointer event data
-     */
-    private triggerOnExit(object: Object3D, args: PointerEventData, newObject: Object3D | null) {
+    /** Propagate up in hierarchy and call onPointerExit */
+    private propagatePointerExit(object: Object3D, args: PointerEventData, newObject: Object3D | null) {
         this.propagate(object, (behaviour) => {
             if (!behaviour.gameObject || behaviour.destroyed) return;
@@ -596,10 +589,7 @@
                 if (newObject && this.isChild(newObject, behaviour.gameObject)) {
                     return;
                 }
-                if (inst[this.pointerExitSymbol]) return;
-                inst[this.pointerExitSymbol] = true;
-                delete inst[this.pointerEnterSymbol];
-                inst.onPointerExit(args);
+                this.handlePointerExit(inst, args);
             }
         });
     }
@@ -610,6 +600,53 @@
         this.releasePointerCapture(evt, handler);
     }
+    /** Responsible for invoking onPointerEnter (and updating onPointerExit). We invoke onPointerEnter once per active pointerId */
+    private handlePointerEnter(comp: IComponentCanMaybeReceiveEvents, args: PointerEventData) {
+        if (comp.onPointerEnter) {
+            if (this.updatePointerState(comp, args.pointerId, this.pointerEnterSymbol, true)) {
+                comp.onPointerEnter(args);
+            }
+        }
+        this.updatePointerState(comp, args.pointerId, this.pointerExitSymbol, false);
+    }
+    /** Responsible for invoking onPointerExit (and updating onPointerEnter). We invoke onPointerExit once per active pointerId */
+    private handlePointerExit(comp: IComponentCanMaybeReceiveEvents, evt: PointerEventData) {
+        if (comp.onPointerExit) {
+            if (this.updatePointerState(comp, evt.pointerId, this.pointerExitSymbol, true)) {
+                comp.onPointerExit(evt);
+            }
+        }
+        this.updatePointerState(comp, evt.pointerId, this.pointerEnterSymbol, false);
+    }
+    /** updates the pointer state list for a component
+     * @param comp the component to update
+     * @param pointerId the pointerId to update
+     * @param symbol the symbol to use for the state
+     * @param add if true, the pointerId is added to the state list, if false the pointerId will be removed
+     */
+    private updatePointerState(comp: IComponentCanMaybeReceiveEvents, pointerId: number, symbol: symbol, add: boolean) {
+        let state = comp[symbol];
+        if (add) {
+            // the pointer is already in the state list
+            if (state && state.includes(pointerId)) return false;
+            state = state || [];
+            state.push(pointerId);
+            comp[symbol] = state;
+            return true;
+        }
+        else {
+            if (!state || !state.includes(pointerId)) return false;
+            const i = state.indexOf(pointerId);
+            if (i !== -1) {
+                state.splice(i, 1);
+            }
+            return true;
+        }
+    }
     /** the list of component handlers that requested pointerCapture for a specific pointerId */
     private readonly _capturedPointer: { [id: number]: IPointerEventHandler[] } = {};

src/engine/xr/NeedleXRController.ts CHANGED Viewed

@@ -1,17 +1,26 @@
 import { fetchProfile, MotionController } from "@webxr-input-profiles/motion-controllers";
 import { AxesHelper, Int8BufferAttribute, Object3D, Quaternion, Ray, Vector3 } from "three";
+import { RGBAColor } from "../../engine-components/js-extensions/RGBAColor.js";
 import { Context } from "../engine_context.js";
 import { Gizmos } from "../engine_gizmos.js";
 import { InputEventNames, InputEvents, NEPointerEvent, NEPointerEventInit, PointerType } from "../engine_input.js";
 import { getTempQuaternion, getTempVector, getWorldQuaternion } from "../engine_three_utils.js";
-import type { ButtonName, IGameObject, Vec3, XRControllerButtonName } from "../engine_types.js";
+import type { ButtonName, IGameObject, Vec3, XRControllerButtonName, XRGestureName } from "../engine_types.js";
 import { getParam } from "../engine_utils.js";
 import { flipForwardMatrix, flipForwardQuaternion } from "./internal.js";
 import type { NeedleXRHitTestResult, NeedleXRSession } from "./NeedleXRSession.js";
 const debug = getParam("debugwebxr");
+/** when enabled we will not use the browser select event but instead
+ * we will emit the input event based on our own pinch detection
+ * this is a workaround for visionOS not emitting the select events, see https://linear.app/needle/issue/NE-4212
+ */
+const debugCustomGesture = getParam("debugcustomgesture");
+/** true when selectstart was ever received */
+let _didReceiveSelectStartEvent = false;
 // https://github.com/immersive-web/webxr-input-profiles/blob/4484a05e30bcd43fe86bb4e06b7a707861a26796/packages/registry/profiles/meta/meta-quest-touch-plus.json
 declare type ControllerAxes = "xr-standard-thumbstick";
 declare type StickName = "xr-standard-thumbstick";
@@ -62,6 +71,7 @@
 export class NeedleXRController {
     /** the Needle XR Session */
     readonly xr: NeedleXRSession;
+    get context() { return this.xr.context; }
     /**
      * https://developer.mozilla.org/en-US/docs/Web/API/XRInputSource
      */
@@ -90,6 +100,10 @@
      * @link https://developer.mozilla.org/en-US/docs/Web/API/XRHand
      */
     get hand() { return this.inputSource.hand; }
+    /** threejs XRHandSpace, shorthand for `context.renderer.xr.getHand(controllerIndex)`
+     * @link https://threejs.org/docs/#api/en/renderers/webxr/WebXRManager.getHand
+    */
+    get handObject() { return this.context.renderer.xr.getHand(this.index); }
     /** The input source profiles */
     get profiles() { return this.inputSource.profiles; }
     /** The device input layout */
@@ -200,7 +214,7 @@
     get object() { return this._object; }
     private readonly _object: IGameObject;
-    private readonly _debugAxesHelper = new AxesHelper(.03);
+    private readonly _debugAxesHelper = new AxesHelper(.2);
     /** returns the URL of the default controller model */
     async getModelUrl(): Promise<string | null> {
@@ -250,6 +264,13 @@
     onRenderDebug() {
         Gizmos.DrawWireSphere(this.rayWorldPosition, .02);
         Gizmos.DrawDirection(this.rayWorldPosition, getTempVector(0, 0, 10).applyQuaternion(this.rayWorldQuaternion));
+        const debugLabelPosition = this.gripWorldPosition.sub(this.object.worldForward.multiplyScalar(.1));
+        const profileStr = this.inputSource.profiles.join("\n");
+        let debugStr = `Controller[${this.index}] ${this.side}
+C:${this.connected ? "yes" : "no"} T:${this.isTracking ? "yes" : "no"} Hand:${this.inputSource.hand ? "yes" : "no"}`;
+        if (this.inputSource.hand) debugStr += `\nPinch: ${this.getGesture("pinch")?.value.toFixed(3)}`;
+        debugStr += "\n" + profileStr;
+        Gizmos.DrawLabel(debugLabelPosition, debugStr, .01);
     }
     private onUpdateFrame(frame: XRFrame) {
@@ -345,7 +366,7 @@
     /** Called when the input source disconnects */
     onDisconnected() {
-        if (this.connected) return;
+        if (debug) console.warn("Controller disconnected", this.index);
         // move all attached objects into the scene
         for (const child of this._object.children) {
             this.xr.context.scene.attach(child);
@@ -398,7 +419,21 @@
         return undefined;
     }
-    private readonly _needleGamepadButtons = new Array<NeedleGamepadButton>();
+    /** Get a gesture state */
+    getGesture(key: XRGestureName): NeedleGamepadButton | null {
+        const state = this.states[key];
+        if (!state) return null;
+        this.states[key] = state;
+        const needleButton = this._needleGamepadButtons[key] || new NeedleGamepadButton();
+        needleButton.pressed = state.pressed;
+        needleButton.value = state.value;
+        needleButton.isDown = state.isDown;
+        needleButton.isUp = state.isUp;
+        this._needleGamepadButtons[key] = needleButton;
+        return needleButton;
+    }
+    private readonly _needleGamepadButtons: { [key: number | string]: NeedleGamepadButton } = {};
     /** combine the InputState information + the GamepadButton information (since GamepadButtons can not be extended) */
     private toNeedleGamepadButton(index: number): NeedleGamepadButton {
         const button = this.inputSource.gamepad?.buttons[index];
@@ -453,7 +488,6 @@
         return { x: 0, y: 0, z: 0 }
     }
     private readonly _buttonMap = new Map<ButtonName, number>();
     // the motion controller contains the controller scheme, we use this to simplify button access
@@ -520,10 +554,21 @@
         const selectComponentId = this._layout?.selectComponentId;
         const i = this._layout?.components[selectComponentId!]?.gamepadIndices?.button;
         if (i !== undefined) this._selectButtonIndex = i;
+        if (debugCustomGesture) return;
+        if (!_didReceiveSelectStartEvent) {
+            _didReceiveSelectStartEvent = true;
+            // safeguard first pinch event - check if the pinch gesture is already down
+            const pinch = this.getGesture("pinch");
+            if (pinch?.pressed) {
+                console.warn("Select start event was received but the pinch gesture is already down. This might happen the first time you start pinching", this.index, this.side);
+                return;
+            }
+        }
         if (debug) Gizmos.DrawDirection(this.rayWorldPosition, getTempVector(0, .01, 1).applyQuaternion(this.rayWorldQuaternion), 0xff0000, 10);
         this.emitPointerEvent(InputEvents.PointerDown, this._selectButtonIndex || 0, "xr-standard-trigger", true, evt);
     }
     private onSelectEnd = (evt: XRInputSourceEvent) => {
+        if (debugCustomGesture) return;
         if (this.inputSource !== evt.inputSource) return;
         this.emitPointerEvent(InputEvents.PointerUp, this._selectButtonIndex || 0, "xr-standard-trigger", true, evt);
     }
@@ -542,7 +587,7 @@
     };
     /** Index = button index */
-    private readonly states = new Array<InputState>();
+    private readonly states: { [key: number | string]: InputState } = {};
     // If we want to invoke button events for ALL buttons we need to keep track of the previous state
     // instead of using XR input select start events which is only raised for the primary button
     // we should probably do both but then we need to ignore the primary index in the following function (to not raise an event for the same button twice)
@@ -583,11 +628,65 @@
                 if (eventName != null && emitEvent) {
                     const name = this._layout?.gamepad[k];
+                    if (debug || debugCustomGesture) console.log("Emitting pointer event", eventName, k, name, button.value, this.gamepad, this._layout);
                     this.emitPointerEvent(eventName, k, name ?? "none", false, null, button.value);
                 }
             }
         }
+        // update hand gesture states
+        if (this.hand) {
+            const handObject = this.handObject;
+            if (handObject) {
+                // update pinch state
+                const indexTip = handObject.joints["index-finger-tip"];
+                const thumbTip = handObject.joints["thumb-tip"];
+                if (indexTip && thumbTip) {
+                    const pinchThreshold = .02;
+                    const pinchHysteresis = .01;
+                    const distance = indexTip.position.distanceTo(thumbTip.position);
+                    const state = this.states["pinch"] || new InputState();
+                    state.value = distance;
+                    const isPressed = distance < (pinchThreshold - pinchHysteresis);
+                    const isReleased = distance > (pinchThreshold + pinchHysteresis);
+                    if (isPressed && !state.pressed) {
+                        if (debugCustomGesture) console.log("pinch start", distance);
+                        state.isDown = true;
+                        state.isUp = false;
+                        state.pressed = true;
+                    }
+                    else if (isReleased && state.pressed) {
+                        state.isDown = false;
+                        state.isUp = true;
+                        state.pressed = false;
+                    }
+                    else {
+                        state.isDown = false;
+                        state.isUp = false;
+                    }
+                    this.states["pinch"] = state;
+                    /** workaround for visionOS not emitting selectstart. See https://linear.app/needle/issue/NE-4212
+                     * If a select start event was never received we do a manual check here if the user is pinching
+                    */
+                    if (!_didReceiveSelectStartEvent && (state.isDown || state.isUp)) {
+                        const eventName = isPressed ? "pointerdown" : "pointerup";
+                        const pressure = distance / pinchThreshold;
+                        if (debugCustomGesture) {
+                            const p = this.rayWorldPosition.add(this.object.worldForward.multiplyScalar(.2));
+                            p.y += .05;
+                            p.y += Math.random() * .02;
+                            Gizmos.DrawLabel(p, "pinch:" + eventName + ", " + this.index + ", " + this.side + "\n" + handObject.uuid, 0.01, 5, 0x000000, new RGBAColor(1, 1, 1, .1));
+                        }
+                        this.emitPointerEvent(eventName, 0, "pinch", false, null, pressure);
+                    }
+                }
+            }
+        }
     }
     private onUpdateMove() {
         let button = this.xr.context.input.getFirstPressedButtonForPointer(this.index);
         if (button === undefined) button = 0;
@@ -622,6 +721,7 @@
             const prevContext = Context.Current;
             Context.Current = this.xr.context;
+            if (debug && type !== "pointermove") console.warn("Pointer event", type, button, buttonName, { ...this.pointerInit });
             this.xr.context.input.createInputEvent(new NEPointerEvent(type, source, this.pointerInit));
             Context.Current = prevContext;
         }

src/engine-components/export/usdz/ThreeUSDZExporter.ts CHANGED Viewed

@@ -1081,7 +1081,9 @@
 	if ( geometry ) {
 		writer.beginBlock( `def ${objType} "${name}"`, "(", false );
-		if (context.quickLookCompatible && material && material.side === DoubleSide)
+		// NE-4084: To use the doubleSided workaround with skeletal meshes we'd have to
+		// also emit extra data for jointIndices etc., so we're skipping skinned meshes here.
+		if (context.quickLookCompatible && material && material.side === DoubleSide && !isSkinnedMesh)
 			writer.appendLine(`prepend references = @./geometries/Geometry_${geometry.id}.usd@</Geometry_doubleSided>`);
 		else
 			writer.appendLine(`prepend references = @./geometries/Geometry_${geometry.id}.usd@</Geometry>`);

src/engine-components/webxr/controllers/XRControllerModel.ts CHANGED Viewed

@@ -6,6 +6,7 @@
 import { showBalloonWarning } from "../../../engine/debug/index.js";
 import { AssetReference } from "../../../engine/engine_addressables.js";
 import { setDontDestroy } from "../../../engine/engine_gameobject.js";
+import { Gizmos } from "../../../engine/engine_gizmos.js";
 import { addDracoAndKTX2Loaders } from "../../../engine/engine_loaders.js";
 import { serializable } from "../../../engine/engine_serialization_decorator.js";
 import { IGameObject } from "../../../engine/engine_types.js";
@@ -58,7 +59,8 @@
                         res?.handmesh?.controller?.removeFromParent();
                         return;
                     }
-                    this._models[controller.index] = { controller: controller, model: res.handObject, handmesh: res.handmesh };
+                    this._models.push({ controller: controller, model: res.handObject, handmesh: res.handmesh });
+                    this._models.sort((a, b) => a.controller.index - b.controller.index);
                     this.scene.add(res.handObject);
                 }
             }
@@ -68,7 +70,8 @@
                     if (assetUrl) {
                         const model = await this.loadModel(controller, assetUrl);
                         if (!model || !controller.connected) return;
-                        this._models[controller.index] = { controller: controller, model };
+                        this._models.push({ controller: controller, model });
+                        this._models.sort((a, b) => a.controller.index - b.controller.index);
                         this.scene.add(model);
                         // The controller mesh should by default inherit layers.
                         model.traverse(child => {
@@ -187,6 +190,10 @@
         const context = this.context;
         const hand = context.renderer.xr.getHand(controller.index);
+        if (!hand) {
+            if (debug) Gizmos.DrawLabel(controller.rayWorldPosition, "No hand found for index " + controller.index, .05, 5);
+            else console.warn("No hand found for index " + controller.index);
+        }
         const loader = new GLTFLoader();
         addDracoAndKTX2Loaders(loader, context);
@@ -220,6 +227,7 @@
                     this.makeOccluder(child);
             });
             if (!controller.connected) {
+                if(debug) Gizmos.DrawLabel(controller.rayWorldPosition, "Hand is loaded but not connected anymore", .05, 5);
                 object.removeFromParent();
             }
         });
@@ -229,9 +237,7 @@
         if (controller.inputSource.hand) {
             if (debug) console.log(controller.inputSource.hand);
             for (const inputjoint of controller.inputSource.hand.values()) {
                 if (hand.joints[inputjoint.jointName] === undefined) {
                     const joint = new Group();
                     joint.matrixAutoUpdate = false;
                     joint.visible = true;
@@ -243,6 +249,12 @@
                 }
             }
         }
+        else {
+            if(debug) {
+                Gizmos.DrawLabel(controller.rayWorldPosition, "No inputSource.hand found for index " + controller.index, .05, 5);
+            }
+        }
         return { handObject: handObject as IGameObject, handmesh: handmesh };
     }

src/engine-components/webxr/controllers/XRControllerMovement.ts CHANGED Viewed

@@ -122,10 +122,6 @@
             const dir = rotationInput > 0 ? 1 : -1;
             rig.rotateY(dir * Mathf.toRadians(this.rotationStep));
         }
-        const pos = controller.rayWorldPosition;
-        pos.y += .1
-        if (debug) Gizmos.DrawLabel(pos, stick.x.toFixed(2) + ", " + stick.y.toFixed(2), .02, 0)
     }
     protected onHandleTeleport(controller: NeedleXRController, rig: IGameObject) {
@@ -166,20 +162,18 @@
     protected renderRays(session: NeedleXRSession) {
-        if (session.controllers.length < this._lines.length) {
-            for (let i = session.controllers.length; i < this._lines.length; i++) {
-                const line = this._lines[i];
-                line.visible = false;
-            }
+        for (let i = 0; i < this._lines.length; i++) {
+            const line = this._lines[i];
+            line.visible = false;
         }
-        for (const disc of this._hitDiscs) {
-            if (disc) disc.visible = false;
-        }
         for (let i = 0; i < session.controllers.length; i++) {
             const ctrl = session.controllers[i];
             let line = this._lines[i];
+            if (!ctrl.connected || !ctrl.isTracking) {
+                if (line) line.visible = false;
+                continue;
+            }
             if (!line) {
                 line = this.createRayLineObject();
                 line.scale.z = .5;
@@ -209,6 +203,7 @@
         }
         for (let i = 0; i < session.controllers.length; i++) {
             const ctrl = session.controllers[i];
+            if (!ctrl.connected || !ctrl.isTracking) continue;
             const hit = this.context.physics.raycastFromRay(ctrl.ray, { testObject: this.hitPointRaycastFilter })[0];
             this._hitDistances[i] = hit?.distance;
             if (hit) {