Needle Engine

Changes between version 3.34.4-alpha and 3.35.0-alpha
Files changed (16) hide show
  1. src/engine-components/export/usdz/extensions/behavior/BehaviourComponents.ts +33 -29
  2. src/engine-components/DragControls.ts +10 -10
  3. src/engine/engine_audio.ts +9 -7
  4. src/engine/engine_context.ts +0 -1
  5. src/engine/engine_create_objects.ts +5 -1
  6. src/engine/engine_element_loading.ts +4 -1
  7. src/engine/engine_input.ts +12 -3
  8. src/engine/engine_types.ts +1 -1
  9. src/engine-components/ui/EventSystem.ts +9 -3
  10. src/engine/xr/NeedleXRController.ts +33 -6
  11. src/engine/xr/NeedleXRSession.ts +8 -3
  12. src/engine-components/ui/PointerEvents.ts +1 -1
  13. src/engine-components/ui/Raycaster.ts +2 -0
  14. src/engine-components/webxr/controllers/XRControllerFollow.ts +5 -3
  15. src/engine-components/webxr/controllers/XRControllerModel.ts +1 -1
  16. src/engine-components/webxr/controllers/XRControllerMovement.ts +7 -4
src/engine-components/export/usdz/extensions/behavior/BehaviourComponents.ts CHANGED
@@ -173,45 +173,49 @@
173
173
  @serializable()
174
174
  fadeDuration: number = 0;
175
175
 
176
- private _objectsWithThisMaterial: Mesh[] = [];
177
-
178
- awake() {
179
- if (this.variantMaterial && this.materialToSwitch) {
180
- const renderer = GameObject.findObjectsOfType(Renderer);
181
- for (const rend of renderer) {
182
- for (let i = 0; i < rend.sharedMaterials.length; i++) {
183
- const mat = rend.sharedMaterials[i];
184
- if (mat === this.materialToSwitch) {
185
- if (rend.gameObject instanceof Mesh) {
186
- this._objectsWithThisMaterial.push(rend.gameObject);
187
- }
188
- else if (rend.gameObject instanceof Group) {
189
- for (const child of rend.gameObject.children) {
190
- if (child instanceof Mesh && child.material === mat) {
191
- this._objectsWithThisMaterial.push(child);
192
- }
193
- }
194
- }
195
- break;
196
- }
197
- }
198
- }
199
- }
200
- }
201
-
202
176
  start(): void {
177
+ // initialize the object list
178
+ this._objectsWithThisMaterial = this.objectsWithThisMaterial;
203
179
  ensureRaycaster(this.gameObject);
204
180
  }
205
181
 
206
182
  onPointerClick(args: PointerEventData) {
207
183
  args.use();
208
184
  if (!this.variantMaterial) return;
209
- for (let i = 0; i < this._objectsWithThisMaterial.length; i++) {
210
- const obj = this._objectsWithThisMaterial[i];
185
+ for (let i = 0; i < this.objectsWithThisMaterial.length; i++) {
186
+ const obj = this.objectsWithThisMaterial[i];
211
187
  obj.material = this.variantMaterial;
212
188
  }
213
189
  }
214
190
 
191
+ private _objectsWithThisMaterial: Mesh[] | null = null;
192
+ /** Get all objects in the scene that have the assigned materialToSwitch */
193
+ private get objectsWithThisMaterial(): Mesh[] {
194
+ if (this._objectsWithThisMaterial != null) return this._objectsWithThisMaterial;
195
+ this._objectsWithThisMaterial = [];
196
+ if (this.variantMaterial && this.materialToSwitch) {
197
+ this.context.scene.traverse(obj => {
198
+ if (obj instanceof Mesh) {
199
+ if (Array.isArray(obj.material)) {
200
+ for (const mat of obj.material) {
201
+ if (mat === this.materialToSwitch) {
202
+ this.objectsWithThisMaterial.push(obj);
203
+ break;
204
+ }
205
+ }
206
+ }
207
+ else {
208
+ if (obj.material === this.materialToSwitch) {
209
+ this.objectsWithThisMaterial.push(obj);
210
+ }
211
+
212
+ }
213
+ }
214
+ });
215
+ }
216
+ return this._objectsWithThisMaterial;
217
+ }
218
+
215
219
  private selfModel!: USDObject;
216
220
  private targetModels!: USDObject[];
217
221
 
@@ -235,7 +239,7 @@
235
239
 
236
240
  createBehaviours(_ext: BehaviorExtension, model: USDObject, _context) {
237
241
 
238
- const shouldExport = this._objectsWithThisMaterial.find(o => o.uuid === model.uuid);
242
+ const shouldExport = this.objectsWithThisMaterial.find(o => o.uuid === model.uuid);
239
243
  if (shouldExport) {
240
244
  this.targetModels.push(model);
241
245
  }
src/engine-components/DragControls.ts CHANGED
@@ -113,7 +113,7 @@
113
113
  start() {
114
114
  this.orbit = GameObject.findObjectOfType(OrbitControls, this.context);
115
115
  if (!this.gameObject.getComponentInParent(ObjectRaycaster))
116
- this.gameObject.addNewComponent(ObjectRaycaster);
116
+ this.gameObject.addComponent(ObjectRaycaster);
117
117
  }
118
118
 
119
119
  private allowEdit(_obj: Object3D | null = null) {
@@ -125,7 +125,7 @@
125
125
  if (evt.mode !== "screen") return;
126
126
 
127
127
  // get the drag mode and check if we need to abort early here
128
- const isSpatialInput = evt.event.mode === "tracked-pointer";
128
+ const isSpatialInput = evt.event.mode === "tracked-pointer" || evt.event.mode === "transient-pointer";
129
129
  const dragMode = isSpatialInput ? this.xrDragMode : this.dragMode;
130
130
  if (dragMode === DragMode.None) return;
131
131
 
@@ -147,7 +147,7 @@
147
147
  if (args.used) return;
148
148
 
149
149
  // get the drag mode and check if we need to abort early here
150
- const isSpatialInput = args.mode === "tracked-pointer";
150
+ const isSpatialInput = args.mode === "tracked-pointer" || args.mode === "transient-pointer";
151
151
  const dragMode = isSpatialInput ? this.xrDragMode : this.dragMode;
152
152
  if (dragMode === DragMode.None) return;
153
153
 
@@ -282,7 +282,7 @@
282
282
  sync?.requestOwnership();
283
283
  }
284
284
 
285
- this._marker = GameObject.addNewComponent(object, UsageMarker);
285
+ this._marker = GameObject.addComponent(object, UsageMarker);
286
286
 
287
287
  this._draggingRigidbodies.length = 0;
288
288
  const rbs = GameObject.getComponentsInChildren(object, Rigidbody);
@@ -346,7 +346,7 @@
346
346
 
347
347
  private _followObject: GameObject;
348
348
  private _manipulatorObject: GameObject;
349
- private _deviceMode!: XRTargetRayMode;
349
+ private _deviceMode!: XRTargetRayMode | "transient-pointer";
350
350
  private _followObjectStartWorldQuaternion: Quaternion = new Quaternion();
351
351
 
352
352
  constructor(dragControls: DragControls, gameObject: GameObject, pointerA: DragPointerHandler, pointerB: DragPointerHandler) {
@@ -531,7 +531,7 @@
531
531
  targetObject.updateMatrix();
532
532
  targetObject.updateMatrixWorld(true);
533
533
 
534
- const isSpatialInput = this._deviceMode === "tracked-pointer";
534
+ const isSpatialInput = this._deviceMode === "tracked-pointer" || this._deviceMode === "transient-pointer";
535
535
  const keepRotation = isSpatialInput ? this.settings.xrKeepRotation : this.settings.keepRotation;
536
536
 
537
537
  // TODO refactor to a common place
@@ -611,7 +611,7 @@
611
611
  private _totalMovementAlongRayDirection: number = 0;
612
612
  /** Distance between _followObject and its parent at grab start, in local space */
613
613
  private _grabStartDistance: number = 0;
614
- private _deviceMode!: XRTargetRayMode;
614
+ private _deviceMode!: XRTargetRayMode | "transient-pointer";
615
615
  private _followObjectStartPosition: Vector3 = new Vector3();
616
616
  private _followObjectStartQuaternion: Quaternion = new Quaternion();
617
617
  private _followObjectStartWorldQuaternion: Quaternion = new Quaternion();
@@ -722,7 +722,7 @@
722
722
  const dragSource = this._followObject.parent as IGameObject;
723
723
  const rayDirection = dragSource.worldForward;
724
724
 
725
- const isSpatialInput = this._deviceMode === "tracked-pointer";
725
+ const isSpatialInput = this._deviceMode === "tracked-pointer" || this._deviceMode === "transient-pointer";
726
726
  const dragMode = isSpatialInput ? this.settings.xrDragMode : this.settings.dragMode;
727
727
 
728
728
  // set up drag plane; we don't really know the normal yet but we can already set the point
@@ -880,7 +880,7 @@
880
880
 
881
881
 
882
882
  // Actually move and rotate draggedObject
883
- const isSpatialInput = this._deviceMode === "tracked-pointer";
883
+ const isSpatialInput = this._deviceMode === "tracked-pointer" || this._deviceMode === "transient-pointer";
884
884
  const keepRotation = isSpatialInput ? this.settings.xrKeepRotation : this.settings.keepRotation;
885
885
  const dragMode = isSpatialInput ? this.settings.xrDragMode : this.settings.dragMode;
886
886
 
@@ -895,7 +895,7 @@
895
895
  // Acceleration for moving the object - move followObject along the ray distance by _totalMovementAlongRayDirection
896
896
  let currentDist = 1.0;
897
897
  let lerpFactor = 1.0;
898
- if (this._deviceMode === "tracked-pointer" && this._grabStartDistance > 0.5) // hands and controllers, but not touches
898
+ if (isSpatialInput && this._grabStartDistance > 0.5) // hands and controllers, but not touches
899
899
  {
900
900
  const factor = 1 + this._totalMovementAlongRayDirection * (2 * this.settings.xrDistanceDragFactor);
901
901
  currentDist = Math.max(0.0, factor);
src/engine/engine_audio.ts CHANGED
@@ -8,13 +8,15 @@
8
8
  // this is a fix for https://github.com/mrdoob/three.js/issues/27779 & https://linear.app/needle/issue/NE-4257
9
9
  const ctx = AudioContext.getContext();
10
10
  ctx.addEventListener("statechange", () => {
11
- // on iOS the audiocontext can be interrupted: https://developer.mozilla.org/en-US/docs/Web/API/BaseAudioContext/state#resuming_interrupted_play_states_in_ios_safari
12
- const state = ctx.state as AudioContextState | "interrupted";
13
- if (state === "suspended" || state === "interrupted") {
14
- ctx.resume()
15
- .then(() => { console.log("AudioContext resumed successfully"); })
16
- .catch((e) => { console.log("Failed to resume AudioContext: " + e); });
17
- }
11
+ setTimeout(() => {
12
+ // on iOS the audiocontext can be interrupted: https://developer.mozilla.org/en-US/docs/Web/API/BaseAudioContext/state#resuming_interrupted_play_states_in_ios_safari
13
+ const state = ctx.state as AudioContextState | "interrupted";
14
+ if (state === "suspended" || state === "interrupted") {
15
+ ctx.resume()
16
+ .then(() => { console.log("AudioContext resumed successfully"); })
17
+ .catch((e) => { console.log("Failed to resume AudioContext: " + e); });
18
+ }
19
+ }, 500);
18
20
  });
19
21
  });
20
22
  }
src/engine/engine_context.ts CHANGED
@@ -759,7 +759,6 @@
759
759
  })
760
760
  .then(() => {
761
761
  if(debug) console.log("Needle Engine dependencies are ready");
762
- globalThis["needle:dependencies:ready"] = true;
763
762
  });
764
763
  }
765
764
 
src/engine/engine_create_objects.ts CHANGED
@@ -7,6 +7,7 @@
7
7
  Cube = 1,
8
8
  Sphere = 2,
9
9
  }
10
+ export type PrimitiveTypeNames = keyof typeof PrimitiveType;
10
11
 
11
12
  export type ObjectOptions = {
12
13
  name?: string,
@@ -19,20 +20,23 @@
19
20
 
20
21
  export class ObjectUtils {
21
22
 
22
- static createPrimitive(type: PrimitiveType, opts?: ObjectOptions): Mesh {
23
+ static createPrimitive(type: PrimitiveType | PrimitiveTypeNames, opts?: ObjectOptions): Mesh {
23
24
  let obj: Mesh;
24
25
  const color = 0xffffff;
25
26
  switch (type) {
27
+ case "Quad":
26
28
  case PrimitiveType.Quad:
27
29
  const quadGeo = new PlaneGeometry(1, 1, 1, 1);
28
30
  const quadMat = opts?.material ?? new MeshStandardMaterial({ color: color });
29
31
  obj = new Mesh(quadGeo, quadMat);
30
32
  break;
33
+ case "Cube":
31
34
  case PrimitiveType.Cube:
32
35
  const boxGeo = new BoxGeometry(1, 1, 1);
33
36
  const boxMat = opts?.material ?? new MeshStandardMaterial({ color: color });
34
37
  obj = new Mesh(boxGeo, boxMat);
35
38
  break;
39
+ case "Sphere":
36
40
  case PrimitiveType.Sphere:
37
41
  const sphereGeo = new SphereGeometry(.5, 16, 16);
38
42
  const sphereMat = opts?.material ?? new MeshStandardMaterial({ color: color });
src/engine/engine_element_loading.ts CHANGED
@@ -236,7 +236,10 @@
236
236
  const logoSize = 120;
237
237
  logo.style.width = `${logoSize}px`;
238
238
  logo.style.height = `${logoSize}px`;
239
- logo.style.marginBottom = "10px";
239
+ logo.style.borderRadius = "80px";
240
+ logo.style.padding = "20px";
241
+ logo.style.margin = "-20px";
242
+ logo.style.marginBottom = "-10px";
240
243
  logo.style.userSelect = "none";
241
244
  logo.style.objectFit = "contain";
242
245
  logo.style.transition = "transform 2s ease-out, opacity 1s ease-in-out";
src/engine/engine_input.ts CHANGED
@@ -1,4 +1,4 @@
1
- import { Matrix4, Object3D, Ray, Vector2, Vector3 } from 'three';
1
+ import { Intersection,Matrix4, Object3D, Ray, Vector2, Vector3 } from 'three';
2
2
 
3
3
  import { showBalloonMessage, showBalloonWarning } from './debug/debug.js';
4
4
  import { Context } from './engine_setup.js';
@@ -55,6 +55,8 @@
55
55
  buttonName: ButtonName | "none";
56
56
  }
57
57
 
58
+ /** An intersection that is potentially associated with a pointer event */
59
+ export declare type NEPointerEventIntersection = Intersection & { event?: NEPointerEvent };
58
60
 
59
61
  export class NEPointerEvent extends PointerEvent {
60
62
 
@@ -70,7 +72,7 @@
70
72
  readonly source: Event | null;
71
73
 
72
74
  /** Is the pointer event created via a touch on screen or a spatial device like a XR controller or hand tracking? */
73
- readonly mode: XRTargetRayMode;
75
+ readonly mode: XRTargetRayMode | "transient-pointer";
74
76
  /** A ray in worldspace for the event.
75
77
  * If the ray is undefined you can also use `space.worldForward` and `space.worldPosition` */
76
78
  get ray(): Ray {
@@ -116,6 +118,12 @@
116
118
  override get type(): InputEventNames { return this._type; }
117
119
  private readonly _type: InputEventNames;
118
120
 
121
+ /** metadata can be used to associate additional information with the event */
122
+ readonly metadata = {}
123
+
124
+ /** intersections that were generated from this event (or are associated with this event in any way) */
125
+ readonly intersections = new Array<NEPointerEventIntersection>();
126
+
119
127
  constructor(type: InputEvents | InputEventNames, source: Event | null, init: NEPointerEventInit) {
120
128
  super(type, init);
121
129
  // apply the init arguments. Otherwise the arguments will be undefined in the bundled / published version of needle engine
@@ -835,7 +843,8 @@
835
843
  }
836
844
  private onPointerUp = (evt: PointerEvent) => {
837
845
  if (this.context.isInAR) return;
838
- if (this.canReceiveInput(evt) === false) return;
846
+ // the pointer up event should always be handled
847
+ // if (this.canReceiveInput(evt) === false) return;
839
848
  const id = this.getPointerId(evt);
840
849
  // if (!this.isNewEvent(evt.timeStamp, id, this._pointerUpTimestamp)) return;
841
850
  const ne = new NEPointerEvent(InputEvents.PointerUp, evt, { origin: this, mode: "screen", deviceIndex: 0, pointerId: id, button: evt.button, clientX: evt.clientX, clientY: evt.clientY, pointerType: evt.pointerType as PointerTypeNames, buttonName: this.getButtonName(evt), device: this.getAndUpdateSpatialObjectForScreenPosition(id, evt.clientX, evt.clientY), pressure: evt.pressure });
src/engine/engine_types.ts CHANGED
@@ -14,7 +14,7 @@
14
14
  /** Removes all undefined functions */
15
15
  type NoUndefinedNoFunctions<T> = FilterTypes<T, Function | undefined | null>;
16
16
  /* Removes all properties that start with a specific prefix */
17
- type FilterStartingWith<T, Prefix extends string> = { [P in keyof T as P extends (`${Prefix}${infer _X}`) ? never : P]: T[P] };
17
+ type FilterStartingWith<T, Prefix extends string> = { [K in keyof T as K extends string ? (K extends `${Prefix}${string}` ? never : K) : never]: T[K] };
18
18
  /** Removes all properties that start with an underscore */
19
19
  type NoInternals<T> = FilterStartingWith<T, "_">;
20
20
  type NoInternalNeedleEngineState<T> = Omit<T, "destroyed" | "gameObject" | "activeAndEnabled" | "context" | "isComponent" | "scene" | "up" | "forward" | "right" | "worldRotation" | "worldEuler" | "worldPosition" | "worldQuaternion">;
src/engine-components/ui/EventSystem.ts CHANGED
@@ -1,7 +1,7 @@
1
1
  import { type Intersection, Mesh,Object3D } from "three";
2
2
 
3
3
  import { isDevEnvironment, showBalloonMessage } from "../../engine/debug/index.js";
4
- import { type InputEventNames, InputEvents, NEPointerEvent, PointerType } from "../../engine/engine_input.js";
4
+ import { type InputEventNames, InputEvents, NEPointerEvent, NEPointerEventIntersection,PointerType } from "../../engine/engine_input.js";
5
5
  import { Mathf } from "../../engine/engine_math.js";
6
6
  import { RaycastOptions, type RaycastTestObjectReturnType } from "../../engine/engine_physics.js";
7
7
  import { Context } from "../../engine/engine_setup.js";
@@ -181,7 +181,13 @@
181
181
  }
182
182
 
183
183
 
184
- const hits = this.performRaycast(options);
184
+ const hits = this.performRaycast(options) as Array<NEPointerEventIntersection>;
185
+ if (hits) {
186
+ for (const hit of hits) {
187
+ hit.event = pointerEvent;
188
+ pointerEvent.intersections.push(hit);
189
+ }
190
+ }
185
191
 
186
192
  if (debug && data.isClick) {
187
193
  showBalloonMessage("EventSystem: " + data.pointerId + " - " + this.context.time.frame + " - Up:" + data.isUp + ", Down:" + data.isDown)
@@ -502,7 +508,7 @@
502
508
  }
503
509
 
504
510
  /**
505
- * Propagate up in hiearchy and call handlers based on the pointer event data
511
+ * Propagate up in hierarchy and call handlers based on the pointer event data
506
512
  */
507
513
  private handleMainInteraction(object: Object3D, args: PointerEventData, prevHovering: Object3D | null) {
508
514
  const pressedEvent = this.pressedByID.get(args.pointerId);
src/engine/xr/NeedleXRController.ts CHANGED
@@ -18,8 +18,10 @@
18
18
  */
19
19
  const debugCustomGesture = getParam("debugcustomgesture");
20
20
 
21
- /** true when selectstart was ever received */
22
- let _didReceiveSelectStartEvent = false;
21
+ /** true when selectstart was ever received.
22
+ * On VisionOS 1.1 we always have select events (as per the spec), so this is always true
23
+ */
24
+ // let _didReceiveSelectStartEvent = false;
23
25
 
24
26
  // https://github.com/immersive-web/webxr-input-profiles/blob/4484a05e30bcd43fe86bb4e06b7a707861a26796/packages/registry/profiles/meta/meta-quest-touch-plus.json
25
27
  declare type ControllerAxes = "xr-standard-thumbstick";
@@ -113,7 +115,7 @@
113
115
  get layout() { return this._layout; }
114
116
 
115
117
  /** shorthand for `inputSource.targetRayMode` */
116
- get targetRayMode() { return this.inputSource.targetRayMode; }
118
+ get targetRayMode(): (XRTargetRayMode | "transient-pointer") { return this.inputSource.targetRayMode; }
117
119
  /** shorthand for `inputSource.targetRaySpace` */
118
120
  get targetRaySpace() { return this.inputSource.targetRaySpace; }
119
121
  /** shorthand for `inputSource.gripSpace` */
@@ -133,6 +135,8 @@
133
135
  */
134
136
  get hitTestSource() { return this._hitTestSource; }
135
137
  private _hitTestSource: XRTransientInputHitTestSource | undefined = undefined;
138
+ private _hasSelectEvent = false;
139
+ get hasSelectEvent() { return this._hasSelectEvent; }
136
140
 
137
141
  /** Perform a hit test against the XR planes or meshes. shorthand for `xr.getHitTest(controller)`
138
142
  * @returns the hit test result (with position and rotation in worldspace) or null if no hit was found
@@ -357,6 +361,11 @@
357
361
  this._gripQuaternion.set(q.x, q.y, q.z, q.w);
358
362
  }
359
363
  }
364
+ // on VisionOS we get a gripSpace that matches where the controller is for transient input sources
365
+ else if (this.inputSource.gripSpace && this.targetRayMode === "transient-pointer") {
366
+ this._object.position.copy(this._gripPosition);
367
+ this._object.quaternion.copy(this._gripQuaternion).multiply(flipForwardQuaternion);
368
+ }
360
369
  else {
361
370
  this._object.position.copy(this._rayPosition);
362
371
  this._object.quaternion.copy(this._rayQuaternion).multiply(flipForwardQuaternion);
@@ -513,7 +522,12 @@
513
522
  private _layout: InputDeviceLayout | undefined;
514
523
  private getMotionController!: Promise<MotionController>;
515
524
  private initialize() {
525
+ this._hasSelectEvent = this.profiles.includes("generic-hand-select") || this.profiles.some(p => p.startsWith("generic-trigger"));
516
526
  if (!this._layout) {
527
+ // Ignore transient-pointer since we likely don't want to spawn a controller visual just for a temporary pointer.
528
+ // TODO we should check how this is actually handled on Quest Browser when the transient-pointer flag is on.
529
+ if (this.inputSource.targetRayMode as XRTargetRayMode | "transient-pointer" === "transient-pointer") return;
530
+
517
531
  // TODO: we should fetch the profiles or better yet the profile list once and cache it
518
532
  const fetchProfileCall = fetchProfile(this.inputSource, DEFAULT_PROFILES_PATH, DEFAULT_PROFILE);
519
533
  /** @ts-ignore */
@@ -544,7 +558,9 @@
544
558
  // this.getButton("a-button")
545
559
  return this._motioncontroller;
546
560
  }).catch(err => {
547
- console.error(err);
561
+ if (this.inputSource)
562
+ console.warn("Couldn't initialize motion controller profile for ", this.inputSource, err);
563
+ return null;
548
564
  });
549
565
  }
550
566
  }
@@ -569,10 +585,16 @@
569
585
 
570
586
  private onSelectStart = (evt: XRInputSourceEvent) => {
571
587
  if (this.inputSource !== evt.inputSource) return;
588
+ // if a selectstart event happens right after an input source is connected, we may even receive this event before
589
+ // requestAnimationFrame callback with the current session. So, we need to update the frame here.
590
+ this.onUpdateFrame(evt.frame);
591
+ // if we receive a select event we can be true that this device supports select events
592
+ this._hasSelectEvent = true;
572
593
  const selectComponentId = this._layout?.selectComponentId;
573
594
  const i = this._layout?.components[selectComponentId!]?.gamepadIndices?.button;
574
595
  if (i !== undefined) this._selectButtonIndex = i;
575
596
  if (debugCustomGesture) return;
597
+ /*
576
598
  if (!_didReceiveSelectStartEvent) {
577
599
  _didReceiveSelectStartEvent = true;
578
600
  // safeguard first pinch event - check if the pinch gesture is already down
@@ -582,6 +604,7 @@
582
604
  return;
583
605
  }
584
606
  }
607
+ */
585
608
  if (debug) Gizmos.DrawDirection(this.rayWorldPosition, getTempVector(0, .01, 1).applyQuaternion(this.rayWorldQuaternion), 0xff0000, 10);
586
609
  this.emitPointerEvent(InputEvents.PointerDown, this._selectButtonIndex || 0, "xr-standard-trigger", true, evt);
587
610
  }
@@ -683,9 +706,12 @@
683
706
  }
684
707
  this.states["pinch"] = state;
685
708
 
686
- /** workaround for visionOS not emitting selectstart. See https://linear.app/needle/issue/NE-4212
687
- * If a select start event was never received we do a manual check here if the user is pinching
709
+ /** Workaround for visionOS not emitting selectstart. See https://linear.app/needle/issue/NE-4212
710
+ * If a selectstart event was never received we do a manual check here if the user is pinching
711
+ * Update: VisionOS 1.1 now properly emits select events from transient input sources, based on gaze.
712
+ * We're keeping this code commented for now since there may be future changes before VisionOS WebXR ships.
688
713
  */
714
+ /*
689
715
  if (!_didReceiveSelectStartEvent && (state.isDown || state.isUp)) {
690
716
  const eventName = isPressed ? "pointerdown" : "pointerup";
691
717
  const pressure = distance / pinchThreshold;
@@ -697,6 +723,7 @@
697
723
  }
698
724
  this.emitPointerEvent(eventName, 0, "pinch", false, null, pressure);
699
725
  }
726
+ */
700
727
  }
701
728
  }
702
729
  }
src/engine/xr/NeedleXRSession.ts CHANGED
@@ -547,7 +547,10 @@
547
547
  /** shorthand to query the right controller. Use `controllers` to get access to all connected controllers */
548
548
  get rightController() { return this.controllers.find(c => c.isRight); }
549
549
  /** @returns the given controller if it is connected */
550
- getController(side: XRHandedness) { return this.controllers.find(c => c.side === side) || null; }
550
+ getController(side: XRHandedness | number) {
551
+ if (typeof side === "number") return this.controllers[side] || null;
552
+ return this.controllers.find(c => c.side === side) || null;
553
+ }
551
554
 
552
555
  /** Returns true if running in pass through mode in immersive AR (e.g. user is wearing a headset while in AR) */
553
556
  get isPassThrough() {
@@ -875,7 +878,9 @@
875
878
  console.warn("Controller already exists for input source", index);
876
879
  return;
877
880
  }
878
- console.log("Adding controller", index);
881
+ if (debug) console.log("Adding controller", index);
882
+ // TODO: check if this is a transient input source AND we can figure out which existing controller it likely belongs to
883
+ // TODO: do not draw raycasts for controllers that don't have primary input actions / until we know that they have primary input actions
879
884
  const newController = new NeedleXRController(this, newInputSource, index);
880
885
  this._newControllers.push(newController);
881
886
  }
@@ -884,7 +889,7 @@
884
889
  private disconnectInputSource(inputSource: XRInputSource) {
885
890
  const handleController = (oldController: NeedleXRController, _array: Array<NeedleXRController>, i: number) => {
886
891
  if (oldController.inputSource === inputSource) {
887
- console.log("Disconnecting controller", oldController.index);
892
+ if (debug) console.log("Disconnecting controller", oldController.index);
888
893
  this.controllers.splice(i, 1);
889
894
  this.invokeControllerEvent(oldController, this._controllerRemoved, "removed");
890
895
  const args: NeedleXRControllerEventArgs = {
src/engine-components/ui/PointerEvents.ts CHANGED
@@ -88,7 +88,7 @@
88
88
  inputSource: Input | any;
89
89
 
90
90
  /** Returns the input target ray mode e.g. "screen" for 2D mouse and touch events */
91
- get mode(): XRTargetRayMode { return this.event.mode; }
91
+ get mode(): XRTargetRayMode | "transient-pointer" { return this.event.mode; }
92
92
 
93
93
  /** The object this event hit or interacted with */
94
94
  object!: Object3D;
src/engine-components/ui/Raycaster.ts CHANGED
@@ -80,6 +80,8 @@
80
80
  if (!NeedleXRSession.active) return null;
81
81
  if (!_opts?.ray) return null;
82
82
 
83
+ // TODO this raycast should actually start from gripWorldPosition, not the ray origin, for
84
+ // cases like transient-pointer on VisionOS where the ray starts at the head and not the hand
83
85
  const rayOrigin = _opts.ray.origin;
84
86
  const radius = 0.01;
85
87
 
src/engine-components/webxr/controllers/XRControllerFollow.ts CHANGED
@@ -12,9 +12,10 @@
12
12
  return true;
13
13
  }
14
14
 
15
- /** should this object follow a right hand/controller or left hand/controller */
15
+ /** Should this object follow a right hand/controller or left hand/controller.
16
+ * When a number is provided, the controller with that index is followed. */
16
17
  @serializable()
17
- side: XRHandedness = "none";
18
+ side: XRHandedness | number = "none";
18
19
 
19
20
  /** should it follow controllers (the physics controller) */
20
21
  @serializable()
@@ -52,7 +53,8 @@
52
53
  // we're following a controller (or hand)
53
54
  if (this.controlVisibility)
54
55
  this.gameObject.visible = true;
55
- if (this.useGripSpace) {
56
+
57
+ if (this.useGripSpace || ctrl.targetRayMode === "transient-pointer") {
56
58
  this.gameObject.worldPosition = ctrl.gripWorldPosition;
57
59
  this.gameObject.worldQuaternion = ctrl.gripWorldQuaternion;
58
60
  }
src/engine-components/webxr/controllers/XRControllerModel.ts CHANGED
@@ -85,7 +85,7 @@
85
85
  child.layers.set(2);
86
86
  });
87
87
  }
88
- else {
88
+ else if (controller.targetRayMode !== "transient-pointer") {
89
89
  console.warn("XRControllerModel: no model found for " + controller.side);
90
90
  }
91
91
  }
src/engine-components/webxr/controllers/XRControllerMovement.ts CHANGED
@@ -164,13 +164,16 @@
164
164
 
165
165
  for (let i = 0; i < this._lines.length; i++) {
166
166
  const line = this._lines[i];
167
- line.visible = false;
167
+ if (line) line.visible = false;
168
168
  }
169
169
 
170
170
  for (let i = 0; i < session.controllers.length; i++) {
171
171
  const ctrl = session.controllers[i];
172
172
  let line = this._lines[i];
173
- if (!ctrl.connected || !ctrl.isTracking) {
173
+ if (!ctrl.connected || !ctrl.isTracking ||
174
+ !ctrl.ray || ctrl.targetRayMode === "transient-pointer" ||
175
+ !ctrl.hasSelectEvent
176
+ ) {
174
177
  if (line) line.visible = false;
175
178
  continue;
176
179
  }
@@ -203,7 +206,7 @@
203
206
  }
204
207
  for (let i = 0; i < session.controllers.length; i++) {
205
208
  const ctrl = session.controllers[i];
206
- if (!ctrl.connected || !ctrl.isTracking) continue;
209
+ if (!ctrl.connected || !ctrl.isTracking || !ctrl.ray || !ctrl.hasSelectEvent) continue;
207
210
 
208
211
  // save performance by only raycasting every nth frame
209
212
  if (this.context.time.frame % 2 !== 0) {
@@ -294,7 +297,7 @@
294
297
  );
295
298
  disc2.layers.disableAll();
296
299
  disc2.layers.enable(2);
297
- disc2.position.z -= .01;
300
+ disc2.position.y = .01;
298
301
  container.add(disc2);
299
302
  return container;
300
303
  }