Needle Engine

Changes between version 3.45.0-beta.1 and 3.45.1-beta
Files changed (8) hide show
  1. src/engine-components/webxr/Avatar.ts +26 -5
  2. src/engine-components/Camera.ts +19 -2
  3. src/engine/engine_serialization_builtin_serializer.ts +21 -1
  4. src/engine-components/GroundProjection.ts +23 -15
  5. src/engine/webcomponents/needle menu/needle-menu.ts +2 -2
  6. src/engine/xr/NeedleXRController.ts +6 -0
  7. src/engine-components/OrbitControls.ts +3 -6
  8. src/engine-components/webxr/controllers/XRControllerModel.ts +7 -1
src/engine-components/webxr/Avatar.ts CHANGED
@@ -1,16 +1,17 @@
1
- import { Object3D, Quaternion, Vector3 } from "three";
1
+ import { Mesh, Object3D, Quaternion, Vector3 } from "three";
2
2
 
3
3
  import { AssetReference } from "../../engine/engine_addressables.js";
4
4
  import { ObjectUtils, PrimitiveType } from "../../engine/engine_create_objects.js";
5
5
  import { serializable } from "../../engine/engine_serialization_decorator.js";
6
6
  import type { IGameObject } from "../../engine/engine_types.js";
7
- import { getParam,PromiseAllWithErrors } from "../../engine/engine_utils.js";
8
- import { type NeedleXREventArgs, NeedleXRSession, NeedleXRUtils } from "../../engine/xr/api.js";
7
+ import { getParam, PromiseAllWithErrors } from "../../engine/engine_utils.js";
8
+ import { NeedleXRController, type NeedleXREventArgs, NeedleXRSession, NeedleXRUtils } from "../../engine/xr/api.js";
9
9
  import { PlayerState } from "../../engine-components-experimental/networking/PlayerSync.js";
10
10
  import { Behaviour, GameObject } from "../Component.js";
11
11
  import { SyncedTransform } from "../SyncedTransform.js";
12
12
  import { AvatarMarker } from "./WebXRAvatar.js";
13
13
  import { XRFlag } from "./XRFlag.js";
14
+ import { setCustomVisibility } from "../../engine/js-extensions/Layers.js";
14
15
 
15
16
  const debug = getParam("debugwebxr");
16
17
 
@@ -27,6 +28,9 @@
27
28
  @serializable(AssetReference)
28
29
  rightHand?: AssetReference;
29
30
 
31
+ private _leftHandMeshes?: Mesh[];
32
+ private _rightHandMeshes?: Mesh[];
33
+
30
34
  private _syncTransforms?: SyncedTransform[];
31
35
 
32
36
  async onEnterXR(_args: NeedleXREventArgs) {
@@ -108,6 +112,7 @@
108
112
  leftObj.quaternion.copy(leftCtrl.gripQuaternion);
109
113
  leftObj.quaternion.multiply(flipForwardQuaternion);
110
114
  leftObj.visible = leftCtrl.isTracking;
115
+ this.updateHandVisibility(leftCtrl, leftObj, this._leftHandMeshes);
111
116
  }
112
117
  else if (leftObj && leftObj.visible) {
113
118
  leftObj.visible = false;
@@ -120,6 +125,7 @@
120
125
  rightObj.quaternion.copy(right.gripQuaternion);
121
126
  rightObj.quaternion.multiply(flipForwardQuaternion);
122
127
  rightObj.visible = right.isTracking;
128
+ this.updateHandVisibility(right, rightObj, this._rightHandMeshes);
123
129
  }
124
130
  else if (rightObj && rightObj.visible) {
125
131
  rightObj.visible = false;
@@ -127,10 +133,20 @@
127
133
  }
128
134
 
129
135
  onBeforeRender(): void {
130
- if (this.context.time.frame % 10 === 0)
131
- this.updateRemoteAvatarVisibility();
136
+ if (this.context.xr) {
137
+ if (this.context.time.frame % 10 === 0)
138
+ this.updateRemoteAvatarVisibility();
139
+ }
132
140
  }
133
141
 
142
+ private updateHandVisibility(controller: NeedleXRController, avatarHand: Object3D, meshes: Mesh[] | undefined) {
143
+ if (meshes) {
144
+ // Hide the hand meshes for the local user if another model (e.g. the controller model) is being rendered
145
+ // We don't set the visible flag here because it would also disable SyncedTransforms networking
146
+ const hasOtherRenderingModel = controller.model && controller.model.visible && controller.model !== avatarHand;
147
+ meshes.forEach(mesh => { setCustomVisibility(mesh, !hasOtherRenderingModel); });
148
+ }
149
+ }
134
150
 
135
151
  private updateRemoteAvatarVisibility() {
136
152
  if (this.context.connection.isConnected) {
@@ -219,6 +235,11 @@
219
235
 
220
236
  await this.loadAvatarObjects(this.head, this.leftHand, this.rightHand);
221
237
 
238
+ this._leftHandMeshes = [];
239
+ this.leftHand.asset.traverse((obj) => { if ((obj as Mesh)?.isMesh) this._leftHandMeshes!.push(obj); });
240
+ this._rightHandMeshes = [];
241
+ this.rightHand.asset.traverse((obj) => { if ((obj as Mesh)?.isMesh) this._rightHandMeshes!.push(obj); });
242
+
222
243
  if (PlayerState.isLocalPlayer(this.gameObject)) {
223
244
  this._syncTransforms = GameObject.getComponentsInChildren(this.gameObject, SyncedTransform);
224
245
  }
src/engine-components/Camera.ts CHANGED
@@ -1,4 +1,4 @@
1
- import { EquirectangularReflectionMapping, Frustum, Matrix, Matrix4, OrthographicCamera, PerspectiveCamera, Ray, SRGBColorSpace, Vector3 } from "three";
1
+ import { EquirectangularReflectionMapping, Euler, Frustum, Matrix, Matrix4, OrthographicCamera, PerspectiveCamera, Ray, SRGBColorSpace, Vector3 } from "three";
2
2
  import { Texture } from "three";
3
3
 
4
4
  import { isDevEnvironment, showBalloonMessage, showBalloonWarning } from "../engine/debug/index.js";
@@ -167,6 +167,21 @@
167
167
  }
168
168
  private _backgroundIntensity?: number = 1;
169
169
 
170
+ /** the rotation of the background texture (when using a skybox) */
171
+ @serializable(Euler)
172
+ public set backgroundRotation(val: Euler | undefined) {
173
+ if (val === this._backgroundRotation) return;
174
+ if (val === undefined)
175
+ this._backgroundRotation = undefined;
176
+ else
177
+ this._backgroundRotation = val;
178
+ this.applyClearFlagsIfIsActiveCamera();
179
+ }
180
+ public get backgroundRotation(): Euler | undefined {
181
+ return this._backgroundRotation;
182
+ }
183
+ private _backgroundRotation?: Euler;
184
+
170
185
  /** The intensity of the environment map */
171
186
  @serializable()
172
187
  public set environmentIntensity(val: number | undefined) {
@@ -267,7 +282,7 @@
267
282
  }
268
283
  if (target === this._projScreenMatrix) return target;
269
284
  return target.copy(this._projScreenMatrix);
270
- }
285
+ }
271
286
  private readonly _projScreenMatrix = new Matrix4();
272
287
 
273
288
 
@@ -425,6 +440,8 @@
425
440
  else if (debug) console.warn(`Camera \"${this.name}\" has no background blurriness`)
426
441
  if (this._backgroundIntensity !== undefined)
427
442
  this.context.scene.backgroundIntensity = this._backgroundIntensity;
443
+ if (this._backgroundRotation !== undefined)
444
+ this.context.scene.backgroundRotation = this._backgroundRotation;
428
445
  else if (debug) console.warn(`Camera \"${this.name}\" has no background intensity`)
429
446
 
430
447
  break;
src/engine/engine_serialization_builtin_serializer.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import * as THREE from "three";
2
- import { Color, CompressedTexture, LinearSRGBColorSpace, Object3D, Texture, WebGLRenderTarget } from "three";
2
+ import { Color, CompressedTexture, Euler, LinearSRGBColorSpace, Object3D, Texture, WebGLRenderTarget } from "three";
3
3
 
4
4
  import { isDevEnvironment, showBalloonMessage, showBalloonWarning } from "../engine/debug/index.js";
5
5
  import { Behaviour, Component, GameObject } from "../engine-components/Component.js";
@@ -55,6 +55,26 @@
55
55
  }
56
56
  export const colorSerializer = new ColorSerializer();
57
57
 
58
+ class EulerSerializer extends TypeSerializer {
59
+ constructor() {
60
+ super([Euler], "EulerSerializer");
61
+ }
62
+ onDeserialize(data: any, _context: SerializationContext) {
63
+ if (data === undefined || data === null) return;
64
+ if (data.order) {
65
+ return new Euler(data.x, data.y, data.z, data.order);
66
+ }
67
+ else if (data.x != undefined) {
68
+ return new Euler(data.x, data.y, data.z);
69
+ }
70
+ return undefined;
71
+ }
72
+ onSerialize(data: any, _context: SerializationContext) {
73
+ return { x: data.x, y: data.y, z: data.z, order: data.order };
74
+ }
75
+ }
76
+ export const euler = new EulerSerializer();
77
+
58
78
  declare type ObjectData = {
59
79
  node?: number;
60
80
  guid?: string;
src/engine-components/GroundProjection.ts CHANGED
@@ -14,7 +14,7 @@
14
14
 
15
15
  @serializable()
16
16
  applyOnAwake: boolean = false;
17
-
17
+
18
18
  /**
19
19
  * Radius of the projection sphere. Set it large enough so the camera stays inside (make sure the far plane is also large enough)
20
20
  */
@@ -40,15 +40,16 @@
40
40
  private _lastEnvironment?: Texture;
41
41
  private _lastRadius?: number;
42
42
  private _lastHeight?: number;
43
- private env?: GroundProjection;
43
+ private _projection?: GroundProjection;
44
44
  private _watcher?: Watch;
45
45
 
46
46
 
47
+ /** @internal */
47
48
  awake() {
48
49
  if (this.applyOnAwake)
49
50
  this.updateAndCreate();
50
51
  }
51
-
52
+ /** @internal */
52
53
  onEnable() {
53
54
  // TODO: if we do this in the first frame we can not disable it again. Something buggy with the watch?!
54
55
  if (this.context.time.frameCount > 0) {
@@ -62,18 +63,25 @@
62
63
  });
63
64
  }
64
65
  }
65
-
66
+ /** @internal */
66
67
  onDisable() {
67
68
  this._watcher?.revoke();
68
- this.env?.removeFromParent();
69
+ this._projection?.removeFromParent();
69
70
  }
70
-
71
+ /** @internal */
71
72
  onEnterXR(): void {
72
73
  this.updateProjection();
73
74
  }
75
+ /** @internal */
74
76
  onLeaveXR(): void {
75
77
  this.updateProjection();
76
78
  }
79
+ /** @internal */
80
+ onBeforeRender(): void {
81
+ if (this._projection && this.scene.backgroundRotation) {
82
+ this._projection.rotation.copy(this.scene.backgroundRotation);
83
+ }
84
+ }
77
85
 
78
86
  private updateAndCreate() {
79
87
  this.updateProjection();
@@ -82,21 +90,21 @@
82
90
 
83
91
  updateProjection() {
84
92
  if (!this.context.scene.environment || this.context.xr?.isPassThrough) {
85
- this.env?.removeFromParent();
93
+ this._projection?.removeFromParent();
86
94
  return;
87
95
  }
88
- if (!this.env || this.context.scene.environment !== this._lastEnvironment || this._height !== this._lastHeight || this._radius !== this._lastRadius) {
96
+ if (!this._projection || this.context.scene.environment !== this._lastEnvironment || this._height !== this._lastHeight || this._radius !== this._lastRadius) {
89
97
  if (debug)
90
98
  console.log("Create/Update Ground Projection", this.context.scene.environment.name);
91
- this.env?.removeFromParent();
92
- this.env = new GroundProjection(this.context.scene.environment, this._height, this.radius);
93
- this.env.position.y = this._height - .0001;
99
+ this._projection?.removeFromParent();
100
+ this._projection = new GroundProjection(this.context.scene.environment, this._height, this.radius);
101
+ this._projection.position.y = this._height - .0001;
94
102
  }
95
103
  this._lastEnvironment = this.context.scene.environment;
96
104
  this._lastHeight = this._height;
97
105
  this._lastRadius = this._radius;
98
- if (!this.env.parent)
99
- this.gameObject.add(this.env);
106
+ if (!this._projection.parent)
107
+ this.gameObject.add(this._projection);
100
108
 
101
109
  /* TODO realtime adjustments aren't possible anymore with GroundedSkybox (mesh generation)
102
110
  this.env.scale.setScalar(this._scale);
@@ -105,8 +113,8 @@
105
113
  */
106
114
 
107
115
  // dont make the ground projection raycastable by default
108
- if (this.env.isObject3D === true) {
109
- this.env.layers.set(2);
116
+ if (this._projection.isObject3D === true) {
117
+ this._projection.layers.set(2);
110
118
  }
111
119
  }
112
120
 
src/engine/webcomponents/needle menu/needle-menu.ts CHANGED
@@ -232,7 +232,7 @@
232
232
  transform: translateX(-50%);
233
233
  top: 20px;
234
234
  padding: 0.3rem;
235
- background: #ffffff5c;
235
+ background: rgba(255, 255, 255, .4);
236
236
  display: flex;
237
237
  visibility: visible;
238
238
  flex-direction: row-reverse; /* if we overflow this should be right aligned so the logo is always visible */
@@ -280,7 +280,7 @@
280
280
  font-weight: 500;
281
281
  font-weight: 200;
282
282
  font-variation-settings: "wdth" 100;
283
- color: rgb(40,40,40);
283
+ color: rgb(30,30,30);
284
284
  }
285
285
 
286
286
  a {
src/engine/xr/NeedleXRController.ts CHANGED
@@ -233,6 +233,12 @@
233
233
  get object() { return this._object; }
234
234
  private readonly _object: IGameObject;
235
235
 
236
+
237
+ /** Assigned the model that you use for rendering. This can be used as a hint for other components */
238
+ model: Object3D | null = null;
239
+
240
+
241
+
236
242
  private readonly _debugAxesHelper = new AxesHelper(.2);
237
243
 
238
244
  /** returns the URL of the default controller model */
src/engine-components/OrbitControls.ts CHANGED
@@ -327,11 +327,6 @@
327
327
  }
328
328
  }
329
329
  this._syncedTransform = GameObject.getComponent(this.gameObject, SyncedTransform) ?? undefined;
330
- // if we autofit in onEnable then DragControls will trigger fitting every time (because they disable OrbitControls)
331
- // that's confusing and not what we want
332
- // if (this._didStart) {
333
- // if (this.autoFit) this.fitCamera()
334
- // }
335
330
  this.context.input.addEventListener("pointerup", this._onPointerDown);
336
331
  this.context.pre_render_callbacks.push(this.__onPreRender);
337
332
  }
@@ -433,7 +428,9 @@
433
428
  if (this.debugLog)
434
429
  console.log("NO TARGET");
435
430
  const worldPosition = getWorldPosition(camGo.cam);
436
- const distanceToCenter = worldPosition.length();
431
+ // Handle case where the camera is in 0 0 0 of the scene
432
+ // if the look at target is set to the camera position we can't move at all anymore
433
+ const distanceToCenter = Math.max(.01, worldPosition.length());
437
434
  const forward = new Vector3(0, 0, -distanceToCenter).applyMatrix4(camGo.cam.matrixWorld);
438
435
  this.setLookTargetPosition(forward, true);
439
436
  }
src/engine-components/webxr/controllers/XRControllerModel.ts CHANGED
@@ -66,7 +66,7 @@
66
66
 
67
67
  if (debug) console.warn("Add Controller Model for", controller.side, controller.index)
68
68
 
69
- if (this.createControllerModel) {
69
+ if (this.createControllerModel || this.createHandModel) {
70
70
  if (controller.hand) {
71
71
  if (this.createHandModel) {
72
72
  const res = await this.loadHandModel(controller);
@@ -79,6 +79,7 @@
79
79
  this._models.push({ controller: controller, model: res.handObject, handmesh: res.handmesh });
80
80
  this._models.sort((a, b) => a.controller.index - b.controller.index);
81
81
  this.scene.add(res.handObject);
82
+ controller.model = res.handObject;
82
83
  }
83
84
  }
84
85
  else {
@@ -97,6 +98,7 @@
97
98
  model.traverse(child => {
98
99
  child.layers.set(2);
99
100
  });
101
+ controller.model = model;
100
102
  }
101
103
  else if (controller.targetRayMode !== "transient-pointer") {
102
104
  console.warn("XRControllerModel: no model found for " + controller.side);
@@ -144,6 +146,10 @@
144
146
  for (const entry of this._models) {
145
147
  if (!entry) continue;
146
148
  entry.model?.removeFromParent();
149
+ // Unassign the model from the controller when this script becomes inactive
150
+ if (entry.controller.model === entry.model) {
151
+ entry.controller.model = null;
152
+ }
147
153
  }
148
154
  this._models.length = 0;
149
155
  }