Needle Engine

Changes between version 3.2.4-alpha.2 and 3.2.5-alpha
Files changed (39) hide show
  1. src/engine/codegen/register_types.js +25 -19
  2. src/engine-components/api.ts +1 -0
  3. src/engine/api.ts +1 -0
  4. src/engine-components/avatar/Avatar_Brain_LookAt.ts +1 -1
  5. src/engine-components/avatar/Avatar_MouthShapes.ts +1 -1
  6. src/engine-components/avatar/Avatar_MustacheShake.ts +1 -1
  7. src/engine-components/codegen/components.ts +20 -17
  8. src/engine-components/DragControls.ts +1 -1
  9. src/engine-components/Duplicatable.ts +1 -1
  10. src/engine/engine_element_loading.ts +25 -10
  11. src/engine/engine_networking_auto.ts +1 -1
  12. src/engine-components/ui/EventSystem.ts +2 -2
  13. src/engine-components/ui/InputField.ts +2 -1
  14. src/engine-components/Light.ts +2 -2
  15. src/engine-components/PlayerColor.ts +1 -1
  16. src/engine-components/ShadowCatcher.ts +15 -0
  17. src/engine-components/SpectatorCamera.ts +2 -2
  18. src/engine-components/SyncedCamera.ts +2 -2
  19. src/engine-components/export/usdz/USDZExporter.ts +2 -2
  20. src/engine-components/WebARCameraBackground.ts +0 -215
  21. src/engine-components/WebARSessionRoot.ts +0 -178
  22. src/engine-components/WebXR.ts +0 -712
  23. src/engine-components/WebXRAvatar.ts +0 -356
  24. src/engine-components/WebXRController.ts +0 -1125
  25. src/engine-components/WebXRGrabRendering.ts +0 -151
  26. src/engine-components/WebXRImageTracking.ts +0 -192
  27. src/engine-components/WebXRRig.ts +0 -22
  28. src/engine-components/WebXRSync.ts +0 -463
  29. src/engine-components/webxr/index.ts +2 -0
  30. src/engine-components/webxr/WebARCameraBackground.ts +215 -0
  31. src/engine-components/webxr/WebARSessionRoot.ts +178 -0
  32. src/engine-components/webxr/WebXR.ts +712 -0
  33. src/engine-components/webxr/WebXRAvatar.ts +356 -0
  34. src/engine-components/webxr/WebXRController.ts +1125 -0
  35. src/engine-components/webxr/WebXRGrabRendering.ts +151 -0
  36. src/engine-components/webxr/WebXRImageTracking.ts +193 -0
  37. src/engine-components/webxr/WebXRPlaneTracking.ts +254 -0
  38. src/engine-components/webxr/WebXRRig.ts +22 -0
  39. src/engine-components/webxr/WebXRSync.ts +463 -0
src/engine/codegen/register_types.js CHANGED
@@ -1,5 +1,5 @@
1
1
  import { TypeStore } from "./../engine_typestore"
2
-
2
+
3
3
  // Import types
4
4
  import { __Ignore } from "../../engine-components/codegen/components";
5
5
  import { AlignmentConstraint } from "../../engine-components/AlignmentConstraint";
@@ -10,7 +10,7 @@
10
10
  import { Animator } from "../../engine-components/Animator";
11
11
  import { AnimatorController } from "../../engine-components/AnimatorController";
12
12
  import { Antialiasing } from "../../engine-components/postprocessing/Effects/Antialiasing";
13
- import { AttachedObject } from "../../engine-components/WebXRController";
13
+ import { AttachedObject } from "../../engine-components/webxr/WebXRController";
14
14
  import { AudioListener } from "../../engine-components/AudioListener";
15
15
  import { AudioSource } from "../../engine-components/AudioSource";
16
16
  import { AudioTrackHandler } from "../../engine-components/timeline/TimelineTracks";
@@ -21,11 +21,12 @@
21
21
  import { AvatarBlink_Simple } from "../../engine-components/avatar/AvatarBlink_Simple";
22
22
  import { AvatarEyeLook_Rotation } from "../../engine-components/avatar/AvatarEyeLook_Rotation";
23
23
  import { AvatarLoader } from "../../engine-components/AvatarLoader";
24
- import { AvatarMarker } from "../../engine-components/WebXRAvatar";
24
+ import { AvatarMarker } from "../../engine-components/webxr/WebXRAvatar";
25
25
  import { AvatarModel } from "../../engine-components/AvatarLoader";
26
26
  import { AxesHelper } from "../../engine-components/AxesHelper";
27
27
  import { BaseUIComponent } from "../../engine-components/ui/BaseUIComponent";
28
28
  import { BasicIKConstraint } from "../../engine-components/BasicIKConstraint";
29
+ import { Behaviour } from "../../engine-components/Component";
29
30
  import { Bloom } from "../../engine-components/postprocessing/Effects/Bloom";
30
31
  import { BoxCollider } from "../../engine-components/Collider";
31
32
  import { BoxGizmo } from "../../engine-components/Gizmos";
@@ -43,6 +44,7 @@
43
44
  import { ColorAdjustments } from "../../engine-components/postprocessing/Effects/ColorAdjustments";
44
45
  import { ColorBySpeedModule } from "../../engine-components/ParticleSystemModules";
45
46
  import { ColorOverLifetimeModule } from "../../engine-components/ParticleSystemModules";
47
+ import { Component } from "../../engine-components/Component";
46
48
  import { ControlTrackHandler } from "../../engine-components/timeline/TimelineTracks";
47
49
  import { Deletable } from "../../engine-components/DeleteBox";
48
50
  import { DeleteBox } from "../../engine-components/DeleteBox";
@@ -146,7 +148,7 @@
146
148
  import { SyncedCamera } from "../../engine-components/SyncedCamera";
147
149
  import { SyncedRoom } from "../../engine-components/SyncedRoom";
148
150
  import { SyncedTransform } from "../../engine-components/SyncedTransform";
149
- import { TeleportTarget } from "../../engine-components/WebXRController";
151
+ import { TeleportTarget } from "../../engine-components/webxr/WebXRController";
150
152
  import { TestRunner } from "../../engine-components/TestRunner";
151
153
  import { TestSimulateUserData } from "../../engine-components/TestRunner";
152
154
  import { Text } from "../../engine-components/ui/Text";
@@ -168,23 +170,24 @@
168
170
  import { Volume } from "../../engine-components/postprocessing/Volume";
169
171
  import { VolumeParameter } from "../../engine-components/postprocessing/VolumeParameter";
170
172
  import { VolumeProfile } from "../../engine-components/postprocessing/VolumeProfile";
171
- import { VRUserState } from "../../engine-components/WebXRSync";
172
- import { WebAR } from "../../engine-components/WebXR";
173
- import { WebARCameraBackground } from "../../engine-components/WebARCameraBackground";
174
- import { WebARSessionRoot } from "../../engine-components/WebARSessionRoot";
175
- import { WebXR } from "../../engine-components/WebXR";
176
- import { WebXRAvatar } from "../../engine-components/WebXRAvatar";
177
- import { WebXRController } from "../../engine-components/WebXRController";
178
- import { WebXRImageTracking } from "../../engine-components/WebXRImageTracking";
179
- import { WebXRImageTrackingModel } from "../../engine-components/WebXRImageTracking";
180
- import { WebXRSync } from "../../engine-components/WebXRSync";
181
- import { WebXRTrackedImage } from "../../engine-components/WebXRImageTracking";
173
+ import { VRUserState } from "../../engine-components/webxr/WebXRSync";
174
+ import { WebAR } from "../../engine-components/webxr/WebXR";
175
+ import { WebARCameraBackground } from "../../engine-components/webxr/WebARCameraBackground";
176
+ import { WebARSessionRoot } from "../../engine-components/webxr/WebARSessionRoot";
177
+ import { WebXR } from "../../engine-components/webxr/WebXR";
178
+ import { WebXRAvatar } from "../../engine-components/webxr/WebXRAvatar";
179
+ import { WebXRController } from "../../engine-components/webxr/WebXRController";
180
+ import { WebXRImageTracking } from "../../engine-components/webxr/WebXRImageTracking";
181
+ import { WebXRImageTrackingModel } from "../../engine-components/webxr/WebXRImageTracking";
182
+ import { WebXRPlaneTracking } from "../../engine-components/webxr/WebXRPlaneTracking";
183
+ import { WebXRSync } from "../../engine-components/webxr/WebXRSync";
184
+ import { WebXRTrackedImage } from "../../engine-components/webxr/WebXRImageTracking";
182
185
  import { XRFlag } from "../../engine-components/XRFlag";
183
- import { XRGrabModel } from "../../engine-components/WebXRGrabRendering";
184
- import { XRGrabRendering } from "../../engine-components/WebXRGrabRendering";
185
- import { XRRig } from "../../engine-components/WebXRRig";
186
+ import { XRGrabModel } from "../../engine-components/webxr/WebXRGrabRendering";
187
+ import { XRGrabRendering } from "../../engine-components/webxr/WebXRGrabRendering";
188
+ import { XRRig } from "../../engine-components/webxr/WebXRRig";
186
189
  import { XRState } from "../../engine-components/XRFlag";
187
-
190
+
188
191
  // Register types
189
192
  TypeStore.add("__Ignore", __Ignore);
190
193
  TypeStore.add("AlignmentConstraint", AlignmentConstraint);
@@ -211,6 +214,7 @@
211
214
  TypeStore.add("AxesHelper", AxesHelper);
212
215
  TypeStore.add("BaseUIComponent", BaseUIComponent);
213
216
  TypeStore.add("BasicIKConstraint", BasicIKConstraint);
217
+ TypeStore.add("Behaviour", Behaviour);
214
218
  TypeStore.add("Bloom", Bloom);
215
219
  TypeStore.add("BoxCollider", BoxCollider);
216
220
  TypeStore.add("BoxGizmo", BoxGizmo);
@@ -228,6 +232,7 @@
228
232
  TypeStore.add("ColorAdjustments", ColorAdjustments);
229
233
  TypeStore.add("ColorBySpeedModule", ColorBySpeedModule);
230
234
  TypeStore.add("ColorOverLifetimeModule", ColorOverLifetimeModule);
235
+ TypeStore.add("Component", Component);
231
236
  TypeStore.add("ControlTrackHandler", ControlTrackHandler);
232
237
  TypeStore.add("Deletable", Deletable);
233
238
  TypeStore.add("DeleteBox", DeleteBox);
@@ -362,6 +367,7 @@
362
367
  TypeStore.add("WebXRController", WebXRController);
363
368
  TypeStore.add("WebXRImageTracking", WebXRImageTracking);
364
369
  TypeStore.add("WebXRImageTrackingModel", WebXRImageTrackingModel);
370
+ TypeStore.add("WebXRPlaneTracking", WebXRPlaneTracking);
365
371
  TypeStore.add("WebXRSync", WebXRSync);
366
372
  TypeStore.add("WebXRTrackedImage", WebXRTrackedImage);
367
373
  TypeStore.add("XRFlag", XRFlag);
src/engine-components/api.ts CHANGED
@@ -9,5 +9,6 @@
9
9
  export * from "./postprocessing"
10
10
  export * from "./timeline"
11
11
  export * from "./ui"
12
+ export * from "./webxr"
12
13
 
13
14
  export { ClearFlags } from "./Camera"
src/engine/api.ts CHANGED
@@ -18,6 +18,7 @@
18
18
  export * from "./engine_hot_reload";
19
19
  export * from "./engine_gameobject";
20
20
  export * from "./engine_networking";
21
+ export * from "./engine_networking_types";
21
22
  export { syncField } from "./engine_networking_auto";
22
23
  export * from "./engine_networking_files";
23
24
  export * from "./engine_networking_instantiate";
src/engine-components/avatar/Avatar_Brain_LookAt.ts CHANGED
@@ -1,7 +1,7 @@
1
1
  import * as THREE from "three";
2
2
  import { TypeStore } from "../../engine/engine_typestore";
3
3
  import { Behaviour, GameObject } from "../Component";
4
- import { AvatarMarker } from "../WebXRAvatar";
4
+ import { AvatarMarker } from "../webxr/WebXRAvatar";
5
5
  import * as utils from "../../engine/engine_three_utils";
6
6
  import { OwnershipModel } from "../../engine/engine_networking";
7
7
  import { Int8BufferAttribute } from "three";
src/engine-components/avatar/Avatar_MouthShapes.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import { Behaviour, GameObject } from "../Component";
2
2
  import { Voip } from "../Voip";
3
- import { AvatarMarker } from "../WebXRAvatar";
3
+ import { AvatarMarker } from "../webxr/WebXRAvatar";
4
4
  import * as utils from "../../engine/engine_utils";
5
5
  import { Object3D } from "three";
6
6
  import { serializable } from "../../engine/engine_serialization_decorator";
src/engine-components/avatar/Avatar_MustacheShake.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import { Behaviour, GameObject } from "../Component";
2
2
  import { Voip } from "../Voip";
3
- import { AvatarMarker } from "../WebXRAvatar";
3
+ import { AvatarMarker } from "../webxr/WebXRAvatar";
4
4
 
5
5
  export class Avatar_MustacheShake extends Behaviour {
6
6
  private voip: Voip | null = null;
src/engine-components/codegen/components.ts CHANGED
@@ -8,7 +8,7 @@
8
8
  export { Animator } from "../Animator";
9
9
  export { AnimatorController } from "../AnimatorController";
10
10
  export { Antialiasing } from "../postprocessing/Effects/Antialiasing";
11
- export { AttachedObject } from "../WebXRController";
11
+ export { AttachedObject } from "../webxr/WebXRController";
12
12
  export { AudioListener } from "../AudioListener";
13
13
  export { AudioSource } from "../AudioSource";
14
14
  export { AudioTrackHandler } from "../timeline/TimelineTracks";
@@ -19,11 +19,12 @@
19
19
  export { AvatarBlink_Simple } from "../avatar/AvatarBlink_Simple";
20
20
  export { AvatarEyeLook_Rotation } from "../avatar/AvatarEyeLook_Rotation";
21
21
  export { AvatarLoader } from "../AvatarLoader";
22
- export { AvatarMarker } from "../WebXRAvatar";
22
+ export { AvatarMarker } from "../webxr/WebXRAvatar";
23
23
  export { AvatarModel } from "../AvatarLoader";
24
24
  export { AxesHelper } from "../AxesHelper";
25
25
  export { BaseUIComponent } from "../ui/BaseUIComponent";
26
26
  export { BasicIKConstraint } from "../BasicIKConstraint";
27
+ export { Behaviour } from "../Component";
27
28
  export { Bloom } from "../postprocessing/Effects/Bloom";
28
29
  export { BoxCollider } from "../Collider";
29
30
  export { BoxGizmo } from "../Gizmos";
@@ -41,6 +42,7 @@
41
42
  export { ColorAdjustments } from "../postprocessing/Effects/ColorAdjustments";
42
43
  export { ColorBySpeedModule } from "../ParticleSystemModules";
43
44
  export { ColorOverLifetimeModule } from "../ParticleSystemModules";
45
+ export { Component } from "../Component";
44
46
  export { ControlTrackHandler } from "../timeline/TimelineTracks";
45
47
  export { Deletable } from "../DeleteBox";
46
48
  export { DeleteBox } from "../DeleteBox";
@@ -141,7 +143,7 @@
141
143
  export { SyncedCamera } from "../SyncedCamera";
142
144
  export { SyncedRoom } from "../SyncedRoom";
143
145
  export { SyncedTransform } from "../SyncedTransform";
144
- export { TeleportTarget } from "../WebXRController";
146
+ export { TeleportTarget } from "../webxr/WebXRController";
145
147
  export { TestRunner } from "../TestRunner";
146
148
  export { TestSimulateUserData } from "../TestRunner";
147
149
  export { Text } from "../ui/Text";
@@ -163,19 +165,20 @@
163
165
  export { Volume } from "../postprocessing/Volume";
164
166
  export { VolumeParameter } from "../postprocessing/VolumeParameter";
165
167
  export { VolumeProfile } from "../postprocessing/VolumeProfile";
166
- export { VRUserState } from "../WebXRSync";
167
- export { WebAR } from "../WebXR";
168
- export { WebARCameraBackground } from "../WebARCameraBackground";
169
- export { WebARSessionRoot } from "../WebARSessionRoot";
170
- export { WebXR } from "../WebXR";
171
- export { WebXRAvatar } from "../WebXRAvatar";
172
- export { WebXRController } from "../WebXRController";
173
- export { WebXRImageTracking } from "../WebXRImageTracking";
174
- export { WebXRImageTrackingModel } from "../WebXRImageTracking";
175
- export { WebXRSync } from "../WebXRSync";
176
- export { WebXRTrackedImage } from "../WebXRImageTracking";
168
+ export { VRUserState } from "../webxr/WebXRSync";
169
+ export { WebAR } from "../webxr/WebXR";
170
+ export { WebARCameraBackground } from "../webxr/WebARCameraBackground";
171
+ export { WebARSessionRoot } from "../webxr/WebARSessionRoot";
172
+ export { WebXR } from "../webxr/WebXR";
173
+ export { WebXRAvatar } from "../webxr/WebXRAvatar";
174
+ export { WebXRController } from "../webxr/WebXRController";
175
+ export { WebXRImageTracking } from "../webxr/WebXRImageTracking";
176
+ export { WebXRImageTrackingModel } from "../webxr/WebXRImageTracking";
177
+ export { WebXRPlaneTracking } from "../webxr/WebXRPlaneTracking";
178
+ export { WebXRSync } from "../webxr/WebXRSync";
179
+ export { WebXRTrackedImage } from "../webxr/WebXRImageTracking";
177
180
  export { XRFlag } from "../XRFlag";
178
- export { XRGrabModel } from "../WebXRGrabRendering";
179
- export { XRGrabRendering } from "../WebXRGrabRendering";
180
- export { XRRig } from "../WebXRRig";
181
+ export { XRGrabModel } from "../webxr/WebXRGrabRendering";
182
+ export { XRGrabRendering } from "../webxr/WebXRGrabRendering";
183
+ export { XRRig } from "../webxr/WebXRRig";
181
184
  export { XRState } from "../XRFlag";
src/engine-components/DragControls.ts CHANGED
@@ -4,7 +4,7 @@
4
4
  import { Context } from "../engine/engine_setup";
5
5
  import { Interactable, UsageMarker } from "./Interactable";
6
6
  import { Rigidbody } from "./RigidBody";
7
- import { WebXR } from "./WebXR";
7
+ import { WebXR } from "./webxr/WebXR";
8
8
  import { Avatar_POI } from "./avatar/Avatar_Brain_LookAt";
9
9
  import { RaycastOptions } from "../engine/engine_physics";
10
10
  import { getWorldPosition, setWorldPosition } from "../engine/engine_three_utils";
src/engine-components/Duplicatable.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import { Behaviour, GameObject } from "./Component";
2
- import { WebXRController, ControllerEvents } from "./WebXRController";
2
+ import { WebXRController, ControllerEvents } from "./webxr/WebXRController";
3
3
  import { DragControls, DragEvents } from "./DragControls";
4
4
  import { Interactable } from "./Interactable";
5
5
  import { Animation } from "./Animation";
src/engine/engine_element_loading.ts CHANGED
@@ -8,6 +8,8 @@
8
8
  const debug = getParam("debugloadingbar");
9
9
  const debugRendering = getParam("debugloadingbarrendering");
10
10
 
11
+ declare type LoadingStyleOption = "dark" | "light";
12
+
11
13
  export class LoadingElementOptions {
12
14
  className?: string;
13
15
  additionalClasses?: string[];
@@ -172,6 +174,10 @@
172
174
  private createLoadingElement(existing?: HTMLElement): HTMLElement {
173
175
  if (debug && !existing) console.log("Creating loading element");
174
176
  this._loadingElement = existing || document.createElement("div");
177
+
178
+ const loadingStyle: LoadingStyleOption = this._element.getAttribute("loading-style") as LoadingStyleOption;
179
+ console.log(loadingStyle);
180
+
175
181
  const hasLicense = hasProLicense();
176
182
  if (!existing) {
177
183
  this._loadingElement.style.position = "fixed";
@@ -179,7 +185,10 @@
179
185
  this._loadingElement.style.height = "100%";
180
186
  this._loadingElement.style.left = "0";
181
187
  this._loadingElement.style.top = "0";
182
- this._loadingElement.style.backgroundColor = "#000000";
188
+ if (loadingStyle === "light")
189
+ this._loadingElement.style.backgroundColor = "#ddd";
190
+ else
191
+ this._loadingElement.style.backgroundColor = "#000";
183
192
  this._loadingElement.style.display = "flex";
184
193
  this._loadingElement.style.alignItems = "center";
185
194
  this._loadingElement.style.justifyContent = "center";
@@ -187,6 +196,10 @@
187
196
  this._loadingElement.style.flexDirection = "column";
188
197
  this._loadingElement.style.pointerEvents = "none";
189
198
  this._loadingElement.style.color = "white";
199
+ if (loadingStyle === "light")
200
+ this._loadingElement.style.color = "rgba(0,0,0,.8)";
201
+ else
202
+ this._loadingElement.style.color = "rgba(255,255,255,.5)";
190
203
  if (hasLicense && this._element) {
191
204
  const loadingBackgroundColor = this._element.getAttribute("loading-background-color");
192
205
  if (loadingBackgroundColor) {
@@ -213,7 +226,10 @@
213
226
  loadingBarContainer.style.display = "flex";
214
227
  loadingBarContainer.style.width = maxWidth + "%";
215
228
  loadingBarContainer.style.height = "2px";
216
- loadingBarContainer.style.background = "rgba(255,255,255,.2)"
229
+ if (loadingStyle === "light")
230
+ loadingBarContainer.style.backgroundColor = "rgba(0,0,0,.2)"
231
+ else
232
+ loadingBarContainer.style.backgroundColor = "rgba(255,255,255,.2)"
217
233
  // loadingBarContainer.style.alignItems = "center";
218
234
  this._loadingElement.appendChild(loadingBarContainer);
219
235
 
@@ -244,20 +260,20 @@
244
260
  return Mathf.lerp(maxWidth * .5, 100 - maxWidth * .5, t) + "%";
245
261
  }
246
262
  this._loadingBar.style.background =
247
- `linear-gradient(90deg, #02022B ${getGradientPos(0)}, #0BA398 ${getGradientPos(.4)}, #99CC33 ${getGradientPos(.5)}, #D7DB0A ${getGradientPos(1)})`;
263
+ `linear-gradient(90deg, #02022B ${getGradientPos(0)}, #0BA398 ${getGradientPos(.4)}, #99CC33 ${getGradientPos(.5)}, #D7DB0A ${getGradientPos(1)})`;
248
264
  this._loadingBar.style.backgroundAttachment = "fixed";
249
265
  this._loadingBar.style.width = "0%";
250
266
  this._loadingBar.style.height = "100%";
251
- if(hasLicense && this._element){
267
+ if (hasLicense && this._element) {
252
268
  const primaryColor = this._element.getAttribute("primary-color");
253
269
  const secondaryColor = this._element.getAttribute("secondary-color");
254
- if(primaryColor && secondaryColor){
270
+ if (primaryColor && secondaryColor) {
255
271
  this._loadingBar.style.background = `linear-gradient(90deg, ${primaryColor} ${getGradientPos(0)}, ${secondaryColor} ${getGradientPos(1)})`;
256
272
  }
257
- else if(primaryColor){
273
+ else if (primaryColor) {
258
274
  this._loadingBar.style.background = primaryColor;
259
275
  }
260
- else if(secondaryColor){
276
+ else if (secondaryColor) {
261
277
  this._loadingBar.style.background = secondaryColor;
262
278
  }
263
279
  }
@@ -273,16 +289,15 @@
273
289
  messageContainer.style.display = "flex";
274
290
  messageContainer.style.fontSize = ".8em";
275
291
  messageContainer.style.paddingTop = ".5em";
276
- messageContainer.style.color = "rgba(255,255,255,.5)";
277
292
  messageContainer.style.fontWeight = "200";
278
293
  messageContainer.style.fontFamily = "Roboto, sans-serif";
279
294
  // messageContainer.style.border = "1px solid rgba(255,255,255,.1)";
280
295
  messageContainer.style.justifyContent = "center";
281
296
  this._loadingElement.appendChild(messageContainer);
282
297
 
283
- if(hasLicense && this._element){
298
+ if (hasLicense && this._element) {
284
299
  const loadingTextColor = this._element.getAttribute("loading-text-color");
285
- if(loadingTextColor){
300
+ if (loadingTextColor) {
286
301
  messageContainer.style.color = loadingTextColor;
287
302
  }
288
303
  }
src/engine/engine_networking_auto.ts CHANGED
@@ -121,8 +121,8 @@
121
121
  this._isReceiving = true;
122
122
  for (const key in val) {
123
123
  if (key === "guid") continue;
124
+ // TODO: maybe use serializable here?!
124
125
  const value = val[key];
125
- // console.log("SET", key, value, this.comp.guid, this.comp);
126
126
  this.comp[key] = value;
127
127
  }
128
128
  }
src/engine-components/ui/EventSystem.ts CHANGED
@@ -1,7 +1,7 @@
1
1
  import { RaycastOptions } from "../../engine/engine_physics";
2
2
  import { Behaviour, Component, GameObject } from "../Component";
3
- import { WebXR } from "../WebXR";
4
- import { ControllerEvents, WebXRController } from "../WebXRController";
3
+ import { WebXR } from "../webxr/WebXR";
4
+ import { ControllerEvents, WebXRController } from "../webxr/WebXRController";
5
5
  import * as ThreeMeshUI from 'three-mesh-ui'
6
6
  import { Context } from "../../engine/engine_setup";
7
7
  import { OrbitControls } from "../OrbitControls";
src/engine-components/ui/InputField.ts CHANGED
@@ -144,7 +144,8 @@
144
144
  if (this.placeholder && (!this.textComponent || this.textComponent.text.length <= 0))
145
145
  GameObject.setActive(this.placeholder.gameObject, true);
146
146
 
147
- this.onEndEdit?.invoke();
147
+ if (InputField.htmlField)
148
+ this.onEndEdit?.invoke(InputField.htmlField.value);
148
149
  }
149
150
 
150
151
 
src/engine-components/Light.ts CHANGED
@@ -5,8 +5,8 @@
5
5
  import { FrameEvent } from "../engine/engine_setup";
6
6
  import { serializable } from "../engine/engine_serialization_decorator";
7
7
  import { Color, DirectionalLight, OrthographicCamera } from "three";
8
- import { WebXR, WebXREvent } from "./WebXR";
9
- import { WebARSessionRoot } from "./WebARSessionRoot";
8
+ import { WebXR, WebXREvent } from "./webxr/WebXR";
9
+ import { WebARSessionRoot } from "./webxr/WebARSessionRoot";
10
10
  import { ILight } from "../engine/engine_types";
11
11
  import { Mathf } from "../engine/engine_math";
12
12
  import { isLocalNetwork } from "../engine/engine_networking_utils";
src/engine-components/PlayerColor.ts CHANGED
@@ -1,7 +1,7 @@
1
1
  import { RoomEvents } from "../engine/engine_networking";
2
2
  import { Behaviour, GameObject } from "./Component";
3
3
  import * as THREE from "three";
4
- import { AvatarMarker } from "./WebXRAvatar";
4
+ import { AvatarMarker } from "./webxr/WebXRAvatar";
5
5
  import { WaitForSeconds } from "../engine/engine_coroutine";
6
6
 
7
7
 
src/engine-components/ShadowCatcher.ts CHANGED
@@ -7,6 +7,7 @@
7
7
  enum ShadowMode {
8
8
  ShadowMask = 0,
9
9
  Additive = 1,
10
+ Occluder = 2,
10
11
  }
11
12
 
12
13
  export class ShadowCatcher extends Behaviour {
@@ -28,6 +29,9 @@
28
29
  case ShadowMode.Additive:
29
30
  this.applyLightBlendMaterial();
30
31
  break;
32
+ case ShadowMode.Occluder:
33
+ this.applyOccluderMaterial();
34
+ break;
31
35
  }
32
36
 
33
37
  }
@@ -88,6 +92,17 @@
88
92
  }
89
93
  }
90
94
 
95
+ applyOccluderMaterial() {
96
+ const renderer = GameObject.getComponent(this.gameObject, Renderer);
97
+ if (renderer) {
98
+ const material = renderer.sharedMaterial;
99
+ material.depthWrite = true;
100
+ material.stencilWrite = true;
101
+ material.colorWrite = false;
102
+ this.gameObject.renderOrder = -100;
103
+ }
104
+ }
105
+
91
106
  private applyMaterialOptions(material: Material) {
92
107
  if (material) {
93
108
  material.depthWrite = false;
src/engine-components/SpectatorCamera.ts CHANGED
@@ -2,8 +2,8 @@
2
2
  import { Camera } from "./Camera";
3
3
  import * as THREE from "three";
4
4
  import { OrbitControls } from "./OrbitControls";
5
- import { WebXR, WebXREvent } from "./WebXR";
6
- import { AvatarMarker } from "./WebXRAvatar";
5
+ import { WebXR, WebXREvent } from "./webxr/WebXR";
6
+ import { AvatarMarker } from "./webxr/WebXRAvatar";
7
7
  import { XRStateFlag } from "./XRFlag";
8
8
  import { SmoothFollow } from "./SmoothFollow";
9
9
  import { Object3D } from "three";
src/engine-components/SyncedCamera.ts CHANGED
@@ -2,7 +2,7 @@
2
2
  import { Behaviour, GameObject } from "./Component";
3
3
  import { Camera } from "./Camera";
4
4
  import * as utils from "../engine/engine_three_utils"
5
- import { WebXR } from "./WebXR";
5
+ import { WebXR } from "./webxr/WebXR";
6
6
  import { Builder } from "flatbuffers";
7
7
  import { SyncedCameraModel } from "../engine-schemes/synced-camera-model";
8
8
  import { Vec3 } from "../engine-schemes/vec3";
@@ -10,7 +10,7 @@
10
10
  import { InstancingUtil } from "../engine/engine_instancing";
11
11
  import { serializable } from "../engine/engine_serialization_decorator";
12
12
  import { Object3D } from "three";
13
- import { AvatarMarker } from "./WebXRAvatar";
13
+ import { AvatarMarker } from "./webxr/WebXRAvatar";
14
14
  import { AssetReference } from "../engine/engine_addressables";
15
15
  import { ViewDevice } from "../engine/engine_playerview";
16
16
  import { InstantiateOptions } from "../engine/engine_gameobject";
src/engine-components/export/usdz/USDZExporter.ts CHANGED
@@ -8,11 +8,11 @@
8
8
  import { registerAnimatorsImplictly } from "./utils/animationutils";
9
9
  import { IUSDZExporterExtension } from "./Extension";
10
10
  import { Behaviour, GameObject } from "../../Component";
11
- import { WebXR } from "../../WebXR"
11
+ import { WebXR } from "../../webxr/WebXR"
12
12
  import { serializable } from "../../../engine/engine_serialization";
13
13
  import { showBalloonWarning } from "../../../engine/debug/debug";
14
14
  import { Context } from "../../../engine/engine_setup";
15
- import { WebARSessionRoot } from "../../WebARSessionRoot";
15
+ import { WebARSessionRoot } from "../../webxr/WebARSessionRoot";
16
16
 
17
17
  const debug = getParam("debugusdz");
18
18
 
src/engine-components/WebARCameraBackground.ts DELETED
@@ -1,215 +0,0 @@
1
- import { Behaviour } from "./Component";
2
- import { serializable } from "../engine/engine_serialization_decorator";
3
- import { RGBAColor } from "../engine-components/js-extensions/RGBAColor"
4
- import { WebXR } from "../engine-components/WebXR";
5
- import {
6
- Camera as ThreeCamera,
7
- Scene,
8
- Texture,
9
- Mesh, MeshBasicMaterial,
10
- UniformsUtils,
11
- PlaneGeometry,
12
- ShaderLib,
13
- ShaderMaterial,
14
- DoubleSide
15
- } from "three";
16
-
17
- export class WebARCameraBackground extends Behaviour {
18
-
19
- awake(): void {
20
- WebXR.OptionalFeatures_AR.push('camera-access');
21
- }
22
-
23
- @serializable()
24
- public backgroundTint: RGBAColor = new RGBAColor(1,1,1,1);
25
-
26
- public get background() {
27
- return this.backgroundPlane;
28
- }
29
-
30
- private _preRender;
31
-
32
- onEnable(): void {
33
- this._preRender = this.preRender.bind(this);
34
- this.context.pre_render_callbacks.push(this._preRender);
35
-
36
- if (this.backgroundPlane) {
37
- this.gameObject.add(this.backgroundPlane);
38
- this.backgroundPlane.visible = false;
39
- }
40
- }
41
-
42
- onDisable(): void {
43
- this.context.pre_render_callbacks = this.context.pre_render_callbacks.filter(cb => cb !== this._preRender);
44
-
45
- if (this.backgroundPlane)
46
- this.gameObject.remove(this.backgroundPlane);
47
- }
48
-
49
- private backgroundPlane?: Mesh;
50
- private threeTexture?: Texture;
51
- private forceTextureInitialization = function() {
52
- const material = new MeshBasicMaterial();
53
- const geometry = new PlaneGeometry();
54
- const scene = new Scene();
55
- scene.add(new Mesh(geometry, material));
56
- const camera = new ThreeCamera();
57
-
58
- return function forceTextureInitialization(renderer, texture) {
59
- material.map = texture;
60
- renderer.render(scene, camera);
61
- };
62
- }();
63
-
64
- // TODO should only attach on session start, and detach on session end
65
- private preRender() {
66
- if (!this || !this.gameObject) return;
67
-
68
- const xr = this.context.renderer.xr;
69
- const frame = xr.getFrame();
70
-
71
- if (frame) {
72
-
73
- // We're generating a new texture here, and force three to initialize it
74
- // from https://stackoverflow.com/a/55084367 to inject a custom texture into three.js
75
- if (!this.threeTexture && this.context.renderer) {
76
- this.threeTexture = new Texture();
77
- // this.threeTexture.encoding = LinearEncoding;
78
- this.forceTextureInitialization(this.context.renderer, this.threeTexture);
79
- }
80
-
81
- // simple mesh and fullscreen shader to display the camera texture
82
- // from three: WebGLBackground
83
- if (this.backgroundPlane === undefined) {
84
- this.backgroundPlane = makeFullscreenPlane(this.backgroundTint);
85
- this.gameObject.add(this.backgroundPlane);
86
- }
87
-
88
- // WebXR Raw Camera Access -
89
- // we composite the camera texture into the scene background by rendering it first.
90
- this.updateFromFrame(frame);
91
- }
92
-
93
- /*
94
- if (this.planeMesh) {
95
- this.planeMesh.visible = frame != null;
96
- }
97
- */
98
- }
99
-
100
- onBeforeRender(frame: XRFrame | null) {
101
- this.updateFromFrame(frame);
102
- }
103
-
104
- updateFromFrame(frame: XRFrame | null) {
105
- if (!frame) return;
106
-
107
- // https://chromium.googlesource.com/chromium/src/+/7c5ac3c0f95b97cf12be95a5c1c0a8ff163246d8/third_party/webxr_test_pages/webxr-samples/proposals/camera-access-barebones.html
108
- const pose = frame.getViewerPose(this.context.renderer.xr.getReferenceSpace()!);
109
- if (pose) {
110
- for( const view of pose.views) {
111
- // @ts-ignore
112
- if ('camera' in view && view.camera) {
113
- const xrManager = this.context.renderer.xr;
114
- let binding = xrManager.getBinding();
115
- // not sure how / why this can be null, but we can recreate it here
116
- if (!binding) binding = new XRWebGLBinding( frame.session, this.context.renderer.getContext() );
117
-
118
- if (binding) {
119
- let glImage: WebGLTexture | null = null;
120
- if ('getCameraImage' in binding) {
121
- // @ts-ignore
122
- glImage = binding.getCameraImage(view.camera);
123
-
124
- // discussion on exactly this:
125
- // https://discourse.threejs.org/t/using-a-webgltexture-as-texture-for-three-js/46245/8
126
- // HACK from https://stackoverflow.com/a/55084367 to inject a custom texture into three.js
127
- const texProps = this.context.renderer.properties.get(this.threeTexture);
128
- texProps.__webglTexture = glImage;
129
-
130
- if (this.backgroundPlane) {
131
- //@ts-ignore
132
- this.backgroundPlane.setTexture(this.threeTexture);
133
- this.backgroundPlane.visible = true;
134
- }
135
- }
136
- }
137
- else {
138
- // console.error(view.camera, xrManager)
139
- }
140
- }
141
- else {
142
- // console.error("NO CAMERA IN VIEW")
143
- }
144
- }
145
- }
146
- else {
147
- // console.error(this.context.renderer.xr.getReferenceSpace(), frame);
148
- }
149
- }
150
- }
151
- // TODO tint could be an uniform
152
- let backgroundFragment: string = /* glsl */`
153
- uniform sampler2D t2D;
154
-
155
- varying vec2 vUv;
156
-
157
- void main() {
158
-
159
- vec4 texColor = texture2D( t2D, vUv );
160
- texColor.w = 1.0;
161
-
162
- // inline sRGB decode
163
- texColor = vec4( mix( pow( texColor.rgb * 0.9478672986 + vec3( 0.0521327014 ), vec3( 2.4 ) ), texColor.rgb * 0.0773993808, vec3( lessThanEqual( texColor.rgb, vec3( 0.04045 ) ) ) ), texColor.w );
164
-
165
- gl_FragColor = texColor * <backgroundTint>;
166
-
167
- #include <tonemapping_fragment>
168
- #include <encodings_fragment>
169
-
170
- }
171
- `;
172
-
173
- // not sure where we want to move this and in which form is best (extends Object3D?)
174
- export function makeFullscreenPlane(tint: RGBAColor ) {
175
- const replacementTint = "vec4(" + tint.r.toFixed(3) + "," + tint.g.toFixed(3) + "," + tint.b.toFixed(3) + "," + tint.a.toFixed(3) + ")";
176
- console.log(replacementTint);
177
- const planeMesh = new Mesh(
178
- new PlaneGeometry(2, 2),
179
- // @ts-ignore
180
- new ShaderMaterial({
181
- name: 'BackgroundMaterial',
182
- uniforms: UniformsUtils.clone( ShaderLib.background.uniforms ),
183
- vertexShader: ShaderLib.background.vertexShader,
184
- fragmentShader: backgroundFragment.replace("<backgroundTint>", replacementTint), // ShaderLib.background.fragmentShader,
185
- side: DoubleSide,
186
- depthTest: false,
187
- depthWrite: false,
188
- fog: false
189
- })
190
- );
191
-
192
- planeMesh.geometry.deleteAttribute( 'normal' );
193
-
194
- // add "map" material property so the renderer can evaluate it like for built-in materials
195
- Object.defineProperty( planeMesh.material, 'map', {
196
- get: function () {
197
- return this.threeTexture;
198
- }
199
- } );
200
-
201
- // Option 1: add the planeMesh to our scene for rendering.
202
- // This is useful for applying custom shader effects on the background (instead of using the system composite)
203
- planeMesh.renderOrder = -10000; // render first
204
- planeMesh.layers.disableAll();
205
- planeMesh.layers.enable(2); // ignore raycasts
206
- planeMesh.frustumCulled = false;
207
-
208
- // should be a class, for now lets just define a method for the weird way the texture needs to be set
209
- // @ts-ignore
210
- planeMesh.setTexture = function(texture) {
211
- planeMesh.material.uniforms.t2D.value = texture;
212
- }
213
-
214
- return planeMesh;
215
- }
src/engine-components/WebARSessionRoot.ts DELETED
@@ -1,178 +0,0 @@
1
- import { Behaviour, GameObject } from "./Component";
2
- import { Matrix4, Object3D } from "three";
3
- import { WebAR, WebXR } from "./WebXR";
4
- import { InstancingUtil } from "../engine/engine_instancing";
5
- import { serializable } from "../engine/engine_serialization_decorator";
6
-
7
- // https://github.com/takahirox/takahirox.github.io/blob/master/js.mmdeditor/examples/js/controls/DeviceOrientationControls.js
8
-
9
- export class WebARSessionRoot extends Behaviour {
10
-
11
- webAR: WebAR | null = null;
12
-
13
- get rig(): Object3D | undefined {
14
- return this.webAR?.webxr.Rig;
15
- }
16
-
17
- @serializable()
18
- invertForward: boolean = false;
19
-
20
- @serializable()
21
- get arScale(): number {
22
- return this._arScale;
23
- }
24
- set arScale(val: number) {
25
- if (val === this._arScale) return;
26
- this._arScale = val;
27
- this.setScale(val);
28
- }
29
-
30
- private readonly _initalMatrix = new Matrix4();
31
- private readonly _selectStartFn = this.onSelectStart.bind(this);
32
- private readonly _selectEndFn = this.onSelectEnd.bind(this);
33
-
34
- start() {
35
- const xr = GameObject.findObjectOfType(WebXR);
36
- if (xr) {
37
- xr.Rig.updateMatrix();
38
- this._initalMatrix.copy(xr.Rig.matrix);
39
- }
40
- }
41
-
42
- private _arScale: number = 5;
43
- private _rig: Object3D | null = null;
44
- private _startPose: Matrix4 | null = null;
45
- private _placementPose: Matrix4 | null = null;
46
- private _isTouching: boolean = false;
47
- private _rigStartPose: Matrix4 | undefined | null = null;
48
- private _gotFirstHitTestResult: boolean = false;
49
-
50
- onBegin(session: XRSession) {
51
- this._placementPose = null;
52
- this.gameObject.visible = false;
53
- this.gameObject.matrixAutoUpdate = false;
54
- this._startPose = this.gameObject.matrix.clone();
55
- this._rigStartPose = this.rig?.matrix.clone();
56
- this._gotFirstHitTestResult = false;
57
- session.addEventListener('selectstart', this._selectStartFn);
58
- session.addEventListener('selectend', this._selectEndFn);
59
- // setTimeout(() => this.gameObject.visible = false, 1000); // TODO test on phone AR and Hololens if this was still needed
60
-
61
- // console.log(this.rig?.position, this.rig?.quaternion, this.rig?.scale);
62
- this.gameObject.visible = false;
63
-
64
- if (this.rig) {
65
- // reset rig to initial pose, this is helping the mix of immersive AR and immersive VR that we now have on quest
66
- // where the rig can be moved and scaled by the user in VR mode and we use the rig position when entering
67
- // immersive Ar right now to place the user/offset the session
68
- this.rig.matrixAutoUpdate = true;
69
- this._initalMatrix.decompose(this.rig.position, this.rig.quaternion, this.rig.scale);
70
- }
71
-
72
- // TODO this is duplicate to WebXR events AND engine events, would be better in one place
73
- this.dispatchEvent(new CustomEvent('onBeginSession'));
74
- }
75
-
76
- onUpdate(rig: Object3D | null, _session: XRSession, pose: XRPose | null | undefined): boolean {
77
-
78
- if (pose && !this._placementPose) {
79
-
80
- if (!this._gotFirstHitTestResult) {
81
- this._gotFirstHitTestResult = true;
82
- this.dispatchEvent(new CustomEvent('canPlaceSession', { detail: { pose: pose } }));
83
- }
84
-
85
- if (this._isTouching) {
86
- // callbacks
87
- const poseMatrix = new Matrix4().fromArray(pose.transform.matrix).invert();
88
- this.dispatchEvent(new CustomEvent('placedSession', { detail: { pose, poseMatrix } }));
89
-
90
- if (this.webAR) this.webAR.setReticleActive(false);
91
- this.placeAt(rig, poseMatrix);
92
- return true;
93
- }
94
- }
95
- return false;
96
-
97
- // if (this._placementPose) {
98
- // this.gameObject.matrixAutoUpdate = false;
99
- // const matrix = pose?.transform.matrix;
100
- // if (matrix) {
101
- // this.gameObject.matrix.fromArray(matrix);
102
- // }
103
- // this.gameObject.visible = true;
104
- // }
105
- }
106
-
107
- placeAt(rig: Object3D | null, mat: Matrix4) {
108
- if (!this._placementPose) this._placementPose = new Matrix4();
109
- this._placementPose.copy(mat);
110
- // apply session root offset
111
- const invertedSessionRoot = this.gameObject.matrixWorld.clone().invert();
112
- this._placementPose.premultiply(invertedSessionRoot);
113
- if (rig) {
114
-
115
- if (this.invertForward) {
116
- const rot = new Matrix4().makeRotationY(Math.PI);
117
- this._placementPose.premultiply(rot);
118
- }
119
- this._rig = rig;
120
-
121
- this.setScale(this.arScale);
122
- }
123
- else this._rig = null;
124
- // this.gameObject.matrix.copy(this._placementPose);
125
- // if (rig) {
126
- // this.gameObject.matrix.premultiply(rig.matrixWorld)
127
- // }
128
- this.gameObject.visible = true;
129
- }
130
-
131
- onEnd(rig: Object3D | null, _session: XRSession) {
132
- this._placementPose = null;
133
- this.gameObject.visible = false;
134
- this.gameObject.matrixAutoUpdate = false;
135
- if (this._startPose) {
136
- this.gameObject.matrix.copy(this._startPose);
137
- }
138
- if (rig) {
139
- rig.matrixAutoUpdate = true;
140
- if (this._rigStartPose) {
141
- this._rigStartPose.decompose(rig.position, rig.quaternion, rig.scale);
142
- // console.log(rig.position, rig.quaternion, rig.scale);
143
- }
144
- }
145
- InstancingUtil.markDirty(this.gameObject, true);
146
- // HACK to fix physics being not in correct place after exiting AR
147
- setTimeout(() => {
148
- this.gameObject.matrixAutoUpdate = true;
149
- this.gameObject.visible = true;
150
- }, 100);
151
- }
152
-
153
-
154
- private onSelectStart() {
155
- this._isTouching = true;
156
- }
157
-
158
- private onSelectEnd() {
159
- this._isTouching = false;
160
- }
161
-
162
- private setScale(scale) {
163
- const rig = this._rig;
164
- if (!rig || !this._placementPose) {
165
- return;
166
- }
167
- // Capture the rig position before the first time we move it during a session
168
- if (!this._rigStartPose) {
169
- this._rigStartPose = rig.matrix.clone();
170
- }
171
- // we apply the transform to the rig because we want to move the user's position for easy networking
172
- rig.matrixAutoUpdate = false;
173
- rig.matrix.multiplyMatrices(new Matrix4().makeScale(scale, scale, scale), this._placementPose);
174
- rig.matrix.decompose(rig.position, rig.quaternion, rig.scale);
175
- rig.updateMatrixWorld();
176
- console.log("Place", rig.position);
177
- }
178
- }
src/engine-components/WebXR.ts DELETED
@@ -1,712 +0,0 @@
1
- import { ArrayCamera, Color, Euler, EventDispatcher, Group, Matrix4, Mesh, MeshBasicMaterial, Object3D, Quaternion, RingGeometry, Texture, Vector3 } from 'three';
2
- import { ARButton } from '../include/three/ARButton.js';
3
- import { VRButton } from '../include/three/VRButton.js';
4
-
5
- import { AssetReference } from "../engine/engine_addressables";
6
- import { serializable } from "../engine/engine_serialization_decorator";
7
- import { XRSessionMode } from "../engine/engine_setup";
8
- import { getWorldPosition, getWorldQuaternion, setWorldPosition, setWorldQuaternion } from "../engine/engine_three_utils";
9
- import { INeedleEngineComponent } from "../engine/engine_types";
10
- import { getParam, isMozillaXR, setOrAddParamsToUrl } from "../engine/engine_utils";
11
-
12
- import { Behaviour, GameObject } from "./Component";
13
- import { noVoip } from "./Voip";
14
- import { WebARSessionRoot } from "./WebARSessionRoot";
15
- import { ControllerType, WebXRController } from "./WebXRController";
16
- import { XRRig } from "./WebXRRig";
17
- import { WebXRSync } from "./WebXRSync";
18
- import { XRFlag, XRState, XRStateFlag } from "./XRFlag";
19
- import { showBalloonWarning } from '../engine/debug';
20
-
21
-
22
- export async function detectARSupport() {
23
- if(isMozillaXR()) return true;
24
- if ("xr" in navigator) {
25
- //@ts-ignore
26
- return (await navigator["xr"].isSessionSupported('immersive-ar')) === true;
27
- }
28
- return false;
29
- }
30
- export async function detectVRSupport() {
31
- if ("xr" in navigator) {
32
- //@ts-ignore
33
- return (await navigator["xr"].isSessionSupported('immersive-vr')) === true;
34
- }
35
- return false;
36
- }
37
-
38
- let arSupported = false;
39
- let vrSupported = false;
40
- detectARSupport().then(res => arSupported = res);
41
- detectVRSupport().then(res => vrSupported = res);
42
-
43
- // import TeleportVR from "teleportvr";
44
-
45
- export enum WebXREvent {
46
- XRStarted = "xrStarted",
47
- XRStopped = "xrStopped",
48
- XRUpdate = "xrUpdate",
49
- RequestVRSession = "requestVRSession",
50
- ModifyAROptions = "modify-ar-options",
51
- }
52
-
53
- export declare type CreateButtonOptions = {
54
- registerClick: boolean
55
- };
56
-
57
- export class WebXR extends Behaviour {
58
-
59
- @serializable()
60
- enableVR = true;
61
- @serializable()
62
- enableAR = true;
63
-
64
- @serializable(AssetReference)
65
- defaultAvatar?: AssetReference;
66
- @serializable()
67
- handModelPath: string = "";
68
-
69
- @serializable()
70
- createVRButton: boolean = true;
71
- @serializable()
72
- createARButton: boolean = true;
73
-
74
- private static _isInXr: boolean = false;
75
- private static events: EventDispatcher = new EventDispatcher();
76
-
77
- public static get IsInWebXR(): boolean { return this._isInXr; }
78
- public static get XRSupported(): boolean { return 'xr' in navigator && (arSupported || vrSupported); }
79
- public static get IsARSupported(): boolean { return arSupported; }
80
- public static get IsVRSupported(): boolean { return vrSupported; }
81
-
82
- private static _optionalFeatures_VR: string[] = ['local-floor', 'bounded-floor', 'hand-tracking', 'high-fixed-foveation-level', 'layers'];
83
- private static _optionalFeatures_AR: string[] = ['plane-detection', 'anchors', 'local-floor', 'hand-tracking', 'layers'];
84
- public static get OptionalFeatures_VR(): string[] { return this._optionalFeatures_VR; }
85
- public static get OptionalFeatures_AR(): string[] { return this._optionalFeatures_AR; }
86
-
87
- public static addEventListener(type: string, listener: any): any {
88
- this.events.addEventListener(type, listener);
89
- return listener;
90
- }
91
- public static removeEventListener(type: string, listener: any): any {
92
- this.events.removeEventListener(type, listener);
93
- return listener;
94
- }
95
- private static dispatchEvent(type: string, event: any): void {
96
- this.events.dispatchEvent({ type, detail: event });
97
- }
98
-
99
- public static createVRButton(webXR: WebXR, opts?: CreateButtonOptions): HTMLButtonElement | HTMLAnchorElement {
100
- if (!WebXR.XRSupported) {
101
- console.warn("WebXR is not supported on this device");
102
- }
103
- else
104
- webXR.__internalAwake();
105
- const options = { optionalFeatures: WebXR.OptionalFeatures_VR };
106
- const vrButton = VRButton.createButton(webXR.context.renderer, options);
107
- vrButton.classList.add('webxr-ar-button');
108
- vrButton.classList.add('webxr-button');
109
- this.resetButtonStyles(vrButton);
110
- // if (this.enableAR) vrButton.style.marginLeft = "60px";
111
- if (opts?.registerClick ?? true)
112
- vrButton.addEventListener('click', webXR.onClickedVRButton.bind(webXR));
113
- return vrButton;
114
- }
115
-
116
- public static createARButton(webXR: WebXR, opts?: CreateButtonOptions): HTMLButtonElement | HTMLAnchorElement {
117
- webXR.__internalAwake();
118
- const domOverlayRoot = webXR.webAR?.getAROverlayContainer();
119
- const options: any = { optionalFeatures: [...this.OptionalFeatures_AR] };
120
- if (domOverlayRoot) {
121
- options.domOverlay = { root: domOverlayRoot };
122
- options.optionalFeatures.push('dom-overlay')
123
- options.optionalFeatures.push('hit-test');
124
- }
125
- else {
126
- console.warn("No dom overlay root found, HTML overlays on top of screen-based AR will not work.");
127
- }
128
-
129
- const arButton = ARButton.createButton(webXR.context.renderer, options, this.onModifyAROptions.bind(this));
130
- arButton.classList.add('webxr-ar-button');
131
- arButton.classList.add('webxr-button');
132
- WebXR.resetButtonStyles(arButton);
133
- if (opts?.registerClick ?? true)
134
- arButton.addEventListener('click', webXR.onClickedARButton.bind(webXR));
135
- return arButton;
136
- }
137
-
138
- private static onModifyAROptions(options){
139
- WebXR.dispatchEvent(WebXREvent.ModifyAROptions, options);
140
- }
141
-
142
- public static resetButtonStyles(button) {
143
- if (!button) return;
144
- button.style.position = "";
145
- button.style.bottom = "";
146
- button.style.left = "";
147
- }
148
-
149
- public endSession() {
150
- const session = this.context.renderer.xr.getSession();
151
- if (session) session.end();
152
- }
153
-
154
- public get Rig(): Object3D {
155
- if (!this.rig) this.ensureRig();
156
- return this.rig;
157
- }
158
-
159
-
160
- private controllers: WebXRController[] = [];
161
- public get Controllers(): WebXRController[] {
162
- return this.controllers;
163
- }
164
-
165
- public get LeftController(): WebXRController | null {
166
- if (this.controllers.length > 0 && this.controllers[0].input?.handedness === "left") return this.controllers[0];
167
- if (this.controllers.length > 1 && this.controllers[1].input?.handedness === "left") return this.controllers[1];
168
- return null;
169
- }
170
-
171
- public get RightController(): WebXRController | null {
172
- if (this.controllers.length > 0 && this.controllers[0].input?.handedness === "right") return this.controllers[0];
173
- if (this.controllers.length > 1 && this.controllers[1].input?.handedness === "right") return this.controllers[1];
174
- return null;
175
- }
176
-
177
- public get ARButton(): HTMLButtonElement | undefined {
178
- return this._arButton;
179
- }
180
-
181
- public get VRButton(): HTMLButtonElement | undefined {
182
- return this._vrButton;
183
- }
184
-
185
- public get IsInVR() { return this._isInVR; }
186
- public get IsInAR() { return this._isInAR; }
187
-
188
- private rig!: Object3D;
189
- private isInit: boolean = false;
190
-
191
- private _requestedAR: boolean = false;
192
- private _requestedVR: boolean = false;
193
- private _isInAR: boolean = false;
194
- private _isInVR: boolean = false;
195
-
196
- private _arButton?: HTMLButtonElement;
197
- private _vrButton?: HTMLButtonElement;
198
-
199
- private webAR: WebAR | null = null;
200
-
201
- awake(): void {
202
- // as the webxr component is most of the times currently loaded as part of the scene
203
- // and not part of the glTF directly and thus does not go through the whole serialization process currently
204
- // we need to to manuall make sure it is of the correct type here
205
- if (this.defaultAvatar) {
206
- if (typeof (this.defaultAvatar) === "string") {
207
- this.defaultAvatar = AssetReference.getOrCreate(this.sourceId ?? "/", this.defaultAvatar, this.context);
208
- }
209
- }
210
- if (!GameObject.findObjectOfType(WebXRSync, this.context)) {
211
- const sync = GameObject.addNewComponent(this.gameObject, WebXRSync, false) as WebXRSync;
212
- sync.webXR = this;
213
- }
214
- this.webAR = new WebAR(this);
215
- }
216
-
217
- start() {
218
- if (location.protocol == 'http:' && location.host.indexOf('localhost') < 0) {
219
- showBalloonWarning("WebXR only works on https");
220
- console.warn("WebXR only works on https. https://engine.needle.tools/docs/xr.html");
221
- }
222
- }
223
-
224
- onEnable() {
225
- if (this.isInit) return;
226
- if (!this.enableAR && !this.enableVR) return;
227
- this.isInit = true;
228
-
229
- this.context.renderer.xr.enabled = true;
230
-
231
- // general WebXR support?
232
- const browserSupportsXR = WebXR.XRSupported;
233
-
234
-
235
- // TODO: move the whole buttons positioning out of here and make it configureable from css
236
- // better set proper classes so user code can react to it instead
237
- // of this hardcoded stuff
238
- let arButton, vrButton;
239
- const buttonsContainer = document.createElement('div');
240
- buttonsContainer.classList.add("webxr-buttons");
241
- this.context.domElement.append(buttonsContainer);
242
-
243
- // AR support
244
- // if (this.enableAR && this.createARButton && arSupported)
245
- {
246
- arButton = WebXR.createARButton(this);
247
- this._arButton = arButton;
248
- buttonsContainer.appendChild(arButton);
249
- }
250
-
251
- // VR support
252
- if (this.createVRButton && this.enableVR && vrSupported) {
253
- vrButton = WebXR.createVRButton(this);
254
- this._vrButton = vrButton;
255
- buttonsContainer.appendChild(vrButton);
256
- }
257
-
258
- setTimeout(() => {
259
- WebXR.resetButtonStyles(vrButton);
260
- WebXR.resetButtonStyles(arButton);
261
- }, 1000);
262
- }
263
-
264
- private _transformOrientation: Quaternion = new Quaternion();
265
- public get TransformOrientation(): Quaternion { return this._transformOrientation; }
266
-
267
- private _currentHeadPose: XRViewerPose | null = null;
268
- public get HeadPose(): XRViewerPose | null { return this._currentHeadPose; }
269
-
270
- onBeforeRender(frame) {
271
- if (!frame) return;
272
- // TODO: figure out why screen is black if we enable the code written here
273
- // const referenceSpace = renderer.xr.getReferenceSpace();
274
- const session = this.context.renderer.xr.getSession();
275
-
276
-
277
- if (session) {
278
- const pose = frame.getViewerPose(this.context.renderer.xr.getReferenceSpace());
279
- if(!pose) return;
280
- this._currentHeadPose = pose;
281
- const transform: XRRigidTransform = pose?.transform;
282
- if (transform) {
283
- this._transformOrientation.set(transform.orientation.x, transform.orientation.y, transform.orientation.z, transform.orientation.w);
284
- }
285
-
286
- if (WebXR._isInXr === false && session) {
287
- this.onEnterXR(session, frame);
288
- }
289
-
290
- for (const ctrl of this.controllers) {
291
- ctrl.onUpdate(session);
292
- }
293
-
294
- if (this._isInAR) {
295
- this.webAR?.onUpdate(session, frame);
296
- }
297
- }
298
-
299
- WebXR.events.dispatchEvent({ type: WebXREvent.XRUpdate, frame: frame, xr: this.context.renderer.xr, rig: this.rig });
300
- }
301
-
302
- private onClickedARButton() {
303
- if (!this._isInAR) {
304
- this._requestedAR = true;
305
- this._requestedVR = false;
306
-
307
- // if we do this on enter xr the state has already been changed in AR mode
308
- // so we need to to this before session has started
309
- this.captureStateBeforeXR();
310
- }
311
- }
312
-
313
- private onClickedVRButton() {
314
- if (!this._isInVR) {
315
-
316
- // happens e.g. when headset is off and xr session never actually started
317
- if (this._requestedVR) {
318
- this.onExitXR(null);
319
- return;
320
- }
321
-
322
- this._requestedAR = false;
323
- this._requestedVR = true;
324
- this.captureStateBeforeXR();
325
-
326
- // build controllers before session begins - this seems to fix issue with controller models not appearing/not getting connection event
327
- this.ensureRig();
328
- for (let i = 0; i < 2; i++) {
329
- WebXRController.Create(this, i, this.gameObject as GameObject, ControllerType.PhysicalDevice);
330
- }
331
-
332
- WebXR.events.dispatchEvent({ type: WebXREvent.RequestVRSession });
333
- }
334
- }
335
-
336
- private captureStateBeforeXR() {
337
- if (this.context.mainCamera) {
338
- this._originalCameraPosition.copy(getWorldPosition(this.context.mainCamera));
339
- this._originalCameraRotation.copy(getWorldQuaternion(this.context.mainCamera));
340
- this._originalCameraParent = this.context.mainCamera.parent;
341
- }
342
- if(this.Rig){
343
- this._originalXRRigParent = this.Rig.parent;
344
- this._originalXRRigPosition.copy(this.Rig.position);
345
- this._originalXRRigRotation.copy(this.Rig.quaternion);
346
- }
347
- }
348
-
349
- private ensureRig() {
350
- if (!this.rig) {
351
- // currently just used for pose
352
- const xrRig = GameObject.findObjectOfType(XRRig, this.context);
353
- if (xrRig) {
354
- // make it match unity forward
355
- this.rig = xrRig.gameObject;
356
- // this.rig.rotateY(Math.PI);
357
- // this.rig.position.copy(existing.worldPosition);
358
- // this.rig.quaternion.premultiply(existing.worldQuaternion);
359
- }
360
- else {
361
- this.rig = new Group();
362
- this.rig.rotateY(Math.PI);
363
- this.rig.name = "XRRig";
364
- this.context.scene.add(this.rig);
365
- }
366
- }
367
- }
368
-
369
-
370
- private _originalCameraParent: Object3D | null = null;
371
- private _originalCameraPosition: Vector3 = new Vector3();
372
- private _originalCameraRotation: Quaternion = new Quaternion();
373
-
374
- private _originalXRRigParent: Object3D | null = null;
375
- private _originalXRRigPosition: Vector3 = new Vector3();
376
- private _originalXRRigRotation: Quaternion = new Quaternion();
377
-
378
- private onEnterXR(session: XRSession, frame: XRFrame) {
379
- console.log("[XR] session begin", session);
380
- WebXR._isInXr = true;
381
-
382
- this.ensureRig();
383
-
384
- const space = this.context.renderer.xr.getReferenceSpace();
385
- if (space && this.rig) {
386
- const pose = frame.getViewerPose(space);
387
- const rot = pose?.transform.orientation;
388
- if (rot) {
389
- const quat = new Quaternion(rot.x, rot.y, rot.z, rot.w);
390
- const eu = new Euler().setFromQuaternion(quat);
391
- this.rig.rotateY(eu.y);
392
- // this.rig.quaternion.multiply(quat);
393
- }
394
- }
395
-
396
- // when we set unity layers objects will only be rendered on one eye
397
- // we set layers to sync raycasting and have a similar behaviour to unity
398
- const xr = this.context.renderer.xr;
399
- if (this.context.mainCamera) {
400
- //@ts-ignore
401
- const cam = xr.getCamera(this.context.mainCamera) as ArrayCamera;
402
- const cull = this.context.mainCameraComponent?.cullingMask;
403
- if(cull !== undefined){
404
- for (const c of cam.cameras) {
405
- c.layers.mask = cull;
406
- }
407
- cam.layers.mask = cull;
408
- }
409
- else {
410
- for (const c of cam.cameras) {
411
- c.layers.enableAll();
412
- }
413
- cam.layers.enableAll();
414
- }
415
- this.rig.add(this.context.mainCamera);
416
- if (this._requestedAR) {
417
- this.context.scene.add(this.rig);
418
- }
419
- }
420
-
421
- const flag = this._requestedAR ? XRStateFlag.AR : XRStateFlag.VR;
422
-
423
- XRState.Global.Set(flag);
424
-
425
- switch (flag) {
426
- case XRStateFlag.AR:
427
- this.context.xrSessionMode = XRSessionMode.ImmersiveAR;
428
- this._isInAR = true;
429
- this.webAR?.onBegin(session);
430
- break;
431
- case XRStateFlag.VR:
432
- this.context.xrSessionMode = XRSessionMode.ImmersiveVR;
433
- this._isInVR = true;
434
- this.onEnterVR(session);
435
- break;
436
- }
437
-
438
- session.addEventListener('end', () => {
439
- console.log("[XR] session end");
440
- WebXR._isInXr = false;
441
- this.onExitXR(session);
442
- });
443
-
444
- this.onEnterXR_HandleMirrorWindow(session);
445
-
446
- WebXR.events.dispatchEvent({ type: WebXREvent.XRStarted, session: session });
447
- }
448
-
449
- private onExitXR(session: XRSession | null) {
450
-
451
- const wasInAR = this._isInAR;
452
-
453
- if (this._isInAR && session) {
454
- this.webAR?.onEnd(session);
455
- }
456
-
457
- this._isInAR = false;
458
- this._isInVR = false;
459
- this._requestedAR = false;
460
- this._requestedVR = false;
461
- this.context.xrSessionMode = undefined;
462
-
463
- if (this.xrMirrorWindow) {
464
- this.xrMirrorWindow.close();
465
- this.xrMirrorWindow = null;
466
- }
467
-
468
- this.destroyControllers();
469
-
470
- if (this.context.mainCamera) {
471
- this._originalCameraParent?.add(this.context.mainCamera);
472
- setWorldPosition(this.context.mainCamera, this._originalCameraPosition);
473
- setWorldQuaternion(this.context.mainCamera, this._originalCameraRotation);
474
- this.context.mainCamera.scale.set(1, 1, 1);
475
- }
476
-
477
- if(wasInAR){
478
- this._originalXRRigParent?.add(this.rig);
479
- this.rig.position.copy(this._originalXRRigPosition);
480
- this.rig.quaternion.copy(this._originalXRRigRotation);
481
- }
482
-
483
- XRState.Global.Set(XRStateFlag.Browser | XRStateFlag.ThirdPerson);
484
- WebXR.events.dispatchEvent({ type: WebXREvent.XRStopped, session: session });
485
- }
486
-
487
- private onEnterVR(_session: XRSession) {
488
- }
489
-
490
- private destroyControllers() {
491
- for (let i = this.controllers.length - 1; i >= 0; i -= 1) {
492
- this.controllers[i]?.destroy();
493
- }
494
- this.controllers.length = 0;
495
- }
496
-
497
- private xrMirrorWindow: Window | null = null;
498
-
499
- private onEnterXR_HandleMirrorWindow(session: XRSession) {
500
- if (!getParam("mirror")) return;
501
- setTimeout(() => {
502
- if (!WebXR.IsInWebXR) return;
503
- const url = new URL(window.location.href);
504
- setOrAddParamsToUrl(url.searchParams, noVoip, 1);
505
- setOrAddParamsToUrl(url.searchParams, "isMirror", 1);
506
- const str = url.toString();
507
- this.xrMirrorWindow = window.open(str, "webxr sync", "popup=yes");
508
- if (this.xrMirrorWindow) {
509
- this.xrMirrorWindow.onload = () => {
510
- if (this.xrMirrorWindow)
511
- this.xrMirrorWindow.onbeforeunload = () => {
512
- if (WebXR.IsInWebXR)
513
- session.end();
514
- };
515
- }
516
- }
517
- }, 1000);
518
- }
519
- }
520
-
521
-
522
- // not sure if this should be a behaviour.
523
- // for now we dont really need it to go through the usual update loop
524
- export class WebAR {
525
-
526
- get webxr(): WebXR { return this._webxr; }
527
-
528
- private _webxr: WebXR;
529
-
530
- private reticle: Object3D | null = null;
531
- private reticleParent: Object3D | null = null;
532
- private hitTestSource: XRHitTestSource | null = null;
533
- private reticleActive: boolean = true;
534
-
535
- // scene.background before entering AR
536
- private previousBackground: Color | null | Texture = null;
537
- private previousEnvironment: Texture | null = null;
538
-
539
- private sessionRoot: WebARSessionRoot | null = null;
540
- private _previousParent: Object3D | null = null;
541
- // we need this in case the session root is on the same object as the webxr component
542
- // so if we disable the session root we attach the webxr component to this temporary object
543
- // to still receive updates
544
- private static tempWebXRObject: Object3D;
545
-
546
- private get context() { return this.webxr.context; }
547
-
548
- constructor(webxr: WebXR) {
549
- this._webxr = webxr;
550
- }
551
-
552
- private arDomOverlay: HTMLElement | null = null;
553
- private arOverlayElement: INeedleEngineComponent | HTMLElement | null = null;
554
- private noHitTestAvailable: boolean = false;
555
- private didPlaceARSessionRoot: boolean = false;
556
-
557
- getAROverlayContainer(): HTMLElement | null {
558
- this.arDomOverlay = this.webxr.context.domElement as HTMLElement;
559
- // for react cases we dont have an Engine Element
560
- const element: any = this.arDomOverlay;
561
- if (element.getAROverlayContainer)
562
- this.arOverlayElement = element.getAROverlayContainer();
563
- else this.arOverlayElement = this.arDomOverlay;
564
- return this.arOverlayElement;
565
- }
566
-
567
- setReticleActive(active: boolean) {
568
- this.reticleActive = active;
569
- }
570
-
571
- async onBegin(session: XRSession) {
572
- const context = this.webxr.context;
573
- this.reticleActive = true;
574
- this.didPlaceARSessionRoot = false;
575
- this.getAROverlayContainer();
576
-
577
- const deviceType = navigator.userAgent?.includes("OculusBrowser") ? ControllerType.PhysicalDevice : ControllerType.Touch;
578
- const controllerCount = deviceType === ControllerType.Touch ? 4 : 2;
579
- for (let i = 0; i < controllerCount; i++) {
580
- WebXRController.Create(this.webxr, i, this.webxr.gameObject as GameObject, deviceType)
581
- }
582
-
583
- if (!this.sessionRoot || this.sessionRoot.destroyed || !this.sessionRoot.activeAndEnabled)
584
- this.sessionRoot = GameObject.findObjectOfType(WebARSessionRoot, context);
585
-
586
- this.previousBackground = context.scene.background;
587
- this.previousEnvironment = context.scene.environment;
588
- context.scene.background = null;
589
-
590
- session.requestReferenceSpace('viewer').then((referenceSpace) => {
591
- session.requestHitTestSource?.call(session, { space: referenceSpace })?.then((source) => {
592
- this.hitTestSource = source;
593
- }).catch((err) => {
594
- this.noHitTestAvailable = true;
595
- console.warn("WebXR: Hit test not supported", err);
596
- });
597
- });
598
-
599
- if (!this.reticle && this.sessionRoot) {
600
- this.reticle = new Mesh(
601
- new RingGeometry(0.07, 0.09, 32).rotateX(- Math.PI / 2),
602
- new MeshBasicMaterial()
603
- );
604
- this.reticle.name = "AR Placement reticle";
605
- this.reticle.matrixAutoUpdate = false;
606
- this.reticle.visible = false;
607
-
608
- // create AR reticle parent to allow WebXRSessionRoot to be translated, rotated or scaled
609
- this.reticleParent = new Object3D();
610
- this.reticleParent.name = "AR Reticle Parent";
611
- this.reticleParent.matrixAutoUpdate = false;
612
- this.reticleParent.add(this.reticle);
613
- // this.reticleParent.matrix.copy(this.sessionRoot.gameObject.matrixWorld);
614
-
615
- if (this.webxr.scene) {
616
- this.context.scene.add(this.reticleParent);
617
- // this.context.scene.add(this.reticle);
618
- this.context.scene.visible = true;
619
- }
620
- else console.warn("Could not found WebXR Rig");
621
- }
622
-
623
- this._previousParent = this.webxr.gameObject;
624
- if (!WebAR.tempWebXRObject) WebAR.tempWebXRObject = new Object3D();
625
- this.context.scene.add(WebAR.tempWebXRObject);
626
- GameObject.addComponent(WebAR.tempWebXRObject as GameObject, this.webxr);
627
-
628
- if (this.sessionRoot) {
629
- this.sessionRoot.webAR = this;
630
- this.sessionRoot?.onBegin(session);
631
- }
632
- else console.warn("No WebARSessionRoot found in scene")
633
-
634
- const eng = this.context.domElement as INeedleEngineComponent;
635
- eng?.onEnterAR?.call(eng, session, this.arOverlayElement!);
636
-
637
- this.context.mainCameraComponent?.applyClearFlagsIfIsActiveCamera();
638
- }
639
-
640
- onEnd(session: XRSession) {
641
- if (this._previousParent) {
642
- GameObject.addComponent(this._previousParent as GameObject, this.webxr);
643
- this._previousParent = null;
644
- }
645
- this.hitTestSource = null;
646
- const context = this.webxr.context;
647
- context.scene.background = this.previousBackground;
648
- context.scene.environment = this.previousEnvironment;
649
- if (this.sessionRoot) {
650
- this.sessionRoot.onEnd(this.webxr.Rig, session);
651
- }
652
-
653
- const el = this.context.domElement as INeedleEngineComponent;
654
- el.onExitAR?.call(el, session);
655
-
656
- this.context.mainCameraComponent?.applyClearFlagsIfIsActiveCamera();
657
- }
658
-
659
- onUpdate(session: XRSession, frame: XRFrame) {
660
-
661
- if (this.noHitTestAvailable === true) {
662
- if (this.reticle)
663
- this.reticle.visible = false;
664
- if (!this.didPlaceARSessionRoot) {
665
- this.didPlaceARSessionRoot = true;
666
- const rig = this.webxr.Rig;
667
- const placementMatrix = arPlacementWithoutHitTestMatrix.clone();
668
- // if (rig) {
669
- // const positionFromRig = new Vector3(0, 0, 0).add(rig.position).divideScalar(this.sessionRoot?.arScale ?? 1);
670
- // placementMatrix.multiply(new Matrix4().makeTranslation(positionFromRig.x, positionFromRig.y, positionFromRig.z));
671
- // // placementMatrix.setPosition(positionFromRig);
672
- // }
673
- this.sessionRoot?.placeAt(rig, placementMatrix);
674
- }
675
- return;
676
- }
677
-
678
- if (!this.hitTestSource) return;
679
- const hitTestResults = frame.getHitTestResults(this.hitTestSource);
680
- if (hitTestResults.length) {
681
- const hit = hitTestResults[0];
682
- const referenceSpace = this.webxr.context.renderer.xr.getReferenceSpace();
683
- if (referenceSpace) {
684
- const pose = hit.getPose(referenceSpace);
685
-
686
- if (this.sessionRoot) {
687
- const didPlace = this.sessionRoot.onUpdate(this.webxr.Rig, session, pose);
688
- this.didPlaceARSessionRoot = didPlace;
689
- }
690
-
691
- if (this.reticle) {
692
- this.reticle.visible = this.reticleActive;
693
- if (this.reticleActive) {
694
- if (pose) {
695
- const matrix = pose.transform.matrix;
696
- this.reticle.matrix.fromArray(matrix);
697
- if (this.webxr.Rig)
698
- this.reticle.matrix.premultiply(this.webxr.Rig.matrix);
699
- }
700
- }
701
- }
702
- }
703
-
704
- } else {
705
- this.sessionRoot?.onUpdate(this.webxr.Rig, session, null);
706
- if (this.reticle)
707
- this.reticle.visible = false;
708
- }
709
- }
710
- }
711
-
712
- const arPlacementWithoutHitTestMatrix = new Matrix4().identity().makeTranslation(0, 0, 0);
src/engine-components/WebXRAvatar.ts DELETED
@@ -1,356 +0,0 @@
1
- import { Behaviour, GameObject } from "./Component";
2
- import { WebXR } from "./WebXR";
3
- import { Quaternion, Vector3 } from "three";
4
- import { AvatarLoader } from "./AvatarLoader";
5
- import { XRFlag, XRStateFlag } from "./XRFlag";
6
- import { Avatar_POI } from "./avatar/Avatar_Brain_LookAt";
7
- import { Context } from "../engine/engine_setup";
8
- import { AssetReference } from "../engine/engine_addressables";
9
- import { Object3D } from "three";
10
- import { VRUserState } from "./WebXRSync";
11
- import { getParam } from "../engine/engine_utils";
12
- import { ViewDevice } from "../engine/engine_playerview";
13
- import { InstancingUtil } from "../engine/engine_instancing";
14
-
15
- export const debug = getParam("debugavatar");
16
-
17
- export type AvatarMarkerEventArgs = {
18
- avatarMarker: AvatarMarker;
19
- gameObject: Object3D;
20
- }
21
-
22
- export class AvatarMarker extends Behaviour {
23
-
24
- public static getAvatar(index: number): AvatarMarker | null {
25
- if (index >= 0 && index < AvatarMarker.instances.length)
26
- return AvatarMarker.instances[index];
27
- return null;
28
- }
29
-
30
- public static instances: AvatarMarker[] = [];
31
-
32
- public static onAvatarMarkerCreated(cb: (args: AvatarMarkerEventArgs) => void): Function {
33
- AvatarMarker._onNewAvatarMarkerAdded.push(cb);
34
- return cb;
35
- }
36
-
37
- public static onAvatarMarkerDestroyed(cb: (args: AvatarMarkerEventArgs) => void): Function {
38
- AvatarMarker._onAvatarMarkerDestroyed.push(cb);
39
- return cb;
40
- }
41
-
42
- private static _onNewAvatarMarkerAdded: Array<(args: AvatarMarkerEventArgs) => void> = [];
43
- private static _onAvatarMarkerDestroyed: Array<(args: AvatarMarkerEventArgs) => void> = [];
44
-
45
-
46
- public connectionId!: string;
47
- public avatar?: WebXRAvatar | Object3D;
48
-
49
- awake() {
50
- AvatarMarker.instances.push(this);
51
- if (debug)
52
- console.log(this);
53
-
54
- for (const cb of AvatarMarker._onNewAvatarMarkerAdded)
55
- cb({ avatarMarker: this, gameObject: this.gameObject });
56
- }
57
-
58
- onDestroy() {
59
- AvatarMarker.instances.splice(AvatarMarker.instances.indexOf(this), 1);
60
-
61
- for (const cb of AvatarMarker._onAvatarMarkerDestroyed)
62
- cb({ avatarMarker: this, gameObject: this.gameObject });
63
- }
64
-
65
- isLocalAvatar() {
66
- return this.connectionId === this.context.connection.connectionId;
67
- }
68
-
69
- setVisible(visible: boolean) {
70
- if (this.avatar) {
71
- if ("setVisible" in this.avatar)
72
- this.avatar.setVisible(visible);
73
- else {
74
- GameObject.setActive(this.avatar, visible);
75
- }
76
- }
77
- }
78
- }
79
-
80
-
81
- export class WebXRAvatar {
82
- private static loader: AvatarLoader = new AvatarLoader();
83
-
84
- private _isVisible: boolean = true;
85
- setVisible(visible: boolean) {
86
- this._isVisible = visible;
87
- this.updateVisibility();
88
- }
89
-
90
- get isWebXRAvatar() { return true; }
91
-
92
- // TODO: set layers on all avatars
93
- /** the user id */
94
- public guid: string;
95
-
96
- private root: Object3D | null = null;
97
- public head: Object3D | null = null;
98
- public handLeft: Object3D | null = null;
99
- public handRight: Object3D | null = null;
100
- public lastUpdate: number = -1;
101
- public isLocalAvatar: boolean = false;
102
- public flags: XRFlag[] | null = null;
103
- private headScale: Vector3 = new Vector3(1, 1, 1);
104
- private handLeftScale: Vector3 = new Vector3(1, 1, 1);
105
- private handRightScale: Vector3 = new Vector3(1, 1, 1);
106
-
107
- private readonly webxr: WebXR;
108
-
109
- private lastAvatarId: string | null = null;
110
- private hasAvatarOverride: boolean = false;
111
-
112
-
113
- private context: Context;
114
- private avatarMarker: AvatarMarker | null = null;
115
-
116
- constructor(context: Context, guid: string, webXR: WebXR) {
117
- this.context = context;
118
- this.guid = guid;
119
- this.webxr = webXR;
120
- this.setupCustomAvatar(this.webxr.defaultAvatar as AssetReference);
121
- }
122
-
123
- public updateFlags() {
124
- if (!this.flags)
125
- return;
126
- let mask = this.isLocalAvatar ? XRStateFlag.FirstPerson : XRStateFlag.ThirdPerson;
127
- if (this.context.isInVR)
128
- mask |= XRStateFlag.VR;
129
- else if (this.context.isInAR)
130
- mask |= XRStateFlag.AR;
131
- else
132
- mask |= XRStateFlag.Browser;
133
- for (const f of this.flags) {
134
- f.gameObject.visible = true;
135
- f.UpdateVisible(mask);
136
- }
137
- }
138
-
139
- public async setAvatarOverride(avatarId: string | null): Promise<boolean | null> {
140
- this.hasAvatarOverride = avatarId !== null;
141
- if (this.hasAvatarOverride && this.lastAvatarId !== avatarId) {
142
- this.lastAvatarId = avatarId;
143
- if (avatarId != null && avatarId.length > 0)
144
- return await this.setupCustomAvatar(avatarId);
145
- }
146
- return null;
147
- }
148
-
149
- private _headTarget: Object3D = new Object3D();
150
- private _handLeftTarget: Object3D = new Object3D();
151
- private _handRightTarget: Object3D = new Object3D();
152
- private _canInterpolate: boolean = false;
153
-
154
- private static invertRotation: Quaternion = new Quaternion().setFromAxisAngle(new Vector3(0, 1, 0), Math.PI);
155
-
156
- public tryUpdate(state: VRUserState, _timeDiff: number) {
157
- if (state.guid === this.guid) {
158
-
159
- if (this.lastAvatarId !== state.avatarId && state.avatarId && state.avatarId.length > 0) {
160
- this.lastAvatarId = state.avatarId;
161
- this.setupCustomAvatar(state.avatarId);
162
- }
163
-
164
- this.lastUpdate = state.time;
165
- if (this.head) {
166
-
167
- const device = this.webxr.IsInAR ? ViewDevice.Handheld : ViewDevice.Headset;
168
- let viewObj = this.head;
169
- // if (this.isLocalAvatar) {
170
- // if (this.context.mainCamera && this.context.isInXR) {
171
- // viewObj = this.context.renderer.xr.getCamera(this.context.mainCamera);
172
- // }
173
- // }
174
- this.context.players.setPlayerView(state.guid, viewObj, device);
175
-
176
- InstancingUtil.markDirty(this.head);
177
-
178
- this._canInterpolate = true;
179
- const ht = this.isLocalAvatar ? this.head : this._headTarget;
180
- ht.position.set(state.position.x, state.position.y, state.position.z);
181
- // not sure how position in local space can be correct but rotation is wrong / offset when parent rotates
182
- ht.quaternion.set(state.rotation.x, state.rotation.y, state.rotation.z, state.rotation.w);
183
- ht.scale.set(state.scale, state.scale, state.scale);
184
- ht.scale.multiply(this.headScale);
185
-
186
- if (this.handLeft) {
187
- const ht = this.isLocalAvatar ? this.handLeft : this._handLeftTarget;
188
- ht.position.set(state.posLeftHand.x, state.posLeftHand.y, state.posLeftHand.z);
189
- ht.quaternion.set(state.rotLeftHand["_x"], state.rotLeftHand["_y"], state.rotLeftHand["_z"], state.rotLeftHand["_w"]);
190
- ht.quaternion.multiply(WebXRAvatar.invertRotation);
191
- ht.scale.set(state.scale, state.scale, state.scale);
192
- ht.scale.multiply(this.handLeftScale);
193
- InstancingUtil.markDirty(this.handLeft);
194
- }
195
-
196
- if (this.handRight) {
197
- const ht = this.isLocalAvatar ? this.handRight : this._handRightTarget;
198
- ht.position.set(state.posRightHand.x, state.posRightHand.y, state.posRightHand.z);
199
- ht.quaternion.set(state.rotRightHand["_x"], state.rotRightHand["_y"], state.rotRightHand["_z"], state.rotRightHand["_w"]);
200
- ht.quaternion.multiply(WebXRAvatar.invertRotation);
201
- ht.scale.set(state.scale, state.scale, state.scale);
202
- ht.scale.multiply(this.handRightScale);
203
- InstancingUtil.markDirty(this.handRight);
204
- }
205
- }
206
- }
207
- }
208
-
209
- public update() {
210
- if (this.isLocalAvatar)
211
- return;
212
- if (!this._canInterpolate)
213
- return;
214
- const t = this.context.time.deltaTime / .1;
215
- if (this.head) {
216
- this.head.position.lerp(this._headTarget.position, t);
217
- this.head.quaternion.slerp(this._headTarget.quaternion, t);
218
- this.head.scale.lerp(this._headTarget.scale, t);
219
- }
220
- if (this.handLeft && this._handLeftTarget) {
221
- this.handLeft.position.lerp(this._handLeftTarget.position, t);
222
- this.handLeft.quaternion.slerp(this._handLeftTarget.quaternion, t);
223
- this.handLeft.scale.lerp(this._handLeftTarget.scale, t);
224
- }
225
- if (this.handRight && this._handRightTarget) {
226
- this.handRight.position.lerp(this._handRightTarget.position, t);
227
- this.handRight.quaternion.slerp(this._handRightTarget.quaternion, t);
228
- this.handRight.scale.lerp(this._handRightTarget.scale, t);
229
- }
230
- }
231
-
232
- public destroy() {
233
- if (debug)
234
- console.log("Destroy avatar", this.guid);
235
- this.root?.removeFromParent();
236
- this.avatarMarker?.destroy();
237
- this.lastAvatarId = null;
238
-
239
- if (this.head) {
240
- Avatar_POI.Remove(this.context, this.head);
241
- }
242
- // this.head?.removeFromParent();
243
- // this.handLeft?.removeFromParent();
244
- // this.handRight?.removeFromParent();
245
- }
246
-
247
- private updateVisibility() {
248
- const root = this.root;
249
- if (root) {
250
- GameObject.setActive(root, this._isVisible);
251
- }
252
- }
253
-
254
- private async setupCustomAvatar(avatarId: string | Object3D | AssetReference): Promise<boolean> {
255
- if (debug)
256
- console.log("LOAD", avatarId, this);
257
-
258
- if (!avatarId || (typeof avatarId === "string" && avatarId.length <= 0))
259
- return false;
260
-
261
- if (this.head) {
262
- Avatar_POI.Remove(this.context, this.head);
263
- }
264
-
265
- const reference = avatarId as AssetReference;
266
- if (reference?.loadAssetAsync !== undefined) {
267
- await reference.loadAssetAsync();
268
- const prefab = reference.asset as Object3D;
269
- GameObject.setActive(prefab, false);
270
- avatarId = GameObject.instantiate(prefab as Object3D) as Object3D;
271
- GameObject.setActive(avatarId, true);
272
- // console.log("Avatar", avatarId);
273
- }
274
- if (debug)
275
- console.log(avatarId);
276
-
277
- const model = await WebXRAvatar.loader.getOrCreateNewAvatarInstance(this.context, avatarId as (Object3D | string));
278
- if (debug)
279
- console.log(model, model?.isValid, this.lastAvatarId, avatarId);
280
- // if (this.lastAvatarId !== avatarId) {
281
- // // avatar id changed in the meantime
282
- // return true;
283
- // }
284
- if (model?.isValid) {
285
- this.root = model.root;
286
-
287
- this.root.position.set(0, 0, 0);
288
- this.root.quaternion.set(0, 0, 0, 1);
289
- this.root.scale.set(1, 1, 1); // should we allow a scaled avatar root?!
290
-
291
- this.avatarMarker = GameObject.addNewComponent(this.root as GameObject, AvatarMarker) as AvatarMarker;
292
- this.avatarMarker.connectionId = this.guid;
293
- this.avatarMarker.avatar = this;
294
-
295
- if (this.head && this.head !== model.head)
296
- this.head?.removeFromParent();
297
- this.head = model.head;
298
- this.headScale.copy(this.head.scale);
299
-
300
- if (this.head && !this.isLocalAvatar) {
301
- Avatar_POI.Add(this.context, this.head, this.avatarMarker);
302
- }
303
-
304
- if (model.leftHand)
305
- this.handLeft?.removeFromParent();
306
- this.handLeft = model.leftHand ?? this.handLeft;
307
- if (this.handLeft)
308
- this.handLeftScale.copy(this.handLeft.scale);
309
- else
310
- this.handLeftScale.set(1, 1, 1);
311
-
312
- if (model.rigthHand)
313
- this.handRight?.removeFromParent();
314
- this.handRight = model.rigthHand ?? this.handRight;
315
- if (this.handRight)
316
- this.handRightScale.copy(this.handRight.scale);
317
- else
318
- this.handRightScale.set(1, 1, 1);
319
-
320
-
321
- this.context.scene.add(this.root);
322
- // scene.add(this.handLeft);
323
- // scene.add(this.handRight);
324
- // this.mouthShapes = null;
325
- // this.needSearchEyes = true;
326
- if (this.flags == null)
327
- this.flags = [];
328
- this.flags.length = 0;
329
- this.flags.push(...GameObject.getComponentsInChildren(this.root as GameObject, XRFlag));
330
- // if no flags are found add at least a head flag to hide head in first person VR
331
- if (this.flags.length <= 0) {
332
- if (this.head) {
333
- const flag = GameObject.addNewComponent(this.head, XRFlag) as XRFlag;
334
- // TODO: the defaults are wrong? should be Desktop | ThirdPerson ?
335
- flag.visibleIn = XRStateFlag.ThirdPerson | XRStateFlag.VR;
336
- this.flags.push(flag);
337
- if (debug)
338
- console.log("Added flag to head: " + flag.visibleIn, this.head.name);
339
- }
340
- }
341
-
342
- if (debug)
343
- console.log("[Avatar], is Local? ", this.isLocalAvatar, this.root);
344
- this.updateFlags();
345
-
346
- this.updateVisibility();
347
-
348
- return true;
349
- }
350
- else {
351
- if (debug)
352
- console.warn("build avatar failed");
353
- return false;
354
- }
355
- }
356
- }
src/engine-components/WebXRController.ts DELETED
@@ -1,1125 +0,0 @@
1
- import { BoxHelper, BufferGeometry, Color, Euler, Group, Intersection, Layers, Line, LineBasicMaterial, Material, Mesh, MeshBasicMaterial, Object3D, PerspectiveCamera, Quaternion, Ray, SphereGeometry, Vector2, Vector3 } from "three";
2
- import { OculusHandModel } from 'three/examples/jsm/webxr/OculusHandModel.js';
3
- import { OculusHandPointerModel } from 'three/examples/jsm/webxr/OculusHandPointerModel.js';
4
- import { XRControllerModel, XRControllerModelFactory } from 'three/examples/jsm/webxr/XRControllerModelFactory.js';
5
- import { GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader.js'
6
-
7
- import { InstancingUtil } from "../engine/engine_instancing";
8
- import { Mathf } from "../engine/engine_math";
9
- import { RaycastOptions } from "../engine/engine_physics";
10
- import { getWorldPosition, getWorldQuaternion, setWorldPosition, setWorldQuaternion } from "../engine/engine_three_utils";
11
- import { getParam, resolveUrl } from "../engine/engine_utils";
12
- import { addDracoAndKTX2Loaders } from "../engine/engine_loaders";
13
-
14
- import { Avatar_POI } from "./avatar/Avatar_Brain_LookAt";
15
- import { Behaviour, GameObject } from "./Component";
16
- import { Interactable, UsageMarker } from "./Interactable";
17
- import { Rigidbody } from "./RigidBody";
18
- import { SyncedTransform } from "./SyncedTransform";
19
- import { UIRaycastUtils } from "./ui/RaycastUtils";
20
- import { WebXR } from "./WebXR";
21
-
22
- const debug = getParam("debugwebxrcontroller");
23
-
24
- export enum ControllerType {
25
- PhysicalDevice = 0,
26
- Touch = 1,
27
- }
28
-
29
- export enum ControllerEvents {
30
- SelectStart = "select-start",
31
- SelectEnd = "select-end",
32
- Update = "update",
33
- }
34
-
35
- export class TeleportTarget extends Behaviour {
36
-
37
- }
38
-
39
- export class WebXRController extends Behaviour {
40
-
41
- public static Factory: XRControllerModelFactory = new XRControllerModelFactory();
42
-
43
- private static raycastColor: Color = new Color(.9, .3, .3);
44
- private static raycastNoHitColor: Color = new Color(.6, .6, .6);
45
- private static geometry = new BufferGeometry().setFromPoints([new Vector3(0, 0, 0), new Vector3(0, 0, -1)]);
46
- private static handModels: { [index: number]: OculusHandPointerModel } = {};
47
-
48
- private static CreateRaycastLine(): Line {
49
- const line = new Line(this.geometry);
50
- const mat = line.material as LineBasicMaterial;
51
- mat.color = this.raycastColor;
52
- // mat.linewidth = 10;
53
- line.layers.set(2);
54
- line.name = 'line';
55
- line.scale.z = 1;
56
- return line;
57
- }
58
-
59
- private static CreateRaycastHitPoint(): Mesh {
60
- const geometry = new SphereGeometry(.5, 22, 22);
61
- const material = new MeshBasicMaterial({ color: this.raycastColor });
62
- const sphere = new Mesh(geometry, material);
63
- sphere.visible = false;
64
- sphere.layers.set(2);
65
- return sphere;
66
- }
67
-
68
- public static Create(owner: WebXR, index: number, addTo: GameObject, type: ControllerType): WebXRController {
69
- const ctrl = addTo ? GameObject.addNewComponent(addTo, WebXRController, false) : new WebXRController();
70
-
71
- ctrl.webXR = owner;
72
- ctrl.index = index;
73
- ctrl.type = type;
74
-
75
- const context = owner.context;
76
- // from https://github.com/mrdoob/js/blob/master/examples/webxr_vr_dragging.html
77
- // controllers
78
- ctrl.controller = context.renderer.xr.getController(index);
79
- ctrl.controllerGrip = context.renderer.xr.getControllerGrip(index);
80
- ctrl.controllerModel = this.Factory.createControllerModel(ctrl.controller);
81
- ctrl.controllerGrip.add(ctrl.controllerModel);
82
-
83
- ctrl.hand = context.renderer.xr.getHand(index);
84
-
85
- const loader = new GLTFLoader();
86
- addDracoAndKTX2Loaders(loader, context);
87
- if (ctrl.webXR.handModelPath && ctrl.webXR.handModelPath !== "")
88
- loader.setPath(resolveUrl(owner.sourceId, ctrl.webXR.handModelPath));
89
- else
90
- // from XRHandMeshModel.js
91
- loader.setPath('https://cdn.jsdelivr.net/npm/@webxr-input-profiles/[email protected]/dist/profiles/generic-hand/');
92
- //@ts-ignore
93
- const hand = new OculusHandModel(ctrl.hand, loader);
94
-
95
- ctrl.hand.add(hand);
96
- ctrl.hand.traverse(x => x.layers.set(2));
97
-
98
- ctrl.handPointerModel = new OculusHandPointerModel(ctrl.hand, ctrl.controller);
99
-
100
-
101
- // TODO remove all these once https://github.com/mrdoob/js/pull/23279 lands
102
- ctrl.controller.addEventListener('connected', (_) => {
103
- ctrl.setControllerLayers(ctrl.controllerModel, 2);
104
- ctrl.setControllerLayers(ctrl.controllerGrip, 2);
105
- ctrl.setControllerLayers(ctrl.hand, 2);
106
- setTimeout(() => {
107
- ctrl.setControllerLayers(ctrl.controllerModel, 2);
108
- ctrl.setControllerLayers(ctrl.controllerGrip, 2);
109
- ctrl.setControllerLayers(ctrl.hand, 2);
110
- }, 1000);
111
- });
112
-
113
- // TODO: unsubscribe! this should be moved into onenable and ondisable!
114
- // TODO remove all these once https://github.com/mrdoob/js/pull/23279 lands
115
- ctrl.hand.addEventListener('connected', (event) => {
116
- const xrInputSource = event.data;
117
- if (xrInputSource.hand) {
118
- if (owner.Rig) owner.Rig.add(ctrl.hand);
119
- ctrl.type = ControllerType.PhysicalDevice;
120
- ctrl.handPointerModel.traverse(x => x.layers.set(2)); // ignore raycast
121
- ctrl.handPointerModel.pointerObject?.traverse(x => x.layers.set(2));
122
-
123
- // when exiting and re-entering xr the joints are not parented to the hand anymore
124
- // this is a workaround to fix that temporarely
125
- // see https://github.com/needle-tools/needle-tiny-playground/issues/123
126
- const jnts = ctrl.hand["joints"];
127
- if (jnts) {
128
- for (const key of Object.keys(jnts)) {
129
- const joint = jnts[key];
130
- if (joint.parent) continue;
131
- ctrl.hand.add(joint);
132
- }
133
- }
134
- }
135
- });
136
-
137
- return ctrl;
138
- }
139
-
140
- // TODO: replace with component events
141
- public static addEventListener(evt: ControllerEvents, callback: (controller: WebXRController, args: any) => void) {
142
- const list = this.eventSubs[evt] ?? [];
143
- list.push(callback);
144
- this.eventSubs[evt] = list;
145
- }
146
-
147
- // TODO: replace with component events
148
- public static removeEventListener(evt: ControllerEvents, callback: (controller: WebXRController, args: any) => void) {
149
- if (!callback) return;
150
- const list = this.eventSubs[evt] ?? [];
151
- const idx = list.indexOf(callback);
152
- if (idx >= 0) list.splice(idx, 1);
153
- this.eventSubs[evt] = list;
154
- }
155
-
156
- private static eventSubs: { [key: string]: Function[] } = {};
157
-
158
- public webXR!: WebXR;
159
- public index: number = -1;
160
- public controllerModel!: XRControllerModel;
161
- public controller!: Group;
162
- public controllerGrip!: Group;
163
- public hand!: Group;
164
- public handPointerModel!: OculusHandPointerModel;
165
- public grabbed: AttachedObject | null = null;
166
- public input: XRInputSource | null = null;
167
- public type: ControllerType = ControllerType.PhysicalDevice;
168
- public showRaycastLine : boolean = true;
169
-
170
- get isUsingHands(): boolean {
171
- const r = this.input?.hand;
172
- return r !== null && r !== undefined;
173
- }
174
-
175
- get wrist(): Object3D | null {
176
- if (!this.hand) return null;
177
- const jnts = this.hand["joints"];
178
- if (!jnts) return null;
179
- return jnts["wrist"];
180
- }
181
-
182
- private _wristQuaternion: Quaternion | null = null;
183
- getWristQuaternion(): Quaternion | null {
184
- const wrist = this.wrist;
185
- if (!wrist) return null;
186
- if (!this._wristQuaternion) this._wristQuaternion = new Quaternion();
187
- const wr = getWorldQuaternion(wrist).multiply(this._wristQuaternion.setFromEuler(new Euler(-Math.PI / 4, 0, 0)));
188
- return wr;
189
- }
190
-
191
- private movementVector: Vector3 = new Vector3();
192
- private worldRot: Quaternion = new Quaternion();
193
- private joystick: Vector2 = new Vector2();
194
- private didRotate: boolean = false;
195
- private didTeleport: boolean = false;
196
- private didChangeScale: boolean = false;
197
- private static PreviousCameraFarDistance: number | undefined = undefined;
198
- private static MovementSpeedFactor: number = 1;
199
-
200
- private lastHit: Intersection | null = null;
201
-
202
- private raycastLine: Line | null = null;
203
- private _raycastHitPoint: Object3D | null = null;
204
- private _connnectedCallback: any | null = null;
205
- private _disconnectedCallback: any | null = null;
206
- private _selectStartEvt: any | null = null;
207
- private _selectEndEvt: any | null = null;
208
-
209
- public get selectionDown(): boolean { return this._selectionPressed && !this._selectionPressedLastFrame; }
210
- public get selectionUp(): boolean { return !this._selectionPressed && this._selectionPressedLastFrame; }
211
- public get selectionPressed(): boolean { return this._selectionPressed; }
212
- public get selectionClick(): boolean { return this._selectionEndTime - this._selectionStartTime < 0.3; }
213
- public get raycastHitPoint(): Object3D | null { return this._raycastHitPoint; }
214
-
215
- private _selectionPressed: boolean = false;
216
- private _selectionPressedLastFrame: boolean = false;
217
- private _selectionStartTime: number = 0;
218
- private _selectionEndTime: number = 0;
219
-
220
- public get useSmoothing(): boolean { return this._useSmoothing };
221
- private _useSmoothing: boolean = true;
222
-
223
- awake(): void {
224
- if (!this.controller) {
225
- console.warn("Missing Controller!!!", this);
226
- return;
227
- }
228
- this._connnectedCallback = this.onSourceConnected.bind(this);
229
- this._disconnectedCallback = this.onSourceDisconnected.bind(this);
230
- this._selectStartEvt = this.onSelectStart.bind(this);
231
- this._selectEndEvt = this.onSelectEnd.bind(this);
232
- if (this.type === ControllerType.Touch) {
233
- this.controllerGrip.addEventListener("connected", this._connnectedCallback);
234
- this.controllerGrip.addEventListener("disconnected", this._disconnectedCallback);
235
- this.controller.addEventListener('selectstart', this._selectStartEvt);
236
- this.controller.addEventListener('selectend', this._selectEndEvt);
237
- }
238
- if (this.type === ControllerType.PhysicalDevice) {
239
- this.controller.addEventListener('selectstart', this._selectStartEvt);
240
- this.controller.addEventListener('selectend', this._selectEndEvt);
241
- }
242
- }
243
-
244
- onDestroy(): void {
245
- if (this.type === ControllerType.Touch) {
246
- this.controllerGrip.removeEventListener("connected", this._connnectedCallback);
247
- this.controllerGrip.removeEventListener("disconnected", this._disconnectedCallback);
248
- this.controller.removeEventListener('selectstart', this._selectStartEvt);
249
- this.controller.removeEventListener('selectend', this._selectEndEvt);
250
- }
251
- if (this.type === ControllerType.PhysicalDevice) {
252
- this.controller.removeEventListener('selectstart', this._selectStartEvt);
253
- this.controller.removeEventListener('selectend', this._selectEndEvt);
254
- }
255
-
256
- this.hand?.clear();
257
- this.controllerGrip?.clear();
258
- this.controller?.clear();
259
- }
260
-
261
- public onEnable(): void {
262
- if (this.hand)
263
- this.hand.name = "Hand";
264
- if (this.controllerGrip)
265
- this.controllerGrip.name = "ControllerGrip";
266
- if (this.controller)
267
- this.controller.name = "Controller";
268
- if (this.raycastLine)
269
- this.raycastLine.name = "RaycastLine;"
270
-
271
- if (this.webXR.Controllers.indexOf(this) < 0)
272
- this.webXR.Controllers.push(this);
273
-
274
- if (!this.raycastLine)
275
- this.raycastLine = WebXRController.CreateRaycastLine();
276
- if (!this._raycastHitPoint)
277
- this._raycastHitPoint = WebXRController.CreateRaycastHitPoint();
278
-
279
- this.webXR.Rig?.add(this.hand);
280
- this.webXR.Rig?.add(this.controllerGrip);
281
- this.webXR.Rig?.add(this.controller);
282
- this.webXR.Rig?.add(this.raycastLine);
283
- this.raycastLine?.add(this._raycastHitPoint);
284
- this._raycastHitPoint.visible = false;
285
- this.hand.add(this.handPointerModel);
286
- if (debug)
287
- console.log("ADDED TO RIG", this.webXR.Rig);
288
-
289
- // // console.log("enable", this.index, this.controllerGrip.uuid)
290
- }
291
-
292
- onDisable(): void {
293
- // console.log("XR controller disabled", this);
294
- this.hand?.removeFromParent();
295
- this.controllerGrip?.removeFromParent();
296
- this.controller?.removeFromParent();
297
- this.raycastLine?.removeFromParent();
298
- this._raycastHitPoint?.removeFromParent();
299
- // console.log("Disable", this._connnectedCallback, this._disconnectedCallback);
300
- // this.controllerGrip.removeEventListener("connected", this._connnectedCallback);
301
- // this.controllerGrip.removeEventListener("disconnected", this._disconnectedCallback);
302
-
303
- const i = this.webXR.Controllers.indexOf(this);
304
- if (i >= 0)
305
- this.webXR.Controllers.splice(i, 1);
306
- }
307
-
308
- // onDestroy(): void {
309
- // console.log("destroyed", this.index);
310
- // }
311
-
312
- private _isConnected: boolean = false;
313
-
314
- private onSourceConnected(e: { data: XRInputSource, target: any }) {
315
- if (this._isConnected) {
316
- console.warn("Received connected event for controller that is already connected", this.index, e);
317
- return;
318
- }
319
- this._isConnected = true;
320
- this.input = e.data;
321
-
322
- if (this.type === ControllerType.Touch) {
323
- this.onSelectStart();
324
- this.createPointerEvent("down");
325
- }
326
- }
327
-
328
- private onSourceDisconnected(_e: any) {
329
- if (!this._isConnected) {
330
- console.warn("Received discnnected event for controller that is not connected", _e);
331
- return;
332
- }
333
- this._isConnected = false;
334
- if (this.type === ControllerType.Touch) {
335
- this.onSelectEnd();
336
- this.createPointerEvent("up");
337
- }
338
- this.input = null;
339
- }
340
-
341
- private createPointerEvent(type: string) {
342
- switch (type) {
343
- case "down":
344
- this.context.input.createPointerDown({ clientX: 0, clientY: 0, button: this.index, pointerType: "touch" });
345
- break;
346
- case "move":
347
- break;
348
- case "up":
349
- this.context.input.createPointerUp({ clientX: 0, clientY: 0, button: this.index, pointerType: "touch" });
350
- break;
351
- }
352
- }
353
-
354
- rayRotation: Quaternion = new Quaternion();
355
-
356
- update(): void {
357
-
358
- // TODO: we should wait until we actually have models, this is just a workaround
359
- if (this.context.time.frameCount % 60 === 0) {
360
- this.setControllerLayers(this.controller, 2);
361
- this.setControllerLayers(this.controllerGrip, 2);
362
- this.setControllerLayers(this.hand, 2);
363
- }
364
-
365
- const subs = WebXRController.eventSubs[ControllerEvents.Update];
366
- if (subs && subs.length > 0) {
367
- for (const sub of subs) {
368
- sub(this);
369
- }
370
- }
371
-
372
- let t = 1;
373
- if (this.type === ControllerType.PhysicalDevice) t = this.context.time.deltaTime / .1;
374
- else if (this.isUsingHands && this.handPointerModel.pinched) t = this.context.time.deltaTime / .3;
375
- this.rayRotation.slerp(getWorldQuaternion(this.controller), this.useSmoothing ? t : 1.0);
376
- const wp = getWorldPosition(this.controller);
377
-
378
- // hide hand pointer model, it's giant and doesn't really help
379
- if (this.isUsingHands && this.handPointerModel.cursorObject) {
380
- this.handPointerModel.cursorObject.visible = false;
381
- }
382
-
383
- if (this.raycastLine) {
384
- const allowRaycastLineVisible = this.showRaycastLine && this.type !== ControllerType.Touch;
385
- if (this.type === ControllerType.Touch) {
386
- this.raycastLine.visible = false;
387
- }
388
- else if (this.isUsingHands) {
389
- this.raycastLine.visible = !this.grabbed && allowRaycastLineVisible;
390
- setWorldPosition(this.raycastLine, wp);
391
- const jnts = this.hand!['joints'];
392
- if (jnts) {
393
- const wrist = jnts['wrist'];
394
- if (wrist && this.grabbed && this.grabbed.isCloseGrab) {
395
- const wr = this.getWristQuaternion();
396
- if (wr)
397
- this.rayRotation.copy(wr);
398
- // this.rayRotation.slerp(wr, this.useSmoothing ? t * 2 : 1);
399
- }
400
- }
401
- setWorldQuaternion(this.raycastLine, this.rayRotation);
402
- }
403
- else {
404
- this.raycastLine.visible = allowRaycastLineVisible;
405
- setWorldQuaternion(this.raycastLine, this.rayRotation);
406
- setWorldPosition(this.raycastLine, wp);
407
- }
408
- }
409
-
410
- this.lastHit = this.updateLastHit();
411
-
412
- if (this.grabbed) {
413
- this.grabbed.update();
414
- }
415
-
416
- this._selectionPressedLastFrame = this._selectionPressed;
417
-
418
- if (this.selectStartCallback) {
419
- this.selectStartCallback();
420
- }
421
- }
422
-
423
- private _pinchStartTime: number | undefined = undefined;
424
-
425
- onUpdate(session: XRSession) {
426
- this.lastHit = null;
427
-
428
- if (!session || session.inputSources.length <= this.index) {
429
- this.input = null;
430
- return;
431
- }
432
- if (this.type === ControllerType.PhysicalDevice)
433
- this.input = session.inputSources[this.index];
434
- if (!this.input) return;
435
- const rig = this.webXR.Rig;
436
- if (!rig) return;
437
-
438
- if (this._didNotEndSelection && !this.handPointerModel.pinched) {
439
- this._didNotEndSelection = false;
440
- this.onSelectEnd();
441
- }
442
-
443
- this.updateStick(this.input);
444
-
445
- const buttons = this.input?.gamepad?.buttons;
446
-
447
- switch (this.input.handedness) {
448
- case "left":
449
- const speedFactor = 3 * WebXRController.MovementSpeedFactor;
450
- const powFactor = 2;
451
- const speed = Mathf.clamp01(this.joystick.length() * 2);
452
-
453
- const sideDir = this.joystick.x > 0 ? 1 : -1;
454
- let side = Math.pow(this.joystick.x, powFactor);
455
- side *= sideDir;
456
- side *= speed;
457
-
458
-
459
- const forwardDir = this.joystick.y > 0 ? 1 : -1;
460
- let forward = Math.pow(this.joystick.y, powFactor);
461
- forward *= forwardDir;
462
- side *= speed;
463
-
464
- rig.getWorldQuaternion(this.worldRot);
465
- this.movementVector.set(side, 0, forward);
466
- this.movementVector.applyQuaternion(this.webXR.TransformOrientation);
467
- this.movementVector.y = 0;
468
- this.movementVector.applyQuaternion(this.worldRot);
469
- this.movementVector.multiplyScalar(speedFactor * this.context.time.deltaTime);
470
- rig.position.add(this.movementVector);
471
-
472
- if (this.isUsingHands)
473
- this.runTeleport(rig, buttons);
474
- break;
475
-
476
- case "right":
477
- const rotate = this.joystick.x;
478
- const rotAbs = Math.abs(rotate);
479
- if (rotAbs < 0.4) {
480
- this.didRotate = false;
481
- }
482
- else if (rotAbs > .5 && !this.didRotate) {
483
- const dir = rotate > 0 ? -1 : 1;
484
- rig.rotateY(Mathf.toRadians(30 * dir));
485
- this.didRotate = true;
486
- }
487
-
488
- this.runTeleport(rig, buttons);
489
-
490
- break;
491
- }
492
- }
493
-
494
- private runTeleport(rig, buttons) {
495
- let teleport = -this.joystick.y;
496
- if (this.hand?.visible && !this.grabbed) {
497
- const pinched = this.handPointerModel.isPinched();
498
- if (pinched && this._pinchStartTime === undefined) {
499
- this._pinchStartTime = this.context.time.time;
500
- }
501
- if (pinched && this._pinchStartTime && this.context.time.time - this._pinchStartTime > .8) {
502
- // hacky approach for basic hand teleportation -
503
- // we teleport if we pinch and the back of the hand points down (open hand gesture)
504
- // const v1 = new Vector3();
505
- // const worldQuaternion = new Quaternion();
506
- // this.controller.getWorldQuaternion(worldQuaternion);
507
- // v1.copy(this.controller.up).applyQuaternion(worldQuaternion);
508
- // const dotPr = -v1.dot(this.controller.up);
509
- teleport = this.handPointerModel.isPinched() ? 1 : 0;
510
- }
511
- if (!pinched) this._pinchStartTime = undefined;
512
- }
513
- else this._pinchStartTime = undefined;
514
-
515
- let doTeleport = teleport > .5 && this.webXR.IsInVR;
516
- let isInMiniatureMode = this.webXR.Rig ? this.webXR.Rig?.scale?.x < .999 : false;
517
- let newRigScale: number | null = null;
518
-
519
- if (buttons && this.input && !this.input.hand) {
520
- for (let i = 0; i < buttons.length; i++) {
521
- const btn = buttons[i];
522
- // button[4] seems to be the A button if it exists. On hololens it's randomly pressed though for hands
523
- // see https://www.w3.org/TR/webxr-gamepads-module-1/#xr-standard-gamepad-mapping
524
- if (i === 4) {
525
- if (btn.pressed && !this.didChangeScale && this.webXR.IsInVR) {
526
- this.didChangeScale = true;
527
- const rig = this.webXR.Rig;
528
- if (rig) {
529
- if (!isInMiniatureMode) {
530
- isInMiniatureMode = true;
531
- doTeleport = true;
532
- newRigScale = .1;
533
- WebXRController.MovementSpeedFactor = newRigScale * 2;
534
- const cam = this.context.mainCamera as PerspectiveCamera;
535
- WebXRController.PreviousCameraFarDistance = cam.far;
536
- cam.far /= newRigScale;
537
- }
538
- else {
539
- isInMiniatureMode = false;
540
- rig.scale.set(1, 1, 1);
541
- newRigScale = 1;
542
- WebXRController.MovementSpeedFactor = 1;
543
- const cam = this.context.mainCamera as PerspectiveCamera;
544
- if (WebXRController.PreviousCameraFarDistance)
545
- cam.far = WebXRController.PreviousCameraFarDistance;
546
- }
547
- }
548
- }
549
- else if (!btn.pressed)
550
- this.didChangeScale = false;
551
- }
552
- }
553
- }
554
-
555
- if (doTeleport) {
556
- if (!this.didTeleport) {
557
- const rc = this.raycast();
558
- this.didTeleport = true;
559
- if (rc && rc.length > 0) {
560
- const hit = rc[0];
561
- if (isInMiniatureMode || this.isValidTeleportTarget(hit.object)) {
562
- const point = hit.point;
563
- setWorldPosition(rig, point);
564
- }
565
- }
566
- }
567
- }
568
- else if (teleport < .1) {
569
- this.didTeleport = false;
570
- }
571
-
572
- if (newRigScale !== null) {
573
- rig.scale.set(newRigScale, newRigScale, newRigScale);
574
- rig.updateMatrixWorld();
575
- }
576
- }
577
-
578
- private isValidTeleportTarget(obj: Object3D): boolean {
579
- return GameObject.getComponentInParent(obj, TeleportTarget) != null;
580
- }
581
-
582
- private updateStick(inputSource: XRInputSource) {
583
- if (!inputSource || !inputSource.gamepad || inputSource.gamepad.axes?.length < 4) return;
584
- this.joystick.x = inputSource.gamepad.axes[2];
585
- this.joystick.y = inputSource.gamepad.axes[3];
586
- }
587
-
588
- private updateLastHit(): Intersection | null {
589
- const rc = this.raycast();
590
- const hit = rc ? rc[0] : null;
591
- this.lastHit = hit;
592
- let factor = 1;
593
- if (this.webXR.Rig) {
594
- factor /= this.webXR.Rig.scale.x;
595
- }
596
- // if (!hit) factor = 0;
597
-
598
- if (this.raycastLine) {
599
- this.raycastLine.scale.z = factor * (this.lastHit?.distance ?? 9999);
600
- const mat = this.raycastLine.material as LineBasicMaterial;
601
- if (hit != null) mat.color = WebXRController.raycastColor;
602
- else mat.color = WebXRController.raycastNoHitColor;
603
- }
604
- if (this._raycastHitPoint) {
605
- if (this.lastHit != null) {
606
- this._raycastHitPoint.position.z = -1;// -this.lastHit.distance;
607
- const scale = Mathf.clamp(this.lastHit.distance * .01 * factor, .015, .1);
608
- this._raycastHitPoint.scale.set(scale, scale, scale);
609
- }
610
- this._raycastHitPoint.visible = this.lastHit !== null && this.lastHit !== undefined;
611
- }
612
- return hit;
613
- }
614
-
615
- private onSelectStart() {
616
- if (!this.context.connection.allowEditing) return;
617
- // console.log("SELECT START", _event);
618
- // if we process the event immediately the controller
619
- // world positions are not yet correctly updated and we have info from the last frame
620
- // so we delay the event processing one frame
621
- // only necessary for AR - ideally we can get it to work right here
622
- // but should be fine as a workaround for now
623
- this.selectStartCallback = () => this.onHandleSelectStart();
624
- }
625
-
626
- private selectStartCallback: Function | null = null;
627
- private lastSelectStartObject: Object3D | null = null;;
628
-
629
- private onHandleSelectStart() {
630
- this.selectStartCallback = null;
631
- this._selectionPressed = true;
632
- this._selectionStartTime = this.context.time.time;
633
- this._selectionEndTime = 1000;
634
- // console.log("DOWN", this.index, WebXRController.eventSubs);
635
-
636
- // let maxDistance = this.isUsingHands ? .1 : undefined;
637
- let intersections: Intersection[] | null = null;
638
- let closeGrab: boolean = false;
639
- if (this.isUsingHands) {
640
- intersections = this.overlap();
641
- if (intersections.length <= 0) {
642
- intersections = this.raycast();
643
- closeGrab = false;
644
- }
645
- else {
646
- closeGrab = true;
647
- }
648
- }
649
- else intersections = this.raycast();
650
-
651
- if (debug)
652
- console.log("onHandleSelectStart", "close grab? " + closeGrab, "intersections", intersections);
653
-
654
- if (intersections && intersections.length > 0) {
655
- for (const intersection of intersections) {
656
- const object = intersection.object;
657
- this.lastSelectStartObject = object;
658
- const args = { selected: object, grab: object };
659
- const subs = WebXRController.eventSubs[ControllerEvents.SelectStart];
660
- if (subs && subs.length > 0) {
661
- for (const sub of subs) {
662
- sub(this, args);
663
- }
664
- }
665
- if (args.grab !== object && debug)
666
- console.log("Grabbed object changed", "original", object, "new", args.grab);
667
- if (args.grab) {
668
- this.grabbed = AttachedObject.TryTake(this, args.grab, intersection, closeGrab);
669
- }
670
- break;
671
- }
672
- }
673
- else {
674
- const subs = WebXRController.eventSubs[ControllerEvents.SelectStart];
675
- const args = { selected: null, grab: null };
676
- if (subs && subs.length > 0) {
677
- for (const sub of subs) {
678
- sub(this, args);
679
- }
680
- }
681
- }
682
- }
683
-
684
- private _didNotEndSelection: boolean = false;
685
-
686
- private onSelectEnd() {
687
- if (this.isUsingHands) {
688
- if (this.handPointerModel.pinched) {
689
- this._didNotEndSelection = true;
690
- return;
691
- }
692
- }
693
-
694
- if (!this._selectionPressed) return;
695
- this.selectStartCallback = null;
696
- this._selectionPressed = false;
697
- this._selectionEndTime = this.context.time.time;
698
-
699
- const args = { grab: this.grabbed?.selected ?? this.lastSelectStartObject };
700
- const subs = WebXRController.eventSubs[ControllerEvents.SelectEnd];
701
- if (subs && subs.length > 0) {
702
- for (const sub of subs) {
703
- sub(this, args);
704
- }
705
- }
706
-
707
- if (this.grabbed) {
708
- this.grabbed.free();
709
- this.grabbed = null;
710
- }
711
- }
712
-
713
- private testIsVisible(obj: Object3D | null): boolean {
714
- if (!obj) return false;
715
- if (GameObject.isActiveInHierarchy(obj) === false) return false;
716
- if (UIRaycastUtils.isInteractable(obj) === false) {
717
- return false;
718
- }
719
- return true;
720
- // if (!obj.visible) return false;
721
- // return this.testIsVisible(obj.parent);
722
- }
723
-
724
- private setControllerLayers(obj: Object3D, layer: number) {
725
- if (!obj) return;
726
- obj.layers.set(layer);
727
- if (obj.children) {
728
- for (const ch of obj.children) {
729
- if (this.grabbed?.selected === ch || this.grabbed?.selectedMesh === ch) {
730
- continue;
731
- }
732
- this.setControllerLayers(ch, layer);
733
- }
734
- }
735
- }
736
-
737
- public getRay(): Ray {
738
- const ray = new Ray();
739
- // this.tempMatrix.identity().extractRotation(this.controller.matrixWorld);
740
- // ray.origin.setFromMatrixPosition(this.controller.matrixWorld);
741
- ray.origin.copy(getWorldPosition(this.controller));
742
- ray.direction.set(0, 0, -1).applyQuaternion(this.rayRotation);
743
- return ray;
744
- }
745
-
746
- private closeGrabBoundingBoxHelper?: BoxHelper;
747
-
748
- public overlap(): Intersection[] {
749
- const overlapCenter = (this.isUsingHands && this.handPointerModel) ? this.handPointerModel.pointerObject : this.controllerGrip;
750
-
751
- if (debug) {
752
- if (!this.closeGrabBoundingBoxHelper && overlapCenter) {
753
- this.closeGrabBoundingBoxHelper = new BoxHelper(overlapCenter, 0xffff00);
754
- this.scene.add(this.closeGrabBoundingBoxHelper);
755
- }
756
-
757
- if (this.closeGrabBoundingBoxHelper && overlapCenter) {
758
- this.closeGrabBoundingBoxHelper.setFromObject(overlapCenter);
759
- }
760
- }
761
-
762
- if (!overlapCenter)
763
- return new Array<Intersection>();
764
-
765
- const wp = getWorldPosition(overlapCenter).clone();
766
- return this.context.physics.sphereOverlap(wp, .02);
767
- }
768
-
769
- public raycast(): Intersection[] {
770
- const opts = new RaycastOptions();
771
- opts.layerMask = new Layers();
772
- opts.layerMask.enableAll();
773
- opts.layerMask.disable(2);
774
- opts.ray = this.getRay();
775
- const hits = this.context.physics.raycast(opts);
776
- for (let i = 0; i < hits.length; i++) {
777
- const hit = hits[i];
778
- const obj = hit.object;
779
- if (!this.testIsVisible(obj)) {
780
- hits.splice(i, 1);
781
- i--;
782
- continue;
783
- }
784
- hit.object = UIRaycastUtils.getObject(obj);
785
- break;
786
- }
787
- // console.log(...hits);
788
- return hits;
789
- }
790
- }
791
-
792
-
793
- export enum AttachedObjectEvents {
794
- WillTake = "WillTake",
795
- DidTake = "DidTake",
796
- WillFree = "WillFree",
797
- DidFree = "DidFree",
798
- }
799
-
800
- export class AttachedObject {
801
-
802
- public static Events: { [key: string]: Function[] } = {};
803
- public static AddEventListener(event: AttachedObjectEvents, callback: Function): Function {
804
- if (!AttachedObject.Events[event]) AttachedObject.Events[event] = [];
805
- AttachedObject.Events[event].push(callback);
806
- return callback;
807
- }
808
- public static RemoveEventListener(event: AttachedObjectEvents, callback: Function | null) {
809
- if (!callback) return;
810
- if (!AttachedObject.Events[event]) return;
811
- const idx = AttachedObject.Events[event].indexOf(callback);
812
- if (idx >= 0) AttachedObject.Events[event].splice(idx, 1);
813
- }
814
-
815
-
816
- public static Current: AttachedObject[] = [];
817
-
818
- private static Register(obj: AttachedObject) {
819
-
820
- if (!this.Current.find(x => x === obj)) {
821
- this.Current.push(obj);
822
- }
823
- }
824
-
825
- private static Remove(obj: AttachedObject) {
826
- const i = this.Current.indexOf(obj);
827
- if (i >= 0) {
828
- this.Current.splice(i, 1);
829
- }
830
- }
831
-
832
- public static TryTake(controller: WebXRController, candidate: Object3D, intersection: Intersection, closeGrab: boolean): AttachedObject | null {
833
- const interactable = GameObject.getComponentInParent(candidate, Interactable);
834
- if (!interactable) {
835
- if (debug)
836
- console.warn("Prevented taking object that is not interactable", candidate);
837
- return null;
838
- }
839
- else candidate = interactable.gameObject;
840
-
841
-
842
- let objectToAttach = candidate;
843
- const sync = GameObject.getComponentInParent(candidate, SyncedTransform);
844
- if (sync) {
845
- sync.requestOwnership();
846
- objectToAttach = sync.gameObject;
847
- }
848
-
849
- for (const o of this.Current) {
850
- if (o.selected === objectToAttach) {
851
- if (o.controller === controller) return o;
852
- o.free();
853
- o.Take(controller, objectToAttach, candidate, sync, interactable, intersection, closeGrab);
854
- return o;
855
- }
856
- }
857
-
858
- const att = new AttachedObject();
859
- att.Take(controller, objectToAttach, candidate, sync, interactable, intersection, closeGrab);
860
- return att;
861
- }
862
-
863
-
864
- public sync: SyncedTransform | null = null;
865
- public selected: Object3D | null = null;
866
- public selectedParent: Object3D | null = null;
867
- public selectedMesh: Mesh | null = null;
868
- public controller: WebXRController | null = null;
869
- public grabTime: number = 0;
870
- public grabUUID: string = "";
871
- public isCloseGrab: boolean = false; // when taken via sphere cast with hands
872
-
873
- private originalMaterial: Material | Material[] | null = null;
874
- private usageMarker: UsageMarker | null = null;
875
- private rigidbodies: Rigidbody[] | null = null;
876
- private didReparent: boolean = false;
877
- private grabDistance: number = 0;
878
- private interactable: Interactable | null = null;
879
- private positionSource: Object3D | null = null;
880
-
881
- private Take(controller: WebXRController, take: Object3D, hit: Object3D, sync: SyncedTransform | null, _interactable: Interactable,
882
- intersection: Intersection, closeGrab: boolean)
883
- : AttachedObject {
884
- console.assert(take !== null, "Expected object to be taken but was", take);
885
-
886
- if (controller.isUsingHands) {
887
- this.positionSource = closeGrab ? controller.wrist : controller.controller;
888
- }
889
- else {
890
- this.positionSource = controller.controller;
891
- }
892
- if (!this.positionSource) {
893
- console.warn("No position source");
894
- return this;
895
- }
896
-
897
- const args = { controller, take, hit, sync, interactable: _interactable };
898
- AttachedObject.Events.WillTake?.forEach(x => x(this, args));
899
-
900
-
901
- const mesh = hit as Mesh;
902
- if (mesh?.material) {
903
- this.originalMaterial = mesh.material;
904
- if (!Array.isArray(mesh.material)) {
905
- mesh.material = (mesh.material as Material).clone();
906
- if (mesh.material && mesh.material["emissive"])
907
- mesh.material["emissive"].b = .2;
908
- }
909
- }
910
-
911
- this.selected = take;
912
- if (!this.selectedParent) {
913
- this.selectedParent = take.parent;
914
- }
915
- this.selectedMesh = mesh;
916
- this.controller = controller;
917
- this.interactable = _interactable;
918
- this.isCloseGrab = closeGrab;
919
- // if (interactable.canGrab) {
920
- // this.didReparent = true;
921
- // this.device.controller.attach(take);
922
- // }
923
- // else
924
- this.didReparent = false;
925
-
926
-
927
- this.sync = sync;
928
- this.grabTime = controller.context.time.time;
929
- this.grabUUID = Date.now().toString();
930
- this.usageMarker = GameObject.addNewComponent(this.selected, UsageMarker);
931
- this.rigidbodies = GameObject.getComponentsInChildren(this.selected, Rigidbody);
932
- getWorldPosition(this.positionSource, this.lastControllerWorldPos);
933
- const getGrabPoint = () => closeGrab ? this.lastControllerWorldPos.clone() : intersection.point.clone();
934
- this.grabDistance = getGrabPoint().distanceTo(this.lastControllerWorldPos);
935
- this.totalChangeAlongDirection = 0.0;
936
-
937
- // we're storing position relative to the grab point
938
- // we're storing rotation relative to the ray
939
- this.localPositionOffsetToGrab = this.selected.worldToLocal(getGrabPoint());
940
- const rot = controller.isUsingHands && closeGrab ? this.controller.getWristQuaternion()!.clone() : controller.rayRotation.clone();
941
- getWorldQuaternion(this.selected, this.localQuaternionToGrab).premultiply(rot.invert());
942
-
943
- const rig = this.controller.webXR.Rig;
944
- if (rig)
945
- this.rigPositionLastFrame.copy(getWorldPosition(rig))
946
-
947
- Avatar_POI.Add(controller.context, this.selected);
948
- AttachedObject.Register(this);
949
-
950
- if (this.sync) {
951
- this.sync.fastMode = true;
952
- }
953
-
954
- AttachedObject.Events.DidTake?.forEach(x => x(this, args));
955
-
956
- return this;
957
- }
958
-
959
- public free(): void {
960
- if (!this.selected) return;
961
-
962
- const args = { controller: this.controller, take: this.selected, hit: this.selected, sync: this.sync, interactable: null };
963
- AttachedObject.Events.WillFree?.forEach(x => x(this, args));
964
-
965
- Avatar_POI.Remove(this.controller!.context, this.selected);
966
- AttachedObject.Remove(this);
967
-
968
- if (this.sync) {
969
- this.sync.fastMode = false;
970
- }
971
-
972
- const mesh = this.selectedMesh;
973
- if (mesh && this.originalMaterial && mesh.material) {
974
- mesh.material = this.originalMaterial;
975
- }
976
-
977
- const object = this.selected;
978
- // only attach the object back if it has a parent
979
- // no parent means it was destroyed while holding it!
980
- if (this.didReparent && object.parent) {
981
- const prevParent = this.selectedParent;
982
- if (prevParent) prevParent.attach(object);
983
- else this.controller?.context.scene.attach(object);
984
- }
985
-
986
- this.usageMarker?.destroy();
987
-
988
- if (this.controller)
989
- this.controller.grabbed = null;
990
- this.selected = null;
991
- this.selectedParent = null;
992
- this.selectedMesh = null;
993
- this.sync = null;
994
-
995
-
996
- // TODO: make throwing work again
997
- if (this.rigidbodies) {
998
- for (const rb of this.rigidbodies) {
999
- rb.wakeUp();
1000
- rb.setVelocity(rb.smoothedVelocity);
1001
- }
1002
- }
1003
- this.rigidbodies = null;
1004
-
1005
- this.localPositionOffsetToGrab = null;
1006
- this.quaternionLerp = null;
1007
-
1008
- AttachedObject.Events.DidFree?.forEach(x => x(this, args));
1009
- }
1010
-
1011
- public grabPoint: Vector3 = new Vector3();
1012
-
1013
- private localPositionOffsetToGrab: Vector3 | null = null;
1014
- private localPositionOffsetToGrab_worldSpace: Vector3 = new Vector3();
1015
- private localQuaternionToGrab: Quaternion = new Quaternion(0, 0, 0, 1);
1016
- private targetDir: Vector3 | null = null;
1017
- private quaternionLerp: Quaternion | null = null;
1018
-
1019
- private controllerDir = new Vector3();
1020
- private controllerWorldPos = new Vector3();
1021
- private lastControllerWorldPos = new Vector3();
1022
- private controllerPosDelta = new Vector3();
1023
- private totalChangeAlongDirection = 0.0;
1024
- private rigPositionLastFrame = new Vector3();
1025
-
1026
- private controllerMovementSinceLastFrame() {
1027
- if (!this.positionSource || !this.controller) return 0.0;
1028
-
1029
- // controller direction
1030
- this.controllerDir.set(0, 0, -1);
1031
- this.controllerDir.applyQuaternion(this.controller.rayRotation);
1032
-
1033
- // controller delta
1034
- getWorldPosition(this.positionSource, this.controllerWorldPos);
1035
- this.controllerPosDelta.copy(this.controllerWorldPos);
1036
- this.controllerPosDelta.sub(this.lastControllerWorldPos);
1037
- this.lastControllerWorldPos.copy(this.controllerWorldPos);
1038
- const rig = this.controller.webXR.Rig;
1039
- if (rig) {
1040
- const rigPos = getWorldPosition(rig);
1041
- const rigDelta = this.rigPositionLastFrame.sub(rigPos);
1042
- this.controllerPosDelta.add(rigDelta);
1043
- this.rigPositionLastFrame.copy(rigPos);
1044
- }
1045
-
1046
- // calculate delta along direction
1047
- const changeAlongControllerDirection = this.controllerDir.dot(this.controllerPosDelta);
1048
-
1049
- return changeAlongControllerDirection;
1050
- }
1051
-
1052
- public update() {
1053
- if (this.rigidbodies)
1054
- for (const rb of this.rigidbodies)
1055
- rb.resetVelocities();
1056
- // TODO: add/use sync lost ownership event
1057
- if (this.sync && this.controller && this.controller.context.connection.isInRoom) {
1058
- const td = this.controller.context.time.time - this.grabTime;
1059
- // if (time.frameCount % 60 === 0) {
1060
- // console.log("ownership?", this.selected.name, this.sync.hasOwnership(), td)
1061
- // }
1062
- if (td > 3) {
1063
- // if (time.frameCount % 60 === 0) {
1064
- // console.log(this.sync.hasOwnership())
1065
- // }
1066
- if (this.sync.hasOwnership() === false) {
1067
- console.log("no ownership, will leave", this.sync.guid);
1068
- this.free();
1069
- }
1070
- }
1071
- }
1072
- if (this.interactable && !this.interactable.canGrab) return;
1073
-
1074
- if (!this.didReparent && this.selected && this.controller) {
1075
-
1076
- const rigScale = this.controller.webXR.Rig?.scale.x ?? 1.0;
1077
-
1078
- this.totalChangeAlongDirection += this.controllerMovementSinceLastFrame();
1079
- // console.log(this.totalChangeAlongDirection);
1080
-
1081
- // alert("yo: " + this.controller.webXR.Rig?.scale.x); // this is 0.1 on Hololens
1082
- let currentDist = 1.0;
1083
- if (this.controller.type === ControllerType.PhysicalDevice) // only for controllers and not on touch (AR touches are controllers)
1084
- {
1085
- currentDist = Math.max(0.0, 1 + this.totalChangeAlongDirection * 2.0 / rigScale);
1086
- currentDist = currentDist * currentDist * currentDist;
1087
- }
1088
- if (this.grabDistance / rigScale < 0.8) currentDist = 1.0; // don't accelerate if this is a close grab, want full control
1089
-
1090
- if (!this.targetDir) {
1091
- this.targetDir = new Vector3();
1092
- }
1093
- this.targetDir.set(0, 0, -this.grabDistance * currentDist);
1094
- const target = this.targetDir.applyQuaternion(this.controller.rayRotation).add(this.controllerWorldPos);
1095
-
1096
- // apply rotation
1097
- const targetQuat = this.controller.rayRotation.clone().multiplyQuaternions(this.controller.rayRotation, this.localQuaternionToGrab);
1098
- if (!this.quaternionLerp) {
1099
- this.quaternionLerp = targetQuat.clone();
1100
- }
1101
- this.quaternionLerp.slerp(targetQuat, this.controller.useSmoothing ? this.controller.context.time.deltaTime / .03 : 1.0);
1102
- setWorldQuaternion(this.selected, this.quaternionLerp);
1103
- this.selected.updateWorldMatrix(false, false); // necessary so that rotation is correct for the following position update
1104
-
1105
- // apply position
1106
- this.grabPoint.copy(target);
1107
- // apply local grab offset
1108
- if (this.localPositionOffsetToGrab) {
1109
- this.localPositionOffsetToGrab_worldSpace.copy(this.localPositionOffsetToGrab);
1110
- this.selected.localToWorld(this.localPositionOffsetToGrab_worldSpace).sub(getWorldPosition(this.selected));
1111
- target.sub(this.localPositionOffsetToGrab_worldSpace);
1112
- }
1113
- setWorldPosition(this.selected, target);
1114
- }
1115
-
1116
-
1117
- if (this.rigidbodies != null) {
1118
- for (const rb of this.rigidbodies) {
1119
- rb.wakeUp();
1120
- }
1121
- }
1122
-
1123
- InstancingUtil.markDirty(this.selected, true);
1124
- }
1125
- }
src/engine-components/WebXRGrabRendering.ts DELETED
@@ -1,151 +0,0 @@
1
- import { getWorldPosition, setWorldPosition, setWorldPositionXYZ } from "../engine/engine_three_utils";
2
- import { Behaviour, GameObject } from "./Component";
3
- import { AttachedObject, AttachedObjectEvents } from "./WebXRController";
4
- import { Object3D, Vector3 } from "three";
5
- import { PlayerColor } from "./PlayerColor";
6
- import { Context } from "../engine/engine_setup";
7
- import { IModel, SendQueue } from "../engine/engine_networking_types";
8
-
9
- enum XRGrabEvent {
10
- StartOrUpdate = "xr-grab-visual-start-or-update",
11
- End = "xr-grab-visual-end",
12
- }
13
-
14
- export class XRGrabModel implements IModel {
15
- guid!: any;
16
- dontSave: boolean = true;
17
-
18
- userId : string | null | undefined;
19
- point: { x: number, y: number, z: number } = { x: 0, y: 0, z: 0 };
20
- source: { x: number, y: number, z: number } = { x: 0, y: 0, z: 0 };
21
- target: string | undefined;
22
-
23
- update(context : Context, point: Vector3, source: Vector3, target: string | undefined = undefined) {
24
- this.userId = context.connection.connectionId;
25
- this.point.x = point.x;
26
- this.point.y = point.y;
27
- this.point.z = point.z;
28
- this.source.x = source.x;
29
- this.source.y = source.y;
30
- this.source.z = source.z;
31
- this.target = target;
32
- }
33
- }
34
-
35
- // sends grab info to other users and creates rendering instances
36
- export class XRGrabRendering extends Behaviour {
37
- prefab: Object3D | null = null;
38
-
39
- private _grabModels: Array<XRGrabModel> = [];
40
- private _grabModelsUpdateTime: Array<number> = [];
41
- private _addOrUpdateSub: Function | null = null;
42
- private _endSub: Function | null = null;
43
- private _freeSub: Function | null = null;
44
- private _instances: { [key: string]: {instance:Object3D, model:XRGrabModel} } = {};
45
-
46
- awake(): void {
47
- if(this.prefab) this.prefab.visible = false;
48
- }
49
-
50
- onEnable(): void {
51
- this._addOrUpdateSub = this.context.connection.beginListen(XRGrabEvent.StartOrUpdate, this.onRemoteGrabStartOrUpdate.bind(this));
52
- this._endSub = this.context.connection.beginListen(XRGrabEvent.End, this.onRemoteGrabEnd.bind(this));
53
- this._freeSub = AttachedObject.AddEventListener(AttachedObjectEvents.WillFree, this.onAttachedObjectFree.bind(this));
54
- }
55
-
56
- onDisable(): void {
57
- this.context.connection.stopListen(XRGrabEvent.StartOrUpdate, this._addOrUpdateSub);
58
- this.context.connection.stopListen(XRGrabEvent.End, this._endSub);
59
- AttachedObject.RemoveEventListener(AttachedObjectEvents.WillFree, this._freeSub);
60
- }
61
-
62
- addOrUpdateGrab(model: XRGrabModel) {
63
- this.context.connection.send(XRGrabEvent.StartOrUpdate, model, SendQueue.Queued);
64
- }
65
-
66
- endGrab(model: XRGrabModel) {
67
- this.context.connection.send(XRGrabEvent.End, model, SendQueue.Queued);
68
- }
69
-
70
- private onRemoteGrabStartOrUpdate(data: XRGrabModel) {
71
- if(!this.prefab) return;
72
- const inst = this._instances[data.guid];
73
- if(!inst)
74
- {
75
- const instance = GameObject.instantiate(this.prefab) as Object3D;
76
- instance.visible = true;
77
- this._instances[data.guid] = {instance, model:data};
78
- if(data.userId){
79
- const playerColor = GameObject.getComponentsInChildren(instance, PlayerColor);
80
- if(playerColor?.length > 0)
81
- {
82
- for(const pl of playerColor){
83
- pl.assignUserColor(data.userId)
84
- }
85
- }
86
- }
87
- return;
88
- }
89
- inst.model = data;
90
- }
91
-
92
- private onRemoteGrabEnd(data: XRGrabModel) {
93
- if (!data) return;
94
- const id = data.guid;
95
- if(this._instances[id])
96
- {
97
- GameObject.destroy(this._instances[id].instance);
98
- delete this._instances[id];
99
- }
100
- }
101
-
102
- private onAttachedObjectFree(att: AttachedObject) {
103
- if (this._grabModels.length <= 0) return;
104
- const mod = this._grabModels[0];
105
- this.updateModel(mod, att);
106
- this.endGrab(mod);
107
- }
108
-
109
- onBeforeRender() {
110
- this.updateRendering();
111
-
112
- if (!this.prefab) return;
113
- this.prefab.visible = false;
114
- if (this.context.time.frameCount % 10 !== 0) return;
115
- for (let i = 0; i < AttachedObject.Current.length; i++) {
116
- const att = AttachedObject.Current[i];
117
-
118
- if (!att.controller || !att.selected) continue;
119
-
120
- if (this._grabModels.length <= i) {
121
- this._grabModels.push(new XRGrabModel());
122
- this._grabModelsUpdateTime.push(0);
123
- }
124
- this._grabModelsUpdateTime[i] = this.context.time.time;
125
- const model = this._grabModels[i];
126
- this.updateModel(model, att);
127
- this.addOrUpdateGrab(model);
128
- }
129
- }
130
-
131
- private updateModel(model: XRGrabModel, att: AttachedObject) {
132
- if (!att.controller || !att.selected) return;
133
- model.guid = att.grabUUID;
134
- const targetObject = att.selected["guid"];
135
- model.update(this.context, att.grabPoint, att.controller.worldPosition, targetObject);
136
- }
137
-
138
- private temp : Vector3 = new Vector3();
139
- private updateRendering() {
140
- const step = this.context.time.deltaTime / .5;
141
- for(const key in this._instances){
142
- const { instance, model } = this._instances[key];
143
- if(!instance || !model) continue;
144
- const { point } = model;
145
- const wp = getWorldPosition(instance);
146
- this.temp.set(point.x, point.y, point.z);
147
- wp.lerp(this.temp, step);
148
- setWorldPosition(instance, wp);
149
- }
150
- }
151
- }
src/engine-components/WebXRImageTracking.ts DELETED
@@ -1,192 +0,0 @@
1
- import { WebXR } from "./WebXR";
2
- import { serializable } from "../engine/engine_serialization";
3
- import { Behaviour } from "./Component";
4
- import { Matrix4, Object3D, Quaternion, Vector, Vector3 } from "three";
5
- import { CircularBuffer, getParam } from "../engine/engine_utils";
6
-
7
- // https://github.com/immersive-web/marker-tracking/blob/main/explainer.md
8
-
9
- const debug = getParam("debugimagetracking");
10
-
11
- const _scaleTemp = new Vector3();
12
-
13
- export class WebXRTrackedImage {
14
-
15
-
16
- get url(): string { return this._trackedImage.image ?? ""; }
17
- get widthInMeters() { return this._trackedImage.widthInMeters ?? undefined; }
18
- get bitmap(): ImageBitmap { return this._bitmap; }
19
- readonly measuredSize: number;
20
- readonly state: "tracked" | "emulated";
21
-
22
- // private _matrix: Matrix4 | null = null;
23
- // private get matrix(): Matrix4 {
24
- // if (!this._matrix) {
25
- // // this._matrix = WebXRTrackedImage._matrixBuffer.get();
26
- // // const matrix = this._pose.transform.matrix;
27
- // // this._matrix.fromArray(matrix);
28
- // }
29
- // return this._matrix!;
30
- // }
31
-
32
- /** Copy the image position to a vector */
33
- getPosition(vec: Vector3) {
34
- this.ensureTransformData();
35
- vec.copy(this._position);
36
- return vec;
37
- }
38
-
39
- /** Copy the image rotation to a quaternion */
40
- getQuaternion(quat: Quaternion) {
41
- this.ensureTransformData();
42
- quat.copy(this._rotation);
43
- return quat;
44
- }
45
-
46
- applyToObject(object: Object3D) {
47
- this.ensureTransformData();
48
- object.position.copy(this._position);
49
- object.quaternion.copy(this._rotation);
50
- }
51
-
52
- // private static _matrixBuffer: CircularBuffer<Matrix4> = new CircularBuffer(() => new Matrix4(), 20);
53
- private static _positionBuffer: CircularBuffer<Vector3> = new CircularBuffer(() => new Vector3(), 20);
54
- private static _rotationBuffer: CircularBuffer<Quaternion> = new CircularBuffer(() => new Quaternion(), 20);
55
- private _position!: Vector3;
56
- private _rotation!: Quaternion;
57
- private ensureTransformData() {
58
- if (!this._position) {
59
- this._position = WebXRTrackedImage._positionBuffer.get();
60
- this._rotation = WebXRTrackedImage._rotationBuffer.get();
61
- const t = this._pose.transform;
62
- this._position.set(-t.position.x, t.position.y, -t.position.z);
63
- this._rotation.set(-t.orientation.x, t.orientation.y, -t.orientation.z, t.orientation.w);
64
- }
65
- }
66
-
67
- private readonly _trackingComponent: WebXRImageTracking;;
68
- private readonly _trackedImage: WebXRImageTrackingModel;
69
- private readonly _bitmap: ImageBitmap;
70
- private readonly _pose: any;
71
-
72
- constructor(context: WebXRImageTracking, trackedImage: WebXRImageTrackingModel, bitmap: ImageBitmap, measuredSize: number, state: "tracked" | "emulated", pose: any) {
73
- this._trackingComponent = context;;
74
- this._trackedImage = trackedImage;
75
- this._bitmap = bitmap;
76
- this.measuredSize = measuredSize;
77
- this.state = state;
78
- this._pose = pose;
79
- }
80
-
81
- }
82
-
83
- declare type WebXRImageTrackingEvent = (images: WebXRImageTrackingEvent[]) => void;
84
-
85
- export class WebXRImageTrackingModel {
86
-
87
- @serializable(URL)
88
- image?: string;
89
-
90
- @serializable()
91
- widthInMeters!: number;
92
-
93
- }
94
-
95
- export class WebXRImageTracking extends Behaviour {
96
-
97
- @serializable(WebXRImageTrackingModel)
98
- trackedImages!: WebXRImageTrackingModel[];
99
-
100
-
101
-
102
-
103
- private readonly trackedImageIndexMap: Map<number, WebXRImageTrackingModel> = new Map();
104
-
105
- private static _imageElements: Map<string, ImageBitmap | null> = new Map();
106
-
107
- awake(): void {
108
- for (const trackedImage of this.trackedImages) {
109
- if (trackedImage.image) {
110
- if (WebXRImageTracking._imageElements.has(trackedImage.image)) {
111
- }
112
- else {
113
- const url = trackedImage.image;
114
- WebXRImageTracking._imageElements.set(url, null);
115
- const imageElement = document.createElement("img") as HTMLImageElement;
116
- imageElement.src = url;
117
- imageElement.addEventListener("load", async () => {
118
- const img = await createImageBitmap(imageElement);
119
- WebXRImageTracking._imageElements.set(url, img);
120
- });
121
- }
122
- }
123
- }
124
- }
125
-
126
- onEnable(): void {
127
- WebXR.addEventListener("modify-ar-options", this.onModifyAROptions);
128
- }
129
-
130
- onDisable(): void {
131
- WebXR.removeEventListener("modify-ar-options", this.onModifyAROptions);
132
- }
133
-
134
-
135
- private onModifyAROptions = (event: any) => {
136
- const options = event.detail;
137
- const features = options.optionalFeatures || [];
138
- features.push("image-tracking");
139
- options.optionalFeatures = features;
140
-
141
- options.trackedImages = [];
142
- for (const trackedImage of this.trackedImages) {
143
- if (trackedImage.image?.length && trackedImage.widthInMeters > 0) {
144
- const bitmap = WebXRImageTracking._imageElements.get(trackedImage.image);
145
- if (bitmap) {
146
- this.trackedImageIndexMap.set(options.trackedImages.length, trackedImage);
147
- options.trackedImages.push({
148
- image: bitmap,
149
- widthInMeters: trackedImage.widthInMeters
150
- });
151
- }
152
- }
153
- }
154
- }
155
-
156
- onBeforeRender(frame: XRFrame | null): void {
157
- //@ts-ignore
158
- if (frame?.session && typeof frame.getImageTrackingResults === "function") {
159
- //@ts-ignore
160
- const results = frame.getImageTrackingResults();
161
- if (results.length) {
162
- const space = this.context.renderer.xr.getReferenceSpace();
163
- if (space) {
164
- const images: WebXRTrackedImage[] = [];
165
- for (const result of results) {
166
- const imageIndex = result.index;
167
- const trackedImage = this.trackedImageIndexMap.get(imageIndex);
168
- if (trackedImage) {
169
- const pose = frame.getPose(result.imageSpace, space);
170
- const state = result.trackingState;
171
- const imageData = new WebXRTrackedImage(this, trackedImage, result.image, result.measuredSize, state, pose);
172
- images.push(imageData);
173
- }
174
- else {
175
- if (debug) {
176
- console.warn("No tracked image for index", imageIndex);
177
- }
178
- }
179
- }
180
- if (images.length > 0) {
181
- try {
182
- this.dispatchEvent(new CustomEvent("image-tracking", { detail: images }));
183
- }
184
- catch (e) {
185
- console.error(e);
186
- }
187
- }
188
- }
189
- }
190
- }
191
- }
192
- }
src/engine-components/WebXRRig.ts DELETED
@@ -1,22 +0,0 @@
1
- import { Object3D } from "three";
2
- import { IGameObject } from "../engine/engine_types";
3
- import { getParam } from "../engine/engine_utils";
4
- import { Behaviour, GameObject } from "./Component";
5
- import { BoxGizmo } from "./Gizmos";
6
-
7
- const debug = getParam("debugrig");
8
-
9
- export class XRRig extends Behaviour {
10
- awake(): void {
11
- // const helper = new AxesHelper(.1);
12
- // this.gameObject.add(helper);
13
- if (debug) {
14
- const gizmoObj = new Object3D() as IGameObject;
15
- gizmoObj.position.y += .5;
16
- this.gameObject.add(gizmoObj);
17
- const gizmo = gizmoObj.addNewComponent(BoxGizmo);
18
- if (gizmo)
19
- gizmo.isGizmo = false;
20
- }
21
- }
22
- }
src/engine-components/WebXRSync.ts DELETED
@@ -1,463 +0,0 @@
1
- import { Behaviour, GameObject } from "./Component";
2
- import { RoomEvents, OwnershipModel, NetworkConnection } from "../engine/engine_networking";
3
- import { WebXR, WebXREvent } from "./WebXR";
4
- import { Group, Quaternion, Vector3, Vector4, WebXRManager } from "three";
5
- import { getParam } from "../engine/engine_utils";
6
- import { Voip } from "./Voip";
7
- import { Builder, Long } from "flatbuffers";
8
- import { VrUserStateBuffer } from "../engine-schemes/vr-user-state-buffer";
9
- import { Vec3 } from "../engine-schemes/vec3";
10
- import { registerType } from "../engine-schemes/schemes";
11
- import { Vec4 } from "../engine-schemes/vec4";
12
- import { WebXRAvatar } from "./WebXRAvatar";
13
-
14
- // for debug GUI
15
- // import { GUI } from "three/examples/jsm/libs/lil-gui.module.min.js";
16
- // import { HTMLMesh } from 'three/examples/jsm/interactive/HTMLMesh.js';
17
- // import { InteractiveGroup } from 'three/examples/jsm/interactive/InteractiveGroup.js';
18
- // import { renderer, sceneData } from "../engine/engine_setup";
19
-
20
- const debugLogs = getParam("debugxr");
21
- const debugAvatar = getParam("debugavatar");
22
- // const debugAvatarVoip = getParam("debugavatarvoip");
23
-
24
- enum WebXRSyncEvent {
25
- WebXR_UserJoined = "webxr-user-joined",
26
- WebXR_UserLeft = "webxr-user-left",
27
- VRSessionStart = "vr-session-started",
28
- VRSessionEnd = "vr-session-ended",
29
- VRSessionUpdate = "vr-session-update",
30
- }
31
-
32
- enum XRMode {
33
- VR = "vr",
34
- AR = "ar",
35
- }
36
-
37
- const VRUserStateBufferIdentifier = "VRUS";
38
- registerType(VRUserStateBufferIdentifier, VrUserStateBuffer.getRootAsVrUserStateBuffer);
39
-
40
- function getTimeStampNow() {
41
- return new Date().getTime(); // avoid sending millis in flatbuffer
42
- }
43
-
44
- function flatbuffers_long_from_number(num: number): Long {
45
- let low = num & 0xffffffff
46
- let high = (num / Math.pow(2, 32)) & 0xfffff
47
- return Long.create(low, high);
48
- }
49
-
50
- export class VRUserState {
51
- public guid: string;
52
- public time!: number;
53
- public avatarId!: string;
54
- public position: Vector3 = new Vector3();
55
- public rotation: Vector4 = new Vector4();
56
- public scale: number = 1;
57
-
58
- public posLeftHand = new Vector3();
59
- public posRightHand = new Vector3();
60
-
61
- public rotLeftHand = new Quaternion();
62
- public rotRightHand = new Quaternion();
63
-
64
- public constructor(guid: string) {
65
- this.guid = guid;
66
- }
67
-
68
- private static invertRotation: Quaternion = new Quaternion().setFromAxisAngle(new Vector3(0, 1, 0), Math.PI);
69
-
70
- public update(rig: Group, pos: DOMPointReadOnly, rot: DOMPointReadOnly, webXR: WebXR, avatarId: string) {
71
- this.time = getTimeStampNow();
72
- this.avatarId = avatarId;
73
- this.position.set(pos.x, pos.y, pos.z);
74
- if (rig)
75
- this.position.applyMatrix4(rig.matrixWorld);
76
-
77
- let q0 = VRUserState.quat0;
78
- const q1 = VRUserState.quat1;
79
- q0.set(rot.x, rot.y, rot.z, rot.w);
80
- q0 = q0.multiplyQuaternions(q0, VRUserState.invertRotation);
81
-
82
- if (rig) {
83
- rig.getWorldQuaternion(q1);
84
- q0.multiplyQuaternions(q1, q0);
85
- }
86
-
87
- this.rotation.set(q0.x, q0.y, q0.z, q0.w);
88
- this.scale = rig.scale.x;
89
-
90
- // for controllers, it seems we need grip pose
91
- const ctrl0 = webXR.LeftController?.controllerGrip;
92
- if (ctrl0) {
93
- ctrl0.getWorldPosition(this.posLeftHand);
94
- ctrl0.getWorldQuaternion(this.rotLeftHand);
95
- }
96
- const ctrl1 = webXR.RightController?.controllerGrip;
97
- if (ctrl1) {
98
- ctrl1.getWorldPosition(this.posRightHand);
99
- ctrl1.getWorldQuaternion(this.rotRightHand);
100
- }
101
-
102
- // if this is a hand, we need to get the root bone of that / use that for position/rotation
103
- if (webXR.LeftController?.hand?.visible) {
104
- const wrist = webXR.LeftController.wrist;
105
- if (wrist) {
106
- wrist.getWorldPosition(this.posLeftHand);
107
- wrist.getWorldQuaternion(this.rotLeftHand);
108
- }
109
- }
110
-
111
- if (webXR.RightController?.hand?.visible) {
112
- const wrist = webXR.RightController.wrist;
113
- if (wrist) {
114
- wrist.getWorldPosition(this.posRightHand);
115
- wrist.getWorldQuaternion(this.rotRightHand);
116
- }
117
- }
118
- }
119
-
120
- private static quat0: Quaternion = new Quaternion();
121
- private static quat1: Quaternion = new Quaternion();
122
-
123
- public sendAsBuffer(builder: Builder, net: NetworkConnection) {
124
- builder.clear();
125
- const guid = builder.createString(this.guid);
126
- const id = builder.createString(this.avatarId);
127
- VrUserStateBuffer.startVrUserStateBuffer(builder);
128
- VrUserStateBuffer.addGuid(builder, guid);
129
- VrUserStateBuffer.addTime(builder, flatbuffers_long_from_number(this.time));
130
- VrUserStateBuffer.addAvatarId(builder, id);
131
- VrUserStateBuffer.addPosition(builder, Vec3.createVec3(builder, this.position.x, this.position.y, this.position.z));
132
- VrUserStateBuffer.addRotation(builder, Vec4.createVec4(builder, this.rotation.x, this.rotation.y, this.rotation.z, this.rotation.w));
133
- VrUserStateBuffer.addScale(builder, this.scale);
134
- VrUserStateBuffer.addPosLeftHand(builder, Vec3.createVec3(builder, this.posLeftHand.x, this.posLeftHand.y, this.posLeftHand.z));
135
- VrUserStateBuffer.addPosRightHand(builder, Vec3.createVec3(builder, this.posRightHand.x, this.posRightHand.y, this.posRightHand.z));
136
- VrUserStateBuffer.addRotLeftHand(builder, Vec4.createVec4(builder, this.rotLeftHand.x, this.rotLeftHand.y, this.rotLeftHand.z, this.rotLeftHand.w));
137
- VrUserStateBuffer.addRotRightHand(builder, Vec4.createVec4(builder, this.rotRightHand.x, this.rotRightHand.y, this.rotRightHand.z, this.rotRightHand.w));
138
- const res = VrUserStateBuffer.endVrUserStateBuffer(builder);
139
- builder.finish(res, VRUserStateBufferIdentifier);
140
- const arr = builder.asUint8Array();
141
- net.sendBinary(arr);
142
- }
143
-
144
- public setFromBuffer(guid: string, state: VrUserStateBuffer) {
145
- if (!guid) return;
146
- this.guid = guid;
147
- this.time = state.time().toFloat64();
148
- const id = state.avatarId();
149
- if (id)
150
- this.avatarId = id;
151
- const pos = state.position();
152
- if (pos)
153
- this.position.set(pos.x(), pos.y(), pos.z());
154
- // TODO: maybe just send one float more instead of converting back and forth
155
- const rot = state.rotation();
156
- if (rot)
157
- this.rotation.set(rot.x(), rot.y(), rot.z(), rot.w());
158
- const posLeftHand = state.posLeftHand();
159
- if (posLeftHand)
160
- this.posLeftHand.set(posLeftHand.x(), posLeftHand.y(), posLeftHand.z());
161
- const posRightHand = state.posRightHand();
162
- if (posRightHand)
163
- this.posRightHand.set(posRightHand.x(), posRightHand.y(), posRightHand.z());
164
- const rotLeftHand = state.rotLeftHand();
165
- if (rotLeftHand)
166
- this.rotLeftHand.set(rotLeftHand.x(), rotLeftHand.y(), rotLeftHand.z(), rotLeftHand.w());
167
- const rotRightHand = state.rotRightHand();
168
- if (rotRightHand)
169
- this.rotRightHand.set(rotRightHand.x(), rotRightHand.y(), rotRightHand.z(), rotRightHand.w());
170
- this.scale = state.scale();
171
- }
172
- }
173
-
174
- export class WebXRSync extends Behaviour {
175
-
176
- webXR: WebXR | null = null;
177
-
178
- // private allowCustomAvatars: boolean | null = true;
179
-
180
- private debugAvatarUser: WebXRAvatar | null = null;
181
- private voip: Voip | null = null;
182
-
183
- async awake() {
184
-
185
- if(!this.webXR) this.webXR = GameObject.getComponent(this.gameObject, WebXR);
186
- if(!this.webXR) this.webXR = GameObject.findObjectOfType(WebXR, this.context);
187
-
188
- if(!this.webXR)
189
- {
190
- console.log("Missing webxr component");
191
- this.webXR = GameObject.findObjectOfType(WebXR, this.context);
192
- if(!this.webXR) {
193
- console.error("Could not find webxr component");
194
- return;
195
- }
196
- }
197
-
198
- if (!this.voip) this.voip = GameObject.findObjectOfType(Voip, this.context);
199
-
200
- if (debugAvatar) {
201
- const debugGuid = "debug-avatar-" + debugAvatar;
202
- const newUser = new WebXRAvatar(this.context, debugGuid, this.webXR);
203
- // newUser.isLocalAvatar = true;
204
- this.debugAvatarUser = newUser;
205
- if (typeof debugAvatar === "string" && debugAvatar.length > 0) {
206
- if (await newUser.setAvatarOverride(debugAvatar)) {
207
- const debugState = new VRUserState(debugGuid);
208
- debugState.position.y += 1;
209
- const off = .5;
210
- debugState.posLeftHand.y += off;
211
- debugState.posLeftHand.x += off;
212
- debugState.posRightHand.y += off;
213
- debugState.posRightHand.x -= off;
214
- newUser.tryUpdate(debugState, 0);
215
- }
216
- else {
217
- newUser.destroy();
218
- }
219
- }
220
- }
221
- }
222
-
223
- onEnable() {
224
- // const debugUser = new WebXRAvatar(this.context, "sorry-no-guid", this.webXR!);
225
-
226
- if (!this.webXR) {
227
- this.webXR = GameObject.getComponent(this.gameObject, WebXR);
228
- if (!this.webXR) {
229
- console.warn("Missing webxr component on " + this.gameObject.name);
230
- return;
231
- }
232
- }
233
-
234
- this.eventSub_WebXRStartEvent = this.onXRSessionStart.bind(this);
235
- WebXR.addEventListener(WebXREvent.XRStarted, this.eventSub_WebXRStartEvent);
236
- this.eventSub_WebXRUpdateEvent = this.onXRSessionUpdate.bind(this);
237
- WebXR.addEventListener(WebXREvent.XRUpdate, this.eventSub_WebXRUpdateEvent);
238
- this.eventSub_WebXREndEvent = this.onXRSessionEnded.bind(this);
239
- WebXR.addEventListener(WebXREvent.XRStopped, this.eventSub_WebXREndEvent);
240
-
241
- this.eventSub_ConnectionEvent = this.onConnected.bind(this);
242
- this.context.connection.beginListen(RoomEvents.JoinedRoom, this.eventSub_ConnectionEvent);
243
- this.context.connection.beginListen(WebXRSyncEvent.WebXR_UserJoined, _evt => {
244
- console.log("webxr user joined evt");
245
- });
246
- this.context.connection.beginListen(WebXRSyncEvent.WebXR_UserLeft, evt => {
247
- const hasId = evt.id !== null && evt.id !== undefined;
248
- if (!hasId) return;
249
- console.log("webxr user left evt");
250
- if (hasId) {
251
- const avatar = this.avatars[evt.id];
252
- avatar?.destroy();
253
- this.avatars[evt.id] = undefined;
254
- }
255
- });
256
- this.context.connection.beginListenBinary(VRUserStateBufferIdentifier, (state: VrUserStateBuffer) => {
257
- // console.log("BUFFER", state);
258
- const guid = state.guid();
259
- if (!guid) return;
260
- const time = state.time().toFloat64();
261
- const temp = this.tempState;
262
- temp.setFromBuffer(guid, state);
263
- // console.log(temp);
264
- const user = this.onTryGetAvatar(guid, time);
265
- user?.tryUpdate(temp, time);
266
- });
267
- this.context.connection.beginListen(WebXRSyncEvent.VRSessionUpdate, (state: VRUserState) => {
268
- const guid = state.guid;
269
- const time = state.time;
270
- const user = this.onTryGetAvatar(guid, time);
271
- user?.tryUpdate(state, time);
272
- });
273
- }
274
-
275
- private tempState: VRUserState = new VRUserState("");
276
-
277
- private onTryGetAvatar(guid: string, time: number) {
278
- if (guid === this.context.connection.connectionId) return null; // ignore self in case we receive that also!
279
- const timeDiff = new Date().getTime() - time;
280
- if (timeDiff > 5000) {
281
- if (debugLogs)
282
- console.log("old data", timeDiff, guid)
283
- return null;
284
- }
285
- let user = this.avatars[guid];
286
- if (user === undefined) {
287
- try {
288
- console.log("create new avatar");
289
- const newUser = new WebXRAvatar(this.context, guid, this.webXR!);
290
- user = newUser;
291
- this.avatars[guid] = newUser;
292
- } catch (err) {
293
- this.avatars[guid] = null;
294
- console.error(err);
295
- }
296
- }
297
- return user;
298
- }
299
-
300
- onDisable() {
301
- if (this.eventSub_ConnectionEvent)
302
- this.context.connection.stopListen(RoomEvents.JoinedRoom, this.eventSub_ConnectionEvent);
303
- WebXR.removeEventListener(WebXREvent.XRStarted, this.eventSub_WebXRStartEvent);
304
- WebXR.removeEventListener(WebXREvent.XRUpdate, this.eventSub_WebXRUpdateEvent);
305
- WebXR.removeEventListener(WebXREvent.XRStopped, this.eventSub_WebXREndEvent);
306
- }
307
-
308
- update(): void {
309
-
310
- const now = getTimeStampNow();
311
-
312
- if (this.debugAvatarUser) {
313
- this.debugAvatarUser.lastUpdate = now;
314
- }
315
-
316
- this.detectPotentiallyDisconnectedAvatarsAndRemove();
317
-
318
- for (const key in this.avatars) {
319
- const avatar = this.avatars[key];
320
- if (!avatar) continue;
321
- avatar.update();
322
- }
323
- }
324
-
325
-
326
- private _removeAvatarsList: string[] = [];
327
- private detectPotentiallyDisconnectedAvatarsAndRemove() {
328
- const utcnow = getTimeStampNow();
329
- for (const key in this.avatars) {
330
- const avatar = this.avatars[key];
331
- if (!avatar) {
332
- this._removeAvatarsList.push(key);
333
- continue;
334
- }
335
- if (utcnow - avatar.lastUpdate > 10_000) {
336
- console.log("avatar timed out (didnt receive any updates in a while) - destroying it now");
337
- avatar.destroy();
338
- this.avatars[key] = undefined;
339
- }
340
- }
341
- for (const rem of this._removeAvatarsList) {
342
- delete this.avatars[rem];
343
- }
344
- this._removeAvatarsList.length = 0;
345
- }
346
-
347
- private buildLocalAvatar() {
348
- if (this.localAvatar) return;
349
- const connectionId = this.context.connection?.connectionId ?? this.k_LocalAvatarNoNetworkingGuid;
350
- this.localAvatar = new WebXRAvatar(this.context, connectionId, this.webXR!);
351
- this.localAvatar.isLocalAvatar = true;
352
- this.localAvatar.setAvatarOverride(this.getAvatarId());
353
- this.avatars[this.localAvatar.guid] = this.localAvatar;
354
- }
355
-
356
-
357
- private eventSub_ConnectionEvent: Function | null = null;
358
- private eventSub_WebXRStartEvent: Function | null = null;
359
- private eventSub_WebXREndEvent: Function | null = null;
360
- private eventSub_WebXRUpdateEvent: Function | null = null;
361
- private avatars: { [key: string]: WebXRAvatar | undefined | null } = {}
362
- private localAvatar: WebXRAvatar | null = null;
363
- private k_LocalAvatarNoNetworkingGuid = "local";
364
-
365
- private onConnected() {
366
- // this event gets fired when we have joined a room and are ready to update
367
- if (debugLogs)
368
- console.log("Hey you are connected as " + this.context.connection.connectionId);
369
-
370
- if (this.localAvatar?.guid === this.k_LocalAvatarNoNetworkingGuid) {
371
- if (this.localAvatar) {
372
- this.localAvatar?.destroy();
373
- this.avatars[this.localAvatar.guid] = undefined;
374
- }
375
- this.localAvatar = null;
376
- this.xrState = null;
377
- this.ownership?.freeOwnership();
378
- this.ownership = null;
379
- }
380
- }
381
-
382
- private onXRSessionStart(_evt: { session: XRSession }) {
383
- console.log("XR session started");
384
- this.context.connection.send(WebXRSyncEvent.WebXR_UserJoined, { id: this.context.connection.connectionId, mode: XRMode.VR });
385
-
386
- if (this.localAvatar) {
387
- this.localAvatar?.destroy();
388
- this.avatars[this.localAvatar.guid] = undefined;
389
- this.localAvatar = null;
390
- }
391
- this.xrState = null;
392
- this.ownership?.freeOwnership();
393
- this.ownership = null;
394
-
395
- if (this.avatars) {
396
- for (const key in this.avatars) {
397
- this.avatars[key]?.updateFlags();
398
- }
399
- }
400
- }
401
-
402
- private onXRSessionEnded(_evt: { session: XRSession }) {
403
- console.log("XR session ended");
404
- this.context.connection.send(WebXRSyncEvent.WebXR_UserLeft, { id: this.context.connection.connectionId, mode: XRMode.VR });
405
- if(this.localAvatar){
406
- this.localAvatar?.destroy();
407
- this.avatars[this.localAvatar.guid] = undefined;
408
- this.localAvatar = null;
409
- }
410
- }
411
-
412
- private ownership: OwnershipModel | null = null;
413
- private xrState: VRUserState | null = null;
414
- private builder: Builder = new Builder(1024);
415
-
416
- private onXRSessionUpdate(evt: { rig: Group, frame: XRFrame, xr: WebXRManager, input: XRInputSource[] }) {
417
-
418
- this.xrState ??= new VRUserState(this.context.connection.connectionId ?? this.k_LocalAvatarNoNetworkingGuid);
419
- this.ownership ??= new OwnershipModel(this.context.connection, this.context.connection.connectionId ?? this.k_LocalAvatarNoNetworkingGuid);
420
- this.ownership.guid = this.context.connection.connectionId ?? this.k_LocalAvatarNoNetworkingGuid;
421
- this.buildLocalAvatar();
422
-
423
-
424
- const { frame, xr, rig } = evt;
425
- const pose = frame.getViewerPose(xr.getReferenceSpace()!);
426
- if (!pose) return; // e.g. if user is not wearing headset
427
- const transform: XRRigidTransform = pose?.transform;
428
- const pos = transform.position;
429
- const rot = transform.orientation;
430
- this.xrState.update(rig, pos, rot, this.webXR!, this.getAvatarId());
431
-
432
- if (this.localAvatar) {
433
- if (this.context.connection.connectionId) {
434
- this.localAvatar.guid = this.context.connection.connectionId;
435
- }
436
- this.localAvatar.tryUpdate(this.xrState, 0);
437
- }
438
-
439
- if (this.ownership && !this.ownership.hasOwnership && this.context.connection.isConnected) {
440
- if (this.context.time.frameCount % 120 === 0)
441
- this.ownership.requestOwnership();
442
- if (!this.ownership.hasOwnership) {
443
- // console.log("NO OWNERSHIP", this.ownership.guid);
444
- return;
445
- }
446
- }
447
-
448
- if (!this.context.connection.isConnected || !this.context.connection.connectionId) {
449
- return;
450
- }
451
-
452
- this.xrState.sendAsBuffer(this.builder, this.context.connection);
453
-
454
- // this.context.connection.send(WebXRSyncEvent.VRSessionUpdate, this.xrState);
455
-
456
- }
457
-
458
- private getAvatarId() {
459
- const urlAvatar = getParam("avatar") as string;
460
- const avatarId = urlAvatar ?? null;
461
- return avatarId;
462
- }
463
- }
src/engine-components/webxr/index.ts ADDED
@@ -0,0 +1,2 @@
1
+ export * from "./WebXR";
2
+ export * from "./WebXRPlaneTracking";
src/engine-components/webxr/WebARCameraBackground.ts ADDED
@@ -0,0 +1,215 @@
1
+ import { Behaviour } from "../Component";
2
+ import { serializable } from "../../engine/engine_serialization_decorator";
3
+ import { RGBAColor } from "../js-extensions/RGBAColor"
4
+ import { WebXR } from "./WebXR";
5
+ import {
6
+ Camera as ThreeCamera,
7
+ Scene,
8
+ Texture,
9
+ Mesh, MeshBasicMaterial,
10
+ UniformsUtils,
11
+ PlaneGeometry,
12
+ ShaderLib,
13
+ ShaderMaterial,
14
+ DoubleSide
15
+ } from "three";
16
+
17
+ export class WebARCameraBackground extends Behaviour {
18
+
19
+ awake(): void {
20
+ WebXR.OptionalFeatures_AR.push('camera-access');
21
+ }
22
+
23
+ @serializable()
24
+ public backgroundTint: RGBAColor = new RGBAColor(1,1,1,1);
25
+
26
+ public get background() {
27
+ return this.backgroundPlane;
28
+ }
29
+
30
+ private _preRender;
31
+
32
+ onEnable(): void {
33
+ this._preRender = this.preRender.bind(this);
34
+ this.context.pre_render_callbacks.push(this._preRender);
35
+
36
+ if (this.backgroundPlane) {
37
+ this.gameObject.add(this.backgroundPlane);
38
+ this.backgroundPlane.visible = false;
39
+ }
40
+ }
41
+
42
+ onDisable(): void {
43
+ this.context.pre_render_callbacks = this.context.pre_render_callbacks.filter(cb => cb !== this._preRender);
44
+
45
+ if (this.backgroundPlane)
46
+ this.gameObject.remove(this.backgroundPlane);
47
+ }
48
+
49
+ private backgroundPlane?: Mesh;
50
+ private threeTexture?: Texture;
51
+ private forceTextureInitialization = function() {
52
+ const material = new MeshBasicMaterial();
53
+ const geometry = new PlaneGeometry();
54
+ const scene = new Scene();
55
+ scene.add(new Mesh(geometry, material));
56
+ const camera = new ThreeCamera();
57
+
58
+ return function forceTextureInitialization(renderer, texture) {
59
+ material.map = texture;
60
+ renderer.render(scene, camera);
61
+ };
62
+ }();
63
+
64
+ // TODO should only attach on session start, and detach on session end
65
+ private preRender() {
66
+ if (!this || !this.gameObject) return;
67
+
68
+ const xr = this.context.renderer.xr;
69
+ const frame = xr.getFrame();
70
+
71
+ if (frame) {
72
+
73
+ // We're generating a new texture here, and force three to initialize it
74
+ // from https://stackoverflow.com/a/55084367 to inject a custom texture into three.js
75
+ if (!this.threeTexture && this.context.renderer) {
76
+ this.threeTexture = new Texture();
77
+ // this.threeTexture.encoding = LinearEncoding;
78
+ this.forceTextureInitialization(this.context.renderer, this.threeTexture);
79
+ }
80
+
81
+ // simple mesh and fullscreen shader to display the camera texture
82
+ // from three: WebGLBackground
83
+ if (this.backgroundPlane === undefined) {
84
+ this.backgroundPlane = makeFullscreenPlane(this.backgroundTint);
85
+ this.gameObject.add(this.backgroundPlane);
86
+ }
87
+
88
+ // WebXR Raw Camera Access -
89
+ // we composite the camera texture into the scene background by rendering it first.
90
+ this.updateFromFrame(frame);
91
+ }
92
+
93
+ /*
94
+ if (this.planeMesh) {
95
+ this.planeMesh.visible = frame != null;
96
+ }
97
+ */
98
+ }
99
+
100
+ onBeforeRender(frame: XRFrame | null) {
101
+ this.updateFromFrame(frame);
102
+ }
103
+
104
+ updateFromFrame(frame: XRFrame | null) {
105
+ if (!frame) return;
106
+
107
+ // https://chromium.googlesource.com/chromium/src/+/7c5ac3c0f95b97cf12be95a5c1c0a8ff163246d8/third_party/webxr_test_pages/webxr-samples/proposals/camera-access-barebones.html
108
+ const pose = frame.getViewerPose(this.context.renderer.xr.getReferenceSpace()!);
109
+ if (pose) {
110
+ for( const view of pose.views) {
111
+ // @ts-ignore
112
+ if ('camera' in view && view.camera) {
113
+ const xrManager = this.context.renderer.xr;
114
+ let binding = xrManager.getBinding();
115
+ // not sure how / why this can be null, but we can recreate it here
116
+ if (!binding) binding = new XRWebGLBinding( frame.session, this.context.renderer.getContext() );
117
+
118
+ if (binding) {
119
+ let glImage: WebGLTexture | null = null;
120
+ if ('getCameraImage' in binding) {
121
+ // @ts-ignore
122
+ glImage = binding.getCameraImage(view.camera);
123
+
124
+ // discussion on exactly this:
125
+ // https://discourse.threejs.org/t/using-a-webgltexture-as-texture-for-three-js/46245/8
126
+ // HACK from https://stackoverflow.com/a/55084367 to inject a custom texture into three.js
127
+ const texProps = this.context.renderer.properties.get(this.threeTexture);
128
+ texProps.__webglTexture = glImage;
129
+
130
+ if (this.backgroundPlane) {
131
+ //@ts-ignore
132
+ this.backgroundPlane.setTexture(this.threeTexture);
133
+ this.backgroundPlane.visible = true;
134
+ }
135
+ }
136
+ }
137
+ else {
138
+ // console.error(view.camera, xrManager)
139
+ }
140
+ }
141
+ else {
142
+ // console.error("NO CAMERA IN VIEW")
143
+ }
144
+ }
145
+ }
146
+ else {
147
+ // console.error(this.context.renderer.xr.getReferenceSpace(), frame);
148
+ }
149
+ }
150
+ }
151
+ // TODO tint could be an uniform
152
+ let backgroundFragment: string = /* glsl */`
153
+ uniform sampler2D t2D;
154
+
155
+ varying vec2 vUv;
156
+
157
+ void main() {
158
+
159
+ vec4 texColor = texture2D( t2D, vUv );
160
+ texColor.w = 1.0;
161
+
162
+ // inline sRGB decode
163
+ texColor = vec4( mix( pow( texColor.rgb * 0.9478672986 + vec3( 0.0521327014 ), vec3( 2.4 ) ), texColor.rgb * 0.0773993808, vec3( lessThanEqual( texColor.rgb, vec3( 0.04045 ) ) ) ), texColor.w );
164
+
165
+ gl_FragColor = texColor * <backgroundTint>;
166
+
167
+ #include <tonemapping_fragment>
168
+ #include <encodings_fragment>
169
+
170
+ }
171
+ `;
172
+
173
+ // not sure where we want to move this and in which form is best (extends Object3D?)
174
+ export function makeFullscreenPlane(tint: RGBAColor ) {
175
+ const replacementTint = "vec4(" + tint.r.toFixed(3) + "," + tint.g.toFixed(3) + "," + tint.b.toFixed(3) + "," + tint.a.toFixed(3) + ")";
176
+ console.log(replacementTint);
177
+ const planeMesh = new Mesh(
178
+ new PlaneGeometry(2, 2),
179
+ // @ts-ignore
180
+ new ShaderMaterial({
181
+ name: 'BackgroundMaterial',
182
+ uniforms: UniformsUtils.clone( ShaderLib.background.uniforms ),
183
+ vertexShader: ShaderLib.background.vertexShader,
184
+ fragmentShader: backgroundFragment.replace("<backgroundTint>", replacementTint), // ShaderLib.background.fragmentShader,
185
+ side: DoubleSide,
186
+ depthTest: false,
187
+ depthWrite: false,
188
+ fog: false
189
+ })
190
+ );
191
+
192
+ planeMesh.geometry.deleteAttribute( 'normal' );
193
+
194
+ // add "map" material property so the renderer can evaluate it like for built-in materials
195
+ Object.defineProperty( planeMesh.material, 'map', {
196
+ get: function () {
197
+ return this.threeTexture;
198
+ }
199
+ } );
200
+
201
+ // Option 1: add the planeMesh to our scene for rendering.
202
+ // This is useful for applying custom shader effects on the background (instead of using the system composite)
203
+ planeMesh.renderOrder = -10000; // render first
204
+ planeMesh.layers.disableAll();
205
+ planeMesh.layers.enable(2); // ignore raycasts
206
+ planeMesh.frustumCulled = false;
207
+
208
+ // should be a class, for now lets just define a method for the weird way the texture needs to be set
209
+ // @ts-ignore
210
+ planeMesh.setTexture = function(texture) {
211
+ planeMesh.material.uniforms.t2D.value = texture;
212
+ }
213
+
214
+ return planeMesh;
215
+ }
src/engine-components/webxr/WebARSessionRoot.ts ADDED
@@ -0,0 +1,178 @@
1
+ import { Behaviour, GameObject } from "../Component";
2
+ import { Matrix4, Object3D } from "three";
3
+ import { WebAR, WebXR } from "./WebXR";
4
+ import { InstancingUtil } from "../../engine/engine_instancing";
5
+ import { serializable } from "../../engine/engine_serialization_decorator";
6
+
7
+ // https://github.com/takahirox/takahirox.github.io/blob/master/js.mmdeditor/examples/js/controls/DeviceOrientationControls.js
8
+
9
+ export class WebARSessionRoot extends Behaviour {
10
+
11
+ webAR: WebAR | null = null;
12
+
13
+ get rig(): Object3D | undefined {
14
+ return this.webAR?.webxr.Rig;
15
+ }
16
+
17
+ @serializable()
18
+ invertForward: boolean = false;
19
+
20
+ @serializable()
21
+ get arScale(): number {
22
+ return this._arScale;
23
+ }
24
+ set arScale(val: number) {
25
+ if (val === this._arScale) return;
26
+ this._arScale = val;
27
+ this.setScale(val);
28
+ }
29
+
30
+ private readonly _initalMatrix = new Matrix4();
31
+ private readonly _selectStartFn = this.onSelectStart.bind(this);
32
+ private readonly _selectEndFn = this.onSelectEnd.bind(this);
33
+
34
+ start() {
35
+ const xr = GameObject.findObjectOfType(WebXR);
36
+ if (xr) {
37
+ xr.Rig.updateMatrix();
38
+ this._initalMatrix.copy(xr.Rig.matrix);
39
+ }
40
+ }
41
+
42
+ private _arScale: number = 5;
43
+ private _rig: Object3D | null = null;
44
+ private _startPose: Matrix4 | null = null;
45
+ private _placementPose: Matrix4 | null = null;
46
+ private _isTouching: boolean = false;
47
+ private _rigStartPose: Matrix4 | undefined | null = null;
48
+ private _gotFirstHitTestResult: boolean = false;
49
+
50
+ onBegin(session: XRSession) {
51
+ this._placementPose = null;
52
+ this.gameObject.visible = false;
53
+ this.gameObject.matrixAutoUpdate = false;
54
+ this._startPose = this.gameObject.matrix.clone();
55
+ this._rigStartPose = this.rig?.matrix.clone();
56
+ this._gotFirstHitTestResult = false;
57
+ session.addEventListener('selectstart', this._selectStartFn);
58
+ session.addEventListener('selectend', this._selectEndFn);
59
+ // setTimeout(() => this.gameObject.visible = false, 1000); // TODO test on phone AR and Hololens if this was still needed
60
+
61
+ // console.log(this.rig?.position, this.rig?.quaternion, this.rig?.scale);
62
+ this.gameObject.visible = false;
63
+
64
+ if (this.rig) {
65
+ // reset rig to initial pose, this is helping the mix of immersive AR and immersive VR that we now have on quest
66
+ // where the rig can be moved and scaled by the user in VR mode and we use the rig position when entering
67
+ // immersive Ar right now to place the user/offset the session
68
+ this.rig.matrixAutoUpdate = true;
69
+ this._initalMatrix.decompose(this.rig.position, this.rig.quaternion, this.rig.scale);
70
+ }
71
+
72
+ // TODO this is duplicate to WebXR events AND engine events, would be better in one place
73
+ this.dispatchEvent(new CustomEvent('onBeginSession'));
74
+ }
75
+
76
+ onUpdate(rig: Object3D | null, _session: XRSession, pose: XRPose | null | undefined): boolean {
77
+
78
+ if (pose && !this._placementPose) {
79
+
80
+ if (!this._gotFirstHitTestResult) {
81
+ this._gotFirstHitTestResult = true;
82
+ this.dispatchEvent(new CustomEvent('canPlaceSession', { detail: { pose: pose } }));
83
+ }
84
+
85
+ if (this._isTouching) {
86
+ // callbacks
87
+ const poseMatrix = new Matrix4().fromArray(pose.transform.matrix).invert();
88
+ this.dispatchEvent(new CustomEvent('placedSession', { detail: { pose, poseMatrix } }));
89
+
90
+ if (this.webAR) this.webAR.setReticleActive(false);
91
+ this.placeAt(rig, poseMatrix);
92
+ return true;
93
+ }
94
+ }
95
+ return false;
96
+
97
+ // if (this._placementPose) {
98
+ // this.gameObject.matrixAutoUpdate = false;
99
+ // const matrix = pose?.transform.matrix;
100
+ // if (matrix) {
101
+ // this.gameObject.matrix.fromArray(matrix);
102
+ // }
103
+ // this.gameObject.visible = true;
104
+ // }
105
+ }
106
+
107
+ placeAt(rig: Object3D | null, mat: Matrix4) {
108
+ if (!this._placementPose) this._placementPose = new Matrix4();
109
+ this._placementPose.copy(mat);
110
+ // apply session root offset
111
+ const invertedSessionRoot = this.gameObject.matrixWorld.clone().invert();
112
+ this._placementPose.premultiply(invertedSessionRoot);
113
+ if (rig) {
114
+
115
+ if (this.invertForward) {
116
+ const rot = new Matrix4().makeRotationY(Math.PI);
117
+ this._placementPose.premultiply(rot);
118
+ }
119
+ this._rig = rig;
120
+
121
+ this.setScale(this.arScale);
122
+ }
123
+ else this._rig = null;
124
+ // this.gameObject.matrix.copy(this._placementPose);
125
+ // if (rig) {
126
+ // this.gameObject.matrix.premultiply(rig.matrixWorld)
127
+ // }
128
+ this.gameObject.visible = true;
129
+ }
130
+
131
+ onEnd(rig: Object3D | null, _session: XRSession) {
132
+ this._placementPose = null;
133
+ this.gameObject.visible = false;
134
+ this.gameObject.matrixAutoUpdate = false;
135
+ if (this._startPose) {
136
+ this.gameObject.matrix.copy(this._startPose);
137
+ }
138
+ if (rig) {
139
+ rig.matrixAutoUpdate = true;
140
+ if (this._rigStartPose) {
141
+ this._rigStartPose.decompose(rig.position, rig.quaternion, rig.scale);
142
+ // console.log(rig.position, rig.quaternion, rig.scale);
143
+ }
144
+ }
145
+ InstancingUtil.markDirty(this.gameObject, true);
146
+ // HACK to fix physics being not in correct place after exiting AR
147
+ setTimeout(() => {
148
+ this.gameObject.matrixAutoUpdate = true;
149
+ this.gameObject.visible = true;
150
+ }, 100);
151
+ }
152
+
153
+
154
+ private onSelectStart() {
155
+ this._isTouching = true;
156
+ }
157
+
158
+ private onSelectEnd() {
159
+ this._isTouching = false;
160
+ }
161
+
162
+ private setScale(scale) {
163
+ const rig = this._rig;
164
+ if (!rig || !this._placementPose) {
165
+ return;
166
+ }
167
+ // Capture the rig position before the first time we move it during a session
168
+ if (!this._rigStartPose) {
169
+ this._rigStartPose = rig.matrix.clone();
170
+ }
171
+ // we apply the transform to the rig because we want to move the user's position for easy networking
172
+ rig.matrixAutoUpdate = false;
173
+ rig.matrix.multiplyMatrices(new Matrix4().makeScale(scale, scale, scale), this._placementPose);
174
+ rig.matrix.decompose(rig.position, rig.quaternion, rig.scale);
175
+ rig.updateMatrixWorld();
176
+ console.log("Place", rig.position);
177
+ }
178
+ }
src/engine-components/webxr/WebXR.ts ADDED
@@ -0,0 +1,712 @@
1
+ import { ArrayCamera, Color, Euler, EventDispatcher, Group, Matrix4, Mesh, MeshBasicMaterial, Object3D, Quaternion, RingGeometry, Texture, Vector3 } from 'three';
2
+ import { ARButton } from '../../include/three/ARButton.js';
3
+ import { VRButton } from '../../include/three/VRButton.js';
4
+
5
+ import { AssetReference } from "../../engine/engine_addressables";
6
+ import { serializable } from "../../engine/engine_serialization_decorator";
7
+ import { XRSessionMode } from "../../engine/engine_setup";
8
+ import { getWorldPosition, getWorldQuaternion, setWorldPosition, setWorldQuaternion } from "../../engine/engine_three_utils";
9
+ import { INeedleEngineComponent } from "../../engine/engine_types";
10
+ import { getParam, isMozillaXR, setOrAddParamsToUrl } from "../../engine/engine_utils";
11
+
12
+ import { Behaviour, GameObject } from "../Component";
13
+ import { noVoip } from "../Voip";
14
+ import { WebARSessionRoot } from "./WebARSessionRoot";
15
+ import { ControllerType, WebXRController } from "./WebXRController";
16
+ import { XRRig } from "./WebXRRig";
17
+ import { WebXRSync } from "./WebXRSync";
18
+ import { XRFlag, XRState, XRStateFlag } from "../XRFlag";
19
+ import { showBalloonWarning } from '../../engine/debug';
20
+
21
+
22
+ export async function detectARSupport() {
23
+ if(isMozillaXR()) return true;
24
+ if ("xr" in navigator) {
25
+ //@ts-ignore
26
+ return (await navigator["xr"].isSessionSupported('immersive-ar')) === true;
27
+ }
28
+ return false;
29
+ }
30
+ export async function detectVRSupport() {
31
+ if ("xr" in navigator) {
32
+ //@ts-ignore
33
+ return (await navigator["xr"].isSessionSupported('immersive-vr')) === true;
34
+ }
35
+ return false;
36
+ }
37
+
38
+ let arSupported = false;
39
+ let vrSupported = false;
40
+ detectARSupport().then(res => arSupported = res);
41
+ detectVRSupport().then(res => vrSupported = res);
42
+
43
+ // import TeleportVR from "teleportvr";
44
+
45
+ export enum WebXREvent {
46
+ XRStarted = "xrStarted",
47
+ XRStopped = "xrStopped",
48
+ XRUpdate = "xrUpdate",
49
+ RequestVRSession = "requestVRSession",
50
+ ModifyAROptions = "modify-ar-options",
51
+ }
52
+
53
+ export declare type CreateButtonOptions = {
54
+ registerClick: boolean
55
+ };
56
+
57
+ export class WebXR extends Behaviour {
58
+
59
+ @serializable()
60
+ enableVR = true;
61
+ @serializable()
62
+ enableAR = true;
63
+
64
+ @serializable(AssetReference)
65
+ defaultAvatar?: AssetReference;
66
+ @serializable()
67
+ handModelPath: string = "";
68
+
69
+ @serializable()
70
+ createVRButton: boolean = true;
71
+ @serializable()
72
+ createARButton: boolean = true;
73
+
74
+ private static _isInXr: boolean = false;
75
+ private static events: EventDispatcher = new EventDispatcher();
76
+
77
+ public static get IsInWebXR(): boolean { return this._isInXr; }
78
+ public static get XRSupported(): boolean { return 'xr' in navigator && (arSupported || vrSupported); }
79
+ public static get IsARSupported(): boolean { return arSupported; }
80
+ public static get IsVRSupported(): boolean { return vrSupported; }
81
+
82
+ private static _optionalFeatures_VR: string[] = ['local-floor', 'bounded-floor', 'hand-tracking', 'high-fixed-foveation-level', 'layers'];
83
+ private static _optionalFeatures_AR: string[] = ['anchors', 'local-floor', 'hand-tracking', 'layers'];
84
+ public static get OptionalFeatures_VR(): string[] { return this._optionalFeatures_VR; }
85
+ public static get OptionalFeatures_AR(): string[] { return this._optionalFeatures_AR; }
86
+
87
+ public static addEventListener(type: string, listener: any): any {
88
+ this.events.addEventListener(type, listener);
89
+ return listener;
90
+ }
91
+ public static removeEventListener(type: string, listener: any): any {
92
+ this.events.removeEventListener(type, listener);
93
+ return listener;
94
+ }
95
+ private static dispatchEvent(type: string, event: any): void {
96
+ this.events.dispatchEvent({ type, detail: event });
97
+ }
98
+
99
+ public static createVRButton(webXR: WebXR, opts?: CreateButtonOptions): HTMLButtonElement | HTMLAnchorElement {
100
+ if (!WebXR.XRSupported) {
101
+ console.warn("WebXR is not supported on this device");
102
+ }
103
+ else
104
+ webXR.__internalAwake();
105
+ const options = { optionalFeatures: WebXR.OptionalFeatures_VR };
106
+ const vrButton = VRButton.createButton(webXR.context.renderer, options);
107
+ vrButton.classList.add('webxr-ar-button');
108
+ vrButton.classList.add('webxr-button');
109
+ this.resetButtonStyles(vrButton);
110
+ // if (this.enableAR) vrButton.style.marginLeft = "60px";
111
+ if (opts?.registerClick ?? true)
112
+ vrButton.addEventListener('click', webXR.onClickedVRButton.bind(webXR));
113
+ return vrButton;
114
+ }
115
+
116
+ public static createARButton(webXR: WebXR, opts?: CreateButtonOptions): HTMLButtonElement | HTMLAnchorElement {
117
+ webXR.__internalAwake();
118
+ const domOverlayRoot = webXR.webAR?.getAROverlayContainer();
119
+ const options: any = { optionalFeatures: [...this.OptionalFeatures_AR] };
120
+ if (domOverlayRoot) {
121
+ options.domOverlay = { root: domOverlayRoot };
122
+ options.optionalFeatures.push('dom-overlay')
123
+ options.optionalFeatures.push('hit-test');
124
+ }
125
+ else {
126
+ console.warn("No dom overlay root found, HTML overlays on top of screen-based AR will not work.");
127
+ }
128
+
129
+ const arButton = ARButton.createButton(webXR.context.renderer, options, this.onModifyAROptions.bind(this));
130
+ arButton.classList.add('webxr-ar-button');
131
+ arButton.classList.add('webxr-button');
132
+ WebXR.resetButtonStyles(arButton);
133
+ if (opts?.registerClick ?? true)
134
+ arButton.addEventListener('click', webXR.onClickedARButton.bind(webXR));
135
+ return arButton;
136
+ }
137
+
138
+ private static onModifyAROptions(options){
139
+ WebXR.dispatchEvent(WebXREvent.ModifyAROptions, options);
140
+ }
141
+
142
+ public static resetButtonStyles(button) {
143
+ if (!button) return;
144
+ button.style.position = "";
145
+ button.style.bottom = "";
146
+ button.style.left = "";
147
+ }
148
+
149
+ public endSession() {
150
+ const session = this.context.renderer.xr.getSession();
151
+ if (session) session.end();
152
+ }
153
+
154
+ public get Rig(): Object3D {
155
+ if (!this.rig) this.ensureRig();
156
+ return this.rig;
157
+ }
158
+
159
+
160
+ private controllers: WebXRController[] = [];
161
+ public get Controllers(): WebXRController[] {
162
+ return this.controllers;
163
+ }
164
+
165
+ public get LeftController(): WebXRController | null {
166
+ if (this.controllers.length > 0 && this.controllers[0].input?.handedness === "left") return this.controllers[0];
167
+ if (this.controllers.length > 1 && this.controllers[1].input?.handedness === "left") return this.controllers[1];
168
+ return null;
169
+ }
170
+
171
+ public get RightController(): WebXRController | null {
172
+ if (this.controllers.length > 0 && this.controllers[0].input?.handedness === "right") return this.controllers[0];
173
+ if (this.controllers.length > 1 && this.controllers[1].input?.handedness === "right") return this.controllers[1];
174
+ return null;
175
+ }
176
+
177
+ public get ARButton(): HTMLButtonElement | undefined {
178
+ return this._arButton;
179
+ }
180
+
181
+ public get VRButton(): HTMLButtonElement | undefined {
182
+ return this._vrButton;
183
+ }
184
+
185
+ public get IsInVR() { return this._isInVR; }
186
+ public get IsInAR() { return this._isInAR; }
187
+
188
+ private rig!: Object3D;
189
+ private isInit: boolean = false;
190
+
191
+ private _requestedAR: boolean = false;
192
+ private _requestedVR: boolean = false;
193
+ private _isInAR: boolean = false;
194
+ private _isInVR: boolean = false;
195
+
196
+ private _arButton?: HTMLButtonElement;
197
+ private _vrButton?: HTMLButtonElement;
198
+
199
+ private webAR: WebAR | null = null;
200
+
201
+ awake(): void {
202
+ // as the webxr component is most of the times currently loaded as part of the scene
203
+ // and not part of the glTF directly and thus does not go through the whole serialization process currently
204
+ // we need to to manuall make sure it is of the correct type here
205
+ if (this.defaultAvatar) {
206
+ if (typeof (this.defaultAvatar) === "string") {
207
+ this.defaultAvatar = AssetReference.getOrCreate(this.sourceId ?? "/", this.defaultAvatar, this.context);
208
+ }
209
+ }
210
+ if (!GameObject.findObjectOfType(WebXRSync, this.context)) {
211
+ const sync = GameObject.addNewComponent(this.gameObject, WebXRSync, false) as WebXRSync;
212
+ sync.webXR = this;
213
+ }
214
+ this.webAR = new WebAR(this);
215
+ }
216
+
217
+ start() {
218
+ if (location.protocol == 'http:' && location.host.indexOf('localhost') < 0) {
219
+ showBalloonWarning("WebXR only works on https");
220
+ console.warn("WebXR only works on https. https://engine.needle.tools/docs/xr.html");
221
+ }
222
+ }
223
+
224
+ onEnable() {
225
+ if (this.isInit) return;
226
+ if (!this.enableAR && !this.enableVR) return;
227
+ this.isInit = true;
228
+
229
+ this.context.renderer.xr.enabled = true;
230
+
231
+ // general WebXR support?
232
+ const browserSupportsXR = WebXR.XRSupported;
233
+
234
+
235
+ // TODO: move the whole buttons positioning out of here and make it configureable from css
236
+ // better set proper classes so user code can react to it instead
237
+ // of this hardcoded stuff
238
+ let arButton, vrButton;
239
+ const buttonsContainer = document.createElement('div');
240
+ buttonsContainer.classList.add("webxr-buttons");
241
+ this.context.domElement.append(buttonsContainer);
242
+
243
+ // AR support
244
+ // if (this.enableAR && this.createARButton && arSupported)
245
+ {
246
+ arButton = WebXR.createARButton(this);
247
+ this._arButton = arButton;
248
+ buttonsContainer.appendChild(arButton);
249
+ }
250
+
251
+ // VR support
252
+ if (this.createVRButton && this.enableVR && vrSupported) {
253
+ vrButton = WebXR.createVRButton(this);
254
+ this._vrButton = vrButton;
255
+ buttonsContainer.appendChild(vrButton);
256
+ }
257
+
258
+ setTimeout(() => {
259
+ WebXR.resetButtonStyles(vrButton);
260
+ WebXR.resetButtonStyles(arButton);
261
+ }, 1000);
262
+ }
263
+
264
+ private _transformOrientation: Quaternion = new Quaternion();
265
+ public get TransformOrientation(): Quaternion { return this._transformOrientation; }
266
+
267
+ private _currentHeadPose: XRViewerPose | null = null;
268
+ public get HeadPose(): XRViewerPose | null { return this._currentHeadPose; }
269
+
270
+ onBeforeRender(frame) {
271
+ if (!frame) return;
272
+ // TODO: figure out why screen is black if we enable the code written here
273
+ // const referenceSpace = renderer.xr.getReferenceSpace();
274
+ const session = this.context.renderer.xr.getSession();
275
+
276
+
277
+ if (session) {
278
+ const pose = frame.getViewerPose(this.context.renderer.xr.getReferenceSpace());
279
+ if(!pose) return;
280
+ this._currentHeadPose = pose;
281
+ const transform: XRRigidTransform = pose?.transform;
282
+ if (transform) {
283
+ this._transformOrientation.set(transform.orientation.x, transform.orientation.y, transform.orientation.z, transform.orientation.w);
284
+ }
285
+
286
+ if (WebXR._isInXr === false && session) {
287
+ this.onEnterXR(session, frame);
288
+ }
289
+
290
+ for (const ctrl of this.controllers) {
291
+ ctrl.onUpdate(session);
292
+ }
293
+
294
+ if (this._isInAR) {
295
+ this.webAR?.onUpdate(session, frame);
296
+ }
297
+ }
298
+
299
+ WebXR.events.dispatchEvent({ type: WebXREvent.XRUpdate, frame: frame, xr: this.context.renderer.xr, rig: this.rig });
300
+ }
301
+
302
+ private onClickedARButton() {
303
+ if (!this._isInAR) {
304
+ this._requestedAR = true;
305
+ this._requestedVR = false;
306
+
307
+ // if we do this on enter xr the state has already been changed in AR mode
308
+ // so we need to to this before session has started
309
+ this.captureStateBeforeXR();
310
+ }
311
+ }
312
+
313
+ private onClickedVRButton() {
314
+ if (!this._isInVR) {
315
+
316
+ // happens e.g. when headset is off and xr session never actually started
317
+ if (this._requestedVR) {
318
+ this.onExitXR(null);
319
+ return;
320
+ }
321
+
322
+ this._requestedAR = false;
323
+ this._requestedVR = true;
324
+ this.captureStateBeforeXR();
325
+
326
+ // build controllers before session begins - this seems to fix issue with controller models not appearing/not getting connection event
327
+ this.ensureRig();
328
+ for (let i = 0; i < 2; i++) {
329
+ WebXRController.Create(this, i, this.gameObject as GameObject, ControllerType.PhysicalDevice);
330
+ }
331
+
332
+ WebXR.events.dispatchEvent({ type: WebXREvent.RequestVRSession });
333
+ }
334
+ }
335
+
336
+ private captureStateBeforeXR() {
337
+ if (this.context.mainCamera) {
338
+ this._originalCameraPosition.copy(getWorldPosition(this.context.mainCamera));
339
+ this._originalCameraRotation.copy(getWorldQuaternion(this.context.mainCamera));
340
+ this._originalCameraParent = this.context.mainCamera.parent;
341
+ }
342
+ if(this.Rig){
343
+ this._originalXRRigParent = this.Rig.parent;
344
+ this._originalXRRigPosition.copy(this.Rig.position);
345
+ this._originalXRRigRotation.copy(this.Rig.quaternion);
346
+ }
347
+ }
348
+
349
+ private ensureRig() {
350
+ if (!this.rig) {
351
+ // currently just used for pose
352
+ const xrRig = GameObject.findObjectOfType(XRRig, this.context);
353
+ if (xrRig) {
354
+ // make it match unity forward
355
+ this.rig = xrRig.gameObject;
356
+ // this.rig.rotateY(Math.PI);
357
+ // this.rig.position.copy(existing.worldPosition);
358
+ // this.rig.quaternion.premultiply(existing.worldQuaternion);
359
+ }
360
+ else {
361
+ this.rig = new Group();
362
+ this.rig.rotateY(Math.PI);
363
+ this.rig.name = "XRRig";
364
+ this.context.scene.add(this.rig);
365
+ }
366
+ }
367
+ }
368
+
369
+
370
+ private _originalCameraParent: Object3D | null = null;
371
+ private _originalCameraPosition: Vector3 = new Vector3();
372
+ private _originalCameraRotation: Quaternion = new Quaternion();
373
+
374
+ private _originalXRRigParent: Object3D | null = null;
375
+ private _originalXRRigPosition: Vector3 = new Vector3();
376
+ private _originalXRRigRotation: Quaternion = new Quaternion();
377
+
378
+ private onEnterXR(session: XRSession, frame: XRFrame) {
379
+ console.log("[XR] session begin", session);
380
+ WebXR._isInXr = true;
381
+
382
+ this.ensureRig();
383
+
384
+ const space = this.context.renderer.xr.getReferenceSpace();
385
+ if (space && this.rig) {
386
+ const pose = frame.getViewerPose(space);
387
+ const rot = pose?.transform.orientation;
388
+ if (rot) {
389
+ const quat = new Quaternion(rot.x, rot.y, rot.z, rot.w);
390
+ const eu = new Euler().setFromQuaternion(quat);
391
+ this.rig.rotateY(eu.y);
392
+ // this.rig.quaternion.multiply(quat);
393
+ }
394
+ }
395
+
396
+ // when we set unity layers objects will only be rendered on one eye
397
+ // we set layers to sync raycasting and have a similar behaviour to unity
398
+ const xr = this.context.renderer.xr;
399
+ if (this.context.mainCamera) {
400
+ //@ts-ignore
401
+ const cam = xr.getCamera(this.context.mainCamera) as ArrayCamera;
402
+ const cull = this.context.mainCameraComponent?.cullingMask;
403
+ if(cull !== undefined){
404
+ for (const c of cam.cameras) {
405
+ c.layers.mask = cull;
406
+ }
407
+ cam.layers.mask = cull;
408
+ }
409
+ else {
410
+ for (const c of cam.cameras) {
411
+ c.layers.enableAll();
412
+ }
413
+ cam.layers.enableAll();
414
+ }
415
+ this.rig.add(this.context.mainCamera);
416
+ if (this._requestedAR) {
417
+ this.context.scene.add(this.rig);
418
+ }
419
+ }
420
+
421
+ const flag = this._requestedAR ? XRStateFlag.AR : XRStateFlag.VR;
422
+
423
+ XRState.Global.Set(flag);
424
+
425
+ switch (flag) {
426
+ case XRStateFlag.AR:
427
+ this.context.xrSessionMode = XRSessionMode.ImmersiveAR;
428
+ this._isInAR = true;
429
+ this.webAR?.onBegin(session);
430
+ break;
431
+ case XRStateFlag.VR:
432
+ this.context.xrSessionMode = XRSessionMode.ImmersiveVR;
433
+ this._isInVR = true;
434
+ this.onEnterVR(session);
435
+ break;
436
+ }
437
+
438
+ session.addEventListener('end', () => {
439
+ console.log("[XR] session end");
440
+ WebXR._isInXr = false;
441
+ this.onExitXR(session);
442
+ });
443
+
444
+ this.onEnterXR_HandleMirrorWindow(session);
445
+
446
+ WebXR.events.dispatchEvent({ type: WebXREvent.XRStarted, session: session });
447
+ }
448
+
449
+ private onExitXR(session: XRSession | null) {
450
+
451
+ const wasInAR = this._isInAR;
452
+
453
+ if (this._isInAR && session) {
454
+ this.webAR?.onEnd(session);
455
+ }
456
+
457
+ this._isInAR = false;
458
+ this._isInVR = false;
459
+ this._requestedAR = false;
460
+ this._requestedVR = false;
461
+ this.context.xrSessionMode = undefined;
462
+
463
+ if (this.xrMirrorWindow) {
464
+ this.xrMirrorWindow.close();
465
+ this.xrMirrorWindow = null;
466
+ }
467
+
468
+ this.destroyControllers();
469
+
470
+ if (this.context.mainCamera) {
471
+ this._originalCameraParent?.add(this.context.mainCamera);
472
+ setWorldPosition(this.context.mainCamera, this._originalCameraPosition);
473
+ setWorldQuaternion(this.context.mainCamera, this._originalCameraRotation);
474
+ this.context.mainCamera.scale.set(1, 1, 1);
475
+ }
476
+
477
+ if(wasInAR){
478
+ this._originalXRRigParent?.add(this.rig);
479
+ this.rig.position.copy(this._originalXRRigPosition);
480
+ this.rig.quaternion.copy(this._originalXRRigRotation);
481
+ }
482
+
483
+ XRState.Global.Set(XRStateFlag.Browser | XRStateFlag.ThirdPerson);
484
+ WebXR.events.dispatchEvent({ type: WebXREvent.XRStopped, session: session });
485
+ }
486
+
487
+ private onEnterVR(_session: XRSession) {
488
+ }
489
+
490
+ private destroyControllers() {
491
+ for (let i = this.controllers.length - 1; i >= 0; i -= 1) {
492
+ this.controllers[i]?.destroy();
493
+ }
494
+ this.controllers.length = 0;
495
+ }
496
+
497
+ private xrMirrorWindow: Window | null = null;
498
+
499
+ private onEnterXR_HandleMirrorWindow(session: XRSession) {
500
+ if (!getParam("mirror")) return;
501
+ setTimeout(() => {
502
+ if (!WebXR.IsInWebXR) return;
503
+ const url = new URL(window.location.href);
504
+ setOrAddParamsToUrl(url.searchParams, noVoip, 1);
505
+ setOrAddParamsToUrl(url.searchParams, "isMirror", 1);
506
+ const str = url.toString();
507
+ this.xrMirrorWindow = window.open(str, "webxr sync", "popup=yes");
508
+ if (this.xrMirrorWindow) {
509
+ this.xrMirrorWindow.onload = () => {
510
+ if (this.xrMirrorWindow)
511
+ this.xrMirrorWindow.onbeforeunload = () => {
512
+ if (WebXR.IsInWebXR)
513
+ session.end();
514
+ };
515
+ }
516
+ }
517
+ }, 1000);
518
+ }
519
+ }
520
+
521
+
522
+ // not sure if this should be a behaviour.
523
+ // for now we dont really need it to go through the usual update loop
524
+ export class WebAR {
525
+
526
+ get webxr(): WebXR { return this._webxr; }
527
+
528
+ private _webxr: WebXR;
529
+
530
+ private reticle: Object3D | null = null;
531
+ private reticleParent: Object3D | null = null;
532
+ private hitTestSource: XRHitTestSource | null = null;
533
+ private reticleActive: boolean = true;
534
+
535
+ // scene.background before entering AR
536
+ private previousBackground: Color | null | Texture = null;
537
+ private previousEnvironment: Texture | null = null;
538
+
539
+ private sessionRoot: WebARSessionRoot | null = null;
540
+ private _previousParent: Object3D | null = null;
541
+ // we need this in case the session root is on the same object as the webxr component
542
+ // so if we disable the session root we attach the webxr component to this temporary object
543
+ // to still receive updates
544
+ private static tempWebXRObject: Object3D;
545
+
546
+ private get context() { return this.webxr.context; }
547
+
548
+ constructor(webxr: WebXR) {
549
+ this._webxr = webxr;
550
+ }
551
+
552
+ private arDomOverlay: HTMLElement | null = null;
553
+ private arOverlayElement: INeedleEngineComponent | HTMLElement | null = null;
554
+ private noHitTestAvailable: boolean = false;
555
+ private didPlaceARSessionRoot: boolean = false;
556
+
557
+ getAROverlayContainer(): HTMLElement | null {
558
+ this.arDomOverlay = this.webxr.context.domElement as HTMLElement;
559
+ // for react cases we dont have an Engine Element
560
+ const element: any = this.arDomOverlay;
561
+ if (element.getAROverlayContainer)
562
+ this.arOverlayElement = element.getAROverlayContainer();
563
+ else this.arOverlayElement = this.arDomOverlay;
564
+ return this.arOverlayElement;
565
+ }
566
+
567
+ setReticleActive(active: boolean) {
568
+ this.reticleActive = active;
569
+ }
570
+
571
+ async onBegin(session: XRSession) {
572
+ const context = this.webxr.context;
573
+ this.reticleActive = true;
574
+ this.didPlaceARSessionRoot = false;
575
+ this.getAROverlayContainer();
576
+
577
+ const deviceType = navigator.userAgent?.includes("OculusBrowser") ? ControllerType.PhysicalDevice : ControllerType.Touch;
578
+ const controllerCount = deviceType === ControllerType.Touch ? 4 : 2;
579
+ for (let i = 0; i < controllerCount; i++) {
580
+ WebXRController.Create(this.webxr, i, this.webxr.gameObject as GameObject, deviceType)
581
+ }
582
+
583
+ if (!this.sessionRoot || this.sessionRoot.destroyed || !this.sessionRoot.activeAndEnabled)
584
+ this.sessionRoot = GameObject.findObjectOfType(WebARSessionRoot, context);
585
+
586
+ this.previousBackground = context.scene.background;
587
+ this.previousEnvironment = context.scene.environment;
588
+ context.scene.background = null;
589
+
590
+ session.requestReferenceSpace('viewer').then((referenceSpace) => {
591
+ session.requestHitTestSource?.call(session, { space: referenceSpace })?.then((source) => {
592
+ this.hitTestSource = source;
593
+ }).catch((err) => {
594
+ this.noHitTestAvailable = true;
595
+ console.warn("WebXR: Hit test not supported", err);
596
+ });
597
+ });
598
+
599
+ if (!this.reticle && this.sessionRoot) {
600
+ this.reticle = new Mesh(
601
+ new RingGeometry(0.07, 0.09, 32).rotateX(- Math.PI / 2),
602
+ new MeshBasicMaterial()
603
+ );
604
+ this.reticle.name = "AR Placement reticle";
605
+ this.reticle.matrixAutoUpdate = false;
606
+ this.reticle.visible = false;
607
+
608
+ // create AR reticle parent to allow WebXRSessionRoot to be translated, rotated or scaled
609
+ this.reticleParent = new Object3D();
610
+ this.reticleParent.name = "AR Reticle Parent";
611
+ this.reticleParent.matrixAutoUpdate = false;
612
+ this.reticleParent.add(this.reticle);
613
+ // this.reticleParent.matrix.copy(this.sessionRoot.gameObject.matrixWorld);
614
+
615
+ if (this.webxr.scene) {
616
+ this.context.scene.add(this.reticleParent);
617
+ // this.context.scene.add(this.reticle);
618
+ this.context.scene.visible = true;
619
+ }
620
+ else console.warn("Could not found WebXR Rig");
621
+ }
622
+
623
+ this._previousParent = this.webxr.gameObject;
624
+ if (!WebAR.tempWebXRObject) WebAR.tempWebXRObject = new Object3D();
625
+ this.context.scene.add(WebAR.tempWebXRObject);
626
+ GameObject.addComponent(WebAR.tempWebXRObject as GameObject, this.webxr);
627
+
628
+ if (this.sessionRoot) {
629
+ this.sessionRoot.webAR = this;
630
+ this.sessionRoot?.onBegin(session);
631
+ }
632
+ else console.warn("No WebARSessionRoot found in scene")
633
+
634
+ const eng = this.context.domElement as INeedleEngineComponent;
635
+ eng?.onEnterAR?.call(eng, session, this.arOverlayElement!);
636
+
637
+ this.context.mainCameraComponent?.applyClearFlagsIfIsActiveCamera();
638
+ }
639
+
640
+ onEnd(session: XRSession) {
641
+ if (this._previousParent) {
642
+ GameObject.addComponent(this._previousParent as GameObject, this.webxr);
643
+ this._previousParent = null;
644
+ }
645
+ this.hitTestSource = null;
646
+ const context = this.webxr.context;
647
+ context.scene.background = this.previousBackground;
648
+ context.scene.environment = this.previousEnvironment;
649
+ if (this.sessionRoot) {
650
+ this.sessionRoot.onEnd(this.webxr.Rig, session);
651
+ }
652
+
653
+ const el = this.context.domElement as INeedleEngineComponent;
654
+ el.onExitAR?.call(el, session);
655
+
656
+ this.context.mainCameraComponent?.applyClearFlagsIfIsActiveCamera();
657
+ }
658
+
659
+ onUpdate(session: XRSession, frame: XRFrame) {
660
+
661
+ if (this.noHitTestAvailable === true) {
662
+ if (this.reticle)
663
+ this.reticle.visible = false;
664
+ if (!this.didPlaceARSessionRoot) {
665
+ this.didPlaceARSessionRoot = true;
666
+ const rig = this.webxr.Rig;
667
+ const placementMatrix = arPlacementWithoutHitTestMatrix.clone();
668
+ // if (rig) {
669
+ // const positionFromRig = new Vector3(0, 0, 0).add(rig.position).divideScalar(this.sessionRoot?.arScale ?? 1);
670
+ // placementMatrix.multiply(new Matrix4().makeTranslation(positionFromRig.x, positionFromRig.y, positionFromRig.z));
671
+ // // placementMatrix.setPosition(positionFromRig);
672
+ // }
673
+ this.sessionRoot?.placeAt(rig, placementMatrix);
674
+ }
675
+ return;
676
+ }
677
+
678
+ if (!this.hitTestSource) return;
679
+ const hitTestResults = frame.getHitTestResults(this.hitTestSource);
680
+ if (hitTestResults.length) {
681
+ const hit = hitTestResults[0];
682
+ const referenceSpace = this.webxr.context.renderer.xr.getReferenceSpace();
683
+ if (referenceSpace) {
684
+ const pose = hit.getPose(referenceSpace);
685
+
686
+ if (this.sessionRoot) {
687
+ const didPlace = this.sessionRoot.onUpdate(this.webxr.Rig, session, pose);
688
+ this.didPlaceARSessionRoot = didPlace;
689
+ }
690
+
691
+ if (this.reticle) {
692
+ this.reticle.visible = this.reticleActive;
693
+ if (this.reticleActive) {
694
+ if (pose) {
695
+ const matrix = pose.transform.matrix;
696
+ this.reticle.matrix.fromArray(matrix);
697
+ if (this.webxr.Rig)
698
+ this.reticle.matrix.premultiply(this.webxr.Rig.matrix);
699
+ }
700
+ }
701
+ }
702
+ }
703
+
704
+ } else {
705
+ this.sessionRoot?.onUpdate(this.webxr.Rig, session, null);
706
+ if (this.reticle)
707
+ this.reticle.visible = false;
708
+ }
709
+ }
710
+ }
711
+
712
+ const arPlacementWithoutHitTestMatrix = new Matrix4().identity().makeTranslation(0, 0, 0);
src/engine-components/webxr/WebXRAvatar.ts ADDED
@@ -0,0 +1,356 @@
1
+ import { Behaviour, GameObject } from "../Component";
2
+ import { WebXR } from "./WebXR";
3
+ import { Quaternion, Vector3 } from "three";
4
+ import { AvatarLoader } from "../AvatarLoader";
5
+ import { XRFlag, XRStateFlag } from "../XRFlag";
6
+ import { Avatar_POI } from "../avatar/Avatar_Brain_LookAt";
7
+ import { Context } from "../../engine/engine_setup";
8
+ import { AssetReference } from "../../engine/engine_addressables";
9
+ import { Object3D } from "three";
10
+ import { VRUserState } from "./WebXRSync";
11
+ import { getParam } from "../../engine/engine_utils";
12
+ import { ViewDevice } from "../../engine/engine_playerview";
13
+ import { InstancingUtil } from "../../engine/engine_instancing";
14
+
15
+ export const debug = getParam("debugavatar");
16
+
17
+ export type AvatarMarkerEventArgs = {
18
+ avatarMarker: AvatarMarker;
19
+ gameObject: Object3D;
20
+ }
21
+
22
+ export class AvatarMarker extends Behaviour {
23
+
24
+ public static getAvatar(index: number): AvatarMarker | null {
25
+ if (index >= 0 && index < AvatarMarker.instances.length)
26
+ return AvatarMarker.instances[index];
27
+ return null;
28
+ }
29
+
30
+ public static instances: AvatarMarker[] = [];
31
+
32
+ public static onAvatarMarkerCreated(cb: (args: AvatarMarkerEventArgs) => void): Function {
33
+ AvatarMarker._onNewAvatarMarkerAdded.push(cb);
34
+ return cb;
35
+ }
36
+
37
+ public static onAvatarMarkerDestroyed(cb: (args: AvatarMarkerEventArgs) => void): Function {
38
+ AvatarMarker._onAvatarMarkerDestroyed.push(cb);
39
+ return cb;
40
+ }
41
+
42
+ private static _onNewAvatarMarkerAdded: Array<(args: AvatarMarkerEventArgs) => void> = [];
43
+ private static _onAvatarMarkerDestroyed: Array<(args: AvatarMarkerEventArgs) => void> = [];
44
+
45
+
46
+ public connectionId!: string;
47
+ public avatar?: WebXRAvatar | Object3D;
48
+
49
+ awake() {
50
+ AvatarMarker.instances.push(this);
51
+ if (debug)
52
+ console.log(this);
53
+
54
+ for (const cb of AvatarMarker._onNewAvatarMarkerAdded)
55
+ cb({ avatarMarker: this, gameObject: this.gameObject });
56
+ }
57
+
58
+ onDestroy() {
59
+ AvatarMarker.instances.splice(AvatarMarker.instances.indexOf(this), 1);
60
+
61
+ for (const cb of AvatarMarker._onAvatarMarkerDestroyed)
62
+ cb({ avatarMarker: this, gameObject: this.gameObject });
63
+ }
64
+
65
+ isLocalAvatar() {
66
+ return this.connectionId === this.context.connection.connectionId;
67
+ }
68
+
69
+ setVisible(visible: boolean) {
70
+ if (this.avatar) {
71
+ if ("setVisible" in this.avatar)
72
+ this.avatar.setVisible(visible);
73
+ else {
74
+ GameObject.setActive(this.avatar, visible);
75
+ }
76
+ }
77
+ }
78
+ }
79
+
80
+
81
+ export class WebXRAvatar {
82
+ private static loader: AvatarLoader = new AvatarLoader();
83
+
84
+ private _isVisible: boolean = true;
85
+ setVisible(visible: boolean) {
86
+ this._isVisible = visible;
87
+ this.updateVisibility();
88
+ }
89
+
90
+ get isWebXRAvatar() { return true; }
91
+
92
+ // TODO: set layers on all avatars
93
+ /** the user id */
94
+ public guid: string;
95
+
96
+ private root: Object3D | null = null;
97
+ public head: Object3D | null = null;
98
+ public handLeft: Object3D | null = null;
99
+ public handRight: Object3D | null = null;
100
+ public lastUpdate: number = -1;
101
+ public isLocalAvatar: boolean = false;
102
+ public flags: XRFlag[] | null = null;
103
+ private headScale: Vector3 = new Vector3(1, 1, 1);
104
+ private handLeftScale: Vector3 = new Vector3(1, 1, 1);
105
+ private handRightScale: Vector3 = new Vector3(1, 1, 1);
106
+
107
+ private readonly webxr: WebXR;
108
+
109
+ private lastAvatarId: string | null = null;
110
+ private hasAvatarOverride: boolean = false;
111
+
112
+
113
+ private context: Context;
114
+ private avatarMarker: AvatarMarker | null = null;
115
+
116
+ constructor(context: Context, guid: string, webXR: WebXR) {
117
+ this.context = context;
118
+ this.guid = guid;
119
+ this.webxr = webXR;
120
+ this.setupCustomAvatar(this.webxr.defaultAvatar as AssetReference);
121
+ }
122
+
123
+ public updateFlags() {
124
+ if (!this.flags)
125
+ return;
126
+ let mask = this.isLocalAvatar ? XRStateFlag.FirstPerson : XRStateFlag.ThirdPerson;
127
+ if (this.context.isInVR)
128
+ mask |= XRStateFlag.VR;
129
+ else if (this.context.isInAR)
130
+ mask |= XRStateFlag.AR;
131
+ else
132
+ mask |= XRStateFlag.Browser;
133
+ for (const f of this.flags) {
134
+ f.gameObject.visible = true;
135
+ f.UpdateVisible(mask);
136
+ }
137
+ }
138
+
139
+ public async setAvatarOverride(avatarId: string | null): Promise<boolean | null> {
140
+ this.hasAvatarOverride = avatarId !== null;
141
+ if (this.hasAvatarOverride && this.lastAvatarId !== avatarId) {
142
+ this.lastAvatarId = avatarId;
143
+ if (avatarId != null && avatarId.length > 0)
144
+ return await this.setupCustomAvatar(avatarId);
145
+ }
146
+ return null;
147
+ }
148
+
149
+ private _headTarget: Object3D = new Object3D();
150
+ private _handLeftTarget: Object3D = new Object3D();
151
+ private _handRightTarget: Object3D = new Object3D();
152
+ private _canInterpolate: boolean = false;
153
+
154
+ private static invertRotation: Quaternion = new Quaternion().setFromAxisAngle(new Vector3(0, 1, 0), Math.PI);
155
+
156
+ public tryUpdate(state: VRUserState, _timeDiff: number) {
157
+ if (state.guid === this.guid) {
158
+
159
+ if (this.lastAvatarId !== state.avatarId && state.avatarId && state.avatarId.length > 0) {
160
+ this.lastAvatarId = state.avatarId;
161
+ this.setupCustomAvatar(state.avatarId);
162
+ }
163
+
164
+ this.lastUpdate = state.time;
165
+ if (this.head) {
166
+
167
+ const device = this.webxr.IsInAR ? ViewDevice.Handheld : ViewDevice.Headset;
168
+ let viewObj = this.head;
169
+ // if (this.isLocalAvatar) {
170
+ // if (this.context.mainCamera && this.context.isInXR) {
171
+ // viewObj = this.context.renderer.xr.getCamera(this.context.mainCamera);
172
+ // }
173
+ // }
174
+ this.context.players.setPlayerView(state.guid, viewObj, device);
175
+
176
+ InstancingUtil.markDirty(this.head);
177
+
178
+ this._canInterpolate = true;
179
+ const ht = this.isLocalAvatar ? this.head : this._headTarget;
180
+ ht.position.set(state.position.x, state.position.y, state.position.z);
181
+ // not sure how position in local space can be correct but rotation is wrong / offset when parent rotates
182
+ ht.quaternion.set(state.rotation.x, state.rotation.y, state.rotation.z, state.rotation.w);
183
+ ht.scale.set(state.scale, state.scale, state.scale);
184
+ ht.scale.multiply(this.headScale);
185
+
186
+ if (this.handLeft) {
187
+ const ht = this.isLocalAvatar ? this.handLeft : this._handLeftTarget;
188
+ ht.position.set(state.posLeftHand.x, state.posLeftHand.y, state.posLeftHand.z);
189
+ ht.quaternion.set(state.rotLeftHand["_x"], state.rotLeftHand["_y"], state.rotLeftHand["_z"], state.rotLeftHand["_w"]);
190
+ ht.quaternion.multiply(WebXRAvatar.invertRotation);
191
+ ht.scale.set(state.scale, state.scale, state.scale);
192
+ ht.scale.multiply(this.handLeftScale);
193
+ InstancingUtil.markDirty(this.handLeft);
194
+ }
195
+
196
+ if (this.handRight) {
197
+ const ht = this.isLocalAvatar ? this.handRight : this._handRightTarget;
198
+ ht.position.set(state.posRightHand.x, state.posRightHand.y, state.posRightHand.z);
199
+ ht.quaternion.set(state.rotRightHand["_x"], state.rotRightHand["_y"], state.rotRightHand["_z"], state.rotRightHand["_w"]);
200
+ ht.quaternion.multiply(WebXRAvatar.invertRotation);
201
+ ht.scale.set(state.scale, state.scale, state.scale);
202
+ ht.scale.multiply(this.handRightScale);
203
+ InstancingUtil.markDirty(this.handRight);
204
+ }
205
+ }
206
+ }
207
+ }
208
+
209
+ public update() {
210
+ if (this.isLocalAvatar)
211
+ return;
212
+ if (!this._canInterpolate)
213
+ return;
214
+ const t = this.context.time.deltaTime / .1;
215
+ if (this.head) {
216
+ this.head.position.lerp(this._headTarget.position, t);
217
+ this.head.quaternion.slerp(this._headTarget.quaternion, t);
218
+ this.head.scale.lerp(this._headTarget.scale, t);
219
+ }
220
+ if (this.handLeft && this._handLeftTarget) {
221
+ this.handLeft.position.lerp(this._handLeftTarget.position, t);
222
+ this.handLeft.quaternion.slerp(this._handLeftTarget.quaternion, t);
223
+ this.handLeft.scale.lerp(this._handLeftTarget.scale, t);
224
+ }
225
+ if (this.handRight && this._handRightTarget) {
226
+ this.handRight.position.lerp(this._handRightTarget.position, t);
227
+ this.handRight.quaternion.slerp(this._handRightTarget.quaternion, t);
228
+ this.handRight.scale.lerp(this._handRightTarget.scale, t);
229
+ }
230
+ }
231
+
232
+ public destroy() {
233
+ if (debug)
234
+ console.log("Destroy avatar", this.guid);
235
+ this.root?.removeFromParent();
236
+ this.avatarMarker?.destroy();
237
+ this.lastAvatarId = null;
238
+
239
+ if (this.head) {
240
+ Avatar_POI.Remove(this.context, this.head);
241
+ }
242
+ // this.head?.removeFromParent();
243
+ // this.handLeft?.removeFromParent();
244
+ // this.handRight?.removeFromParent();
245
+ }
246
+
247
+ private updateVisibility() {
248
+ const root = this.root;
249
+ if (root) {
250
+ GameObject.setActive(root, this._isVisible);
251
+ }
252
+ }
253
+
254
+ private async setupCustomAvatar(avatarId: string | Object3D | AssetReference): Promise<boolean> {
255
+ if (debug)
256
+ console.log("LOAD", avatarId, this);
257
+
258
+ if (!avatarId || (typeof avatarId === "string" && avatarId.length <= 0))
259
+ return false;
260
+
261
+ if (this.head) {
262
+ Avatar_POI.Remove(this.context, this.head);
263
+ }
264
+
265
+ const reference = avatarId as AssetReference;
266
+ if (reference?.loadAssetAsync !== undefined) {
267
+ await reference.loadAssetAsync();
268
+ const prefab = reference.asset as Object3D;
269
+ GameObject.setActive(prefab, false);
270
+ avatarId = GameObject.instantiate(prefab as Object3D) as Object3D;
271
+ GameObject.setActive(avatarId, true);
272
+ // console.log("Avatar", avatarId);
273
+ }
274
+ if (debug)
275
+ console.log(avatarId);
276
+
277
+ const model = await WebXRAvatar.loader.getOrCreateNewAvatarInstance(this.context, avatarId as (Object3D | string));
278
+ if (debug)
279
+ console.log(model, model?.isValid, this.lastAvatarId, avatarId);
280
+ // if (this.lastAvatarId !== avatarId) {
281
+ // // avatar id changed in the meantime
282
+ // return true;
283
+ // }
284
+ if (model?.isValid) {
285
+ this.root = model.root;
286
+
287
+ this.root.position.set(0, 0, 0);
288
+ this.root.quaternion.set(0, 0, 0, 1);
289
+ this.root.scale.set(1, 1, 1); // should we allow a scaled avatar root?!
290
+
291
+ this.avatarMarker = GameObject.addNewComponent(this.root as GameObject, AvatarMarker) as AvatarMarker;
292
+ this.avatarMarker.connectionId = this.guid;
293
+ this.avatarMarker.avatar = this;
294
+
295
+ if (this.head && this.head !== model.head)
296
+ this.head?.removeFromParent();
297
+ this.head = model.head;
298
+ this.headScale.copy(this.head.scale);
299
+
300
+ if (this.head && !this.isLocalAvatar) {
301
+ Avatar_POI.Add(this.context, this.head, this.avatarMarker);
302
+ }
303
+
304
+ if (model.leftHand)
305
+ this.handLeft?.removeFromParent();
306
+ this.handLeft = model.leftHand ?? this.handLeft;
307
+ if (this.handLeft)
308
+ this.handLeftScale.copy(this.handLeft.scale);
309
+ else
310
+ this.handLeftScale.set(1, 1, 1);
311
+
312
+ if (model.rigthHand)
313
+ this.handRight?.removeFromParent();
314
+ this.handRight = model.rigthHand ?? this.handRight;
315
+ if (this.handRight)
316
+ this.handRightScale.copy(this.handRight.scale);
317
+ else
318
+ this.handRightScale.set(1, 1, 1);
319
+
320
+
321
+ this.context.scene.add(this.root);
322
+ // scene.add(this.handLeft);
323
+ // scene.add(this.handRight);
324
+ // this.mouthShapes = null;
325
+ // this.needSearchEyes = true;
326
+ if (this.flags == null)
327
+ this.flags = [];
328
+ this.flags.length = 0;
329
+ this.flags.push(...GameObject.getComponentsInChildren(this.root as GameObject, XRFlag));
330
+ // if no flags are found add at least a head flag to hide head in first person VR
331
+ if (this.flags.length <= 0) {
332
+ if (this.head) {
333
+ const flag = GameObject.addNewComponent(this.head, XRFlag) as XRFlag;
334
+ // TODO: the defaults are wrong? should be Desktop | ThirdPerson ?
335
+ flag.visibleIn = XRStateFlag.ThirdPerson | XRStateFlag.VR;
336
+ this.flags.push(flag);
337
+ if (debug)
338
+ console.log("Added flag to head: " + flag.visibleIn, this.head.name);
339
+ }
340
+ }
341
+
342
+ if (debug)
343
+ console.log("[Avatar], is Local? ", this.isLocalAvatar, this.root);
344
+ this.updateFlags();
345
+
346
+ this.updateVisibility();
347
+
348
+ return true;
349
+ }
350
+ else {
351
+ if (debug)
352
+ console.warn("build avatar failed");
353
+ return false;
354
+ }
355
+ }
356
+ }
src/engine-components/webxr/WebXRController.ts ADDED
@@ -0,0 +1,1125 @@
1
+ import { BoxHelper, BufferGeometry, Color, Euler, Group, Intersection, Layers, Line, LineBasicMaterial, Material, Mesh, MeshBasicMaterial, Object3D, PerspectiveCamera, Quaternion, Ray, SphereGeometry, Vector2, Vector3 } from "three";
2
+ import { OculusHandModel } from 'three/examples/jsm/webxr/OculusHandModel.js';
3
+ import { OculusHandPointerModel } from 'three/examples/jsm/webxr/OculusHandPointerModel.js';
4
+ import { XRControllerModel, XRControllerModelFactory } from 'three/examples/jsm/webxr/XRControllerModelFactory.js';
5
+ import { GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader.js'
6
+
7
+ import { InstancingUtil } from "../../engine/engine_instancing";
8
+ import { Mathf } from "../../engine/engine_math";
9
+ import { RaycastOptions } from "../../engine/engine_physics";
10
+ import { getWorldPosition, getWorldQuaternion, setWorldPosition, setWorldQuaternion } from "../../engine/engine_three_utils";
11
+ import { getParam, resolveUrl } from "../../engine/engine_utils";
12
+ import { addDracoAndKTX2Loaders } from "../../engine/engine_loaders";
13
+
14
+ import { Avatar_POI } from "../avatar/Avatar_Brain_LookAt";
15
+ import { Behaviour, GameObject } from "../Component";
16
+ import { Interactable, UsageMarker } from "../Interactable";
17
+ import { Rigidbody } from "../RigidBody";
18
+ import { SyncedTransform } from "../SyncedTransform";
19
+ import { UIRaycastUtils } from "../ui/RaycastUtils";
20
+ import { WebXR } from "./WebXR";
21
+
22
+ const debug = getParam("debugwebxrcontroller");
23
+
24
+ export enum ControllerType {
25
+ PhysicalDevice = 0,
26
+ Touch = 1,
27
+ }
28
+
29
+ export enum ControllerEvents {
30
+ SelectStart = "select-start",
31
+ SelectEnd = "select-end",
32
+ Update = "update",
33
+ }
34
+
35
+ export class TeleportTarget extends Behaviour {
36
+
37
+ }
38
+
39
+ export class WebXRController extends Behaviour {
40
+
41
+ public static Factory: XRControllerModelFactory = new XRControllerModelFactory();
42
+
43
+ private static raycastColor: Color = new Color(.9, .3, .3);
44
+ private static raycastNoHitColor: Color = new Color(.6, .6, .6);
45
+ private static geometry = new BufferGeometry().setFromPoints([new Vector3(0, 0, 0), new Vector3(0, 0, -1)]);
46
+ private static handModels: { [index: number]: OculusHandPointerModel } = {};
47
+
48
+ private static CreateRaycastLine(): Line {
49
+ const line = new Line(this.geometry);
50
+ const mat = line.material as LineBasicMaterial;
51
+ mat.color = this.raycastColor;
52
+ // mat.linewidth = 10;
53
+ line.layers.set(2);
54
+ line.name = 'line';
55
+ line.scale.z = 1;
56
+ return line;
57
+ }
58
+
59
+ private static CreateRaycastHitPoint(): Mesh {
60
+ const geometry = new SphereGeometry(.5, 22, 22);
61
+ const material = new MeshBasicMaterial({ color: this.raycastColor });
62
+ const sphere = new Mesh(geometry, material);
63
+ sphere.visible = false;
64
+ sphere.layers.set(2);
65
+ return sphere;
66
+ }
67
+
68
+ public static Create(owner: WebXR, index: number, addTo: GameObject, type: ControllerType): WebXRController {
69
+ const ctrl = addTo ? GameObject.addNewComponent(addTo, WebXRController, false) : new WebXRController();
70
+
71
+ ctrl.webXR = owner;
72
+ ctrl.index = index;
73
+ ctrl.type = type;
74
+
75
+ const context = owner.context;
76
+ // from https://github.com/mrdoob/js/blob/master/examples/webxr_vr_dragging.html
77
+ // controllers
78
+ ctrl.controller = context.renderer.xr.getController(index);
79
+ ctrl.controllerGrip = context.renderer.xr.getControllerGrip(index);
80
+ ctrl.controllerModel = this.Factory.createControllerModel(ctrl.controller);
81
+ ctrl.controllerGrip.add(ctrl.controllerModel);
82
+
83
+ ctrl.hand = context.renderer.xr.getHand(index);
84
+
85
+ const loader = new GLTFLoader();
86
+ addDracoAndKTX2Loaders(loader, context);
87
+ if (ctrl.webXR.handModelPath && ctrl.webXR.handModelPath !== "")
88
+ loader.setPath(resolveUrl(owner.sourceId, ctrl.webXR.handModelPath));
89
+ else
90
+ // from XRHandMeshModel.js
91
+ loader.setPath('https://cdn.jsdelivr.net/npm/@webxr-input-profiles/[email protected]/dist/profiles/generic-hand/');
92
+ //@ts-ignore
93
+ const hand = new OculusHandModel(ctrl.hand, loader);
94
+
95
+ ctrl.hand.add(hand);
96
+ ctrl.hand.traverse(x => x.layers.set(2));
97
+
98
+ ctrl.handPointerModel = new OculusHandPointerModel(ctrl.hand, ctrl.controller);
99
+
100
+
101
+ // TODO remove all these once https://github.com/mrdoob/js/pull/23279 lands
102
+ ctrl.controller.addEventListener('connected', (_) => {
103
+ ctrl.setControllerLayers(ctrl.controllerModel, 2);
104
+ ctrl.setControllerLayers(ctrl.controllerGrip, 2);
105
+ ctrl.setControllerLayers(ctrl.hand, 2);
106
+ setTimeout(() => {
107
+ ctrl.setControllerLayers(ctrl.controllerModel, 2);
108
+ ctrl.setControllerLayers(ctrl.controllerGrip, 2);
109
+ ctrl.setControllerLayers(ctrl.hand, 2);
110
+ }, 1000);
111
+ });
112
+
113
+ // TODO: unsubscribe! this should be moved into onenable and ondisable!
114
+ // TODO remove all these once https://github.com/mrdoob/js/pull/23279 lands
115
+ ctrl.hand.addEventListener('connected', (event) => {
116
+ const xrInputSource = event.data;
117
+ if (xrInputSource.hand) {
118
+ if (owner.Rig) owner.Rig.add(ctrl.hand);
119
+ ctrl.type = ControllerType.PhysicalDevice;
120
+ ctrl.handPointerModel.traverse(x => x.layers.set(2)); // ignore raycast
121
+ ctrl.handPointerModel.pointerObject?.traverse(x => x.layers.set(2));
122
+
123
+ // when exiting and re-entering xr the joints are not parented to the hand anymore
124
+ // this is a workaround to fix that temporarely
125
+ // see https://github.com/needle-tools/needle-tiny-playground/issues/123
126
+ const jnts = ctrl.hand["joints"];
127
+ if (jnts) {
128
+ for (const key of Object.keys(jnts)) {
129
+ const joint = jnts[key];
130
+ if (joint.parent) continue;
131
+ ctrl.hand.add(joint);
132
+ }
133
+ }
134
+ }
135
+ });
136
+
137
+ return ctrl;
138
+ }
139
+
140
+ // TODO: replace with component events
141
+ public static addEventListener(evt: ControllerEvents, callback: (controller: WebXRController, args: any) => void) {
142
+ const list = this.eventSubs[evt] ?? [];
143
+ list.push(callback);
144
+ this.eventSubs[evt] = list;
145
+ }
146
+
147
+ // TODO: replace with component events
148
+ public static removeEventListener(evt: ControllerEvents, callback: (controller: WebXRController, args: any) => void) {
149
+ if (!callback) return;
150
+ const list = this.eventSubs[evt] ?? [];
151
+ const idx = list.indexOf(callback);
152
+ if (idx >= 0) list.splice(idx, 1);
153
+ this.eventSubs[evt] = list;
154
+ }
155
+
156
+ private static eventSubs: { [key: string]: Function[] } = {};
157
+
158
+ public webXR!: WebXR;
159
+ public index: number = -1;
160
+ public controllerModel!: XRControllerModel;
161
+ public controller!: Group;
162
+ public controllerGrip!: Group;
163
+ public hand!: Group;
164
+ public handPointerModel!: OculusHandPointerModel;
165
+ public grabbed: AttachedObject | null = null;
166
+ public input: XRInputSource | null = null;
167
+ public type: ControllerType = ControllerType.PhysicalDevice;
168
+ public showRaycastLine : boolean = true;
169
+
170
+ get isUsingHands(): boolean {
171
+ const r = this.input?.hand;
172
+ return r !== null && r !== undefined;
173
+ }
174
+
175
+ get wrist(): Object3D | null {
176
+ if (!this.hand) return null;
177
+ const jnts = this.hand["joints"];
178
+ if (!jnts) return null;
179
+ return jnts["wrist"];
180
+ }
181
+
182
+ private _wristQuaternion: Quaternion | null = null;
183
+ getWristQuaternion(): Quaternion | null {
184
+ const wrist = this.wrist;
185
+ if (!wrist) return null;
186
+ if (!this._wristQuaternion) this._wristQuaternion = new Quaternion();
187
+ const wr = getWorldQuaternion(wrist).multiply(this._wristQuaternion.setFromEuler(new Euler(-Math.PI / 4, 0, 0)));
188
+ return wr;
189
+ }
190
+
191
+ private movementVector: Vector3 = new Vector3();
192
+ private worldRot: Quaternion = new Quaternion();
193
+ private joystick: Vector2 = new Vector2();
194
+ private didRotate: boolean = false;
195
+ private didTeleport: boolean = false;
196
+ private didChangeScale: boolean = false;
197
+ private static PreviousCameraFarDistance: number | undefined = undefined;
198
+ private static MovementSpeedFactor: number = 1;
199
+
200
+ private lastHit: Intersection | null = null;
201
+
202
+ private raycastLine: Line | null = null;
203
+ private _raycastHitPoint: Object3D | null = null;
204
+ private _connnectedCallback: any | null = null;
205
+ private _disconnectedCallback: any | null = null;
206
+ private _selectStartEvt: any | null = null;
207
+ private _selectEndEvt: any | null = null;
208
+
209
+ public get selectionDown(): boolean { return this._selectionPressed && !this._selectionPressedLastFrame; }
210
+ public get selectionUp(): boolean { return !this._selectionPressed && this._selectionPressedLastFrame; }
211
+ public get selectionPressed(): boolean { return this._selectionPressed; }
212
+ public get selectionClick(): boolean { return this._selectionEndTime - this._selectionStartTime < 0.3; }
213
+ public get raycastHitPoint(): Object3D | null { return this._raycastHitPoint; }
214
+
215
+ private _selectionPressed: boolean = false;
216
+ private _selectionPressedLastFrame: boolean = false;
217
+ private _selectionStartTime: number = 0;
218
+ private _selectionEndTime: number = 0;
219
+
220
+ public get useSmoothing(): boolean { return this._useSmoothing };
221
+ private _useSmoothing: boolean = true;
222
+
223
+ awake(): void {
224
+ if (!this.controller) {
225
+ console.warn("Missing Controller!!!", this);
226
+ return;
227
+ }
228
+ this._connnectedCallback = this.onSourceConnected.bind(this);
229
+ this._disconnectedCallback = this.onSourceDisconnected.bind(this);
230
+ this._selectStartEvt = this.onSelectStart.bind(this);
231
+ this._selectEndEvt = this.onSelectEnd.bind(this);
232
+ if (this.type === ControllerType.Touch) {
233
+ this.controllerGrip.addEventListener("connected", this._connnectedCallback);
234
+ this.controllerGrip.addEventListener("disconnected", this._disconnectedCallback);
235
+ this.controller.addEventListener('selectstart', this._selectStartEvt);
236
+ this.controller.addEventListener('selectend', this._selectEndEvt);
237
+ }
238
+ if (this.type === ControllerType.PhysicalDevice) {
239
+ this.controller.addEventListener('selectstart', this._selectStartEvt);
240
+ this.controller.addEventListener('selectend', this._selectEndEvt);
241
+ }
242
+ }
243
+
244
+ onDestroy(): void {
245
+ if (this.type === ControllerType.Touch) {
246
+ this.controllerGrip.removeEventListener("connected", this._connnectedCallback);
247
+ this.controllerGrip.removeEventListener("disconnected", this._disconnectedCallback);
248
+ this.controller.removeEventListener('selectstart', this._selectStartEvt);
249
+ this.controller.removeEventListener('selectend', this._selectEndEvt);
250
+ }
251
+ if (this.type === ControllerType.PhysicalDevice) {
252
+ this.controller.removeEventListener('selectstart', this._selectStartEvt);
253
+ this.controller.removeEventListener('selectend', this._selectEndEvt);
254
+ }
255
+
256
+ this.hand?.clear();
257
+ this.controllerGrip?.clear();
258
+ this.controller?.clear();
259
+ }
260
+
261
+ public onEnable(): void {
262
+ if (this.hand)
263
+ this.hand.name = "Hand";
264
+ if (this.controllerGrip)
265
+ this.controllerGrip.name = "ControllerGrip";
266
+ if (this.controller)
267
+ this.controller.name = "Controller";
268
+ if (this.raycastLine)
269
+ this.raycastLine.name = "RaycastLine;"
270
+
271
+ if (this.webXR.Controllers.indexOf(this) < 0)
272
+ this.webXR.Controllers.push(this);
273
+
274
+ if (!this.raycastLine)
275
+ this.raycastLine = WebXRController.CreateRaycastLine();
276
+ if (!this._raycastHitPoint)
277
+ this._raycastHitPoint = WebXRController.CreateRaycastHitPoint();
278
+
279
+ this.webXR.Rig?.add(this.hand);
280
+ this.webXR.Rig?.add(this.controllerGrip);
281
+ this.webXR.Rig?.add(this.controller);
282
+ this.webXR.Rig?.add(this.raycastLine);
283
+ this.raycastLine?.add(this._raycastHitPoint);
284
+ this._raycastHitPoint.visible = false;
285
+ this.hand.add(this.handPointerModel);
286
+ if (debug)
287
+ console.log("ADDED TO RIG", this.webXR.Rig);
288
+
289
+ // // console.log("enable", this.index, this.controllerGrip.uuid)
290
+ }
291
+
292
+ onDisable(): void {
293
+ // console.log("XR controller disabled", this);
294
+ this.hand?.removeFromParent();
295
+ this.controllerGrip?.removeFromParent();
296
+ this.controller?.removeFromParent();
297
+ this.raycastLine?.removeFromParent();
298
+ this._raycastHitPoint?.removeFromParent();
299
+ // console.log("Disable", this._connnectedCallback, this._disconnectedCallback);
300
+ // this.controllerGrip.removeEventListener("connected", this._connnectedCallback);
301
+ // this.controllerGrip.removeEventListener("disconnected", this._disconnectedCallback);
302
+
303
+ const i = this.webXR.Controllers.indexOf(this);
304
+ if (i >= 0)
305
+ this.webXR.Controllers.splice(i, 1);
306
+ }
307
+
308
+ // onDestroy(): void {
309
+ // console.log("destroyed", this.index);
310
+ // }
311
+
312
+ private _isConnected: boolean = false;
313
+
314
+ private onSourceConnected(e: { data: XRInputSource, target: any }) {
315
+ if (this._isConnected) {
316
+ console.warn("Received connected event for controller that is already connected", this.index, e);
317
+ return;
318
+ }
319
+ this._isConnected = true;
320
+ this.input = e.data;
321
+
322
+ if (this.type === ControllerType.Touch) {
323
+ this.onSelectStart();
324
+ this.createPointerEvent("down");
325
+ }
326
+ }
327
+
328
+ private onSourceDisconnected(_e: any) {
329
+ if (!this._isConnected) {
330
+ console.warn("Received discnnected event for controller that is not connected", _e);
331
+ return;
332
+ }
333
+ this._isConnected = false;
334
+ if (this.type === ControllerType.Touch) {
335
+ this.onSelectEnd();
336
+ this.createPointerEvent("up");
337
+ }
338
+ this.input = null;
339
+ }
340
+
341
+ private createPointerEvent(type: string) {
342
+ switch (type) {
343
+ case "down":
344
+ this.context.input.createPointerDown({ clientX: 0, clientY: 0, button: this.index, pointerType: "touch" });
345
+ break;
346
+ case "move":
347
+ break;
348
+ case "up":
349
+ this.context.input.createPointerUp({ clientX: 0, clientY: 0, button: this.index, pointerType: "touch" });
350
+ break;
351
+ }
352
+ }
353
+
354
+ rayRotation: Quaternion = new Quaternion();
355
+
356
+ update(): void {
357
+
358
+ // TODO: we should wait until we actually have models, this is just a workaround
359
+ if (this.context.time.frameCount % 60 === 0) {
360
+ this.setControllerLayers(this.controller, 2);
361
+ this.setControllerLayers(this.controllerGrip, 2);
362
+ this.setControllerLayers(this.hand, 2);
363
+ }
364
+
365
+ const subs = WebXRController.eventSubs[ControllerEvents.Update];
366
+ if (subs && subs.length > 0) {
367
+ for (const sub of subs) {
368
+ sub(this);
369
+ }
370
+ }
371
+
372
+ let t = 1;
373
+ if (this.type === ControllerType.PhysicalDevice) t = this.context.time.deltaTime / .1;
374
+ else if (this.isUsingHands && this.handPointerModel.pinched) t = this.context.time.deltaTime / .3;
375
+ this.rayRotation.slerp(getWorldQuaternion(this.controller), this.useSmoothing ? t : 1.0);
376
+ const wp = getWorldPosition(this.controller);
377
+
378
+ // hide hand pointer model, it's giant and doesn't really help
379
+ if (this.isUsingHands && this.handPointerModel.cursorObject) {
380
+ this.handPointerModel.cursorObject.visible = false;
381
+ }
382
+
383
+ if (this.raycastLine) {
384
+ const allowRaycastLineVisible = this.showRaycastLine && this.type !== ControllerType.Touch;
385
+ if (this.type === ControllerType.Touch) {
386
+ this.raycastLine.visible = false;
387
+ }
388
+ else if (this.isUsingHands) {
389
+ this.raycastLine.visible = !this.grabbed && allowRaycastLineVisible;
390
+ setWorldPosition(this.raycastLine, wp);
391
+ const jnts = this.hand!['joints'];
392
+ if (jnts) {
393
+ const wrist = jnts['wrist'];
394
+ if (wrist && this.grabbed && this.grabbed.isCloseGrab) {
395
+ const wr = this.getWristQuaternion();
396
+ if (wr)
397
+ this.rayRotation.copy(wr);
398
+ // this.rayRotation.slerp(wr, this.useSmoothing ? t * 2 : 1);
399
+ }
400
+ }
401
+ setWorldQuaternion(this.raycastLine, this.rayRotation);
402
+ }
403
+ else {
404
+ this.raycastLine.visible = allowRaycastLineVisible;
405
+ setWorldQuaternion(this.raycastLine, this.rayRotation);
406
+ setWorldPosition(this.raycastLine, wp);
407
+ }
408
+ }
409
+
410
+ this.lastHit = this.updateLastHit();
411
+
412
+ if (this.grabbed) {
413
+ this.grabbed.update();
414
+ }
415
+
416
+ this._selectionPressedLastFrame = this._selectionPressed;
417
+
418
+ if (this.selectStartCallback) {
419
+ this.selectStartCallback();
420
+ }
421
+ }
422
+
423
+ private _pinchStartTime: number | undefined = undefined;
424
+
425
+ onUpdate(session: XRSession) {
426
+ this.lastHit = null;
427
+
428
+ if (!session || session.inputSources.length <= this.index) {
429
+ this.input = null;
430
+ return;
431
+ }
432
+ if (this.type === ControllerType.PhysicalDevice)
433
+ this.input = session.inputSources[this.index];
434
+ if (!this.input) return;
435
+ const rig = this.webXR.Rig;
436
+ if (!rig) return;
437
+
438
+ if (this._didNotEndSelection && !this.handPointerModel.pinched) {
439
+ this._didNotEndSelection = false;
440
+ this.onSelectEnd();
441
+ }
442
+
443
+ this.updateStick(this.input);
444
+
445
+ const buttons = this.input?.gamepad?.buttons;
446
+
447
+ switch (this.input.handedness) {
448
+ case "left":
449
+ const speedFactor = 3 * WebXRController.MovementSpeedFactor;
450
+ const powFactor = 2;
451
+ const speed = Mathf.clamp01(this.joystick.length() * 2);
452
+
453
+ const sideDir = this.joystick.x > 0 ? 1 : -1;
454
+ let side = Math.pow(this.joystick.x, powFactor);
455
+ side *= sideDir;
456
+ side *= speed;
457
+
458
+
459
+ const forwardDir = this.joystick.y > 0 ? 1 : -1;
460
+ let forward = Math.pow(this.joystick.y, powFactor);
461
+ forward *= forwardDir;
462
+ side *= speed;
463
+
464
+ rig.getWorldQuaternion(this.worldRot);
465
+ this.movementVector.set(side, 0, forward);
466
+ this.movementVector.applyQuaternion(this.webXR.TransformOrientation);
467
+ this.movementVector.y = 0;
468
+ this.movementVector.applyQuaternion(this.worldRot);
469
+ this.movementVector.multiplyScalar(speedFactor * this.context.time.deltaTime);
470
+ rig.position.add(this.movementVector);
471
+
472
+ if (this.isUsingHands)
473
+ this.runTeleport(rig, buttons);
474
+ break;
475
+
476
+ case "right":
477
+ const rotate = this.joystick.x;
478
+ const rotAbs = Math.abs(rotate);
479
+ if (rotAbs < 0.4) {
480
+ this.didRotate = false;
481
+ }
482
+ else if (rotAbs > .5 && !this.didRotate) {
483
+ const dir = rotate > 0 ? -1 : 1;
484
+ rig.rotateY(Mathf.toRadians(30 * dir));
485
+ this.didRotate = true;
486
+ }
487
+
488
+ this.runTeleport(rig, buttons);
489
+
490
+ break;
491
+ }
492
+ }
493
+
494
+ private runTeleport(rig, buttons) {
495
+ let teleport = -this.joystick.y;
496
+ if (this.hand?.visible && !this.grabbed) {
497
+ const pinched = this.handPointerModel.isPinched();
498
+ if (pinched && this._pinchStartTime === undefined) {
499
+ this._pinchStartTime = this.context.time.time;
500
+ }
501
+ if (pinched && this._pinchStartTime && this.context.time.time - this._pinchStartTime > .8) {
502
+ // hacky approach for basic hand teleportation -
503
+ // we teleport if we pinch and the back of the hand points down (open hand gesture)
504
+ // const v1 = new Vector3();
505
+ // const worldQuaternion = new Quaternion();
506
+ // this.controller.getWorldQuaternion(worldQuaternion);
507
+ // v1.copy(this.controller.up).applyQuaternion(worldQuaternion);
508
+ // const dotPr = -v1.dot(this.controller.up);
509
+ teleport = this.handPointerModel.isPinched() ? 1 : 0;
510
+ }
511
+ if (!pinched) this._pinchStartTime = undefined;
512
+ }
513
+ else this._pinchStartTime = undefined;
514
+
515
+ let doTeleport = teleport > .5 && this.webXR.IsInVR;
516
+ let isInMiniatureMode = this.webXR.Rig ? this.webXR.Rig?.scale?.x < .999 : false;
517
+ let newRigScale: number | null = null;
518
+
519
+ if (buttons && this.input && !this.input.hand) {
520
+ for (let i = 0; i < buttons.length; i++) {
521
+ const btn = buttons[i];
522
+ // button[4] seems to be the A button if it exists. On hololens it's randomly pressed though for hands
523
+ // see https://www.w3.org/TR/webxr-gamepads-module-1/#xr-standard-gamepad-mapping
524
+ if (i === 4) {
525
+ if (btn.pressed && !this.didChangeScale && this.webXR.IsInVR) {
526
+ this.didChangeScale = true;
527
+ const rig = this.webXR.Rig;
528
+ if (rig) {
529
+ if (!isInMiniatureMode) {
530
+ isInMiniatureMode = true;
531
+ doTeleport = true;
532
+ newRigScale = .1;
533
+ WebXRController.MovementSpeedFactor = newRigScale * 2;
534
+ const cam = this.context.mainCamera as PerspectiveCamera;
535
+ WebXRController.PreviousCameraFarDistance = cam.far;
536
+ cam.far /= newRigScale;
537
+ }
538
+ else {
539
+ isInMiniatureMode = false;
540
+ rig.scale.set(1, 1, 1);
541
+ newRigScale = 1;
542
+ WebXRController.MovementSpeedFactor = 1;
543
+ const cam = this.context.mainCamera as PerspectiveCamera;
544
+ if (WebXRController.PreviousCameraFarDistance)
545
+ cam.far = WebXRController.PreviousCameraFarDistance;
546
+ }
547
+ }
548
+ }
549
+ else if (!btn.pressed)
550
+ this.didChangeScale = false;
551
+ }
552
+ }
553
+ }
554
+
555
+ if (doTeleport) {
556
+ if (!this.didTeleport) {
557
+ const rc = this.raycast();
558
+ this.didTeleport = true;
559
+ if (rc && rc.length > 0) {
560
+ const hit = rc[0];
561
+ if (isInMiniatureMode || this.isValidTeleportTarget(hit.object)) {
562
+ const point = hit.point;
563
+ setWorldPosition(rig, point);
564
+ }
565
+ }
566
+ }
567
+ }
568
+ else if (teleport < .1) {
569
+ this.didTeleport = false;
570
+ }
571
+
572
+ if (newRigScale !== null) {
573
+ rig.scale.set(newRigScale, newRigScale, newRigScale);
574
+ rig.updateMatrixWorld();
575
+ }
576
+ }
577
+
578
+ private isValidTeleportTarget(obj: Object3D): boolean {
579
+ return GameObject.getComponentInParent(obj, TeleportTarget) != null;
580
+ }
581
+
582
+ private updateStick(inputSource: XRInputSource) {
583
+ if (!inputSource || !inputSource.gamepad || inputSource.gamepad.axes?.length < 4) return;
584
+ this.joystick.x = inputSource.gamepad.axes[2];
585
+ this.joystick.y = inputSource.gamepad.axes[3];
586
+ }
587
+
588
+ private updateLastHit(): Intersection | null {
589
+ const rc = this.raycast();
590
+ const hit = rc ? rc[0] : null;
591
+ this.lastHit = hit;
592
+ let factor = 1;
593
+ if (this.webXR.Rig) {
594
+ factor /= this.webXR.Rig.scale.x;
595
+ }
596
+ // if (!hit) factor = 0;
597
+
598
+ if (this.raycastLine) {
599
+ this.raycastLine.scale.z = factor * (this.lastHit?.distance ?? 9999);
600
+ const mat = this.raycastLine.material as LineBasicMaterial;
601
+ if (hit != null) mat.color = WebXRController.raycastColor;
602
+ else mat.color = WebXRController.raycastNoHitColor;
603
+ }
604
+ if (this._raycastHitPoint) {
605
+ if (this.lastHit != null) {
606
+ this._raycastHitPoint.position.z = -1;// -this.lastHit.distance;
607
+ const scale = Mathf.clamp(this.lastHit.distance * .01 * factor, .015, .1);
608
+ this._raycastHitPoint.scale.set(scale, scale, scale);
609
+ }
610
+ this._raycastHitPoint.visible = this.lastHit !== null && this.lastHit !== undefined;
611
+ }
612
+ return hit;
613
+ }
614
+
615
+ private onSelectStart() {
616
+ if (!this.context.connection.allowEditing) return;
617
+ // console.log("SELECT START", _event);
618
+ // if we process the event immediately the controller
619
+ // world positions are not yet correctly updated and we have info from the last frame
620
+ // so we delay the event processing one frame
621
+ // only necessary for AR - ideally we can get it to work right here
622
+ // but should be fine as a workaround for now
623
+ this.selectStartCallback = () => this.onHandleSelectStart();
624
+ }
625
+
626
+ private selectStartCallback: Function | null = null;
627
+ private lastSelectStartObject: Object3D | null = null;;
628
+
629
+ private onHandleSelectStart() {
630
+ this.selectStartCallback = null;
631
+ this._selectionPressed = true;
632
+ this._selectionStartTime = this.context.time.time;
633
+ this._selectionEndTime = 1000;
634
+ // console.log("DOWN", this.index, WebXRController.eventSubs);
635
+
636
+ // let maxDistance = this.isUsingHands ? .1 : undefined;
637
+ let intersections: Intersection[] | null = null;
638
+ let closeGrab: boolean = false;
639
+ if (this.isUsingHands) {
640
+ intersections = this.overlap();
641
+ if (intersections.length <= 0) {
642
+ intersections = this.raycast();
643
+ closeGrab = false;
644
+ }
645
+ else {
646
+ closeGrab = true;
647
+ }
648
+ }
649
+ else intersections = this.raycast();
650
+
651
+ if (debug)
652
+ console.log("onHandleSelectStart", "close grab? " + closeGrab, "intersections", intersections);
653
+
654
+ if (intersections && intersections.length > 0) {
655
+ for (const intersection of intersections) {
656
+ const object = intersection.object;
657
+ this.lastSelectStartObject = object;
658
+ const args = { selected: object, grab: object };
659
+ const subs = WebXRController.eventSubs[ControllerEvents.SelectStart];
660
+ if (subs && subs.length > 0) {
661
+ for (const sub of subs) {
662
+ sub(this, args);
663
+ }
664
+ }
665
+ if (args.grab !== object && debug)
666
+ console.log("Grabbed object changed", "original", object, "new", args.grab);
667
+ if (args.grab) {
668
+ this.grabbed = AttachedObject.TryTake(this, args.grab, intersection, closeGrab);
669
+ }
670
+ break;
671
+ }
672
+ }
673
+ else {
674
+ const subs = WebXRController.eventSubs[ControllerEvents.SelectStart];
675
+ const args = { selected: null, grab: null };
676
+ if (subs && subs.length > 0) {
677
+ for (const sub of subs) {
678
+ sub(this, args);
679
+ }
680
+ }
681
+ }
682
+ }
683
+
684
+ private _didNotEndSelection: boolean = false;
685
+
686
+ private onSelectEnd() {
687
+ if (this.isUsingHands) {
688
+ if (this.handPointerModel.pinched) {
689
+ this._didNotEndSelection = true;
690
+ return;
691
+ }
692
+ }
693
+
694
+ if (!this._selectionPressed) return;
695
+ this.selectStartCallback = null;
696
+ this._selectionPressed = false;
697
+ this._selectionEndTime = this.context.time.time;
698
+
699
+ const args = { grab: this.grabbed?.selected ?? this.lastSelectStartObject };
700
+ const subs = WebXRController.eventSubs[ControllerEvents.SelectEnd];
701
+ if (subs && subs.length > 0) {
702
+ for (const sub of subs) {
703
+ sub(this, args);
704
+ }
705
+ }
706
+
707
+ if (this.grabbed) {
708
+ this.grabbed.free();
709
+ this.grabbed = null;
710
+ }
711
+ }
712
+
713
+ private testIsVisible(obj: Object3D | null): boolean {
714
+ if (!obj) return false;
715
+ if (GameObject.isActiveInHierarchy(obj) === false) return false;
716
+ if (UIRaycastUtils.isInteractable(obj) === false) {
717
+ return false;
718
+ }
719
+ return true;
720
+ // if (!obj.visible) return false;
721
+ // return this.testIsVisible(obj.parent);
722
+ }
723
+
724
+ private setControllerLayers(obj: Object3D, layer: number) {
725
+ if (!obj) return;
726
+ obj.layers.set(layer);
727
+ if (obj.children) {
728
+ for (const ch of obj.children) {
729
+ if (this.grabbed?.selected === ch || this.grabbed?.selectedMesh === ch) {
730
+ continue;
731
+ }
732
+ this.setControllerLayers(ch, layer);
733
+ }
734
+ }
735
+ }
736
+
737
+ public getRay(): Ray {
738
+ const ray = new Ray();
739
+ // this.tempMatrix.identity().extractRotation(this.controller.matrixWorld);
740
+ // ray.origin.setFromMatrixPosition(this.controller.matrixWorld);
741
+ ray.origin.copy(getWorldPosition(this.controller));
742
+ ray.direction.set(0, 0, -1).applyQuaternion(this.rayRotation);
743
+ return ray;
744
+ }
745
+
746
+ private closeGrabBoundingBoxHelper?: BoxHelper;
747
+
748
+ public overlap(): Intersection[] {
749
+ const overlapCenter = (this.isUsingHands && this.handPointerModel) ? this.handPointerModel.pointerObject : this.controllerGrip;
750
+
751
+ if (debug) {
752
+ if (!this.closeGrabBoundingBoxHelper && overlapCenter) {
753
+ this.closeGrabBoundingBoxHelper = new BoxHelper(overlapCenter, 0xffff00);
754
+ this.scene.add(this.closeGrabBoundingBoxHelper);
755
+ }
756
+
757
+ if (this.closeGrabBoundingBoxHelper && overlapCenter) {
758
+ this.closeGrabBoundingBoxHelper.setFromObject(overlapCenter);
759
+ }
760
+ }
761
+
762
+ if (!overlapCenter)
763
+ return new Array<Intersection>();
764
+
765
+ const wp = getWorldPosition(overlapCenter).clone();
766
+ return this.context.physics.sphereOverlap(wp, .02);
767
+ }
768
+
769
+ public raycast(): Intersection[] {
770
+ const opts = new RaycastOptions();
771
+ opts.layerMask = new Layers();
772
+ opts.layerMask.enableAll();
773
+ opts.layerMask.disable(2);
774
+ opts.ray = this.getRay();
775
+ const hits = this.context.physics.raycast(opts);
776
+ for (let i = 0; i < hits.length; i++) {
777
+ const hit = hits[i];
778
+ const obj = hit.object;
779
+ if (!this.testIsVisible(obj)) {
780
+ hits.splice(i, 1);
781
+ i--;
782
+ continue;
783
+ }
784
+ hit.object = UIRaycastUtils.getObject(obj);
785
+ break;
786
+ }
787
+ // console.log(...hits);
788
+ return hits;
789
+ }
790
+ }
791
+
792
+
793
+ export enum AttachedObjectEvents {
794
+ WillTake = "WillTake",
795
+ DidTake = "DidTake",
796
+ WillFree = "WillFree",
797
+ DidFree = "DidFree",
798
+ }
799
+
800
+ export class AttachedObject {
801
+
802
+ public static Events: { [key: string]: Function[] } = {};
803
+ public static AddEventListener(event: AttachedObjectEvents, callback: Function): Function {
804
+ if (!AttachedObject.Events[event]) AttachedObject.Events[event] = [];
805
+ AttachedObject.Events[event].push(callback);
806
+ return callback;
807
+ }
808
+ public static RemoveEventListener(event: AttachedObjectEvents, callback: Function | null) {
809
+ if (!callback) return;
810
+ if (!AttachedObject.Events[event]) return;
811
+ const idx = AttachedObject.Events[event].indexOf(callback);
812
+ if (idx >= 0) AttachedObject.Events[event].splice(idx, 1);
813
+ }
814
+
815
+
816
+ public static Current: AttachedObject[] = [];
817
+
818
+ private static Register(obj: AttachedObject) {
819
+
820
+ if (!this.Current.find(x => x === obj)) {
821
+ this.Current.push(obj);
822
+ }
823
+ }
824
+
825
+ private static Remove(obj: AttachedObject) {
826
+ const i = this.Current.indexOf(obj);
827
+ if (i >= 0) {
828
+ this.Current.splice(i, 1);
829
+ }
830
+ }
831
+
832
+ public static TryTake(controller: WebXRController, candidate: Object3D, intersection: Intersection, closeGrab: boolean): AttachedObject | null {
833
+ const interactable = GameObject.getComponentInParent(candidate, Interactable);
834
+ if (!interactable) {
835
+ if (debug)
836
+ console.warn("Prevented taking object that is not interactable", candidate);
837
+ return null;
838
+ }
839
+ else candidate = interactable.gameObject;
840
+
841
+
842
+ let objectToAttach = candidate;
843
+ const sync = GameObject.getComponentInParent(candidate, SyncedTransform);
844
+ if (sync) {
845
+ sync.requestOwnership();
846
+ objectToAttach = sync.gameObject;
847
+ }
848
+
849
+ for (const o of this.Current) {
850
+ if (o.selected === objectToAttach) {
851
+ if (o.controller === controller) return o;
852
+ o.free();
853
+ o.Take(controller, objectToAttach, candidate, sync, interactable, intersection, closeGrab);
854
+ return o;
855
+ }
856
+ }
857
+
858
+ const att = new AttachedObject();
859
+ att.Take(controller, objectToAttach, candidate, sync, interactable, intersection, closeGrab);
860
+ return att;
861
+ }
862
+
863
+
864
+ public sync: SyncedTransform | null = null;
865
+ public selected: Object3D | null = null;
866
+ public selectedParent: Object3D | null = null;
867
+ public selectedMesh: Mesh | null = null;
868
+ public controller: WebXRController | null = null;
869
+ public grabTime: number = 0;
870
+ public grabUUID: string = "";
871
+ public isCloseGrab: boolean = false; // when taken via sphere cast with hands
872
+
873
+ private originalMaterial: Material | Material[] | null = null;
874
+ private usageMarker: UsageMarker | null = null;
875
+ private rigidbodies: Rigidbody[] | null = null;
876
+ private didReparent: boolean = false;
877
+ private grabDistance: number = 0;
878
+ private interactable: Interactable | null = null;
879
+ private positionSource: Object3D | null = null;
880
+
881
+ private Take(controller: WebXRController, take: Object3D, hit: Object3D, sync: SyncedTransform | null, _interactable: Interactable,
882
+ intersection: Intersection, closeGrab: boolean)
883
+ : AttachedObject {
884
+ console.assert(take !== null, "Expected object to be taken but was", take);
885
+
886
+ if (controller.isUsingHands) {
887
+ this.positionSource = closeGrab ? controller.wrist : controller.controller;
888
+ }
889
+ else {
890
+ this.positionSource = controller.controller;
891
+ }
892
+ if (!this.positionSource) {
893
+ console.warn("No position source");
894
+ return this;
895
+ }
896
+
897
+ const args = { controller, take, hit, sync, interactable: _interactable };
898
+ AttachedObject.Events.WillTake?.forEach(x => x(this, args));
899
+
900
+
901
+ const mesh = hit as Mesh;
902
+ if (mesh?.material) {
903
+ this.originalMaterial = mesh.material;
904
+ if (!Array.isArray(mesh.material)) {
905
+ mesh.material = (mesh.material as Material).clone();
906
+ if (mesh.material && mesh.material["emissive"])
907
+ mesh.material["emissive"].b = .2;
908
+ }
909
+ }
910
+
911
+ this.selected = take;
912
+ if (!this.selectedParent) {
913
+ this.selectedParent = take.parent;
914
+ }
915
+ this.selectedMesh = mesh;
916
+ this.controller = controller;
917
+ this.interactable = _interactable;
918
+ this.isCloseGrab = closeGrab;
919
+ // if (interactable.canGrab) {
920
+ // this.didReparent = true;
921
+ // this.device.controller.attach(take);
922
+ // }
923
+ // else
924
+ this.didReparent = false;
925
+
926
+
927
+ this.sync = sync;
928
+ this.grabTime = controller.context.time.time;
929
+ this.grabUUID = Date.now().toString();
930
+ this.usageMarker = GameObject.addNewComponent(this.selected, UsageMarker);
931
+ this.rigidbodies = GameObject.getComponentsInChildren(this.selected, Rigidbody);
932
+ getWorldPosition(this.positionSource, this.lastControllerWorldPos);
933
+ const getGrabPoint = () => closeGrab ? this.lastControllerWorldPos.clone() : intersection.point.clone();
934
+ this.grabDistance = getGrabPoint().distanceTo(this.lastControllerWorldPos);
935
+ this.totalChangeAlongDirection = 0.0;
936
+
937
+ // we're storing position relative to the grab point
938
+ // we're storing rotation relative to the ray
939
+ this.localPositionOffsetToGrab = this.selected.worldToLocal(getGrabPoint());
940
+ const rot = controller.isUsingHands && closeGrab ? this.controller.getWristQuaternion()!.clone() : controller.rayRotation.clone();
941
+ getWorldQuaternion(this.selected, this.localQuaternionToGrab).premultiply(rot.invert());
942
+
943
+ const rig = this.controller.webXR.Rig;
944
+ if (rig)
945
+ this.rigPositionLastFrame.copy(getWorldPosition(rig))
946
+
947
+ Avatar_POI.Add(controller.context, this.selected);
948
+ AttachedObject.Register(this);
949
+
950
+ if (this.sync) {
951
+ this.sync.fastMode = true;
952
+ }
953
+
954
+ AttachedObject.Events.DidTake?.forEach(x => x(this, args));
955
+
956
+ return this;
957
+ }
958
+
959
+ public free(): void {
960
+ if (!this.selected) return;
961
+
962
+ const args = { controller: this.controller, take: this.selected, hit: this.selected, sync: this.sync, interactable: null };
963
+ AttachedObject.Events.WillFree?.forEach(x => x(this, args));
964
+
965
+ Avatar_POI.Remove(this.controller!.context, this.selected);
966
+ AttachedObject.Remove(this);
967
+
968
+ if (this.sync) {
969
+ this.sync.fastMode = false;
970
+ }
971
+
972
+ const mesh = this.selectedMesh;
973
+ if (mesh && this.originalMaterial && mesh.material) {
974
+ mesh.material = this.originalMaterial;
975
+ }
976
+
977
+ const object = this.selected;
978
+ // only attach the object back if it has a parent
979
+ // no parent means it was destroyed while holding it!
980
+ if (this.didReparent && object.parent) {
981
+ const prevParent = this.selectedParent;
982
+ if (prevParent) prevParent.attach(object);
983
+ else this.controller?.context.scene.attach(object);
984
+ }
985
+
986
+ this.usageMarker?.destroy();
987
+
988
+ if (this.controller)
989
+ this.controller.grabbed = null;
990
+ this.selected = null;
991
+ this.selectedParent = null;
992
+ this.selectedMesh = null;
993
+ this.sync = null;
994
+
995
+
996
+ // TODO: make throwing work again
997
+ if (this.rigidbodies) {
998
+ for (const rb of this.rigidbodies) {
999
+ rb.wakeUp();
1000
+ rb.setVelocity(rb.smoothedVelocity);
1001
+ }
1002
+ }
1003
+ this.rigidbodies = null;
1004
+
1005
+ this.localPositionOffsetToGrab = null;
1006
+ this.quaternionLerp = null;
1007
+
1008
+ AttachedObject.Events.DidFree?.forEach(x => x(this, args));
1009
+ }
1010
+
1011
+ public grabPoint: Vector3 = new Vector3();
1012
+
1013
+ private localPositionOffsetToGrab: Vector3 | null = null;
1014
+ private localPositionOffsetToGrab_worldSpace: Vector3 = new Vector3();
1015
+ private localQuaternionToGrab: Quaternion = new Quaternion(0, 0, 0, 1);
1016
+ private targetDir: Vector3 | null = null;
1017
+ private quaternionLerp: Quaternion | null = null;
1018
+
1019
+ private controllerDir = new Vector3();
1020
+ private controllerWorldPos = new Vector3();
1021
+ private lastControllerWorldPos = new Vector3();
1022
+ private controllerPosDelta = new Vector3();
1023
+ private totalChangeAlongDirection = 0.0;
1024
+ private rigPositionLastFrame = new Vector3();
1025
+
1026
+ private controllerMovementSinceLastFrame() {
1027
+ if (!this.positionSource || !this.controller) return 0.0;
1028
+
1029
+ // controller direction
1030
+ this.controllerDir.set(0, 0, -1);
1031
+ this.controllerDir.applyQuaternion(this.controller.rayRotation);
1032
+
1033
+ // controller delta
1034
+ getWorldPosition(this.positionSource, this.controllerWorldPos);
1035
+ this.controllerPosDelta.copy(this.controllerWorldPos);
1036
+ this.controllerPosDelta.sub(this.lastControllerWorldPos);
1037
+ this.lastControllerWorldPos.copy(this.controllerWorldPos);
1038
+ const rig = this.controller.webXR.Rig;
1039
+ if (rig) {
1040
+ const rigPos = getWorldPosition(rig);
1041
+ const rigDelta = this.rigPositionLastFrame.sub(rigPos);
1042
+ this.controllerPosDelta.add(rigDelta);
1043
+ this.rigPositionLastFrame.copy(rigPos);
1044
+ }
1045
+
1046
+ // calculate delta along direction
1047
+ const changeAlongControllerDirection = this.controllerDir.dot(this.controllerPosDelta);
1048
+
1049
+ return changeAlongControllerDirection;
1050
+ }
1051
+
1052
+ public update() {
1053
+ if (this.rigidbodies)
1054
+ for (const rb of this.rigidbodies)
1055
+ rb.resetVelocities();
1056
+ // TODO: add/use sync lost ownership event
1057
+ if (this.sync && this.controller && this.controller.context.connection.isInRoom) {
1058
+ const td = this.controller.context.time.time - this.grabTime;
1059
+ // if (time.frameCount % 60 === 0) {
1060
+ // console.log("ownership?", this.selected.name, this.sync.hasOwnership(), td)
1061
+ // }
1062
+ if (td > 3) {
1063
+ // if (time.frameCount % 60 === 0) {
1064
+ // console.log(this.sync.hasOwnership())
1065
+ // }
1066
+ if (this.sync.hasOwnership() === false) {
1067
+ console.log("no ownership, will leave", this.sync.guid);
1068
+ this.free();
1069
+ }
1070
+ }
1071
+ }
1072
+ if (this.interactable && !this.interactable.canGrab) return;
1073
+
1074
+ if (!this.didReparent && this.selected && this.controller) {
1075
+
1076
+ const rigScale = this.controller.webXR.Rig?.scale.x ?? 1.0;
1077
+
1078
+ this.totalChangeAlongDirection += this.controllerMovementSinceLastFrame();
1079
+ // console.log(this.totalChangeAlongDirection);
1080
+
1081
+ // alert("yo: " + this.controller.webXR.Rig?.scale.x); // this is 0.1 on Hololens
1082
+ let currentDist = 1.0;
1083
+ if (this.controller.type === ControllerType.PhysicalDevice) // only for controllers and not on touch (AR touches are controllers)
1084
+ {
1085
+ currentDist = Math.max(0.0, 1 + this.totalChangeAlongDirection * 2.0 / rigScale);
1086
+ currentDist = currentDist * currentDist * currentDist;
1087
+ }
1088
+ if (this.grabDistance / rigScale < 0.8) currentDist = 1.0; // don't accelerate if this is a close grab, want full control
1089
+
1090
+ if (!this.targetDir) {
1091
+ this.targetDir = new Vector3();
1092
+ }
1093
+ this.targetDir.set(0, 0, -this.grabDistance * currentDist);
1094
+ const target = this.targetDir.applyQuaternion(this.controller.rayRotation).add(this.controllerWorldPos);
1095
+
1096
+ // apply rotation
1097
+ const targetQuat = this.controller.rayRotation.clone().multiplyQuaternions(this.controller.rayRotation, this.localQuaternionToGrab);
1098
+ if (!this.quaternionLerp) {
1099
+ this.quaternionLerp = targetQuat.clone();
1100
+ }
1101
+ this.quaternionLerp.slerp(targetQuat, this.controller.useSmoothing ? this.controller.context.time.deltaTime / .03 : 1.0);
1102
+ setWorldQuaternion(this.selected, this.quaternionLerp);
1103
+ this.selected.updateWorldMatrix(false, false); // necessary so that rotation is correct for the following position update
1104
+
1105
+ // apply position
1106
+ this.grabPoint.copy(target);
1107
+ // apply local grab offset
1108
+ if (this.localPositionOffsetToGrab) {
1109
+ this.localPositionOffsetToGrab_worldSpace.copy(this.localPositionOffsetToGrab);
1110
+ this.selected.localToWorld(this.localPositionOffsetToGrab_worldSpace).sub(getWorldPosition(this.selected));
1111
+ target.sub(this.localPositionOffsetToGrab_worldSpace);
1112
+ }
1113
+ setWorldPosition(this.selected, target);
1114
+ }
1115
+
1116
+
1117
+ if (this.rigidbodies != null) {
1118
+ for (const rb of this.rigidbodies) {
1119
+ rb.wakeUp();
1120
+ }
1121
+ }
1122
+
1123
+ InstancingUtil.markDirty(this.selected, true);
1124
+ }
1125
+ }
src/engine-components/webxr/WebXRGrabRendering.ts ADDED
@@ -0,0 +1,151 @@
1
+ import { getWorldPosition, setWorldPosition, setWorldPositionXYZ } from "../../engine/engine_three_utils";
2
+ import { Behaviour, GameObject } from "../Component";
3
+ import { AttachedObject, AttachedObjectEvents } from "./WebXRController";
4
+ import { Object3D, Vector3 } from "three";
5
+ import { PlayerColor } from "../PlayerColor";
6
+ import { Context } from "../../engine/engine_setup";
7
+ import { IModel, SendQueue } from "../../engine/engine_networking_types";
8
+
9
+ enum XRGrabEvent {
10
+ StartOrUpdate = "xr-grab-visual-start-or-update",
11
+ End = "xr-grab-visual-end",
12
+ }
13
+
14
+ export class XRGrabModel implements IModel {
15
+ guid!: any;
16
+ dontSave: boolean = true;
17
+
18
+ userId : string | null | undefined;
19
+ point: { x: number, y: number, z: number } = { x: 0, y: 0, z: 0 };
20
+ source: { x: number, y: number, z: number } = { x: 0, y: 0, z: 0 };
21
+ target: string | undefined;
22
+
23
+ update(context : Context, point: Vector3, source: Vector3, target: string | undefined = undefined) {
24
+ this.userId = context.connection.connectionId;
25
+ this.point.x = point.x;
26
+ this.point.y = point.y;
27
+ this.point.z = point.z;
28
+ this.source.x = source.x;
29
+ this.source.y = source.y;
30
+ this.source.z = source.z;
31
+ this.target = target;
32
+ }
33
+ }
34
+
35
+ // sends grab info to other users and creates rendering instances
36
+ export class XRGrabRendering extends Behaviour {
37
+ prefab: Object3D | null = null;
38
+
39
+ private _grabModels: Array<XRGrabModel> = [];
40
+ private _grabModelsUpdateTime: Array<number> = [];
41
+ private _addOrUpdateSub: Function | null = null;
42
+ private _endSub: Function | null = null;
43
+ private _freeSub: Function | null = null;
44
+ private _instances: { [key: string]: {instance:Object3D, model:XRGrabModel} } = {};
45
+
46
+ awake(): void {
47
+ if(this.prefab) this.prefab.visible = false;
48
+ }
49
+
50
+ onEnable(): void {
51
+ this._addOrUpdateSub = this.context.connection.beginListen(XRGrabEvent.StartOrUpdate, this.onRemoteGrabStartOrUpdate.bind(this));
52
+ this._endSub = this.context.connection.beginListen(XRGrabEvent.End, this.onRemoteGrabEnd.bind(this));
53
+ this._freeSub = AttachedObject.AddEventListener(AttachedObjectEvents.WillFree, this.onAttachedObjectFree.bind(this));
54
+ }
55
+
56
+ onDisable(): void {
57
+ this.context.connection.stopListen(XRGrabEvent.StartOrUpdate, this._addOrUpdateSub);
58
+ this.context.connection.stopListen(XRGrabEvent.End, this._endSub);
59
+ AttachedObject.RemoveEventListener(AttachedObjectEvents.WillFree, this._freeSub);
60
+ }
61
+
62
+ addOrUpdateGrab(model: XRGrabModel) {
63
+ this.context.connection.send(XRGrabEvent.StartOrUpdate, model, SendQueue.Queued);
64
+ }
65
+
66
+ endGrab(model: XRGrabModel) {
67
+ this.context.connection.send(XRGrabEvent.End, model, SendQueue.Queued);
68
+ }
69
+
70
+ private onRemoteGrabStartOrUpdate(data: XRGrabModel) {
71
+ if(!this.prefab) return;
72
+ const inst = this._instances[data.guid];
73
+ if(!inst)
74
+ {
75
+ const instance = GameObject.instantiate(this.prefab) as Object3D;
76
+ instance.visible = true;
77
+ this._instances[data.guid] = {instance, model:data};
78
+ if(data.userId){
79
+ const playerColor = GameObject.getComponentsInChildren(instance, PlayerColor);
80
+ if(playerColor?.length > 0)
81
+ {
82
+ for(const pl of playerColor){
83
+ pl.assignUserColor(data.userId)
84
+ }
85
+ }
86
+ }
87
+ return;
88
+ }
89
+ inst.model = data;
90
+ }
91
+
92
+ private onRemoteGrabEnd(data: XRGrabModel) {
93
+ if (!data) return;
94
+ const id = data.guid;
95
+ if(this._instances[id])
96
+ {
97
+ GameObject.destroy(this._instances[id].instance);
98
+ delete this._instances[id];
99
+ }
100
+ }
101
+
102
+ private onAttachedObjectFree(att: AttachedObject) {
103
+ if (this._grabModels.length <= 0) return;
104
+ const mod = this._grabModels[0];
105
+ this.updateModel(mod, att);
106
+ this.endGrab(mod);
107
+ }
108
+
109
+ onBeforeRender() {
110
+ this.updateRendering();
111
+
112
+ if (!this.prefab) return;
113
+ this.prefab.visible = false;
114
+ if (this.context.time.frameCount % 10 !== 0) return;
115
+ for (let i = 0; i < AttachedObject.Current.length; i++) {
116
+ const att = AttachedObject.Current[i];
117
+
118
+ if (!att.controller || !att.selected) continue;
119
+
120
+ if (this._grabModels.length <= i) {
121
+ this._grabModels.push(new XRGrabModel());
122
+ this._grabModelsUpdateTime.push(0);
123
+ }
124
+ this._grabModelsUpdateTime[i] = this.context.time.time;
125
+ const model = this._grabModels[i];
126
+ this.updateModel(model, att);
127
+ this.addOrUpdateGrab(model);
128
+ }
129
+ }
130
+
131
+ private updateModel(model: XRGrabModel, att: AttachedObject) {
132
+ if (!att.controller || !att.selected) return;
133
+ model.guid = att.grabUUID;
134
+ const targetObject = att.selected["guid"];
135
+ model.update(this.context, att.grabPoint, att.controller.worldPosition, targetObject);
136
+ }
137
+
138
+ private temp : Vector3 = new Vector3();
139
+ private updateRendering() {
140
+ const step = this.context.time.deltaTime / .5;
141
+ for(const key in this._instances){
142
+ const { instance, model } = this._instances[key];
143
+ if(!instance || !model) continue;
144
+ const { point } = model;
145
+ const wp = getWorldPosition(instance);
146
+ this.temp.set(point.x, point.y, point.z);
147
+ wp.lerp(this.temp, step);
148
+ setWorldPosition(instance, wp);
149
+ }
150
+ }
151
+ }
src/engine-components/webxr/WebXRImageTracking.ts ADDED
@@ -0,0 +1,193 @@
1
+ import { WebXR } from "./WebXR";
2
+ import { serializable } from "../../engine/engine_serialization";
3
+ import { Behaviour } from "../Component";
4
+ import { Matrix4, Object3D, Quaternion, Vector, Vector3 } from "three";
5
+ import { CircularBuffer, getParam } from "../../engine/engine_utils";
6
+
7
+ // https://github.com/immersive-web/marker-tracking/blob/main/explainer.md
8
+
9
+ const debug = getParam("debugimagetracking");
10
+
11
+ const _scaleTemp = new Vector3();
12
+
13
+ export class WebXRTrackedImage {
14
+
15
+
16
+ get url(): string { return this._trackedImage.image ?? ""; }
17
+ get widthInMeters() { return this._trackedImage.widthInMeters ?? undefined; }
18
+ get bitmap(): ImageBitmap { return this._bitmap; }
19
+ readonly measuredSize: number;
20
+ readonly state: "tracked" | "emulated";
21
+
22
+ // private _matrix: Matrix4 | null = null;
23
+ // private get matrix(): Matrix4 {
24
+ // if (!this._matrix) {
25
+ // // this._matrix = WebXRTrackedImage._matrixBuffer.get();
26
+ // // const matrix = this._pose.transform.matrix;
27
+ // // this._matrix.fromArray(matrix);
28
+ // }
29
+ // return this._matrix!;
30
+ // }
31
+
32
+ /** Copy the image position to a vector */
33
+ getPosition(vec: Vector3) {
34
+ this.ensureTransformData();
35
+ vec.copy(this._position);
36
+ return vec;
37
+ }
38
+
39
+ /** Copy the image rotation to a quaternion */
40
+ getQuaternion(quat: Quaternion) {
41
+ this.ensureTransformData();
42
+ quat.copy(this._rotation);
43
+ return quat;
44
+ }
45
+
46
+ applyToObject(object: Object3D) {
47
+ this.ensureTransformData();
48
+ object.position.copy(this._position);
49
+ object.quaternion.copy(this._rotation);
50
+ }
51
+
52
+ // private static _matrixBuffer: CircularBuffer<Matrix4> = new CircularBuffer(() => new Matrix4(), 20);
53
+ private static _positionBuffer: CircularBuffer<Vector3> = new CircularBuffer(() => new Vector3(), 20);
54
+ private static _rotationBuffer: CircularBuffer<Quaternion> = new CircularBuffer(() => new Quaternion(), 20);
55
+ private _position!: Vector3;
56
+ private _rotation!: Quaternion;
57
+ private ensureTransformData() {
58
+ if (!this._position) {
59
+ this._position = WebXRTrackedImage._positionBuffer.get();
60
+ this._rotation = WebXRTrackedImage._rotationBuffer.get();
61
+ const t = this._pose.transform;
62
+ this._position.set(-t.position.x, t.position.y, -t.position.z);
63
+ this._rotation.set(-t.orientation.x, t.orientation.y, -t.orientation.z, t.orientation.w);
64
+ }
65
+ }
66
+
67
+ private readonly _trackingComponent: WebXRImageTracking;;
68
+ private readonly _trackedImage: WebXRImageTrackingModel;
69
+ private readonly _bitmap: ImageBitmap;
70
+ private readonly _pose: any;
71
+
72
+ constructor(context: WebXRImageTracking, trackedImage: WebXRImageTrackingModel, bitmap: ImageBitmap, measuredSize: number, state: "tracked" | "emulated", pose: any) {
73
+ this._trackingComponent = context;;
74
+ this._trackedImage = trackedImage;
75
+ this._bitmap = bitmap;
76
+ this.measuredSize = measuredSize;
77
+ this.state = state;
78
+ this._pose = pose;
79
+ }
80
+
81
+ }
82
+
83
+ declare type WebXRImageTrackingEvent = (images: WebXRImageTrackingEvent[]) => void;
84
+
85
+ export class WebXRImageTrackingModel {
86
+
87
+ @serializable(URL)
88
+ image?: string;
89
+
90
+ @serializable()
91
+ widthInMeters!: number;
92
+
93
+ }
94
+
95
+ export class WebXRImageTracking extends Behaviour {
96
+
97
+ @serializable(WebXRImageTrackingModel)
98
+ trackedImages!: WebXRImageTrackingModel[];
99
+
100
+
101
+
102
+
103
+ private readonly trackedImageIndexMap: Map<number, WebXRImageTrackingModel> = new Map();
104
+
105
+ private static _imageElements: Map<string, ImageBitmap | null> = new Map();
106
+
107
+ awake(): void {
108
+ for (const trackedImage of this.trackedImages) {
109
+ if (trackedImage.image) {
110
+ if (WebXRImageTracking._imageElements.has(trackedImage.image)) {
111
+ }
112
+ else {
113
+ const url = trackedImage.image;
114
+ WebXRImageTracking._imageElements.set(url, null);
115
+ const imageElement = document.createElement("img") as HTMLImageElement;
116
+ imageElement.src = url;
117
+ imageElement.addEventListener("load", async () => {
118
+ const img = await createImageBitmap(imageElement);
119
+ WebXRImageTracking._imageElements.set(url, img);
120
+ });
121
+ }
122
+ }
123
+ }
124
+ }
125
+
126
+ onEnable(): void {
127
+ WebXR.addEventListener("modify-ar-options", this.onModifyAROptions);
128
+ }
129
+
130
+ onDisable(): void {
131
+ WebXR.removeEventListener("modify-ar-options", this.onModifyAROptions);
132
+ }
133
+
134
+
135
+ private onModifyAROptions = (event: any) => {
136
+ const options = event.detail;
137
+ const features = options.optionalFeatures || [];
138
+ if (!features.includes("image-tracking"))
139
+ features.push("image-tracking");
140
+ options.optionalFeatures = features;
141
+
142
+ options.trackedImages = [];
143
+ for (const trackedImage of this.trackedImages) {
144
+ if (trackedImage.image?.length && trackedImage.widthInMeters > 0) {
145
+ const bitmap = WebXRImageTracking._imageElements.get(trackedImage.image);
146
+ if (bitmap) {
147
+ this.trackedImageIndexMap.set(options.trackedImages.length, trackedImage);
148
+ options.trackedImages.push({
149
+ image: bitmap,
150
+ widthInMeters: trackedImage.widthInMeters
151
+ });
152
+ }
153
+ }
154
+ }
155
+ }
156
+
157
+ onBeforeRender(frame: XRFrame | null): void {
158
+ //@ts-ignore
159
+ if (frame?.session && typeof frame.getImageTrackingResults === "function") {
160
+ //@ts-ignore
161
+ const results = frame.getImageTrackingResults();
162
+ if (results.length) {
163
+ const space = this.context.renderer.xr.getReferenceSpace();
164
+ if (space) {
165
+ const images: WebXRTrackedImage[] = [];
166
+ for (const result of results) {
167
+ const imageIndex = result.index;
168
+ const trackedImage = this.trackedImageIndexMap.get(imageIndex);
169
+ if (trackedImage) {
170
+ const pose = frame.getPose(result.imageSpace, space);
171
+ const state = result.trackingState;
172
+ const imageData = new WebXRTrackedImage(this, trackedImage, result.image, result.measuredSize, state, pose);
173
+ images.push(imageData);
174
+ }
175
+ else {
176
+ if (debug) {
177
+ console.warn("No tracked image for index", imageIndex);
178
+ }
179
+ }
180
+ }
181
+ if (images.length > 0) {
182
+ try {
183
+ this.dispatchEvent(new CustomEvent("image-tracking", { detail: images }));
184
+ }
185
+ catch (e) {
186
+ console.error(e);
187
+ }
188
+ }
189
+ }
190
+ }
191
+ }
192
+ }
193
+ }
src/engine-components/webxr/WebXRPlaneTracking.ts ADDED
@@ -0,0 +1,254 @@
1
+ import { BufferAttribute, BufferGeometry, Group, Mesh, Object3D, Vector3 } from "three";
2
+ import { MeshCollider } from "../Collider";
3
+ import { Behaviour, GameObject } from "../Component";
4
+ import { WebXR, WebXREvent } from "./WebXR";
5
+ import { serializable } from "../../engine/engine_serialization";
6
+ import { Vec3 } from "../../engine/engine_types";
7
+ import { disposeObjectResources } from "../../engine/engine_assetdatabase";
8
+ import { getParam } from "../../engine/engine_utils";
9
+
10
+ const debug = getParam("debugplanetracking");
11
+
12
+ declare type XRFramePlanes = XRFrame & {
13
+ detectedPlanes?: Set<XRPlane>;
14
+ }
15
+
16
+ export declare type XRPlaneContext = {
17
+ id: number;
18
+ xrPlane: XRPlane;
19
+ timestamp: number;
20
+ mesh?: Mesh | Group;
21
+ collider?: MeshCollider;
22
+ }
23
+
24
+ export declare type WebXRPlaneTrackingEvent = {
25
+ type: "plane-added" | "plane-updated" | "plane-removed";
26
+ context: XRPlaneContext;
27
+ }
28
+
29
+ export class WebXRPlaneTracking extends Behaviour {
30
+
31
+ /** Optional: if assigned it will be instantiated per tracked plane */
32
+ @serializable(Object3D)
33
+ planeTemplate?: Object3D;
34
+
35
+ get trackedPlanes() { return this._allPlanes.values(); }
36
+
37
+
38
+
39
+
40
+ onEnable(): void {
41
+ WebXR.addEventListener(WebXREvent.XRUpdate, this.onXRUpdate);
42
+ WebXR.addEventListener("modify-ar-options", this.onModifyAROptions);
43
+ }
44
+
45
+ onDisable(): void {
46
+ WebXR.removeEventListener(WebXREvent.XRUpdate, this.onXRUpdate);
47
+ WebXR.removeEventListener("modify-ar-options", this.onModifyAROptions);
48
+ }
49
+
50
+ private onModifyAROptions = (event: any) => {
51
+ const options = event.detail;
52
+ const features = options.optionalFeatures || [];
53
+ if (!features.includes("plane-detection"))
54
+ features.push("plane-detection");
55
+ options.optionalFeatures = features;
56
+ }
57
+
58
+ private onXRUpdate = (evt) => {
59
+ this.processPlanes(evt.rig, evt.frame);
60
+ }
61
+
62
+ private _planeId = 1;
63
+ private readonly _allPlanes = new Map<XRPlane, XRPlaneContext>();
64
+
65
+ private processPlanes(rig: Object3D, frame: XRFramePlanes) {
66
+ const renderer = this.context.renderer;
67
+
68
+ // parenting tracked planes to the XR rig ensures that they synced with the real-world user data;
69
+ // otherwise they would "swim away" when the user rotates / moves / teleports and so on.
70
+ // There may be cases where we want that! E.g. a user walks around on their own table in castle builder
71
+ if (!rig) return;
72
+
73
+ // If we dont have the detectedPlanes field this means the user didnt add the option
74
+ if (frame.detectedPlanes === undefined) return;
75
+
76
+ const referenceSpace = renderer.xr.getReferenceSpace();
77
+ if (!referenceSpace) return;
78
+
79
+ for (const plane of this._allPlanes.keys()) {
80
+ if (!frame.detectedPlanes.has(plane)) {
81
+ const planeContext = this._allPlanes.get(plane)!;
82
+ // plane was removed
83
+ this._allPlanes.delete(plane);
84
+ if (debug) console.log("Plane no longer tracked, id=" + planeContext.id);
85
+ if (planeContext.mesh)
86
+ rig?.remove(planeContext.mesh);
87
+
88
+ const evt = new CustomEvent<WebXRPlaneTrackingEvent>("plane-tracking", {
89
+ detail: {
90
+ type: "plane-removed",
91
+ context: planeContext
92
+ }
93
+ })
94
+ this.dispatchEvent(evt);
95
+ }
96
+ }
97
+
98
+ for (const plane of frame.detectedPlanes) {
99
+ const planePose = frame.getPose(plane.planeSpace, referenceSpace);
100
+
101
+ let planeMesh: Object3D | undefined;
102
+
103
+ // If the plane already existed just update it
104
+ if (this._allPlanes.has(plane)) {
105
+ const planeContext = this._allPlanes.get(plane)!;
106
+ planeMesh = planeContext.mesh;
107
+ if (planeContext.timestamp < plane.lastChangedTime) {
108
+ planeContext.timestamp = plane.lastChangedTime;
109
+
110
+
111
+ // Update the mesh geometry
112
+ if (planeContext.mesh) {
113
+ const geometry = this.createGeometry(plane.polygon);
114
+ if (planeContext.mesh instanceof Mesh) {
115
+ planeContext.mesh.geometry.dispose();
116
+ planeContext.mesh.geometry = geometry;
117
+ }
118
+ else if (planeContext.mesh instanceof Group) {
119
+ for (const ch of planeContext.mesh.children) {
120
+ if (ch instanceof Mesh) {
121
+ ch.geometry.dispose();
122
+ ch.geometry = geometry;
123
+ }
124
+ }
125
+ }
126
+
127
+ // Update the mesh collider if it exists
128
+ if (planeContext.collider) {
129
+ planeContext.collider.sharedMesh = planeContext.mesh as unknown as Mesh;
130
+ planeContext.collider.convex = true;
131
+ planeContext.collider.onDisable();
132
+ planeContext.collider.onEnable();
133
+ }
134
+ }
135
+
136
+ const evt = new CustomEvent<WebXRPlaneTrackingEvent>("plane-tracking", {
137
+ detail: {
138
+ type: "plane-updated",
139
+ context: planeContext
140
+ }
141
+ })
142
+ this.dispatchEvent(evt);
143
+ }
144
+ }
145
+ // Otherwise we create a new plane instance
146
+ else {
147
+
148
+ // if we don't have any template assigned we just use a simple mesh object
149
+ if (!this.planeTemplate) {
150
+ this.planeTemplate = new Mesh();
151
+ }
152
+
153
+ if (this.planeTemplate) {
154
+ // Create instance
155
+ const newPlane = GameObject.instantiate(this.planeTemplate) as GameObject;
156
+ planeMesh = newPlane;
157
+
158
+ if (newPlane instanceof Mesh) {
159
+ disposeObjectResources(newPlane.geometry);
160
+ newPlane.geometry = this.createGeometry(plane.polygon);
161
+ }
162
+
163
+ const mc = newPlane.getComponent(MeshCollider) as MeshCollider;
164
+ if (mc) {
165
+ mc.sharedMesh = newPlane as unknown as Mesh;
166
+ mc.convex = true;
167
+ mc.onDisable();
168
+ mc.onEnable();
169
+ }
170
+
171
+ // doesn't seem to work as MeshCollider doesn't have a clear way to refresh itself
172
+ // after the geometry has changed
173
+ // newPlane.getComponent(MeshCollider)!.sharedMesh = newPlane as unknown as Mesh;
174
+ newPlane.matrixAutoUpdate = false;
175
+ newPlane.matrixWorldNeedsUpdate = true; // force update of rendering settings and so on
176
+ rig.add(newPlane);
177
+
178
+ const planeContext: XRPlaneContext = {
179
+ id: this._planeId++,
180
+ xrPlane: plane,
181
+ timestamp: plane.lastChangedTime,
182
+ mesh: newPlane as unknown as Mesh,
183
+ collider: mc
184
+ };
185
+ this._allPlanes.set(plane, planeContext);
186
+
187
+ if (debug) console.log("New plane detected, id=" + planeContext.id);
188
+
189
+ try {
190
+ const evt = new CustomEvent<WebXRPlaneTrackingEvent>("plane-tracking", {
191
+ detail: {
192
+ type: "plane-added",
193
+ context: planeContext
194
+ }
195
+ })
196
+ this.dispatchEvent(evt);
197
+ }
198
+ catch (e) {
199
+ console.error(e);
200
+ }
201
+ }
202
+ }
203
+
204
+ if (planeMesh) {
205
+ if (planePose) {
206
+ planeMesh.visible = true;
207
+ planeMesh.matrix.fromArray(planePose.transform.matrix);
208
+ } else {
209
+ planeMesh.visible = false;
210
+ }
211
+ }
212
+ };
213
+ }
214
+
215
+
216
+ createGeometry(polygon: Vec3[]) {
217
+ const geometry = new BufferGeometry();
218
+
219
+ const vertices: number[] = [];
220
+ const uvs: number[] = [];
221
+ polygon.forEach(point => {
222
+ vertices.push(point.x, point.y, point.z);
223
+ uvs.push(point.x, point.z);
224
+ })
225
+
226
+ // get the normal of the plane by using the cross product of B-A and C-A
227
+ const a = new Vector3(vertices[0], vertices[1], vertices[2]);
228
+ const b = new Vector3(vertices[3], vertices[4], vertices[5]);
229
+ const c = new Vector3(vertices[6], vertices[7], vertices[8]);
230
+ const ab = new Vector3();
231
+ const ac = new Vector3();
232
+ ab.subVectors(b, a);
233
+ ac.subVectors(c, a);
234
+ ab.cross(ac);
235
+ ab.normalize();
236
+
237
+ const normals: number[] = [];
238
+ for (let i = 0; i < vertices.length / 3; i++) {
239
+ normals.push(ab.x, ab.y, ab.z);
240
+ }
241
+
242
+ const indices: number[] = [];
243
+ for (let i = 2; i < polygon.length; ++i) {
244
+ indices.push(0, i - 1, i);
245
+ }
246
+
247
+ geometry.setAttribute('position', new BufferAttribute(new Float32Array(vertices), 3));
248
+ geometry.setAttribute('uv', new BufferAttribute(new Float32Array(uvs), 2))
249
+ geometry.setAttribute('normal', new BufferAttribute(new Float32Array(normals), 3));
250
+ geometry.setIndex(indices);
251
+
252
+ return geometry;
253
+ }
254
+ }
src/engine-components/webxr/WebXRRig.ts ADDED
@@ -0,0 +1,22 @@
1
+ import { Object3D } from "three";
2
+ import { IGameObject } from "../../engine/engine_types";
3
+ import { getParam } from "../../engine/engine_utils";
4
+ import { Behaviour, GameObject } from "../Component";
5
+ import { BoxGizmo } from "../Gizmos";
6
+
7
+ const debug = getParam("debugrig");
8
+
9
+ export class XRRig extends Behaviour {
10
+ awake(): void {
11
+ // const helper = new AxesHelper(.1);
12
+ // this.gameObject.add(helper);
13
+ if (debug) {
14
+ const gizmoObj = new Object3D() as IGameObject;
15
+ gizmoObj.position.y += .5;
16
+ this.gameObject.add(gizmoObj);
17
+ const gizmo = gizmoObj.addNewComponent(BoxGizmo);
18
+ if (gizmo)
19
+ gizmo.isGizmo = false;
20
+ }
21
+ }
22
+ }
src/engine-components/webxr/WebXRSync.ts ADDED
@@ -0,0 +1,463 @@
1
+ import { Behaviour, GameObject } from "../Component";
2
+ import { RoomEvents, OwnershipModel, NetworkConnection } from "../../engine/engine_networking";
3
+ import { WebXR, WebXREvent } from "./WebXR";
4
+ import { Group, Quaternion, Vector3, Vector4, WebXRManager } from "three";
5
+ import { getParam } from "../../engine/engine_utils";
6
+ import { Voip } from "../Voip";
7
+ import { Builder, Long } from "flatbuffers";
8
+ import { VrUserStateBuffer } from "../../engine-schemes/vr-user-state-buffer";
9
+ import { Vec3 } from "../../engine-schemes/vec3";
10
+ import { registerType } from "../../engine-schemes/schemes";
11
+ import { Vec4 } from "../../engine-schemes/vec4";
12
+ import { WebXRAvatar } from "./WebXRAvatar";
13
+
14
+ // for debug GUI
15
+ // import { GUI } from "three/examples/jsm/libs/lil-gui.module.min.js";
16
+ // import { HTMLMesh } from 'three/examples/jsm/interactive/HTMLMesh.js';
17
+ // import { InteractiveGroup } from 'three/examples/jsm/interactive/InteractiveGroup.js';
18
+ // import { renderer, sceneData } from "../engine/engine_setup";
19
+
20
+ const debugLogs = getParam("debugxr");
21
+ const debugAvatar = getParam("debugavatar");
22
+ // const debugAvatarVoip = getParam("debugavatarvoip");
23
+
24
+ enum WebXRSyncEvent {
25
+ WebXR_UserJoined = "webxr-user-joined",
26
+ WebXR_UserLeft = "webxr-user-left",
27
+ VRSessionStart = "vr-session-started",
28
+ VRSessionEnd = "vr-session-ended",
29
+ VRSessionUpdate = "vr-session-update",
30
+ }
31
+
32
+ enum XRMode {
33
+ VR = "vr",
34
+ AR = "ar",
35
+ }
36
+
37
+ const VRUserStateBufferIdentifier = "VRUS";
38
+ registerType(VRUserStateBufferIdentifier, VrUserStateBuffer.getRootAsVrUserStateBuffer);
39
+
40
+ function getTimeStampNow() {
41
+ return new Date().getTime(); // avoid sending millis in flatbuffer
42
+ }
43
+
44
+ function flatbuffers_long_from_number(num: number): Long {
45
+ let low = num & 0xffffffff
46
+ let high = (num / Math.pow(2, 32)) & 0xfffff
47
+ return Long.create(low, high);
48
+ }
49
+
50
+ export class VRUserState {
51
+ public guid: string;
52
+ public time!: number;
53
+ public avatarId!: string;
54
+ public position: Vector3 = new Vector3();
55
+ public rotation: Vector4 = new Vector4();
56
+ public scale: number = 1;
57
+
58
+ public posLeftHand = new Vector3();
59
+ public posRightHand = new Vector3();
60
+
61
+ public rotLeftHand = new Quaternion();
62
+ public rotRightHand = new Quaternion();
63
+
64
+ public constructor(guid: string) {
65
+ this.guid = guid;
66
+ }
67
+
68
+ private static invertRotation: Quaternion = new Quaternion().setFromAxisAngle(new Vector3(0, 1, 0), Math.PI);
69
+
70
+ public update(rig: Group, pos: DOMPointReadOnly, rot: DOMPointReadOnly, webXR: WebXR, avatarId: string) {
71
+ this.time = getTimeStampNow();
72
+ this.avatarId = avatarId;
73
+ this.position.set(pos.x, pos.y, pos.z);
74
+ if (rig)
75
+ this.position.applyMatrix4(rig.matrixWorld);
76
+
77
+ let q0 = VRUserState.quat0;
78
+ const q1 = VRUserState.quat1;
79
+ q0.set(rot.x, rot.y, rot.z, rot.w);
80
+ q0 = q0.multiplyQuaternions(q0, VRUserState.invertRotation);
81
+
82
+ if (rig) {
83
+ rig.getWorldQuaternion(q1);
84
+ q0.multiplyQuaternions(q1, q0);
85
+ }
86
+
87
+ this.rotation.set(q0.x, q0.y, q0.z, q0.w);
88
+ this.scale = rig.scale.x;
89
+
90
+ // for controllers, it seems we need grip pose
91
+ const ctrl0 = webXR.LeftController?.controllerGrip;
92
+ if (ctrl0) {
93
+ ctrl0.getWorldPosition(this.posLeftHand);
94
+ ctrl0.getWorldQuaternion(this.rotLeftHand);
95
+ }
96
+ const ctrl1 = webXR.RightController?.controllerGrip;
97
+ if (ctrl1) {
98
+ ctrl1.getWorldPosition(this.posRightHand);
99
+ ctrl1.getWorldQuaternion(this.rotRightHand);
100
+ }
101
+
102
+ // if this is a hand, we need to get the root bone of that / use that for position/rotation
103
+ if (webXR.LeftController?.hand?.visible) {
104
+ const wrist = webXR.LeftController.wrist;
105
+ if (wrist) {
106
+ wrist.getWorldPosition(this.posLeftHand);
107
+ wrist.getWorldQuaternion(this.rotLeftHand);
108
+ }
109
+ }
110
+
111
+ if (webXR.RightController?.hand?.visible) {
112
+ const wrist = webXR.RightController.wrist;
113
+ if (wrist) {
114
+ wrist.getWorldPosition(this.posRightHand);
115
+ wrist.getWorldQuaternion(this.rotRightHand);
116
+ }
117
+ }
118
+ }
119
+
120
+ private static quat0: Quaternion = new Quaternion();
121
+ private static quat1: Quaternion = new Quaternion();
122
+
123
+ public sendAsBuffer(builder: Builder, net: NetworkConnection) {
124
+ builder.clear();
125
+ const guid = builder.createString(this.guid);
126
+ const id = builder.createString(this.avatarId);
127
+ VrUserStateBuffer.startVrUserStateBuffer(builder);
128
+ VrUserStateBuffer.addGuid(builder, guid);
129
+ VrUserStateBuffer.addTime(builder, flatbuffers_long_from_number(this.time));
130
+ VrUserStateBuffer.addAvatarId(builder, id);
131
+ VrUserStateBuffer.addPosition(builder, Vec3.createVec3(builder, this.position.x, this.position.y, this.position.z));
132
+ VrUserStateBuffer.addRotation(builder, Vec4.createVec4(builder, this.rotation.x, this.rotation.y, this.rotation.z, this.rotation.w));
133
+ VrUserStateBuffer.addScale(builder, this.scale);
134
+ VrUserStateBuffer.addPosLeftHand(builder, Vec3.createVec3(builder, this.posLeftHand.x, this.posLeftHand.y, this.posLeftHand.z));
135
+ VrUserStateBuffer.addPosRightHand(builder, Vec3.createVec3(builder, this.posRightHand.x, this.posRightHand.y, this.posRightHand.z));
136
+ VrUserStateBuffer.addRotLeftHand(builder, Vec4.createVec4(builder, this.rotLeftHand.x, this.rotLeftHand.y, this.rotLeftHand.z, this.rotLeftHand.w));
137
+ VrUserStateBuffer.addRotRightHand(builder, Vec4.createVec4(builder, this.rotRightHand.x, this.rotRightHand.y, this.rotRightHand.z, this.rotRightHand.w));
138
+ const res = VrUserStateBuffer.endVrUserStateBuffer(builder);
139
+ builder.finish(res, VRUserStateBufferIdentifier);
140
+ const arr = builder.asUint8Array();
141
+ net.sendBinary(arr);
142
+ }
143
+
144
+ public setFromBuffer(guid: string, state: VrUserStateBuffer) {
145
+ if (!guid) return;
146
+ this.guid = guid;
147
+ this.time = state.time().toFloat64();
148
+ const id = state.avatarId();
149
+ if (id)
150
+ this.avatarId = id;
151
+ const pos = state.position();
152
+ if (pos)
153
+ this.position.set(pos.x(), pos.y(), pos.z());
154
+ // TODO: maybe just send one float more instead of converting back and forth
155
+ const rot = state.rotation();
156
+ if (rot)
157
+ this.rotation.set(rot.x(), rot.y(), rot.z(), rot.w());
158
+ const posLeftHand = state.posLeftHand();
159
+ if (posLeftHand)
160
+ this.posLeftHand.set(posLeftHand.x(), posLeftHand.y(), posLeftHand.z());
161
+ const posRightHand = state.posRightHand();
162
+ if (posRightHand)
163
+ this.posRightHand.set(posRightHand.x(), posRightHand.y(), posRightHand.z());
164
+ const rotLeftHand = state.rotLeftHand();
165
+ if (rotLeftHand)
166
+ this.rotLeftHand.set(rotLeftHand.x(), rotLeftHand.y(), rotLeftHand.z(), rotLeftHand.w());
167
+ const rotRightHand = state.rotRightHand();
168
+ if (rotRightHand)
169
+ this.rotRightHand.set(rotRightHand.x(), rotRightHand.y(), rotRightHand.z(), rotRightHand.w());
170
+ this.scale = state.scale();
171
+ }
172
+ }
173
+
174
+ export class WebXRSync extends Behaviour {
175
+
176
+ webXR: WebXR | null = null;
177
+
178
+ // private allowCustomAvatars: boolean | null = true;
179
+
180
+ private debugAvatarUser: WebXRAvatar | null = null;
181
+ private voip: Voip | null = null;
182
+
183
+ async awake() {
184
+
185
+ if(!this.webXR) this.webXR = GameObject.getComponent(this.gameObject, WebXR);
186
+ if(!this.webXR) this.webXR = GameObject.findObjectOfType(WebXR, this.context);
187
+
188
+ if(!this.webXR)
189
+ {
190
+ console.log("Missing webxr component");
191
+ this.webXR = GameObject.findObjectOfType(WebXR, this.context);
192
+ if(!this.webXR) {
193
+ console.error("Could not find webxr component");
194
+ return;
195
+ }
196
+ }
197
+
198
+ if (!this.voip) this.voip = GameObject.findObjectOfType(Voip, this.context);
199
+
200
+ if (debugAvatar) {
201
+ const debugGuid = "debug-avatar-" + debugAvatar;
202
+ const newUser = new WebXRAvatar(this.context, debugGuid, this.webXR);
203
+ // newUser.isLocalAvatar = true;
204
+ this.debugAvatarUser = newUser;
205
+ if (typeof debugAvatar === "string" && debugAvatar.length > 0) {
206
+ if (await newUser.setAvatarOverride(debugAvatar)) {
207
+ const debugState = new VRUserState(debugGuid);
208
+ debugState.position.y += 1;
209
+ const off = .5;
210
+ debugState.posLeftHand.y += off;
211
+ debugState.posLeftHand.x += off;
212
+ debugState.posRightHand.y += off;
213
+ debugState.posRightHand.x -= off;
214
+ newUser.tryUpdate(debugState, 0);
215
+ }
216
+ else {
217
+ newUser.destroy();
218
+ }
219
+ }
220
+ }
221
+ }
222
+
223
+ onEnable() {
224
+ // const debugUser = new WebXRAvatar(this.context, "sorry-no-guid", this.webXR!);
225
+
226
+ if (!this.webXR) {
227
+ this.webXR = GameObject.getComponent(this.gameObject, WebXR);
228
+ if (!this.webXR) {
229
+ console.warn("Missing webxr component on " + this.gameObject.name);
230
+ return;
231
+ }
232
+ }
233
+
234
+ this.eventSub_WebXRStartEvent = this.onXRSessionStart.bind(this);
235
+ WebXR.addEventListener(WebXREvent.XRStarted, this.eventSub_WebXRStartEvent);
236
+ this.eventSub_WebXRUpdateEvent = this.onXRSessionUpdate.bind(this);
237
+ WebXR.addEventListener(WebXREvent.XRUpdate, this.eventSub_WebXRUpdateEvent);
238
+ this.eventSub_WebXREndEvent = this.onXRSessionEnded.bind(this);
239
+ WebXR.addEventListener(WebXREvent.XRStopped, this.eventSub_WebXREndEvent);
240
+
241
+ this.eventSub_ConnectionEvent = this.onConnected.bind(this);
242
+ this.context.connection.beginListen(RoomEvents.JoinedRoom, this.eventSub_ConnectionEvent);
243
+ this.context.connection.beginListen(WebXRSyncEvent.WebXR_UserJoined, _evt => {
244
+ console.log("webxr user joined evt");
245
+ });
246
+ this.context.connection.beginListen(WebXRSyncEvent.WebXR_UserLeft, evt => {
247
+ const hasId = evt.id !== null && evt.id !== undefined;
248
+ if (!hasId) return;
249
+ console.log("webxr user left evt");
250
+ if (hasId) {
251
+ const avatar = this.avatars[evt.id];
252
+ avatar?.destroy();
253
+ this.avatars[evt.id] = undefined;
254
+ }
255
+ });
256
+ this.context.connection.beginListenBinary(VRUserStateBufferIdentifier, (state: VrUserStateBuffer) => {
257
+ // console.log("BUFFER", state);
258
+ const guid = state.guid();
259
+ if (!guid) return;
260
+ const time = state.time().toFloat64();
261
+ const temp = this.tempState;
262
+ temp.setFromBuffer(guid, state);
263
+ // console.log(temp);
264
+ const user = this.onTryGetAvatar(guid, time);
265
+ user?.tryUpdate(temp, time);
266
+ });
267
+ this.context.connection.beginListen(WebXRSyncEvent.VRSessionUpdate, (state: VRUserState) => {
268
+ const guid = state.guid;
269
+ const time = state.time;
270
+ const user = this.onTryGetAvatar(guid, time);
271
+ user?.tryUpdate(state, time);
272
+ });
273
+ }
274
+
275
+ private tempState: VRUserState = new VRUserState("");
276
+
277
+ private onTryGetAvatar(guid: string, time: number) {
278
+ if (guid === this.context.connection.connectionId) return null; // ignore self in case we receive that also!
279
+ const timeDiff = new Date().getTime() - time;
280
+ if (timeDiff > 5000) {
281
+ if (debugLogs)
282
+ console.log("old data", timeDiff, guid)
283
+ return null;
284
+ }
285
+ let user = this.avatars[guid];
286
+ if (user === undefined) {
287
+ try {
288
+ console.log("create new avatar");
289
+ const newUser = new WebXRAvatar(this.context, guid, this.webXR!);
290
+ user = newUser;
291
+ this.avatars[guid] = newUser;
292
+ } catch (err) {
293
+ this.avatars[guid] = null;
294
+ console.error(err);
295
+ }
296
+ }
297
+ return user;
298
+ }
299
+
300
+ onDisable() {
301
+ if (this.eventSub_ConnectionEvent)
302
+ this.context.connection.stopListen(RoomEvents.JoinedRoom, this.eventSub_ConnectionEvent);
303
+ WebXR.removeEventListener(WebXREvent.XRStarted, this.eventSub_WebXRStartEvent);
304
+ WebXR.removeEventListener(WebXREvent.XRUpdate, this.eventSub_WebXRUpdateEvent);
305
+ WebXR.removeEventListener(WebXREvent.XRStopped, this.eventSub_WebXREndEvent);
306
+ }
307
+
308
+ update(): void {
309
+
310
+ const now = getTimeStampNow();
311
+
312
+ if (this.debugAvatarUser) {
313
+ this.debugAvatarUser.lastUpdate = now;
314
+ }
315
+
316
+ this.detectPotentiallyDisconnectedAvatarsAndRemove();
317
+
318
+ for (const key in this.avatars) {
319
+ const avatar = this.avatars[key];
320
+ if (!avatar) continue;
321
+ avatar.update();
322
+ }
323
+ }
324
+
325
+
326
+ private _removeAvatarsList: string[] = [];
327
+ private detectPotentiallyDisconnectedAvatarsAndRemove() {
328
+ const utcnow = getTimeStampNow();
329
+ for (const key in this.avatars) {
330
+ const avatar = this.avatars[key];
331
+ if (!avatar) {
332
+ this._removeAvatarsList.push(key);
333
+ continue;
334
+ }
335
+ if (utcnow - avatar.lastUpdate > 10_000) {
336
+ console.log("avatar timed out (didnt receive any updates in a while) - destroying it now");
337
+ avatar.destroy();
338
+ this.avatars[key] = undefined;
339
+ }
340
+ }
341
+ for (const rem of this._removeAvatarsList) {
342
+ delete this.avatars[rem];
343
+ }
344
+ this._removeAvatarsList.length = 0;
345
+ }
346
+
347
+ private buildLocalAvatar() {
348
+ if (this.localAvatar) return;
349
+ const connectionId = this.context.connection?.connectionId ?? this.k_LocalAvatarNoNetworkingGuid;
350
+ this.localAvatar = new WebXRAvatar(this.context, connectionId, this.webXR!);
351
+ this.localAvatar.isLocalAvatar = true;
352
+ this.localAvatar.setAvatarOverride(this.getAvatarId());
353
+ this.avatars[this.localAvatar.guid] = this.localAvatar;
354
+ }
355
+
356
+
357
+ private eventSub_ConnectionEvent: Function | null = null;
358
+ private eventSub_WebXRStartEvent: Function | null = null;
359
+ private eventSub_WebXREndEvent: Function | null = null;
360
+ private eventSub_WebXRUpdateEvent: Function | null = null;
361
+ private avatars: { [key: string]: WebXRAvatar | undefined | null } = {}
362
+ private localAvatar: WebXRAvatar | null = null;
363
+ private k_LocalAvatarNoNetworkingGuid = "local";
364
+
365
+ private onConnected() {
366
+ // this event gets fired when we have joined a room and are ready to update
367
+ if (debugLogs)
368
+ console.log("Hey you are connected as " + this.context.connection.connectionId);
369
+
370
+ if (this.localAvatar?.guid === this.k_LocalAvatarNoNetworkingGuid) {
371
+ if (this.localAvatar) {
372
+ this.localAvatar?.destroy();
373
+ this.avatars[this.localAvatar.guid] = undefined;
374
+ }
375
+ this.localAvatar = null;
376
+ this.xrState = null;
377
+ this.ownership?.freeOwnership();
378
+ this.ownership = null;
379
+ }
380
+ }
381
+
382
+ private onXRSessionStart(_evt: { session: XRSession }) {
383
+ console.log("XR session started");
384
+ this.context.connection.send(WebXRSyncEvent.WebXR_UserJoined, { id: this.context.connection.connectionId, mode: XRMode.VR });
385
+
386
+ if (this.localAvatar) {
387
+ this.localAvatar?.destroy();
388
+ this.avatars[this.localAvatar.guid] = undefined;
389
+ this.localAvatar = null;
390
+ }
391
+ this.xrState = null;
392
+ this.ownership?.freeOwnership();
393
+ this.ownership = null;
394
+
395
+ if (this.avatars) {
396
+ for (const key in this.avatars) {
397
+ this.avatars[key]?.updateFlags();
398
+ }
399
+ }
400
+ }
401
+
402
+ private onXRSessionEnded(_evt: { session: XRSession }) {
403
+ console.log("XR session ended");
404
+ this.context.connection.send(WebXRSyncEvent.WebXR_UserLeft, { id: this.context.connection.connectionId, mode: XRMode.VR });
405
+ if(this.localAvatar){
406
+ this.localAvatar?.destroy();
407
+ this.avatars[this.localAvatar.guid] = undefined;
408
+ this.localAvatar = null;
409
+ }
410
+ }
411
+
412
+ private ownership: OwnershipModel | null = null;
413
+ private xrState: VRUserState | null = null;
414
+ private builder: Builder = new Builder(1024);
415
+
416
+ private onXRSessionUpdate(evt: { rig: Group, frame: XRFrame, xr: WebXRManager, input: XRInputSource[] }) {
417
+
418
+ this.xrState ??= new VRUserState(this.context.connection.connectionId ?? this.k_LocalAvatarNoNetworkingGuid);
419
+ this.ownership ??= new OwnershipModel(this.context.connection, this.context.connection.connectionId ?? this.k_LocalAvatarNoNetworkingGuid);
420
+ this.ownership.guid = this.context.connection.connectionId ?? this.k_LocalAvatarNoNetworkingGuid;
421
+ this.buildLocalAvatar();
422
+
423
+
424
+ const { frame, xr, rig } = evt;
425
+ const pose = frame.getViewerPose(xr.getReferenceSpace()!);
426
+ if (!pose) return; // e.g. if user is not wearing headset
427
+ const transform: XRRigidTransform = pose?.transform;
428
+ const pos = transform.position;
429
+ const rot = transform.orientation;
430
+ this.xrState.update(rig, pos, rot, this.webXR!, this.getAvatarId());
431
+
432
+ if (this.localAvatar) {
433
+ if (this.context.connection.connectionId) {
434
+ this.localAvatar.guid = this.context.connection.connectionId;
435
+ }
436
+ this.localAvatar.tryUpdate(this.xrState, 0);
437
+ }
438
+
439
+ if (this.ownership && !this.ownership.hasOwnership && this.context.connection.isConnected) {
440
+ if (this.context.time.frameCount % 120 === 0)
441
+ this.ownership.requestOwnership();
442
+ if (!this.ownership.hasOwnership) {
443
+ // console.log("NO OWNERSHIP", this.ownership.guid);
444
+ return;
445
+ }
446
+ }
447
+
448
+ if (!this.context.connection.isConnected || !this.context.connection.connectionId) {
449
+ return;
450
+ }
451
+
452
+ this.xrState.sendAsBuffer(this.builder, this.context.connection);
453
+
454
+ // this.context.connection.send(WebXRSyncEvent.VRSessionUpdate, this.xrState);
455
+
456
+ }
457
+
458
+ private getAvatarId() {
459
+ const urlAvatar = getParam("avatar") as string;
460
+ const avatarId = urlAvatar ?? null;
461
+ return avatarId;
462
+ }
463
+ }