Needle Engine 3.32.24-alpha

Files changed (52) hide show

src/engine-components/export/usdz/extensions/Animation.ts +41 -5
src/engine-components/export/usdz/utils/animationutils.ts +36 -2
src/engine-components/api.ts +1 -0
src/engine/api.ts +1 -0
src/engine-components/export/usdz/extensions/behavior/AudioExtension.ts +46 -22
src/engine-components/AudioSource.ts +1 -0
src/engine-components/webxr/Avatar.ts +2 -2
src/engine-components/export/usdz/extensions/behavior/Behaviour.ts +1 -1
src/engine-components/export/usdz/extensions/behavior/BehaviourComponents.ts +35 -13
src/engine-components/export/usdz/extensions/behavior/BehavioursBuilder.ts +2 -2
src/engine-components/ui/Canvas.ts +1 -1
src/engine-components/Component.ts +3 -3
src/engine/debug/debug_console.ts +3 -3
src/engine/debug/debug.ts +1 -0
src/engine-components/DragControls.ts +20 -2
src/engine-components/Duplicatable.ts +1 -1
src/engine/engine_addressables.ts +1 -1
src/engine/engine_context.ts +1 -1
src/engine/engine_create_objects.ts +2 -2
src/engine/engine_input.ts +15 -5
src/engine/engine_lifecycle_api.ts +1 -1
src/engine/engine_math.ts +33 -0
src/engine/engine_networking_instantiate.ts +1 -1
src/engine/engine_serialization_builtin_serializer.ts +4 -2
src/engine/engine_serialization_core.ts +1 -1
src/engine/engine_types.ts +1 -0
src/engine-components/ui/EventSystem.ts +5 -5
src/engine/extensions/extensions.ts +2 -2
src/engine/xr/internal.ts +2 -2
src/engine-components/Light.ts +1 -1
src/engine/xr/NeedleXRController.ts +1 -1
src/engine/xr/NeedleXRSession.ts +6 -2
src/engine-components/utils/OpenURL.ts +13 -9
src/engine-components-experimental/networking/PlayerSync.ts +1 -1
src/engine-components/ui/PointerEvents.ts +3 -3
src/engine-components/ui/Raycaster.ts +1 -1
src/engine-components/export/usdz/ThreeUSDZExporter.ts +202 -82
src/engine-components/export/usdz/utils/timeutils.ts +0 -20
src/engine-components/export/usdz/USDZExporter.ts +56 -12
src/engine-components/export/usdz/extensions/USDZText.ts +2 -3
src/engine-components/webxr/WebARCameraBackground.ts +1 -1
src/engine-components/webxr/WebARSessionRoot.ts +53 -19
src/engine-components/webxr/WebXR.ts +1 -1
src/engine-components/webxr/WebXRImageTracking.ts +1 -1
src/engine-components/webxr/WebXRPlaneTracking.ts +1 -1
src/engine-components/webxr/WebXRRig.ts +3 -3
src/engine-components/webxr/controllers/XRControllerFollow.ts +1 -1
src/engine-components/webxr/controllers/XRControllerModel.ts +2 -2
src/engine-components/webxr/controllers/XRControllerMovement.ts +4 -4
src/engine/xr/XRRig.ts +1 -1
src/engine/debug/debug_spatial_console.ts +395 -0
src/engine/engine_time_utils.ts +238 -0

src/engine-components/export/usdz/extensions/Animation.ts CHANGED Viewed

@@ -49,7 +49,11 @@
     private duration = 0;
     private useRootMotion = false;
-    static animationDurationPadding = 1;
+    /** This value can theoretically be anything – a value of 1 is good to clearly see animation gaps.
+     * For production, a value of 1/60 is enough, since the files can then still properly play back at 60fps.
+     */
+    static animationDurationPadding = 1 / 60;
+    static restPoseClipDuration = 1 / 60;
     constructor(root: Object3D | null, target: Object3D, clip: AnimationClip | undefined) {
         this.root = root;
@@ -59,7 +63,7 @@
         // this is a rest pose clip.
         // we assume duration 1/60 and no tracks, and when queried for times we just return [0, duration]
         if (!clip) {
-            this.duration = 1 / this.frameRate;
+            this.duration = TransformData.restPoseClipDuration;
         }
         else {
             this.duration = clip.duration;
@@ -171,7 +175,6 @@
 export class AnimationExtension implements IUSDExporterExtension {
     get extensionName(): string { return "animation" }
     private dict: AnimationDict = new Map();
@@ -181,9 +184,9 @@
     private serializers: SerializeAnimation[] = [];
-    // determines if we inject a rest pose clip for each root - only makes sense for QuickLook
+    /** Determines if we inject a rest pose clip for each root - only makes sense for QuickLook */
     injectRestPoses = false;
-    // determines if we inject a PlayAnimationOnClick component with "scenestart" trigger - only makes sense for QuickLook
+    /** Determines if we inject a PlayAnimationOnClick component with "scenestart" trigger - only makes sense for QuickLook */
     injectImplicitBehaviours = false;
     constructor(quickLookCompatible: boolean) {
@@ -191,6 +194,24 @@
         this.injectImplicitBehaviours = quickLookCompatible;
     }
+    getStartTimeCode() {
+        // return the end time + padding of the rest clip, if any exists
+        if (!this.injectRestPoses) return 0;
+        if (this.rootAndClipToRegisteredAnimationMap.size === 0) return 0;
+        return (TransformData.restPoseClipDuration + TransformData.animationDurationPadding) * 60;
+    }
+    /** Returns the end time code, based on 60 frames per second, for all registered animations.
+     * This matches the highest time value in the USDZ file. */
+    getEndTimeCode() {
+        let max = 0;
+        for (const [_, info] of this.rootAndClipToRegisteredAnimationMap) {
+            const end = info.start + info.duration;
+            if (end > max) max = end;
+        }
+        return max * 60;
+    }
     getClipCount(root: Object3D): number {
         // don't count the rest pose
         let currentCount = this.rootToRegisteredClipCount.get(root);
@@ -642,6 +663,17 @@
             const timeSampleObjects = createAllTimeSampleObjects ( boneAndInverse.map( x => x.bone ) );
+            if (debug) {
+                // find the first..last value in the time samples
+                let min = 10000000;
+                let max = 0;
+                for (const key of timeSampleObjects.position?.keys() ?? []) {
+                    min = Math.min(min, key);
+                    max = Math.max(max, key);
+                }
+                console.log("Time samples", min, max, timeSampleObjects);
+            }
             writer.beginBlock( `def SkelAnimation "_anim"` );
             // TODO if we include blendshapes we likely need subdivision?
@@ -739,6 +771,10 @@
         let currentStartTime = 0;
         for (let i = 0; i < arr.length; i++) {
             startTimes.push(currentStartTime);
+            if (arr[i] === undefined) {
+                console.error("Got an undefined transform data, this is likely a bug.", object, arr);
+                continue;
+            }
             currentStartTime += arr[i].getDuration() + TransformData.animationDurationPadding;
         }

src/engine-components/export/usdz/utils/animationutils.ts CHANGED Viewed

@@ -3,9 +3,11 @@
 import { getParam } from "../../../../engine/engine_utils.js";
 import { Animation } from "../../../Animation.js";
 import { Animator } from "../../../Animator.js";
+import { AudioSource } from "../../../AudioSource.js";
 import { Behaviour, GameObject } from "../../../Component.js";
-import { AnimationExtension } from "../extensions/Animation.js";
-import { PlayAnimationOnClick } from "../extensions/behavior/BehaviourComponents.js";
+import type { AnimationExtension } from "../extensions/Animation.js";
+import type { AudioExtension } from "../extensions/behavior/AudioExtension.js";
+import { PlayAnimationOnClick, PlayAudioOnClick } from "../extensions/behavior/BehaviourComponents.js";
 const debug = getParam("debugusdz");
@@ -99,4 +101,36 @@
     }
     return constructedObjects;
+}
+export function registerAudioSourcesImplictly(root: Object3D, _ext: AudioExtension): Array<Object3D> {
+    const audioSources = GameObject.getComponentsInChildren(root, AudioSource);
+    const playAudioOnClicks = GameObject.getComponentsInChildren(root, PlayAudioOnClick);
+    const constructedObjects = new Array<Object3D>();
+    // Remove all audio sources that are already referenced from existing PlayAudioOnClick components
+    for (const player of playAudioOnClicks) {
+        if (!player.target) continue;
+        const index = audioSources.indexOf(player.target);
+        if (index > -1) audioSources.splice(index, 1);
+    }
+    // for the remaning ones, we want to build a PlayAudioOnClick component
+    for (const audioSource of audioSources) {
+        if (!audioSource || !audioSource.clip) continue;
+        if (audioSource.volume <= 0) continue;
+        const newComponent = new PlayAudioOnClick();
+        newComponent.target = audioSource;
+        newComponent.name = "PlayAudioOnClick_implicitAtStart_";
+        const go = new Object3D();
+        GameObject.addComponent(go, newComponent);
+        console.log("implicit PlayAudioOnStart", go, newComponent);
+        constructedObjects.push(go);
+        root.add(go);
+    }
+    return constructedObjects;
 }

src/engine-components/api.ts CHANGED Viewed

@@ -16,5 +16,6 @@
 import "./CameraUtils.js"
 import "./AnimationUtils.js"
+export { DragMode } from "./DragControls.js"
 export { ParticleSystemBaseBehaviour, type QParticle, type QParticleBehaviour } from "./ParticleSystem.js"
 export { ParticleSystemShapeType } from "./ParticleSystemModules.js"

src/engine/api.ts CHANGED Viewed

@@ -43,6 +43,7 @@
 export * from "./engine_texture.js";
 export * from "./engine_three_utils.js";
 export * from "./engine_time.js";
+export * from "./engine_time_utils.js";
 export * from "./engine_types.js";
 export { registerType,TypeStore } from "./engine_typestore.js";
 export { prefix,validate } from "./engine_util_decorator.js";

src/engine-components/export/usdz/extensions/behavior/AudioExtension.ts CHANGED Viewed

@@ -1,17 +1,30 @@
 import { Object3D } from "three";
+import { getParam } from "../../../../../engine/engine_utils.js";
 import { AudioSource } from "../../../../AudioSource.js";
 import { GameObject } from "../../../../Component.js";
 import type { IUSDExporterExtension } from "../../Extension.js";
-import { USDObject, USDWriter, USDZExporterContext } from "../../ThreeUSDZExporter.js";
+import { makeNameSafeForUSD,USDObject, USDWriter, USDZExporterContext } from "../../ThreeUSDZExporter.js";
+const debug = getParam("debugusdz");
 export class AudioExtension implements IUSDExporterExtension {
+    static getName(clip: string) {
+        const clipExt = clip.split(".").pop();
+        const fileWithoutExt = clip.split(".").slice(0, -1).join(".");
+        let clipName = fileWithoutExt.split("/").pop()?.replace(".", "_");
+        if (!clipName) {
+            clipName = "Audio_" + Math.random().toString(36).substring(2, 15);
+        }
+        return makeNameSafeForUSD(clipName) + "." + clipExt;
+    }
     get extensionName(): string {
         return "Audio";
     }
-    private files: string[] = [];
+    private files = new Array<{ path: string, name: string }>();
     onExportObject?(object: Object3D, model : USDObject, _context: USDZExporterContext) {
         // check if this object has an audio source, add the relevant schema in that case.
@@ -27,25 +40,33 @@
                 if (!audioSource.playOnAwake)
                     continue;
-                const clipName = audioSource.clip.split("/").pop();
-                // TODO: ensure audio clip doesnt start with number
+                const clipName = audioSource.clip.split("/").pop() || "Audio";
+                const safeClipNameWithExt = AudioExtension.getName(audioSource.clip);
+                const safeClipName = makeNameSafeForUSD(safeClipNameWithExt);
-                // TODO: store clipname in file and use in onAfterSerialize instead of creating it again!
-                if (!this.files.includes(audioSource.clip)) {
-                    this.files.push(audioSource.clip);
+                if (!this.files.some(f => f.path === audioSource.clip)) {
+                    this.files.push({ path: audioSource.clip, name: safeClipNameWithExt });
                 }
-                model.addEventListener('serialize', (writer: USDWriter, _context: USDZExporterContext) => {
-                    writer.appendLine();
-                    writer.beginBlock(`def SpatialAudio "${model.name}"`);
-                    writer.appendLine(`uniform asset filePath = @audio/${clipName}@`);
-                    writer.appendLine(`uniform token auralMode = "${ audioSource.spatialBlend > 0 ? "spatial" : "nonSpatial" }"`);
-                    // theoretically we could do timeline-like audio sequencing with this.
-                    writer.appendLine(`uniform token playbackMode = "${audioSource.loop ?  "loopFromStage" : "onceFromStart" }"`);
-                    writer.appendLine(`uniform float gain = ${audioSource.volume}`);
-                    writer.closeBlock();
-                });
+                // Turns out that in visionOS versions, using UsdAudio together with preliminary behaviours
+                // is not supported. So we need to NOT export SpatialAudio in those cases, and just rely
+                // on Preliminary_Behavior to play the audio.
+                if (!_context.quickLookCompatible)
+                {
+                    model.addEventListener('serialize', (writer: USDWriter, _context: USDZExporterContext) => {
+                        writer.appendLine();
+                        writer.beginBlock(`def SpatialAudio "${safeClipName}"`, "(", false );
+                        writer.appendLine(`displayName = "${clipName}"`);
+                        writer.closeBlock( ")" );
+                        writer.beginBlock();
+                        writer.appendLine(`uniform asset filePath = @audio/${safeClipNameWithExt}@`);
+                        writer.appendLine(`uniform token auralMode = "${ audioSource.spatialBlend > 0 ? "spatial" : "nonSpatial" }"`);
+                        // theoretically we could do timeline-like audio sequencing with this.
+                        writer.appendLine(`uniform token playbackMode = "${audioSource.loop ?  "loopFromStage" : "onceFromStart" }"`);
+                        writer.appendLine(`uniform float gain = ${audioSource.volume}`);
+                        writer.closeBlock();
+                    });
+                }
             }
         }
     }
@@ -53,17 +74,20 @@
     async onAfterSerialize(context: USDZExporterContext) {
         // write the files to the context.
         for (const file of this.files) {
+            const key = "audio/" + file.name;
+            if (context.files[key]) {
+                if(debug) console.warn("Audio file with name " + key + " already exists in the context. Skipping.");
+                continue;
+            }
-            const clipName = file.split("/").pop();
             // convert file (which is a path) to a blob.
-            const audio = await fetch(file);
+            const audio = await fetch(file.path);
             const audioBlob = await audio.blob();
             const arrayBuffer = await audioBlob.arrayBuffer();
             const audioData: Uint8Array = new Uint8Array(arrayBuffer)
-            context.files["audio/" + clipName] = audioData;
+            context.files[key] = audioData;
         }
     }
 }

src/engine-components/AudioSource.ts CHANGED Viewed

@@ -313,6 +313,7 @@
                 sound.setDistanceModel('linear');
                 break;
             case AudioRolloffMode.Custom:
+                console.warn("Custom rolloff for AudioSource is not supported: " + this.name);
                 break;
         }

src/engine-components/webxr/Avatar.ts CHANGED Viewed

@@ -3,9 +3,9 @@
 import { AssetReference } from "../../engine/engine_addressables.js";
 import { ObjectUtils, PrimitiveType } from "../../engine/engine_create_objects.js";
 import { serializable } from "../../engine/engine_serialization_decorator.js";
-import { IGameObject } from "../../engine/engine_types.js";
+import type { IGameObject } from "../../engine/engine_types.js";
 import { getParam,PromiseAllWithErrors } from "../../engine/engine_utils.js";
-import { NeedleXREventArgs, NeedleXRSession, NeedleXRUtils } from "../../engine/xr/index.js";
+import { type NeedleXREventArgs, NeedleXRSession, NeedleXRUtils } from "../../engine/xr/index.js";
 import { PlayerState } from "../../engine-components-experimental/networking/PlayerSync.js";
 import { Behaviour, GameObject } from "../Component.js";
 import { SyncedTransform } from "../SyncedTransform.js";

src/engine-components/export/usdz/extensions/behavior/Behaviour.ts CHANGED Viewed

@@ -4,7 +4,7 @@
 import { USDObject, USDWriter, USDZExporterContext } from "../../ThreeUSDZExporter.js";
 import { BehaviorModel } from "./BehavioursBuilder.js";
-const debug = getParam("debugusdz");
+const debug = getParam("debugusdzbehaviours");
 export interface UsdzBehaviour {
     createBehaviours?(ext: BehaviorExtension, model: USDObject, context: USDZExporterContext): void;

src/engine-components/export/usdz/extensions/behavior/BehaviourComponents.ts CHANGED Viewed

@@ -3,6 +3,7 @@
 import { isDevEnvironment, showBalloonWarning } from "../../../../../engine/debug/index.js";
 import { serializable } from "../../../../../engine/engine_serialization_decorator.js";
 import { getWorldPosition, getWorldQuaternion, getWorldScale, setWorldPosition, setWorldQuaternion, setWorldScale } from "../../../../../engine/engine_three_utils.js";
+import { getParam } from "../../../../../engine/engine_utils.js";
 import type { State } from "../../../../../engine/extensions/NEEDLE_animator_controller_model.js";
 import { NEEDLE_progressive } from "../../../../../engine/extensions/NEEDLE_progressive.js";
 import { Animator } from "../../../../Animator.js";
@@ -13,9 +14,12 @@
 import { ObjectRaycaster,Raycaster } from "../../../../ui/Raycaster.js";
 import { USDDocument, USDObject, USDZExporterContext } from "../../ThreeUSDZExporter.js";
 import { AnimationExtension, RegisteredAnimationInfo, type UsdzAnimation } from "../Animation.js";
+import { AudioExtension } from "./AudioExtension.js";
 import type { BehaviorExtension, UsdzBehaviour } from "./Behaviour.js";
-import { ActionBuilder, ActionModel, AuralMode, BehaviorModel, GroupActionModel, type IBehaviorElement, MotionType, MultiplePerformOperation,PlayAction, Space, TriggerBuilder } from "./BehavioursBuilder.js";
+import { ActionBuilder, ActionModel, AuralMode, BehaviorModel, type IBehaviorElement, MotionType, MultiplePerformOperation,PlayAction, Space, TriggerBuilder } from "./BehavioursBuilder.js";
+const debug = getParam("debugusdzbehaviours");
 function ensureRaycaster(obj: GameObject) {
     if (!obj) return;
     if (!obj.getComponentInParent(Raycaster)) {
@@ -651,11 +655,16 @@
             if (typeof clipUrl !== "string") return;
             const playbackTarget = this.target ? this.target.gameObject : this.gameObject;
-            const clipName = clipUrl.split("/").pop();
+            const clipName = AudioExtension.getName(clipUrl);
             const volume = this.target ? this.target.volume : 1;
             const auralMode = this.target && this.target.spatialBlend == 0 ? AuralMode.NonSpatial : AuralMode.Spatial;
-            // regular tap trigger
+            // This checks if any child is clickable – if yes, the tap trigger is added; if not, we omit it.
+            let anyChildHasGeometry = false;
+            this.gameObject.traverse(c => {
+                if (c instanceof Mesh && c.visible) anyChildHasGeometry = true;
+            });
+            if (anyChildHasGeometry)
             {
                 let playAction: IBehaviorElement = ActionBuilder.playAudioAction(playbackTarget, "audio/" + clipName, PlayAction.Play, volume, auralMode);
                 // does not seem to work in iOS / QuickLook...
@@ -674,7 +683,7 @@
                 let playAction: IBehaviorElement = ActionBuilder.playAudioAction(playbackTarget, "audio/" + clipName, PlayAction.Play, volume, auralMode);
                 if (this.target.loop)
                     playAction = ActionBuilder.sequence(playAction).makeLooping();
-                const playClipOnStart = new BehaviorModel("playAudioOnStart " + this.name,
+                const playClipOnStart = new BehaviorModel("playAudioOnStart" + (this.name ? "_" + this.name : ""),
                     TriggerBuilder.sceneStartTrigger(),
                     playAction,
                 );
@@ -688,15 +697,17 @@
         const clipUrl = this.clip ? this.clip : this.target ? this.target.clip : undefined;
         if (!clipUrl) return;
         if (typeof clipUrl !== "string") return;
-        const clipName = clipUrl.split("/").pop();
+        const clipName = AudioExtension.getName(clipUrl);
+        const filesKey = "audio/" + clipName;
+        // if the clip was already added, don't add it again
+        if (context.files[filesKey]) return;
         const audio = await fetch(clipUrl);
         const audioBlob = await audio.blob();
         const arrayBuffer = await audioBlob.arrayBuffer();
         const audioData: Uint8Array = new Uint8Array(arrayBuffer)
-        context.files["audio/" + clipName] = audioData;
+        context.files[filesKey] = audioData;
     }
 }
@@ -821,7 +832,7 @@
         });
     }
-    createAnimation(ext, model, _context) {
+    createAnimation(ext: AnimationExtension, model: USDObject, _context: USDZExporterContext) {
         if (!this.target || !this.animator) return;
         // If there's a separate state specified to play after this one, we
@@ -880,12 +891,12 @@
                 const firstStateInLoop = visitedStates.indexOf(currentState);
                 statesUntilLoop = visitedStates.slice(0, firstStateInLoop); // can be empty, which means we're looping all
                 statesLooping = visitedStates.slice(firstStateInLoop); // can be empty, which means nothing is looping
-                console.log("found loop from " + this.stateName, "states until loop", statesUntilLoop, "states looping", statesLooping);
+                if (debug) console.log("found loop from " + this.stateName, "states until loop", statesUntilLoop, "states looping", statesLooping);
             }
             else {
                 statesUntilLoop = visitedStates;
                 statesLooping = [];
-                console.log("found no loop from " + this.stateName, "states", statesUntilLoop);
+                if (debug) console.log("found no loop from " + this.stateName, "states", statesUntilLoop);
             }
         }
@@ -900,17 +911,28 @@
         }
         this.stateAnimationModel = model;
+        const addStateToSequence = (state: State, sequence: Array<RegisteredAnimationInfo>) => {
+            if (!this.target) return;
+            if (!state.motion?.clip) {
+                console.warn("No clip found for state " + state.name + " on " + this.animator?.name + ", can't export animation data");
+                return;
+            }
+            const anim = ext.registerAnimation(this.target, state.motion.clip);
+            if (anim) sequence.push(anim);
+            else console.warn("Couldn't register animation for state " + state.name + " on " + this.animator?.name);
+        };
         // Register all the animation states we found.
         if (statesUntilLoop.length > 0) {
             this.animationSequence = new Array<RegisteredAnimationInfo>();
             for (const state of statesUntilLoop) {
-                this.animationSequence.push(ext.registerAnimation(this.target, state.motion.clip));
+                addStateToSequence(state, this.animationSequence);
             }
         }
         if (statesLooping.length > 0) {
             this.animationLoopAfterSequence = new Array<RegisteredAnimationInfo>();
             for (const state of statesLooping) {
-                this.animationLoopAfterSequence.push(ext.registerAnimation(this.target, state.motion.clip));
+                addStateToSequence(state, this.animationLoopAfterSequence);
             }
         }
     }

src/engine-components/export/usdz/extensions/behavior/BehavioursBuilder.ts CHANGED Viewed

@@ -404,12 +404,12 @@
 export class ActionBuilder {
     static sequence(...params: IBehaviorElement[]) {
-        const group = new GroupActionModel("group_" + GroupActionModel.getId(), params);
+        const group = new GroupActionModel("Group_" + GroupActionModel.getId(), params);
         return group.makeSequence();
     }
     static parallel(...params: IBehaviorElement[]) {
-        const group = new GroupActionModel("group_" + GroupActionModel.getId(), params);
+        const group = new GroupActionModel("Group_" + GroupActionModel.getId(), params);
         return group.makeParallel();
     }

src/engine-components/ui/Canvas.ts CHANGED Viewed

@@ -5,7 +5,7 @@
 import { serializable } from "../../engine/engine_serialization_decorator.js";
 import { FrameEvent } from "../../engine/engine_setup.js";
 import { delayForFrames, getParam } from "../../engine/engine_utils.js";
-import { NeedleXREventArgs } from "../../engine/xr/index.js";
+import { type NeedleXREventArgs } from "../../engine/xr/index.js";
 import { Camera } from "../Camera.js";
 import { GameObject } from "../Component.js";
 import { BaseUIComponent, UIRootComponent } from "./BaseUIComponent.js";

src/engine-components/Component.ts CHANGED Viewed

@@ -3,14 +3,14 @@
 import { isDevEnvironment } from "../engine/debug/index.js";
 import { addNewComponent, destroyComponentInstance, findObjectOfType, findObjectsOfType, getComponent, getComponentInChildren, getComponentInParent, getComponents, getComponentsInChildren, getComponentsInParent, getOrAddComponent, moveComponentInstance, removeComponent } from "../engine/engine_components.js";
 import { activeInHierarchyFieldName } from "../engine/engine_constants.js";
-import { destroy, findByGuid, foreachComponent, HideFlags, IInstantiateOptions, instantiate, isActiveInHierarchy, isActiveSelf, isDestroyed, isUsingInstancing, markAsInstancedRendered, setActive } from "../engine/engine_gameobject.js";
+import { destroy, findByGuid, foreachComponent, HideFlags, type IInstantiateOptions, instantiate, isActiveInHierarchy, isActiveSelf, isDestroyed, isUsingInstancing, markAsInstancedRendered, setActive } from "../engine/engine_gameobject.js";
 import * as main from "../engine/engine_mainloop_utils.js";
 import { syncDestroy, syncInstantiate } from "../engine/engine_networking_instantiate.js";
 import { Context, FrameEvent } from "../engine/engine_setup.js";
 import * as threeutils from "../engine/engine_three_utils.js";
 import type { Collision, Constructor, ConstructorConcrete, GuidsMap, ICollider, IComponent, IGameObject, SourceIdentifier } from "../engine/engine_types.js";
-import { INeedleXRSessionEventReceiver, NeedleXRControllerEventArgs, NeedleXREventArgs } from "../engine/engine_xr.js";
-import { IPointerEventHandler, PointerEventData } from "./ui/PointerEvents.js";
+import type { INeedleXRSessionEventReceiver, NeedleXRControllerEventArgs, NeedleXREventArgs } from "../engine/engine_xr.js";
+import { type IPointerEventHandler, PointerEventData } from "./ui/PointerEvents.js";
 // export interface ISerializationCallbackReceiver {

src/engine/debug/debug_console.ts CHANGED Viewed

@@ -19,9 +19,9 @@
 if (!suppressConsole && (showConsole || isLocalNetwork())) {
     if (isLocalNetwork()) {
-        const currentUrl = new URL(window.location.href);
-        currentUrl.searchParams.set("console", "1");
-        console.log("🌵 Tip: You can add the \"?console\" query parameter to the url to show the debug console (on mobile it will automatically open in the bottom right corner when your get errors during development)", "\nOpen this page console: " + currentUrl.toString());
+        const consoleUrl = new URL(window.location.href);
+        consoleUrl.searchParams.set("console", "1");
+        console.log("🌵 Tip: You can add the \"?console\" query parameter to the url to show the debug console (on mobile it will automatically open in the bottom right corner when your get errors during development. In VR a spatial console will appear.)", "\nOpen this page to get the console: " + consoleUrl.toString());
     }
     const isMobile = isMobileDevice() || (isQuest() && isDevEnvironment());
     if (isMobile) {

src/engine/debug/debug.ts CHANGED Viewed

@@ -5,6 +5,7 @@
 export { showDebugConsole }
 export { LogType, setAllowOverlayMessages };
+export { enableSpatialConsole } from "./debug_spatial_console.js";
 const noDevLogs = getParam("nodevlogs");

src/engine-components/DragControls.ts CHANGED Viewed

@@ -1,4 +1,4 @@
-import { AxesHelper, Box3, BufferGeometry, Camera, Color, Event, Line, LineBasicMaterial, Matrix3, Matrix4, Mesh, MeshBasicMaterial, Object3D, Plane, PlaneHelper, Quaternion, Ray, Raycaster, SphereGeometry, Vector3 } from "three";
+import { AxesHelper, Box3, BufferGeometry, Camera, Color, Line, LineBasicMaterial, Matrix4, Mesh, MeshBasicMaterial, Object3D, Plane, Quaternion, Ray, Raycaster, SphereGeometry, Vector3 } from "three";
 import { Gizmos } from "../engine/engine_gizmos.js";
 import { InstancingUtil } from "../engine/engine_instancing.js";
@@ -7,7 +7,7 @@
 import { serializable } from "../engine/engine_serialization_decorator.js";
 import { Context } from "../engine/engine_setup.js";
 import { getWorldPosition, setWorldPosition } from "../engine/engine_three_utils.js";
-import { IGameObject } from "../engine/engine_types.js";
+import { type IGameObject } from "../engine/engine_types.js";
 import { getParam } from "../engine/engine_utils.js";
 import { NeedleXRSession } from "../engine/engine_xr.js";
 import { Avatar_POI } from "./avatar/Avatar_Brain_LookAt.js";
@@ -32,6 +32,8 @@
     DynamicViewAngle = 3,
     /** The drag plane is adjusted dynamically while dragging. */
     SnapToSurfaces = 4,
+    /** Don't allow dragging the object */
+    None = 5,
 }
 export class DragControls extends Behaviour implements IPointerEventHandler {
@@ -121,6 +123,12 @@
     onPointerEnter(evt: PointerEventData) {
         if (!this.allowEdit(this.gameObject)) return;
         if (evt.mode !== "screen") return;
+        // get the drag mode and check if we need to abort early here
+        const isSpatialInput = evt.event.mode === "tracked-pointer";
+        const dragMode = isSpatialInput ? this.xrDragMode : this.dragMode;
+        if (dragMode === DragMode.None) return;
         const dc = GameObject.getComponentInParent(evt.object, DragControls);
         if (!dc || dc !== this) return;
         DragControls.lastHovered = evt.object;
@@ -137,6 +145,12 @@
     onPointerDown(args: PointerEventData) {
         if (!this.allowEdit(this.gameObject)) return;
         if (args.used) return;
+        // get the drag mode and check if we need to abort early here
+        const isSpatialInput = args.mode === "tracked-pointer";
+        const dragMode = isSpatialInput ? this.xrDragMode : this.dragMode;
+        if (dragMode === DragMode.None) return;
         DragControls.lastHovered = args.object;
         if (args.button === 0) {
@@ -743,6 +757,8 @@
                 else
                     this._dragPlane.setFromNormalAndCoplanarPoint(rayDirection, hitWP);
                 break;
+            case DragMode.None:
+                break;
         }
         // calculate bounding box and snapping points. We want to either snap the "back" point or the "bottom" point.
@@ -864,6 +880,8 @@
         const keepRotation = isSpatialInput ? this.settings.xrKeepRotation : this.settings.keepRotation;
         const dragMode = isSpatialInput ? this.settings.xrDragMode : this.settings.dragMode;
+        if (dragMode === DragMode.None) return;
         const lerpStrength = 10;
         // - keeping rotation constant during dragging
         if (keepRotation) this._followObject.worldQuaternion = this._followObjectStartWorldQuaternion;

src/engine-components/Duplicatable.ts CHANGED Viewed

@@ -5,7 +5,7 @@
 import { serializable } from "../engine/engine_serialization_decorator.js";
 import { Behaviour, GameObject } from "./Component.js";
 import { DragControls } from "./DragControls.js";
-import { IPointerEventHandler, PointerEventData } from "./ui/PointerEvents.js";
+import { type IPointerEventHandler, PointerEventData } from "./ui/PointerEvents.js";
 import { ObjectRaycaster } from "./ui/Raycaster.js";
 export class Duplicatable extends Behaviour implements IPointerEventHandler {

src/engine/engine_addressables.ts CHANGED Viewed

@@ -1,7 +1,7 @@
 import { Group, Object3D, Texture, TextureLoader } from "three";
 import { deepClone, getParam, resolveUrl } from "../engine/engine_utils.js";
-import { destroy, IInstantiateOptions, instantiate, InstantiateOptions, isDestroyed } from "./engine_gameobject.js";
+import { destroy, type IInstantiateOptions, instantiate, InstantiateOptions, isDestroyed } from "./engine_gameobject.js";
 import { getLoader } from "./engine_gltf.js";
 import { processNewScripts } from "./engine_mainloop_utils.js";
 import { registerPrefabProvider, syncInstantiate } from "./engine_networking_instantiate.js";

src/engine/engine_context.ts CHANGED Viewed

@@ -27,7 +27,7 @@
 import { RendererData as SceneLighting } from './engine_scenelighting.js';
 import { logHierarchy } from './engine_three_utils.js';
 import { Time } from './engine_time.js';
-import { type CoroutineData, type GLTF, type ICamera, type IComponent, type IContext, type ILight, INeedleXRSession, type LoadedGLTF } from "./engine_types.js";
+import type { CoroutineData, GLTF, ICamera, IComponent, IContext, ILight, LoadedGLTF } from "./engine_types.js";
 import * as utils from "./engine_utils.js";
 import { delay, getParam } from './engine_utils.js';
 import type { INeedleXRSessionEventReceiver, NeedleXRSession } from './engine_xr.js';

src/engine/engine_create_objects.ts CHANGED Viewed

@@ -1,6 +1,6 @@
-import { BoxGeometry, ColorRepresentation,DoubleSide, Material, Mesh, MeshBasicMaterial, MeshStandardMaterial, PlaneGeometry, SphereGeometry } from "three"
+import { BoxGeometry, Material, Mesh, MeshStandardMaterial, PlaneGeometry, SphereGeometry } from "three"
-import { Vec3 } from "./engine_types.js";
+import type { Vec3 } from "./engine_types.js";
 export enum PrimitiveType {
     Quad = 0,

src/engine/engine_input.ts CHANGED Viewed

@@ -3,7 +3,7 @@
 import { showBalloonMessage, showBalloonWarning } from './debug/debug.js';
 import { Context } from './engine_setup.js';
 import type { ButtonName, IGameObject, IInput, MouseButtonName, Vec2 } from './engine_types.js';
-import { EnumToPrimitiveUnion, getParam, isMozillaXR } from './engine_utils.js';
+import { type EnumToPrimitiveUnion, getParam, isMozillaXR } from './engine_utils.js';
 const debug = getParam("debuginput");
@@ -226,7 +226,10 @@
         }
     }
     private dispatchEvent(evt: NEPointerEvent | NEKeyboardEvent) {
-        let propagationStopped = false;
+        /** True when the next event queue should not be invoked */
+        let preventNextEventQueue = false;
+        // Handle keyboard event
         if (evt instanceof NEKeyboardEvent) {
             const listeners = this._eventListeners[evt.type];
             if (listeners) {
@@ -236,17 +239,24 @@
                 }
             }
         }
-        else if (evt instanceof NEPointerEvent) {
+        // Hnadle pointer event
+        if (evt instanceof NEPointerEvent) {
             const listeners = this._eventListeners[evt.type];
             if (listeners) {
                 for (const queue of listeners) {
-                    if (propagationStopped) break;
+                    if (preventNextEventQueue) break;
                     for (const l of queue.listeners) {
                         if (evt.immediatePropagationStopped) {
-                            propagationStopped = true;
+                            preventNextEventQueue = true;
                             if (debug) console.log("immediatePropagationStopped", evt.type);
                             break;
                         }
+                        else if (evt.propagationStopped) {
+                            preventNextEventQueue = true;
+                            if (debug) console.log("propagationStopped", evt.type);
+                            // we do not break here but continue invoking the listeners in the queue
+                        }
                         (l as PointerEventListener)(evt);
                     }
                 }

src/engine/engine_lifecycle_api.ts CHANGED Viewed

@@ -1,6 +1,6 @@
 import { FrameEvent } from "./engine_context.js";
 import { ContextEvent } from "./engine_context_registry.js";
-import { LifecycleMethod, registerFrameEventCallback } from "./engine_lifecycle_functions_internal.js";
+import { type LifecycleMethod, registerFrameEventCallback } from "./engine_lifecycle_functions_internal.js";
 /**

src/engine/engine_math.ts CHANGED Viewed

@@ -1,5 +1,7 @@
 import type { Vector } from "three";
+import type { Vec3 } from "./engine_types.js";
 class MathHelper {
     random(min?: number, max?: number): number {
@@ -142,6 +144,12 @@
     dx: LowPassFilter;
     lasttime: number | null;
+    /** Create a new OneEuroFilter
+     * @param freq - An estimate of the frequency in Hz of the signal (> 0), if timestamps are not available.
+     * @param minCutOff - Min cutoff frequency in Hz (> 0). Lower values allow to remove more jitter.
+     * @param beta - Parameter to reduce latency (> 0). Higher values make the filter react faster to changes.
+     * @param dCutOff - Used to filter the derivates. 1 Hz by default. Change this parameter if you know what you are doing.
+     */
     constructor(freq: number, minCutOff = 1.0, beta = 0.0, dCutOff = 1.0) {
         if (freq <= 0 || minCutOff <= 0 || dCutOff <= 0) {
             throw new Error();
@@ -161,6 +169,7 @@
         return 1.0 / (1.0 + tau / te);
     }
+    /** Filter your value: call with your value and the current timestamp (e.g. from this.context.time.time) */
     filter(x: number, time: number | null = null) {
         if (this.lasttime && time) {
             this.freq = 1.0 / (time - this.lasttime);
@@ -172,4 +181,28 @@
         const cutOff = this.minCutOff + this.beta * Math.abs(edx);
         return this.x.filter(x, this.alpha(cutOff));
     }
+}
+export class OneEuroFilterXYZ {
+    readonly x: OneEuroFilter;
+    readonly y: OneEuroFilter;
+    readonly z: OneEuroFilter;
+    /** Create a new OneEuroFilter
+     * @param freq - An estimate of the frequency in Hz of the signal (> 0), if timestamps are not available.
+     * @param minCutOff - Min cutoff frequency in Hz (> 0). Lower values allow to remove more jitter.
+     * @param beta - Parameter to reduce latency (> 0). Higher values make the filter react faster to changes.
+     * @param dCutOff - Used to filter the derivates. 1 Hz by default. Change this parameter if you know what you are doing.
+     */
+    constructor(freq: number, minCutOff = 1.0, beta = 0.0, dCutOff = 1.0) {
+        this.x = new OneEuroFilter(freq, minCutOff, beta, dCutOff);
+        this.y = new OneEuroFilter(freq, minCutOff, beta, dCutOff);
+        this.z = new OneEuroFilter(freq, minCutOff, beta, dCutOff);
+    }
+    filter(value: Vec3, target: Vec3, time: number | null = null) {
+        target.x = this.x.filter(value.x, time);
+        target.y = this.y.filter(value.y, time);
+        target.z = this.z.filter(value.z, time);
+    }
 }

src/engine/engine_networking_instantiate.ts CHANGED Viewed

@@ -6,7 +6,7 @@
 import { v5 } from 'uuid';
 import { ContextEvent, ContextRegistry } from "../engine/engine_context_registry.js";
-import { destroy, findByGuid, IInstantiateOptions, instantiate } from "./engine_gameobject.js";
+import { destroy, findByGuid, type IInstantiateOptions, instantiate } from "./engine_gameobject.js";
 import { InstantiateOptions } from "./engine_gameobject.js";
 import type { INetworkConnection } from "./engine_networking_types.js";
 import type { IModel } from "./engine_networking_types.js";

src/engine/engine_serialization_builtin_serializer.ts CHANGED Viewed

@@ -383,15 +383,17 @@
             const tex = data as Texture;
             const rt = new RenderTexture(tex.image.width, tex.image.height, {
                 colorSpace: THREE.LinearSRGBColorSpace,
-            });
+            });
             rt.texture = tex;
             tex.isRenderTargetTexture = true;
             tex.flipY = true;
             tex.offset.y = 1;
             tex.repeat.y = -1;
             tex.needsUpdate = true;
+            // when we have a compressed texture using mipmaps causes error in threejs because the bindframebuffer call will then try to set an array of framebuffers https://linear.app/needle/issue/NE-4294
+            tex.mipmaps = [];
             if (tex instanceof CompressedTexture) {
                 //@ts-ignore
                 tex["isCompressedTexture"] = false;

src/engine/engine_serialization_core.ts CHANGED Viewed

@@ -5,7 +5,7 @@
 import { addLog,LogType } from "./debug/debug_overlay.js";
 import { isLocalNetwork } from "./engine_networking_utils.js";
 import { Context } from "./engine_setup.js";
-import { Constructor, type ConstructorConcrete, type SourceIdentifier } from "./engine_types.js";
+import type { Constructor, ConstructorConcrete, SourceIdentifier } from "./engine_types.js";
 import { $BuiltInTypeFlag } from "./engine_typestore.js";
 import { getParam } from "./engine_utils.js";
 import { isPersistentAsset } from "./extensions/NEEDLE_persistent_assets.js";

src/engine/engine_types.ts CHANGED Viewed

@@ -47,6 +47,7 @@
 export interface ITime {
     get time(): number;
+    get deltaTime(): number;
 }
 export interface IInput {

src/engine-components/ui/EventSystem.ts CHANGED Viewed

@@ -1,16 +1,16 @@
-import { Intersection, Object3D } from "three";
+import { type Intersection, Object3D } from "three";
 import { isDevEnvironment, showBalloonMessage } from "../../engine/debug/index.js";
-import { Input, InputEventNames, InputEvents, NEPointerEvent, PointerType } from "../../engine/engine_input.js";
+import { type InputEventNames, InputEvents, NEPointerEvent, PointerType } from "../../engine/engine_input.js";
 import { Mathf } from "../../engine/engine_math.js";
-import { RaycastOptions, RaycastTestObjectReturnType } from "../../engine/engine_physics.js";
+import { RaycastOptions, type RaycastTestObjectReturnType } from "../../engine/engine_physics.js";
 import { Context } from "../../engine/engine_setup.js";
-import { IComponent } from "../../engine/engine_types.js";
+import type { IComponent } from "../../engine/engine_types.js";
 import { getParam } from "../../engine/engine_utils.js";
 import { Behaviour, GameObject } from "../Component.js";
 import { $shadowDomOwner } from "./BaseUIComponent.js";
 import type { ICanvasGroup } from "./Interfaces.js";
-import { hasPointerEventComponent, type IPointerEventHandler, IPointerUpHandler, PointerEventData } from "./PointerEvents.js";
+import { hasPointerEventComponent, type IPointerEventHandler, type IPointerUpHandler, PointerEventData } from "./PointerEvents.js";
 import { ObjectRaycaster, Raycaster } from "./Raycaster.js";
 import { UIRaycastUtils } from "./RaycastUtils.js";
 import { isUIObject } from "./Utils.js";

src/engine/extensions/extensions.ts CHANGED Viewed

@@ -1,5 +1,5 @@
-import { GLTFExporter, GLTFExporterPlugin, GLTFWriter } from "three/examples/jsm/exporters/GLTFExporter.js";
-import { GLTFLoader, GLTFParser } from "three/examples/jsm/loaders/GLTFLoader.js";
+import { GLTFExporter } from "three/examples/jsm/exporters/GLTFExporter.js";
+import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js";
 import { isDevEnvironment } from "../debug/index.js";
 import { isResourceTrackingEnabled } from "../engine_assetdatabase.js";

src/engine/xr/internal.ts CHANGED Viewed

@@ -1,9 +1,9 @@
 import { Matrix4, Object3D, Quaternion, Scene, Vector3 } from 'three';
 import { CreateWireCube, Gizmos } from '../engine_gizmos.js';
-import { IGameObject } from '../engine_types.js';
+import type { IGameObject } from '../engine_types.js';
 import { getParam } from '../engine_utils.js';
-import { IXRRig } from './XRRig.js';
+import type { IXRRig } from './XRRig.js';
 export const flipForwardMatrix = new Matrix4().makeRotationY(Math.PI);
 export const flipForwardQuaternion = new Quaternion().setFromAxisAngle(new Vector3(0, 1, 0), Math.PI);

src/engine-components/Light.ts CHANGED Viewed

@@ -6,7 +6,7 @@
 import { setWorldPositionXYZ } from "../engine/engine_three_utils.js";
 import type { ILight } from "../engine/engine_types.js";
 import { getParam, isMobileDevice } from "../engine/engine_utils.js";
-import { NeedleXREventArgs } from "../engine/xr/index.js";
+import { type NeedleXREventArgs } from "../engine/xr/index.js";
 import { Behaviour, GameObject } from "./Component.js";
 import { WebARSessionRoot } from "./webxr/WebARSessionRoot.js";

src/engine/xr/NeedleXRController.ts CHANGED Viewed

@@ -4,7 +4,7 @@
 import { RGBAColor } from "../../engine-components/js-extensions/RGBAColor.js";
 import { Context } from "../engine_context.js";
 import { Gizmos } from "../engine_gizmos.js";
-import { InputEventNames, InputEvents, NEPointerEvent, NEPointerEventInit, PointerType } from "../engine_input.js";
+import { type InputEventNames, InputEvents, NEPointerEvent, type NEPointerEventInit } from "../engine_input.js";
 import { getTempQuaternion, getTempVector, getWorldQuaternion } from "../engine_three_utils.js";
 import type { ButtonName, IGameObject, Vec3, XRControllerButtonName, XRGestureName } from "../engine_types.js";
 import { getParam } from "../engine_utils.js";

src/engine/xr/NeedleXRSession.ts CHANGED Viewed

@@ -1,13 +1,13 @@
 import { Camera, Object3D, PerspectiveCamera, Quaternion, Vector3 } from "three";
-import { isDevEnvironment, showBalloonMessage, showBalloonWarning } from "../debug/index.js";
+import { enableSpatialConsole, isDevEnvironment, showBalloonMessage, showBalloonWarning } from "../debug/index.js";
 import { Context, FrameEvent } from "../engine_context.js";
 import { ContextEvent, ContextRegistry } from "../engine_context_registry.js";
 import { isDestroyed } from "../engine_gameobject.js";
 import { Gizmos } from "../engine_gizmos.js";
 import { registerFrameEventCallback, unregisterFrameEventCallback } from "../engine_lifecycle_functions_internal.js";
 import { getTempQuaternion, getTempVector, getWorldPosition, getWorldQuaternion, getWorldScale, setWorldPosition, setWorldQuaternion, setWorldScale } from "../engine_three_utils.js";
-import { ICamera, IComponent, INeedleXRSession } from "../engine_types.js";
+import type { ICamera, IComponent, INeedleXRSession } from "../engine_types.js";
 import { delay, getParam, isDesktop, isQuest } from "../engine_utils.js";
 import { flipForwardMatrix, flipForwardQuaternion, ImplictXRRig } from "./internal.js";
 import { NeedleXRController } from "./NeedleXRController.js";
@@ -714,6 +714,8 @@
         this.mode = mode;
         this.context = context;
+        if(debug || getParam("console")) enableSpatialConsole(true);
         this.context.xr = this;
         this.context.renderer.xr.setSession(this.session).then(this.onRendererSessionSet)
@@ -874,6 +876,8 @@
         this.context.requestSizeUpdate();
         this._defaultRig.gameObject.removeFromParent();
+        enableSpatialConsole(false);
     };
     /** Disconnects the controller, invokes events and notifies previou controller (if any) */

src/engine-components/utils/OpenURL.ts CHANGED Viewed

@@ -1,11 +1,10 @@
 import { isDevEnvironment, showBalloonMessage } from "../../engine/debug/index.js";
 import { serializable } from "../../engine/engine_serialization.js";
-import { isiOS,isSafari } from "../../engine/engine_utils.js";
+import { isiOS, isSafari } from "../../engine/engine_utils.js";
 import { Behaviour } from "../Component.js";
 import { type IPointerClickHandler, PointerEventData } from "../ui/index.js";
 import { ObjectRaycaster, Raycaster } from "../ui/Raycaster.js";
-import { tryGetUIComponent } from "../ui/Utils.js";
 export enum OpenURLMode {
     NewTab = 0,
@@ -32,29 +31,34 @@
         this._validateUrl();
-        if (isDevEnvironment()) showBalloonMessage("Open URL: " + this.url)
+        let url = this.url;
+        if (!url.startsWith("mailto:") && url.includes("@")) {
+            url = "mailto:" + url;
+        }
+        if (isDevEnvironment()) showBalloonMessage("Open URL: " + url)
         switch (this.mode) {
             case OpenURLMode.NewTab:
                 if (isSafari()) {
-                    globalThis.open(this.url, "_blank");
+                    globalThis.open(url, "_blank");
                 }
                 else
-                    globalThis.open(this.url, "_blank");
+                    globalThis.open(url, "_blank");
                 break;
             case OpenURLMode.SameTab:
                 // TODO: test if "same tab" now also works on iOS
                 if (isSafari() && isiOS()) {
-                    globalThis.open(this.url, "_top");
+                    globalThis.open(url, "_top");
                 }
                 else
-                    globalThis.open(this.url, "_self");
+                    globalThis.open(url, "_self");
                 break;
             case OpenURLMode.NewWindow:
                 if (isSafari()) {
-                    globalThis.open(this.url, "_top");
+                    globalThis.open(url, "_top");
                 }
-                else globalThis.open(this.url, "_new");
+                else globalThis.open(url, "_new");
                 break;
         }

src/engine-components-experimental/networking/PlayerSync.ts CHANGED Viewed

@@ -5,7 +5,7 @@
 import { syncField } from "../../engine/engine_networking_auto.js"
 import { syncDestroy } from "../../engine/engine_networking_instantiate.js";
 import { serializable } from "../../engine/engine_serialization_decorator.js";
-import { IGameObject } from "../../engine/engine_types.js";
+import type { IGameObject } from "../../engine/engine_types.js";
 import { delay, getParam } from "../../engine/engine_utils.js";
 import { Behaviour, Component, GameObject } from "../../engine-components/Component.js";
 import { EventList } from "../../engine-components/EventList.js";

src/engine-components/ui/PointerEvents.ts CHANGED Viewed

@@ -1,7 +1,7 @@
-import { Face, Object3D, Vector3 } from "three";
+import { type Face, Object3D, Vector3 } from "three";
-import { Input, InputEventNames, NEPointerEvent } from "../../engine/engine_input.js";
-import { GamepadButtonName, MouseButtonName } from "../../engine/engine_types.js";
+import { Input, type InputEventNames, NEPointerEvent } from "../../engine/engine_input.js";
+import type { GamepadButtonName, MouseButtonName } from "../../engine/engine_types.js";
 import { GameObject } from "../Component.js";
 export interface IInputEventArgs {

src/engine-components/ui/Raycaster.ts CHANGED Viewed

@@ -1,6 +1,6 @@
 import { SkinnedMesh } from "three";
-import { IRaycastOptions, RaycastOptions } from "../../engine/engine_physics.js";
+import { type IRaycastOptions, RaycastOptions } from "../../engine/engine_physics.js";
 import { serializable } from "../../engine/engine_serialization.js";
 import { NeedleXRSession } from "../../engine/engine_xr.js";
 import { Behaviour } from "../Component.js";

src/engine-components/export/usdz/ThreeUSDZExporter.ts CHANGED Viewed

@@ -7,6 +7,7 @@
 	BufferGeometry,
 	Color,
 	DoubleSide,
+	LinearFilter,
 	Material,
 	MathUtils,
 	Matrix4,
@@ -18,19 +19,23 @@
 	OrthographicCamera,
 	PerspectiveCamera,
 	PlaneGeometry,
+	RGBAFormat,
 	Scene,
 	ShaderMaterial,
 	SkinnedMesh,
 	SRGBColorSpace,
 	Texture,
 	Uniform,
+	UnsignedByteType,
 	Vector4,
-	WebGLRenderer} from 'three';
+	WebGLRenderer,
+	WebGLRenderTarget} from 'three';
 import * as fflate from 'three/examples/jsm/libs/fflate.module.js';
 import type { OffscreenCanvasExt } from '../../../engine/engine_shims.js';
-import { GameObject } from '../../Component.js';
-import { Renderer } from '../../Renderer.js';
+import { Progress } from '../../../engine/engine_time_utils.js';
+import type { IUSDExporterExtension } from './Extension.js';
+import type { AnimationExtension } from './extensions/Animation.js';
 function makeNameSafe( str ) {
 	// Remove characters that are not allowed in USD ASCII identifiers
@@ -336,7 +341,7 @@
 	}
-	buildHeader( endTimeCode ) {
+	buildHeader( { startTimeCode, endTimeCode } ) {
 		return `#usda 1.0
 (
@@ -346,7 +351,7 @@
 	defaultPrim = "${makeNameSafe( this.name )}"
 	metersPerUnit = 1
 	upAxis = "Y"
-	startTimeCode = 0
+	startTimeCode = ${startTimeCode}
 	endTimeCode = ${endTimeCode}
 	timeCodesPerSecond = 60
 	framesPerSecond = 60
@@ -452,7 +457,7 @@
 class USDZExporterContext {
 	root: any;
 	exporter: any;
-	extensions: any;
+	extensions: Array<IUSDExporterExtension> = [];
 	quickLookCompatible: boolean;
 	materials: Map<string, Material>;
 	textures: TextureMap;
@@ -502,7 +507,8 @@
 class USDZExporter {
 	debug: boolean;
 	sceneAnchoringOptions: {} = {};
-	extensions: any;
+	extensions: Array<IUSDExporterExtension> = [];
+	keepObject?: (object: Object3D) => boolean;
 	constructor() {
@@ -510,20 +516,6 @@
 	}
-	getEndTimeCode( animations ) {
-		let endTimeCode = 0;
-		for( const animation of animations ) {
-			const currentEndTimeCode = animation.duration * 60;
-			if ( endTimeCode < currentEndTimeCode ) {
-				endTimeCode = currentEndTimeCode;
-			}
-		}
-		return endTimeCode;
-	}
 	async parse( scene, options: USDZExporterOptions = new USDZExporterOptions() ) {
 		options = Object.assign( new USDZExporterOptions(), options );
@@ -541,8 +533,11 @@
 		const materials = context.materials;
 		const textures = context.textures;
+		Progress.report('export-usdz', "Invoking onBeforeBuildDocument");
 		await invokeAll( context, 'onBeforeBuildDocument' );
+		Progress.report('export-usdz', "Done onBeforeBuildDocument");
+		Progress.report('export-usdz', "Reparent bones to common ancestor");
 		// HACK let's find all skeletons and reparent them to their skelroot / armature / uppermost bone parent
 		const reparentings: Array<any> = [];
 		scene.traverseVisible(object => {
@@ -559,20 +554,26 @@
 		for ( const reparenting of reparentings ) {
 			const { object, originalParent, newParent } = reparenting;
-			if (this.debug) console.log("REPARENTING", object, "from", originalParent, "to", newParent);
+			// if (this.debug) console.log("REPARENTING", object, "from", originalParent, "to", newParent);
 			newParent.add( object );
 		}
-		traverseVisible( scene, context.document, context );
+		Progress.report('export-usdz', "Traversing hierarchy");
+		traverseVisible( scene, context.document, context, this.keepObject);
+		Progress.report('export-usdz', "Invoking onAfterBuildDocument");
 		await invokeAll( context, 'onAfterBuildDocument' );
-		parseDocument( context, () => {
+		Progress.report('export-usdz', { message: "Parsing document", autoStep: 10 });
+		await parseDocument( context, () => {
 			// injected after stageRoot.
 			// TODO property use context/writer instead of string concat
-			return buildMaterials( materials, textures, options.quickLookCompatible );
+			Progress.report('export-usdz', "Building materials");
+			const result = buildMaterials( materials, textures, options.quickLookCompatible );
+			return result;
 		} );
+		Progress.report("export-usdz", "Invoking onAfterSerialize");
 		await invokeAll( context, 'onAfterSerialize' );
 		// repair the parenting again
@@ -584,40 +585,57 @@
 		// Moved into parseDocument callback for proper defaultPrim encapsulation
 		// context.output += buildMaterials( materials, textures, options.quickLookCompatible );
-		const endTimeCode = this.getEndTimeCode( context.animations );
+		const animationExtension: AnimationExtension | undefined = context.extensions.find( ext => ext.extensionName === 'animation' ) as AnimationExtension;
+		const startTimeCode = animationExtension?.getStartTimeCode() ?? 0;
+		const endTimeCode = animationExtension?.getEndTimeCode() ?? 0;
-		const header = context.document.buildHeader( endTimeCode );
+		const header = context.document.buildHeader( { startTimeCode, endTimeCode } );
 		const final = header + '\n' + context.output;
 		// full output file
-		if ( this.debug )
-			console.log( final );
+		// if ( this.debug ) console.log( final );
 		files[ modelFileName ] = fflate.strToU8( final );
 		context.output = '';
-		const decompressionRenderer = new WebGLRenderer( { antialias: false, alpha: true } );
+		Progress.report("export-usdz", { message: "Exporting textures", autoStep: 10 });
+		Progress.start("export-usdz-textures", "export-usdz");
+		const decompressionRenderer = new WebGLRenderer( {
+			antialias: false,
+			alpha: true,
+			premultipliedAlpha: false,
+			preserveDrawingBuffer: true
+		} );
-		for ( const id in textures ) {
+		const textureCount = Object.keys(textures).length;
+		Progress.report("export-usdz-textures", { totalSteps: textureCount * 3, currentStep: 0 });
+		const convertTexture = async (id: string) => {
 			const textureData = textures[ id ];
-			let texture = textureData.texture;
+			const texture = textureData.texture;
 			const isRGBA = formatsWithAlphaChannel.includes( texture.format );
-			//@ts-ignore
-			if ( texture.isCompressedTexture || texture.isRenderTargetTexture ) {
-				texture = decompressGpuTexture( texture, options.maxTextureSize, decompressionRenderer );
+			// Change: we need to always read back the texture now, otherwise the unpremultiplied workflow doesn't work.
+			let img: ImageReadbackResult = {
+				imageData: texture.image
+			};
+			Progress.report("export-usdz-textures", { message: "read back texture", autoStep: true });
+			const anyColorScale = textureData.scale !== undefined && textureData.scale.x !== 1 && textureData.scale.y !== 1 && textureData.scale.z !== 1 && textureData.scale.w !== 1;
+			// @ts-ignore
+			if ( texture.isCompressedTexture || texture.isRenderTargetTexture || anyColorScale ) {
+				img = await decompressGpuTexture( texture, options.maxTextureSize, decompressionRenderer, textureData.scale );
 			}
-			const canvas = await imageToCanvas( texture.image, textureData.scale, false, options.maxTextureSize ).catch( err => {
+			Progress.report("export-usdz-textures", { message: "convert texture to canvas", autoStep: true });
+			const canvas = await imageToCanvasUnpremultiplied( img.imageBitmap || img.imageData, options.maxTextureSize ).catch( err => {
 				console.error("Error converting texture to canvas", texture, err);
 			});
 			if ( canvas ) {
+				Progress.report("export-usdz-textures", { message: "convert canvas to blob", autoStep: true });
 				const blob = await canvas.convertToBlob( {type: isRGBA ? 'image/png' : 'image/jpeg', quality: 0.95 } );
 				files[ `textures/${id}.${isRGBA ? 'png' : 'jpg'}` ] = new Uint8Array( await blob.arrayBuffer() );
@@ -626,7 +644,11 @@
 				console.warn( 'Can`t export texture: ', texture );
 			}
+		};
+		for ( const id in textures ) {
+			await convertTexture( id );
 		}
 		decompressionRenderer.dispose();
@@ -634,6 +656,8 @@
 		// 64 byte alignment
 		// https://github.com/101arrowz/fflate/issues/39#issuecomment-777263109
+		Progress.end("export-usdz-textures");
 		let offset = 0;
 		for ( const filename in files ) {
@@ -658,13 +682,14 @@
 		}
+		Progress.report("export-usdz", "zip archive");
 		return fflate.zipSync( files, { level: 0 } );
 	}
 }
-function traverseVisible( object: Object3D, parentModel: USDObject, context: USDZExporterContext ) {
+function traverseVisible( object: Object3D, parentModel: USDObject, context: USDZExporterContext, keepObject?: (object: Object3D) => boolean ) {
 	if ( ! object.visible ) return;
@@ -677,10 +702,7 @@
 		material = object.material;
 	}
-	// TODO what should be do with disabled renderers?
-	// Here we just assume they're off, and don't export them
-	const renderer = GameObject.getComponent( object, Renderer )
-	if (renderer && !renderer.enabled) {
+	if (keepObject && !keepObject(object)) {
 		geometry = undefined;
 		material = undefined;
 	}
@@ -741,7 +763,7 @@
 	for ( const ch of object.children ) {
-		traverseVisible( ch, parentModel, context );
+		traverseVisible( ch, parentModel, context, keepObject );
 	}
@@ -749,12 +771,23 @@
 async function parseDocument( context: USDZExporterContext, afterStageRoot: () => string ) {
+	Progress.start("export-usdz-resources", "export-usdz");
+	const resources: Array<() => void> = [];
 	for ( const child of context.document.children ) {
-		addResources( child, context );
+		addResources( child, context, resources );
 	}
+	// addResources now only collects promises for better progress reporting.
+	// We are resolving them here and reporting progress on that:
+	const total = resources.length;
+	for (let i = 0; i < total; i++) {
+		Progress.report("export-usdz-resources", { totalSteps: total, currentStep: i });
+		await new Promise<void>((resolve, _reject) => {
+			resources[i]();
+			resolve();
+		});
+	}
+	Progress.end("export-usdz-resources");
 	const writer = new USDWriter();
 	writer.beginBlock( `def Xform "${context.document.name}"` );
@@ -780,12 +813,22 @@
 		writer.appendLine( `rel preliminary:imageAnchoring:referenceImage = </${context.document.name}/Scenes/Scene/AnchoringReferenceImage>` );
 	writer.appendLine();
+	const count = (object: USDObject | null) => {
+		if (!object) return 0;
+		let total = 1;
+		for ( const child of object.children ) total += count( child );
+		return total;
+	}
+	const totalXformCount = count(context.document);
+	Progress.start("export-usdz-xforms", "export-usdz");
+	Progress.report("export-usdz-xforms", { totalSteps: totalXformCount, currentStep: 1 });
 	for ( const child of context.document.children ) {
 		buildXform( child, writer, context );
 	}
+	Progress.end("export-usdz-xforms");
+	Progress.report("export-usdz", "invoke onAfterHierarchy");
 	invokeAll( context, 'onAfterHierarchy', writer );
 	writer.closeBlock();
@@ -793,15 +836,15 @@
 	writer.appendLine(afterStageRoot());
 	writer.closeBlock();
+	Progress.report("export-usdz", "write to string")
 	context.output += writer.toString();
 }
-function addResources( object: USDObject | null, context: USDZExporterContext ) {
+function addResources( object: USDObject | null, context: USDZExporterContext, resources: Array<() => void>) {
-	if ( object == null ) {
-		return;
-	}
+	if ( !object ) return;
 	const geometry = object.geometry;
 	const material = object.material;
@@ -813,9 +856,13 @@
 			if ( ! ( geometryFileName in context.files ) ) {
-				const meshObject = buildMeshObject( geometry, object.skinnedMesh?.skeleton?.bones );
-				context.files[ geometryFileName ] = buildUSDFileAsString( meshObject, context );
+				const action = () => {
+					const meshObject = buildMeshObject( geometry, object.skinnedMesh?.skeleton?.bones );
+					context.files[ geometryFileName ] = buildUSDFileAsString( meshObject, context);
+				};
+				resources.push(action);
 			}
 		} else {
@@ -826,7 +873,7 @@
 	}
-	if( material ) {
+	if ( material ) {
 		if ( ! ( material.uuid in context.materials ) ) {
@@ -837,7 +884,7 @@
 	for ( const ch of object.children ) {
-		addResources( ch, context );
+		addResources( ch, context, resources );
 	}
@@ -866,24 +913,40 @@
 }
 let _renderer: WebGLRenderer | null = null;
+let renderTarget: WebGLRenderTarget | null = null;
 let fullscreenQuadGeometry: PlaneGeometry | null;
 let fullscreenQuadMaterial: ShaderMaterial | null;
 let fullscreenQuad: Mesh | null;
-function decompressGpuTexture( texture, maxTextureSize = Infinity, renderer: WebGLRenderer | null = null ) {
+declare type ImageReadbackResult = {
+	imageData: ImageData;
+	imageBitmap?: ImageBitmap;
+}
+/** Reads back a texture from the GPU (can be compressed, a render texture, or aynthing), apply colorScale to it, and return CPU data for further usage.
+ * Note that there are WebGL / WebGPU rules preventing some use of data between WebGL contexts.
+*/
+async function decompressGpuTexture( texture, maxTextureSize = Infinity, renderer: WebGLRenderer | null = null, colorScale: Vector4 | undefined = undefined): Promise<ImageReadbackResult> {
 	if ( ! fullscreenQuadGeometry ) fullscreenQuadGeometry = new PlaneGeometry( 2, 2, 1, 1 );
 	if ( ! fullscreenQuadMaterial ) fullscreenQuadMaterial = new ShaderMaterial( {
-		uniforms: { blitTexture: new Uniform( texture ) },
+		uniforms: {
+			blitTexture: new Uniform( texture ),
+			flipY: new Uniform( false ),
+			scale: new Uniform( new Vector4( 1, 1, 1, 1 ) ),
+		},
 		vertexShader: `
             varying vec2 vUv;
+			uniform bool flipY;
             void main(){
                 vUv = uv;
-				vUv.y = 1. - vUv.y;
+				if (flipY)
+					vUv.y = 1. - vUv.y;
                 gl_Position = vec4(position.xy * 1.0,0.,.999999);
             }`,
 		fragmentShader: `
-            uniform sampler2D blitTexture;
+            uniform sampler2D blitTexture;
+			uniform vec4 scale;
             varying vec2 vUv;
             void main(){
@@ -894,11 +957,18 @@
                 #else
                 gl_FragColor = texture2D( blitTexture, vUv);
                 #endif
-				gl_FragColor.rgb *= gl_FragColor.a;
+				gl_FragColor.rgba *= scale.rgba;
             }`
 	} );
-	fullscreenQuadMaterial.uniforms.blitTexture.value = texture;
+	// update uniforms
+	const uniforms = fullscreenQuadMaterial.uniforms;
+	uniforms.blitTexture.value = texture;
+	uniforms.flipY.value = false;
+	uniforms.scale.value = new Vector4( 1, 1, 1, 1 );
+	if ( colorScale !== undefined ) uniforms.scale.value.copy( colorScale );
 	fullscreenQuadMaterial.defines.IS_SRGB = texture.colorSpace == SRGBColorSpace;
 	fullscreenQuadMaterial.needsUpdate = true;
@@ -915,22 +985,31 @@
 	if ( ! renderer ) {
-		renderer = _renderer = new WebGLRenderer( { antialias: false, alpha: true } );
+		renderer = _renderer = new WebGLRenderer( { antialias: false, alpha: true, premultipliedAlpha: false, preserveDrawingBuffer: true } );
 	}
-	renderer.setSize( Math.min( texture.image.width, maxTextureSize ), Math.min( texture.image.height, maxTextureSize ) );
+	const width = Math.min( texture.image.width, maxTextureSize );
+	const height = Math.min( texture.image.height, maxTextureSize );
+	// dispose render target if the size is wrong
+	if ( renderTarget && ( renderTarget.width !== width || renderTarget.height !== height ) ) {
+		renderTarget.dispose();
+		renderTarget = null;
+	}
+	if ( ! renderTarget ) {
+		renderTarget = new WebGLRenderTarget( width, height, { format: RGBAFormat, type: UnsignedByteType, minFilter: LinearFilter, magFilter: LinearFilter } );
+	}
+	renderer.setRenderTarget( renderTarget );
+	renderer.setSize( width, height );
 	renderer.clear();
 	renderer.render( _scene, _camera );
-	const readableTexture = new Texture( renderer.domElement );
-	readableTexture.minFilter = texture.minFilter;
-	readableTexture.magFilter = texture.magFilter;
-	readableTexture.wrapS = texture.wrapS;
-	readableTexture.wrapT = texture.wrapT;
-	readableTexture.name = texture.name;
 	if ( _renderer ) {
 		_renderer.dispose();
@@ -938,11 +1017,18 @@
 	}
-	return readableTexture;
+	const buffer = new Uint8ClampedArray( renderTarget.width * renderTarget.height * 4 );
+	renderer.readRenderTargetPixels( renderTarget, 0, 0, renderTarget.width, renderTarget.height, buffer );
+	const imageData = new ImageData( buffer, renderTarget.width, renderTarget.height, undefined );
+	const bmp = await createImageBitmap( imageData, { premultiplyAlpha: "none" } );
+	return {
+		imageData,
+		imageBitmap: bmp
+	};
 }
+/** Checks if the given image is of a type with readable data and width/height */
 function isImageBitmap( image ) {
 	return ( typeof HTMLImageElement !== 'undefined' && image instanceof HTMLImageElement ) ||
@@ -952,8 +1038,32 @@
 }
-async function imageToCanvas( image, color: Vector4 | undefined = undefined, flipY = false, maxTextureSize = 4096 ) {
+/** This method uses a 'bitmaprenderer' context and doesn't do any pixel manipulation.
+ * This way, we can keep the alpha channel as it was, but we're losing the ability to do pixel manipulations or resize operations. */
+async function imageToCanvasUnpremultiplied( image: ImageBitmapSource & { width: number, height: number }, maxTextureSize = 4096) {
+	const scale = maxTextureSize / Math.max( image.width, image.height );
+	const width = image.width * Math.min( 1, scale );
+	const height = image.height * Math.min( 1, scale );
+	const canvas = new OffscreenCanvas( width, height );
+	const settings: ImageBitmapOptions = { premultiplyAlpha: "none" };
+	if (image.width !== width) settings.resizeWidth = width;
+	if (image.height !== height) settings.resizeHeight = height;
+	const imageBitmap = await createImageBitmap(image, settings);
+	const ctx = canvas.getContext("bitmaprenderer") as ImageBitmapRenderingContext | null;
+	if (ctx) {
+		ctx.transferFromImageBitmap(imageBitmap);
+	}
+	return canvas as OffscreenCanvasExt;
+}
+/** This method uses a '2d' canvas context for pixel manipulation, and can apply a color scale or Y flip to the given image.
+ * Unfortunately, canvas always uses premultiplied data, and thus images with low alpha values (or multiplying by a=0) will result in black pixels.
+ */
+async function imageToCanvas( image: HTMLImageElement | HTMLCanvasElement | OffscreenCanvas | ImageBitmap, color: Vector4 | undefined = undefined, flipY = false, maxTextureSize = 4096 ) {
 	if ( isImageBitmap( image ) ) {
 		// max. canvas size on Safari is still 4096x4096
@@ -961,7 +1071,7 @@
 		const canvas = new OffscreenCanvas( image.width * Math.min( 1, scale ), image.height * Math.min( 1, scale ) );
-		const context = canvas.getContext( '2d' ) as OffscreenCanvasRenderingContext2D;
+		const context = canvas.getContext( '2d', { alpha: true, premultipliedAlpha: false } ) as OffscreenCanvasRenderingContext2D;
 		if (!context) throw new Error('Could not get canvas 2D context');
 		if ( flipY === true ) {
@@ -1044,9 +1154,15 @@
 }
 function getGeometryName(geometry: BufferGeometry, fallbackName: string) {
-	return makeNameSafe(geometry.name || fallbackName || 'Geometry_') + geometry.id;
+	// TODO using object names here breaks instancing...
+	// Likely need to remove fallbackName again
+	return makeNameSafe(geometry.name || fallbackName || 'Geometry') + "_" + geometry.id;
 }
+function getMaterialName(material: Material) {
+	return makeNameSafe(material.name || 'Material') + "_" + material.id;
+}
 function getPathToSkeleton(bone: Object3D, assumedRoot: Object3D) {
 	let path = getBoneName(bone);
 	let current = bone.parent;
@@ -1064,6 +1180,8 @@
 	if ( model == null)
 		return;
+	Progress.report("export-usdz-xforms", { message: "buildXform " + model.displayName || model.name, autoStep: true });
 	const matrix = model.matrix;
 	const geometry = model.geometry;
 	const material = model.material;
@@ -1109,7 +1227,7 @@
 	writer.beginBlock();
 	if ( geometry && material ) {
-		const materialName = makeNameSafe(material.name) + `_${material.id}`;
+		const materialName = getMaterialName(material);
 		writer.appendLine( `rel material:binding = </StageRoot/Materials/${materialName}>` );
 		// Turns out QuickLook / RealityKit doesn't support the doubleSided attribute, so we
@@ -1469,7 +1587,7 @@
 	const pad = '                ';
 	const inputs: Array<string> = [];
 	const samplers: Array<string> = [];
-	const materialName = makeNameSafe(material.name) + `_${material.id}`;
+	const materialName = getMaterialName(material);
 	function texName(tex: Texture) {
 		return makeNameSafe(tex.name) + '_' + tex.id;
@@ -1837,8 +1955,10 @@
 	decompressGpuTexture,
 	findStructuralNodesInBoneHierarchy,
 	getBoneName,
+	getMaterialName as getMaterialNameForUSD,
 	getPathToSkeleton,
 	imageToCanvas,
+	imageToCanvasUnpremultiplied,
 	makeNameSafe as makeNameSafeForUSD,
 	USDDocument,
 	fn as usdNumberFormatting,

src/engine-components/export/usdz/utils/timeutils.ts DELETED Viewed

@@ -1,20 +0,0 @@
-export function getFormattedDate() {
-    var date = new Date();
-    const month = date.getMonth() + 1;
-    const day = date.getDate();
-    const hour = date.getHours();
-    const min = date.getMinutes();
-    const sec = date.getSeconds();
-    const s_month = (month < 10 ? "0" : "") + month;
-    const s_day = (day < 10 ? "0" : "") + day;
-    const s_hour = (hour < 10 ? "0" : "") + hour;
-    const s_min = (min < 10 ? "0" : "") + min;
-    const s_sec = (sec < 10 ? "0" : "") + sec;
-    return date.getFullYear() + s_month + s_day + "-" + s_hour + s_min + s_sec;
-}

src/engine-components/export/usdz/USDZExporter.ts CHANGED Viewed

@@ -1,15 +1,15 @@
-import { Matrix4,Mesh, Object3D } from "three";
+import { Matrix4, Mesh, Object3D } from "three";
 import { showBalloonMessage, showBalloonWarning } from "../../../engine/debug/index.js";
 import { hasProLicense } from "../../../engine/engine_license.js";
 import { serializable } from "../../../engine/engine_serialization.js";
-import { Context } from "../../../engine/engine_setup.js";
-import { delay, getParam, isiOS, isMobileDevice, isSafari } from "../../../engine/engine_utils.js";
+import { getFormattedDate, Progress } from "../../../engine/engine_time_utils.js";
+import { getParam, isiOS, isMobileDevice, isSafari } from "../../../engine/engine_utils.js";
 import { Behaviour, GameObject } from "../../Component.js";
 import { Renderer } from "../../Renderer.js"
 import { WebARSessionRoot } from "../../webxr/WebARSessionRoot.js";
 import { NeedleWebXRHtmlElement } from "../../webxr/WebXRButtons.js";
-import { XRFlag, XRState, XRStateFlag } from "../../webxr/XRFlag.js";
+import { XRState, XRStateFlag } from "../../webxr/XRFlag.js";
 import type { IUSDExporterExtension } from "./Extension.js";
 import { AnimationExtension } from "./extensions/Animation.js"
 import { AudioExtension } from "./extensions/behavior/AudioExtension.js";
@@ -17,9 +17,8 @@
 import { TextExtension } from "./extensions/USDZText.js";
 import { USDZUIExtension } from "./extensions/USDZUI.js";
 import { USDZExporter as ThreeUSDZExporter } from "./ThreeUSDZExporter.js";
-import { registerAnimatorsImplictly } from "./utils/animationutils.js";
+import { registerAnimatorsImplictly, registerAudioSourcesImplictly } from "./utils/animationutils.js";
 import { ensureQuicklookLinkIsCreated } from "./utils/quicklook.js";
-import { getFormattedDate } from "./utils/timeutils.js";
 const debug = getParam("debugusdz");
@@ -41,10 +40,21 @@
     @serializable(Object3D)
     objectToExport?: Object3D;
+    /** Collect all Animations/Animators automatically on export and emit them as playing at the start.
+     * Animator state chains and loops will automatically be collected and exported in order as well.
+     * If this setting is off, Animators need to be registered by components – for example from PlayAnimationOnClick.
+    */
     @serializable()
     autoExportAnimations: boolean = false;
+    /** Collect all AudioSources automatically on export and emit them as playing at the start.
+     * They will loop according to their settings.
+     * If this setting is off, Audio Sources need to be registered by components – for example from PlayAudioOnClick.
+    */
     @serializable()
+    autoExportAudioSources: boolean = true;
+    @serializable()
     exportFileName?: string;
     @serializable(URL)
@@ -64,6 +74,10 @@
     @serializable()
     planeAnchoringAlignment: "horizontal" | "vertical" | "any" = "horizontal";
+    /** Enabling this option will export QuickLook-specific preliminary behaviours along with the USDZ files.
+     * These extensions are only supported on QuickLook on iOS/visionOS/MacOS.
+     * Keep this option off for general USDZ usage.
+     */
     @serializable()
     interactive: boolean = true;
@@ -194,21 +208,35 @@
         if (!objectToExport)
             return null;
+        Progress.start("export-usdz", { onProgress: (progress) => {
+            this.dispatchEvent(new CustomEvent("export-progress", { detail: { progress } }));
+        }});
+        Progress.report("export-usdz", { message: "Starting export", totalSteps: 40, currentStep: 0 });
+        Progress.report("export-usdz", { message: "Load progressive textures", autoStep: 5 });
+        Progress.start("export-usdz-textures", "export-usdz");
         // trigger progressive textures to be loaded:
         const renderers = GameObject.getComponentsInChildren(objectToExport, Renderer);
         const progressiveLoading = new Array<Promise<any>>();
+        let loadedTextures = 0;
         for (const rend of renderers) {
             for (const mat of rend.sharedMaterials) {
                 if (mat) {
                     const task = rend.loadProgressiveTextures(mat);
                     if (task instanceof Promise)
-                        progressiveLoading.push(task);
+                        progressiveLoading.push(new Promise<void>((resolve, reject) => {
+                            task.then(() => {
+                                loadedTextures++;
+                                Progress.report("export-usdz-textures", { message: "Loaded progressive texture", currentStep: loadedTextures, totalSteps: progressiveLoading.length });
+                                resolve();
+                            }).catch((err) => reject(err));
+                        }));
                 }
             }
         }
         if (debug) showBalloonMessage("Load textures: " + progressiveLoading.length);
         await Promise.all(progressiveLoading);
         if (debug) showBalloonMessage("Load textures: done");
+        Progress.end("export-usdz-textures");
         // apply XRFlags
         const currentXRState = XRState.Global.Mask;
@@ -225,17 +253,31 @@
         extensions.push(animExt);
         const eventArgs = { self: this, exporter: exporter, extensions: extensions, object: objectToExport };
+        Progress.report("export-usdz", "Invoking before-export" );
         this.dispatchEvent(new CustomEvent("before-export", { detail: eventArgs }))
         // Implicit registration and actions for Animators and Animation components
         // Currently, Animators properly build PlayAnimation actions, but Animation components don't.
-        const implicitAnimationBehaviors = new Array<Object3D>();
+        Progress.report("export-usdz", "auto export animations and audio sources");
+        const implicitBehaviors = new Array<Object3D>();
         if (this.autoExportAnimations) {
-            implicitAnimationBehaviors.push(...registerAnimatorsImplictly(objectToExport, animExt));
+            implicitBehaviors.push(...registerAnimatorsImplictly(objectToExport, animExt));
         }
+        const audioExt = this.extensions.find(ext => ext.extensionName === "Audio");
+        if (audioExt && this.autoExportAudioSources)
+            implicitBehaviors.push(...registerAudioSourcesImplictly(objectToExport, audioExt as AudioExtension));
         //@ts-ignore
         exporter.debug = debug;
+        exporter.keepObject = (object) => {
+			// TODO We need to take more care with disabled renderers. This currently breaks when any renderer is disabled
+			// and then enabled at runtime by e.g. SetActiveOnClick, requiring extra work to enable them before export,
+			// cache their state, and then reset their state after export. See
+            const renderer = GameObject.getComponent( object, Renderer )
+			if (renderer && !renderer.enabled) return false;
+			return true;
+        }
         // sanitize anchoring types
         if (this.anchoringType !== "plane" && this.anchoringType !== "none" && this.anchoringType !== "image" && this.anchoringType !== "face")
@@ -243,6 +285,7 @@
         if (this.planeAnchoringAlignment !== "horizontal" && this.planeAnchoringAlignment !== "vertical" && this.planeAnchoringAlignment !== "any")
             this.planeAnchoringAlignment = "horizontal";
+        Progress.report("export-usdz", "Invoking exporter.parse" );
         //@ts-ignore
         const arraybuffer = await exporter.parse(this.objectToExport, {
             ar: {
@@ -260,10 +303,11 @@
         const blob = new Blob([arraybuffer], { type: 'application/octet-stream' });
         // cleanup – implicit animation behaviors need to be removed again
-        for (const go of implicitAnimationBehaviors) {
+        for (const go of implicitBehaviors) {
             GameObject.destroy(go);
         }
+        Progress.report("export-usdz", "Invoking after-export" );
         this.dispatchEvent(new CustomEvent("after-export", { detail: eventArgs }))
         // restore XR flags
@@ -277,6 +321,7 @@
         // this.link.href = URL.createObjectURL(blob2);
         // this.link.click();
+        Progress.end("export-usdz");
         return blob;
     }
@@ -396,7 +441,7 @@
         // that#s the case when no objectToExport is explictly assigned and the whole scene is being exported
         if(!sessionRoot) sessionRoot = GameObject.getComponentInChildren(this.objectToExport, WebARSessionRoot);
-        if(debug) console.log("applyWebARSessionRoot", sessionRoot);
+        if (debug) console.log("applyWebARSessionRoot", sessionRoot);
         if (!sessionRoot) {
             if(debug) console.warn("No WebARSessionRoot found in parent hierarchy", this.objectToExport);
@@ -406,7 +451,6 @@
         // either apply the scale to the object being exported or to the sessionRoot object itself
         const target = hasSessionRootInParentHierarchy ? this.objectToExport : sessionRoot.gameObject;
         const scale = 1 / sessionRoot!.arScale;
-        if (debug) console.log("AR Session Root scale", scale, target);
         target.matrix.makeScale(scale, scale, scale);
         if (sessionRoot.invertForward) {
             target.matrix.multiply(USDZExporter.invertForwardMatrix);

src/engine-components/export/usdz/extensions/USDZText.ts CHANGED Viewed

@@ -6,7 +6,7 @@
 import { TextAnchor } from "../../../ui/Text.js";
 import type { IUSDExporterExtension } from "../Extension.js";
 import type { IBehaviorElement } from "../extensions/behavior/BehavioursBuilder.js";
-import { USDDocument, USDObject, USDWriter, USDZExporterContext } from "../ThreeUSDZExporter.js";
+import { getMaterialNameForUSD,USDDocument, USDObject, USDWriter, USDZExporterContext } from "../ThreeUSDZExporter.js";
 export enum TextWrapMode {
@@ -106,11 +106,10 @@
             writer.appendLine(`token verticalAlignment = "${this.verticalAlignment}"`);
         if (this.material !== undefined) {
-            writer.appendLine(`rel material:binding = </StageRoot/Materials/Material_${this.material.id}>`)
+            writer.appendLine(`rel material:binding = </StageRoot/Materials/${getMaterialNameForUSD(this.material)}>`)
         }
         writer.closeBlock();
     }
 }

src/engine-components/webxr/WebARCameraBackground.ts CHANGED Viewed

@@ -12,7 +12,7 @@
 import { serializable } from "../../engine/engine_serialization_decorator.js";
 import { getParam } from "../../engine/engine_utils.js";
-import { NeedleXREventArgs } from "../../engine/engine_xr.js";
+import type { NeedleXREventArgs } from "../../engine/engine_xr.js";
 import { Behaviour } from "../Component.js";
 import { RGBAColor } from "../js-extensions/RGBAColor.js"

src/engine-components/webxr/WebARSessionRoot.ts CHANGED Viewed

@@ -5,11 +5,11 @@
 import { Context } from "../../engine/engine_context.js";
 import { ObjectUtils, PrimitiveType } from "../../engine/engine_create_objects.js";
 import { destroy, instantiate } from "../../engine/engine_gameobject.js";
-import { NEPointerEvent } from "../../engine/engine_input.js";
+import { InputEventQueue, NEPointerEvent } from "../../engine/engine_input.js";
 import { serializable } from "../../engine/engine_serialization_decorator.js";
-import { IComponent, IGameObject } from "../../engine/engine_types.js";
+import type { IComponent, IGameObject } from "../../engine/engine_types.js";
 import { getParam } from "../../engine/engine_utils.js";
-import { NeedleXRController, NeedleXREventArgs, NeedleXRHitTestResult, NeedleXRSession } from "../../engine/engine_xr.js";
+import { NeedleXRController, type NeedleXREventArgs, type NeedleXRHitTestResult, NeedleXRSession } from "../../engine/engine_xr.js";
 import { Behaviour, GameObject } from "../Component.js";
 // https://github.com/takahirox/takahirox.github.io/blob/master/js.mmdeditor/examples/js/controls/DeviceOrientationControls.js
@@ -135,11 +135,11 @@
         }
         this._reticle.length = 0;
         this._isPlacing = true;
-        this.context.input.addEventListener("pointerup", this.onPlaceScene);
+        this.context.input.addEventListener("pointerup", this.onPlaceScene, InputEventQueue.Early);
     }
     onLeaveXR() {
         // TODO: WebARSessionRoot doesnt work when we enter passthrough and leave XR without having placed the session!!!
-        this.context.input.removeEventListener("pointerup", this.onPlaceScene)
+        this.context.input.removeEventListener("pointerup", this.onPlaceScene, InputEventQueue.Early);
         this.onRevertSceneChanges();
         // this._anchor?.delete();
         this._anchor = null;
@@ -305,6 +305,7 @@
         // if we place the scene we don't want this event to be propagated to any sub-objects (via the EventSystem) anymore and trigger e.g. a click on objects for the "place tap" event
         evt.stopImmediatePropagation();
+        evt.use();
         this._isPlacing = false;
         this.context.input.removeEventListener("pointerup", this.onPlaceScene);
@@ -455,20 +456,7 @@
     }
     private _enabled: boolean = false;
-    enable() {
-        if (this._enabled) return;
-        this._enabled = true;
-        window.addEventListener('touchstart', this.touchStart, { passive: false });
-        window.addEventListener('touchmove', this.touchMove, { passive: false });
-        window.addEventListener('touchend', this.touchEnd, { passive: false });
-    }
-    disable() {
-        if (!this._enabled) return;
-        this._enabled = false;
-        window.removeEventListener('touchstart', this.touchStart);
-        window.removeEventListener('touchmove', this.touchMove);
-        window.removeEventListener('touchend', this.touchEnd);
-    }
     reset() {
         this._scale = 1;
         this.offset.identity();
@@ -494,6 +482,51 @@
         // matrix.premultiply(this.offset).premultiply(this._rotationMatrix)
     }
+    private readonly currentlyUsedPointerIds = new Set<number>();
+    private readonly currentlyUnusedPointerIds = new Set<number>();
+    get isActive() {
+        return this.currentlyUsedPointerIds.size <= 0 && this.currentlyUnusedPointerIds.size > 0;
+    }
+    enable() {
+        if (this._enabled) return;
+        this._enabled = true;
+        this.context.input.addEventListener("pointerdown", this.onPointerDownEarly, InputEventQueue.Early);
+        this.context.input.addEventListener("pointerdown", this.onPointerDownLate, InputEventQueue.Late);
+        this.context.input.addEventListener("pointerup", this.onPointerUpEarly, InputEventQueue.Early);
+        // TODO: refactor the following events to use the input system
+        window.addEventListener('touchstart', this.touchStart, { passive: false });
+        window.addEventListener('touchmove', this.touchMove, { passive: false });
+        window.addEventListener('touchend', this.touchEnd, { passive: false });
+    }
+    disable() {
+        if (!this._enabled) return;
+        this._enabled = false;
+        this.context.input.removeEventListener("pointerdown", this.onPointerDownEarly, InputEventQueue.Early);
+        this.context.input.removeEventListener("pointerdown", this.onPointerDownLate, InputEventQueue.Late);
+        this.context.input.removeEventListener("pointerup", this.onPointerUpEarly, InputEventQueue.Early);
+        window.removeEventListener('touchstart', this.touchStart);
+        window.removeEventListener('touchmove', this.touchMove);
+        window.removeEventListener('touchend', this.touchEnd);
+    }
+    private onPointerDownEarly = (e: NEPointerEvent) => {
+        if (this.isActive) e.stopPropagation();
+    };
+    private onPointerDownLate = (e: NEPointerEvent) => {
+        if (e.used) this.currentlyUsedPointerIds.add(e.pointerId);
+        else if(this.currentlyUsedPointerIds.size <= 0) this.currentlyUnusedPointerIds.add(e.pointerId);
+    };
+    private onPointerUpEarly = (e: NEPointerEvent) => {
+        this.currentlyUsedPointerIds.delete(e.pointerId);
+        this.currentlyUnusedPointerIds.delete(e.pointerId);
+    };
     // private _needsUpdate: boolean = true;
     // private _rotationMatrix: Matrix4 = new Matrix4();
     // private updateMatrix() {
@@ -540,6 +573,7 @@
     }
     private touchMove = (evt: TouchEvent) => {
         if (evt.defaultPrevented) return;
+        if (!this.isActive) return;
         if (evt.touches.length === 1) {
             // if we had multiple touches before due to e.g. pinching / rotating

src/engine-components/webxr/WebXR.ts CHANGED Viewed

@@ -4,7 +4,7 @@
 import { AssetReference } from "../../engine/engine_addressables.js";
 import { serializable } from "../../engine/engine_serialization.js";
 import { getParam, isDesktop, isiOS, isMobileDevice, isQuest, isSafari } from "../../engine/engine_utils.js";
-import { NeedleXREventArgs, NeedleXRSession } from "../../engine/engine_xr.js";
+import { type NeedleXREventArgs, NeedleXRSession } from "../../engine/engine_xr.js";
 import { PlayerSync } from "../../engine-components-experimental/networking/PlayerSync.js";
 import { Behaviour, GameObject } from "../Component.js";
 import { USDZExporter } from "../export/usdz/USDZExporter.js";

src/engine-components/webxr/WebXRImageTracking.ts CHANGED Viewed

@@ -4,7 +4,7 @@
 import { AssetReference } from "../../engine/engine_addressables.js";
 import { serializable } from "../../engine/engine_serialization.js";
 import { CircularBuffer, getParam } from "../../engine/engine_utils.js";
-import { NeedleXREventArgs, NeedleXRSession } from "../../engine/xr/index.js";
+import { type NeedleXREventArgs, NeedleXRSession } from "../../engine/xr/index.js";
 import { imageToCanvas,USDWriter, USDZExporterContext } from "../../engine-components/export/usdz/ThreeUSDZExporter.js";
 import { USDZExporter } from "../../engine-components/export/usdz/USDZExporter.js";
 import { Behaviour, GameObject } from "../Component.js";

src/engine-components/webxr/WebXRPlaneTracking.ts CHANGED Viewed

@@ -6,7 +6,7 @@
 import { serializable } from "../../engine/engine_serialization.js";
 import type { Vec3 } from "../../engine/engine_types.js";
 import { getParam } from "../../engine/engine_utils.js";
-import { NeedleXREventArgs, NeedleXRSession } from "../../engine/engine_xr.js";
+import type { NeedleXREventArgs, NeedleXRSession } from "../../engine/engine_xr.js";
 import { MeshCollider } from "../Collider.js";
 import { Behaviour, GameObject } from "../Component.js";
 // import { SimplifyModifier } from 'three/examples/jsm/modifiers/SimplifyModifier.js';

src/engine-components/webxr/WebXRRig.ts CHANGED Viewed

@@ -1,10 +1,10 @@
-import { AxesHelper, Euler, Object3D, Quaternion, Vector3 } from "three";
+import { AxesHelper, Object3D, Vector3 } from "three";
 import { serializable } from "../../engine/engine_serialization_decorator.js";
 import type { IGameObject } from "../../engine/engine_types.js";
 import { getParam } from "../../engine/engine_utils.js";
-import { NeedleXREventArgs, NeedleXRSession } from "../../engine/engine_xr.js";
-import { IXRRig } from "../../engine/engine_xr.js";
+import type { IXRRig } from "../../engine/engine_xr.js";
+import { type NeedleXREventArgs, NeedleXRSession } from "../../engine/engine_xr.js";
 import { Behaviour } from "../Component.js";
 import { BoxGizmo } from "../Gizmos.js";

src/engine-components/webxr/controllers/XRControllerFollow.ts CHANGED Viewed

@@ -1,6 +1,6 @@
 import { serializable } from "../../../engine/engine_serialization_decorator.js";
-import { NeedleXREventArgs } from "../../../engine/engine_xr.js";
+import type { NeedleXREventArgs } from "../../../engine/engine_xr.js";
 import { Behaviour } from "../../Component.js";

src/engine-components/webxr/controllers/XRControllerModel.ts CHANGED Viewed

@@ -9,9 +9,9 @@
 import { Gizmos } from "../../../engine/engine_gizmos.js";
 import { addDracoAndKTX2Loaders } from "../../../engine/engine_loaders.js";
 import { serializable } from "../../../engine/engine_serialization_decorator.js";
-import { IGameObject } from "../../../engine/engine_types.js";
+import type { IGameObject } from "../../../engine/engine_types.js";
 import { getParam } from "../../../engine/engine_utils.js";
-import { NeedleXRController, NeedleXRControllerEventArgs, NeedleXREventArgs, NeedleXRSession } from "../../../engine/engine_xr.js";
+import { NeedleXRController, type NeedleXRControllerEventArgs, type NeedleXREventArgs, NeedleXRSession } from "../../../engine/engine_xr.js";
 import { Behaviour, GameObject } from "../../Component.js"
 const debug = getParam("debugwebxr");

src/engine-components/webxr/controllers/XRControllerMovement.ts CHANGED Viewed

@@ -5,15 +5,15 @@
 import { Gizmos } from "../../../engine/engine_gizmos.js";
 import { Mathf } from "../../../engine/engine_math.js";
-import { RaycastTestObjectCallback } from "../../../engine/engine_physics.js";
+import type { RaycastTestObjectCallback } from "../../../engine/engine_physics.js";
 import { serializable } from "../../../engine/engine_serialization.js"
 import { getTempVector, getWorldQuaternion, getWorldScale } from "../../../engine/engine_three_utils.js";
-import { IGameObject } from "../../../engine/engine_types.js";
+import type { IGameObject } from "../../../engine/engine_types.js";
 import { getParam } from "../../../engine/engine_utils.js";
-import { NeedleXRController, NeedleXREventArgs, NeedleXRSession } from "../../../engine/engine_xr.js";
+import { NeedleXRController, type NeedleXREventArgs, NeedleXRSession } from "../../../engine/engine_xr.js";
 import { Behaviour, GameObject } from "../../Component.js"
 import { TeleportTarget } from "../TeleportTarget.js";
-import { XRMovementBehaviour } from "../types.js";
+import type { XRMovementBehaviour } from "../types.js";
 const debug = getParam("debugwebxr");

src/engine/xr/XRRig.ts CHANGED Viewed

@@ -1,4 +1,4 @@
-import { IComponent } from "../engine_types.js";
+import type { IComponent } from "../engine_types.js";
 export interface IXRRig extends Pick<IComponent, "gameObject"> {

src/engine/debug/debug_spatial_console.ts ADDED Viewed

@@ -0,0 +1,395 @@
+import { Layers, Material, Matrix3, Matrix4, Mesh, Object3D, PerspectiveCamera, Quaternion, Texture, Vector2, Vector3, Vector4 } from "three";
+import ThreeMeshUI from "three-mesh-ui";
+import type { Options } from "three-mesh-ui/build/types/core/elements/MeshUIBaseElement";
+import { ContextRegistry } from "../engine_context_registry.js";
+import { OneEuroFilterXYZ } from "../engine_math.js";
+import { lookAtObject } from "../engine_three_utils.js";
+import type { IContext, IGameObject } from "../engine_types.js";
+let _isActive = false;
+/** Enable a spatial debug console that follows the camera */
+export function enableSpatialConsole(active: boolean) {
+    if (active) {
+        if (_isActive) return;
+        _isActive = true;
+        onEnable();
+    } else {
+        if (!_isActive) return;
+        _isActive = false;
+        onDisable();
+    }
+}
+const originalConsoleMethods: { [key: string]: Function | undefined } = {
+    "log": undefined,
+    "warn": undefined,
+    "error": undefined,
+};
+class SpatialMessagesHandler {
+    private readonly familyName = "needle-xr";
+    private root: ThreeMeshUI.Block | null = null;
+    private context: IContext | null = null;
+    private readonly defaultFontSize = .06;
+    constructor() {
+        this.ensureFont();
+    }
+    onEnable() {
+        this.context = ContextRegistry.Current || ContextRegistry.All[0];
+        this.context.pre_render_callbacks.push(this.onBeforeRender)
+    }
+    onDisable() {
+        this.context?.pre_render_callbacks.splice(this.context?.pre_render_callbacks.indexOf(this.onBeforeRender), 1);
+    }
+    private readonly targetObject = new Object3D();
+    private readonly oneEuroFilter = new OneEuroFilterXYZ(90, .1);
+    private _lastElementRemoveTime = 0;
+    private onBeforeRender = () => {
+        const cam = this.context?.mainCamera as any as IGameObject;
+        if (this.context && cam instanceof PerspectiveCamera) {
+            const root = this.getRoot() as any as IGameObject;
+            // TODO: need to figure out why this happens when entering VR (in the simulator at least)
+            if (Number.isNaN(root.position.x))
+                root.position.set(0, 0, 0);
+            this.context.scene.add(this.targetObject);
+            const dist = 1.8;
+            const forward = cam.worldForward;
+            forward.y = 0;
+            forward.normalize().multiplyScalar((dist * 100) / cam.fov);
+            this.targetObject.position.copy(cam.worldPosition).sub(forward);
+            lookAtObject(this.targetObject, cam, false, true);
+            this.oneEuroFilter.filter(this.targetObject.position, root.position, this.context.time.time);
+            const step = this.context.time.deltaTime * 10;
+            root.quaternion.slerp(this.targetObject.quaternion, step);
+            this.targetObject.removeFromParent();
+            this.context.scene.add(this.getRoot() as any);
+            if (this.context.time.time - this._lastElementRemoveTime > .3) {
+                this._lastElementRemoveTime = this.context.time.time;
+                const now = Date.now();
+                for (let i = 0; i < root.children.length; i++) {
+                    const el = root.children[i];
+                    if (el instanceof ThreeMeshUI.Text && now - el["_activatedTime"] > 10000) {
+                        el.removeFromParent();
+                        this._textBuffer.push(el);
+                        break;
+                    }
+                }
+            }
+        }
+    }
+    addLog(type: "log" | "warn" | "error", message: string) {
+        const root = this.getRoot();
+        const text = this.getText();
+        let backgroundColor = 0xffffff;
+        let fontColor = 0x000000;
+        switch (type) {
+            case "log":
+                backgroundColor = 0xffffff;
+                fontColor = 0x000000;
+                break;
+            case "warn":
+                backgroundColor = 0xffee99;
+                fontColor = 0x442200;
+                break;
+            case "error":
+                backgroundColor = 0xffaaaa;
+                fontColor = 0xaa0000;
+                break;
+        }
+        text.set({
+            backgroundColor: backgroundColor,
+            color: fontColor,
+        });
+        if (message.length > 1000) message = message.substring(0, 1000) + "...";
+        const hourMinuteSecond = new Date().toLocaleTimeString().split(" ")[0];
+        text.textContent = "[" + hourMinuteSecond + "] " + message;
+        text.visible = true;
+        text["_activatedTime"] = Date.now();
+        root.add(text as any);
+        if (this.context) this.context.scene.add(root as any);
+        ThreeMeshUI.update();
+    }
+    private ensureFont() {
+        let fontFamily = ThreeMeshUI.FontLibrary.getFontFamily(this.familyName);
+        if (!fontFamily) {
+            fontFamily = ThreeMeshUI.FontLibrary.addFontFamily(this.familyName);
+            const variant = fontFamily.addVariant("normal", "normal", "./include/needle/arial-msdf.json", "./include/needle/arial.png") as any as ThreeMeshUI.FontVariant;
+            variant?.addEventListener('ready', () => {
+                ThreeMeshUI.update();
+            });
+        }
+    }
+    private readonly textOptions: Options = {
+        fontSize: this.defaultFontSize,
+        fontFamily: this.familyName,
+        padding: .03,
+        margin: .005,
+        color: 0x000000,
+        backgroundColor: 0xffffff,
+        backgroundOpacity: .4,
+        borderRadius: .03,
+    };
+    private readonly _textBuffer: ThreeMeshUI.Text[] = [];
+    private getText() {
+        const root = this.getRoot();
+        if (this._textBuffer.length > 0) {
+            const text = this._textBuffer.pop() as any as ThreeMeshUI.Text;
+            text.visible = true;
+            setTimeout(() => this.disableDepthTestRecursive(text as any), 100);
+            return text;
+        }
+        if (root.children.length > 20) {
+            return root.children[0] as any as ThreeMeshUI.Text;
+        }
+        const newText = new ThreeMeshUI.Text(this.textOptions);
+        setTimeout(() => this.disableDepthTestRecursive(newText as any), 500);
+        setTimeout(() => this.disableDepthTestRecursive(newText as any), 1500);
+        return newText;
+    }
+    private disableDepthTestRecursive(obj: Object3D) {
+        obj.traverseVisible((t: Object3D) => {
+            t.renderOrder = 1000;
+            t.layers.set(2);
+            const mat = (t as Mesh).material as Material;
+            if (mat) {
+                mat.depthWrite = false;
+                mat.depthTest = false;
+            }
+            ThreeMeshUI.update();
+        });
+    }
+    private getRoot() {
+        if (this.root) {
+            return this.root;
+        }
+        const fontSize = this.defaultFontSize;
+        const defaultOptions: Options = {
+            boxSizing: 'border-box',
+            fontFamily: this.familyName,
+            width: "2.6",
+            fontSize: fontSize,
+            color: 0x000000,
+            lineHeight: 1,
+            backgroundColor: 0xffffff,
+            backgroundOpacity: 0,
+            // borderColor: 0xffffff,
+            // borderOpacity: .5,
+            // borderWidth: 0.01,
+            // padding: 0.01,
+            whiteSpace: 'pre-wrap',
+            flexDirection: 'column-reverse',
+        };
+        this.root = new ThreeMeshUI.Block(defaultOptions);
+        return this.root;
+    }
+}
+let messagesHandler: SpatialMessagesHandler | null = null;
+function onEnable() {
+    // create messages handler
+    if (!messagesHandler) messagesHandler = new SpatialMessagesHandler();
+    messagesHandler.onEnable();
+    // save original console methods
+    for (const key in originalConsoleMethods) {
+        originalConsoleMethods[key] = console[key];
+        let isLogging = false;
+        console[key] = function () {
+            // call original console method
+            originalConsoleMethods[key]?.apply(console, arguments);
+            // prevent circular calls
+            if (isLogging) return;
+            try {
+                isLogging = true;
+                onLog(key as "log" | "warn" | "error", ...arguments);
+            } finally {
+                isLogging = false;
+            }
+        };
+    }
+    console.log("Enabling Spatial Console");
+}
+function onDisable() {
+    messagesHandler?.onDisable();
+    for (const key in originalConsoleMethods) {
+        console[key] = originalConsoleMethods[key];
+    }
+}
+const seen = new Map<string, any>();
+function onLog(key: "log" | "warn" | "error", ...args: any[]) {
+    try {
+        seen.clear();
+        switch (key) {
+            case "log":
+                messagesHandler?.addLog("log", getLogString());
+                break;
+            case "warn":
+                messagesHandler?.addLog("warn", getLogString());
+                break;
+            case "error":
+                messagesHandler?.addLog("error", getLogString());
+                break;
+        }
+    }
+    catch (err) {
+        console.error("Error in spatial console", err);
+    }
+    finally {
+        seen.clear();
+    }
+    function getLogString() {
+        let str = "";
+        for (let i = 0; i < args.length; i++) {
+            const el = args[i];
+            str += serialize(el);
+            if (i < args.length - 1) str += ", ";
+        }
+        return str;
+    }
+    function serialize(value: any, level: number = 0): string {
+        if (typeof value === "string") {
+            return "\"" + value + "\"";
+        }
+        else if (typeof value === "number") {
+            const hasDecimal = value % 1 !== 0;
+            if (hasDecimal) {
+                const str = value.toFixed(5);
+                const dotIndex = str.indexOf(".");
+                let i = str.length - 1;
+                while (i > dotIndex && str[i] === "0") i--;
+                return str.substring(0, i + 1);
+            }
+            return value.toString();
+        }
+        else if (Array.isArray(value)) {
+            let res = "[";
+            for (let i = 0; i < value.length; i++) {
+                const val = value[i];
+                res += serialize(val, level + 1);
+                if (i < value.length - 1) res += ", ";
+            }
+            res += "]";
+            return res;
+        }
+        else if (value === null) {
+            return "null";
+        }
+        else if (value === undefined) {
+            return "undefined";
+        }
+        else if (typeof value === "function") {
+            return value.name + "()";
+        }
+        // if (value instanceof Object3D) {
+        //     const name = value.name?.length > 0 ? value.name : ("object@" + (value["guid"] ?? value.uuid));;
+        //     return name;
+        // }
+        if (value instanceof Vector2) return `(${serialize(value.x)}, ${serialize(value.y)})`;
+        if (value instanceof Vector3) return `(${serialize(value.x)}, ${serialize(value.y)}, ${serialize(value.z)})`;
+        if (value instanceof Vector4) return `(${serialize(value.x)}, ${serialize(value.y)}, ${serialize(value.z)}, ${serialize(value.w)})`;
+        if (value instanceof Quaternion) return `(${serialize(value.x)}, ${serialize(value.y)}, ${serialize(value.z)}, ${serialize(value.w)})`;
+        if (value instanceof Material) return value.name;
+        if (value instanceof Texture) return value.name;
+        if (value instanceof Matrix3) return `[${value.elements.join(", ")}]`;
+        if (value instanceof Matrix4) return `[${value.elements.join(", ")}]`;
+        if (value instanceof Layers) return value.mask.toString();
+        if (typeof value === "object") {
+            if (seen.has(value)) return "*";
+            let res = "{\n";
+            res += pad(level);
+            const keys = Object.keys(value);
+            let line = "";
+            for (let i = 0; i < keys.length; i++) {
+                const key = keys[i];
+                const val = value[key];
+                if (seen.has(val)) {
+                    line += ""
+                    continue;
+                }
+                seen.set(val, true);
+                line += key + ":" + serialize(val, level + 1);
+                if (i < keys.length - 1) line += ", ";
+                if (line.length >= 60) {
+                    line += "\n";
+                    line += pad(level);
+                    res += line;
+                    line = "";
+                }
+            }
+            res += line;
+            res += "\n}";
+            return res;
+            // return JSON.stringify(value, (_key, value) => {
+            //     if (seen.has(value)) return seen.get(value);
+            //     seen.set(value, "-");
+            //     const res = serialize(value);
+            //     seen.set(value, res);
+            //     return _key;
+            // }, 1);
+        }
+        return value;
+    }
+    function pad(spaces: number) {
+        let res = "";
+        for (let i = 0; i < spaces; i++) {
+            res += " ";
+        }
+        return res;
+    }
+}
+// // this is just a hack - the spatial console should be enabled from the user or the NeedleXRSession
+// if (getParam("debugwebxr") || getParam("console"))
+//     setTimeout(() => enableSpatialConsole(true), 1000);

src/engine/engine_time_utils.ts ADDED Viewed

@@ -0,0 +1,238 @@
+import { getParam } from "./engine_utils.js";
+const showProgressLogs = getParam("debugprogress");
+/** Gets the date formatted as 20240220-161993. When no Date is passed in, the current local date is used. */
+export function getFormattedDate(date?: Date) {
+    date = date || new Date();
+    const month = date.getMonth() + 1;
+    const day = date.getDate();
+    const hour = date.getHours();
+    const min = date.getMinutes();
+    const sec = date.getSeconds();
+    const s_month = (month < 10 ? "0" : "") + month;
+    const s_day = (day < 10 ? "0" : "") + day;
+    const s_hour = (hour < 10 ? "0" : "") + hour;
+    const s_min = (min < 10 ? "0" : "") + min;
+    const s_sec = (sec < 10 ? "0" : "") + sec;
+    return date.getFullYear() + s_month + s_day + "-" + s_hour + s_min + s_sec;
+}
+declare type ProgressOptions = {
+    message?: string,
+    progress?: number,
+    autoStep?: boolean | number;
+    currentStep?: number,
+    totalSteps?: number
+};
+declare type ProgressStartOptions = {
+    /** This progress scope will be nested below parentScope */
+    parentScope?: string,
+    /** Callback with progress in 0..1 range. */
+    onProgress?: (progress: number) => void,
+    /** Log timings using console.time() and console.timeLog(). */
+    logTimings?: boolean,
+};
+/** Progress reporting utility.
+ * See `Progress.start` for usage examples.
+ */
+export class Progress {
+    /** Start a new progress reporting scope. Make sure to close it with Progress.end.
+     * @param scope The scope to start progress reporting for.
+     * @param options Parent scope, onProgress callback and logging. If only a string is provided, it's used as parentScope.
+     * @example
+     * // Manual usage:
+     * Progress.start("export-usdz", undefined, (progress) => console.log("Progress: " + progress));
+     * Progress.report("export-usdz", { message: "Exporting object 1", currentStep: 1, totalSteps: 3 });
+     * Progress.report("export-usdz", { message: "Exporting object 2", currentStep: 2, totalSteps: 3 });
+     * Progress.report("export-usdz", { message: "Exporting object 3", currentStep: 3, totalSteps: 3 });
+     *
+     * // Auto step usage:
+     * Progress.start("export-usdz", undefined, (progress) => console.log("Progress: " + progress));
+     * Progress.report("export-usdz", { message: "Exporting objects", autoStep: true, totalSteps: 3 });
+     * Progress.report("export-usdz", "Exporting object 1");
+     * Progress.report("export-usdz", "Exporting object 2");
+     * Progress.report("export-usdz", "Exporting object 3");
+     * Progress.end("export-usdz");
+     *
+     * // Auto step with weights:
+     * Progress.start("export-usdz", undefined, (progress) => console.log("Progress: " + progress));
+     * Progress.report("export-usdz", { message: "Exporting objects", autoStep: true, totalSteps: 10 });
+     * Progress.report("export-usdz", { message: "Exporting object 1", autoStep: 8 }); // will advance to 80% progress
+     * Progress.report("export-usdz", "Exporting object 2"); // 90%
+     * Progress.report("export-usdz", "Exporting object 3"); // 100%
+     *
+     * // Child scopes:
+     * Progress.start("export-usdz", undefined, (progress) => console.log("Progress: " + progress));
+     * Progress.report("export-usdz", { message: "Overall export", autoStep: true, totalSteps: 2 });
+     * Progress.start("export-usdz-objects", "export-usdz");
+     * Progress.report("export-usdz-objects", { message: "Exporting objects", autoStep: true, totalSteps: 3 });
+     * Progress.report("export-usdz-objects", "Exporting object 1");
+     * Progress.report("export-usdz-objects", "Exporting object 2");
+     * Progress.report("export-usdz-objects", "Exporting object 3");
+     * Progress.end("export-usdz-objects");
+     * Progress.report("export-usdz", "Exporting materials");
+     * Progress.end("export-usdz");
+     *
+     * // Enable console logging:
+     * Progress.start("export-usdz", { logTimings: true });
+     */
+    static start(scope: string, options?: ProgressStartOptions | string ) {
+        if (typeof options === "string") options = { parentScope: options };
+        const p = new ProgressEntry(scope, options);
+        progressCache.set(scope, p);
+    }
+    /** Report progress for a formerly started scope.
+     * @param scope The scope to report progress for.
+     * @param options Options for the progress report. If a string is passed, it will be used as the message.
+     * @example
+     * // auto step and show a message
+     * Progress.report("export-usdz", "Exporting object 1");
+     * // same as above
+     * Progress.report("export-usdz", { message: "Exporting object 1", autoStep: true });
+     * // show the current step and total steps and implicitly calculate progress as 10%
+     * Progress.report("export-usdz", { currentStep: 1, totalSteps: 10 });
+     * // enable auto step mode, following calls that have autoStep true will increase currentStep automatically.
+     * Progress.report("export-usdz", { totalSteps: 20, autoStep: true });
+     * // show the progress as 50%
+     * Progress.report("export-usdz", { progress: 0.5 });
+     * // give this step a weight of 20, which changes how progress is calculated. Useful for steps that take longer and/or have child scopes.
+     * Progress.report("export-usdz", { message. "Long process", autoStep: 20 });
+     * // show the current step and total steps and implicitly calculate progress as 10%
+     * Progress.report("export-usdz", { currentStep: 1, totalSteps: 10 });
+     */
+    static report(scope: string, options?: ProgressOptions | string) {
+        const p = progressCache.get(scope);
+        if (!p) {
+            console.warn("Reporting progress for non-existing scope", scope);
+            return;
+        }
+        if (typeof options === "string") options = { message: options, autoStep: true };
+        p.report(options);
+    }
+    /** End a formerly started scope. This will also report the progress as 100%.
+     * @remarks Will warn if any child scope is still running (progress < 1).
+    */
+    static end(scope: string) {
+        const p = progressCache.get(scope);
+        if (!p) return;
+        p.end();
+        progressCache.delete(scope);
+    }
+}
+const progressCache: Map<string, ProgressEntry> = new Map<string, ProgressEntry>();
+/** Internal class that handles Progress instances and their parent/child relationship. */
+class ProgressEntry {
+    private scopeLabel: string;
+    private parentScope?: ProgressEntry;
+    private childScopes: Array<ProgressEntry> = [];
+    private parentDepth = 0;
+    private lastStep? = 0;
+    private lastAutoStepWeight = 1;
+    private lastTotalSteps? = 0;
+    private onProgress?: (progress: number) => void;
+    private showLogs: boolean = false;
+    selfProgress: number = 0;
+    totalProgress: number = 0;
+    selfReports: number = 0;
+    totalReports: number = 0;
+    constructor(scope: string, options?: ProgressStartOptions) {
+        this.parentScope = options?.parentScope ? progressCache.get(options.parentScope) : undefined;
+        if (this.parentScope) {
+            this.parentScope.childScopes.push(this);
+            this.parentDepth = this.parentScope.parentDepth + 1;
+        }
+        this.scopeLabel = " ".repeat(this.parentDepth * 2) + scope;
+        this.showLogs = options?.logTimings ?? showProgressLogs !== undefined;
+        if (this.showLogs) console.time(this.scopeLabel);
+        this.onProgress = options?.onProgress;
+    }
+    report(options?: ProgressOptions, indirect: boolean = false) {
+        if (options) {
+            if (options.totalSteps !== undefined)
+                this.lastTotalSteps = options.totalSteps;
+            if (options.currentStep !== undefined)
+                this.lastStep = options.currentStep;
+            if (options.autoStep !== undefined) {
+                if (options.currentStep === undefined) {
+                    if (this.lastStep === undefined) this.lastStep = 0;
+                    const stepIncrease = typeof options.autoStep === "number" ? options.autoStep : 1;
+                    this.lastStep += this.lastAutoStepWeight;
+                    this.lastAutoStepWeight = stepIncrease;
+                    options.currentStep = this.lastStep;
+                }
+                options.totalSteps = this.lastTotalSteps;
+            }
+            if (options.progress !== undefined)
+                this.selfProgress = options.progress;
+            else if (options.currentStep !== undefined && options.totalSteps !== undefined) {
+                this.selfProgress = options.currentStep / options.totalSteps;
+            }
+        }