Needle Engine 3.51.0

Files changed (18) hide show

src/engine-components/export/usdz/extensions/Animation.ts +183 -45
src/engine-components/export/usdz/extensions/behavior/Behaviour.ts +132 -110
src/engine-components/export/usdz/extensions/behavior/BehaviourComponents.ts +64 -8
src/engine-components/export/usdz/extensions/behavior/BehavioursBuilder.ts +27 -8
src/engine/debug/debug_console.ts +17 -4
src/engine/engine_element.ts +10 -4
src/engine/engine_gameobject.ts +3 -0
src/engine/engine_gltf_builtin_components.ts +1 -1
src/engine/engine_networking_instantiate.ts +21 -6
src/engine/engine_networking.ts +3 -2
src/engine/engine_time_utils.ts +1 -1
src/engine-components/export/usdz/index.ts +1 -1
src/engine-components/export/usdz/extensions/behavior/PhysicsExtension.ts +1 -3
src/engine-components-experimental/networking/PlayerSync.ts +5 -0
src/engine-components/RigidBody.ts +4 -1
src/engine-components/ShadowCatcher.ts +3 -0
src/engine-components/export/usdz/ThreeUSDZExporter.ts +53 -4
src/engine-components/export/usdz/USDZExporter.ts +49 -10

src/engine-components/export/usdz/extensions/Animation.ts CHANGED Viewed

@@ -21,14 +21,17 @@
     private _start?: number;
     get start(): number {
         if (this._start === undefined) {
-            this._start = this.ext.getStartTime01(this.root, this.clip);
+            this._start = this.ext.getStartTimeByClip(this.clip);
         }
         return this._start;
     }
     get duration(): number { return this.clip?.duration ?? TransformData.restPoseClipDuration; }
+    get nearestAnimatedRoot(): Object3D | undefined { return this._nearestAnimatedRoot; }
+    get clipName(): string { return this.clip?.name ?? "rest"; }
     private ext: AnimationExtension;
     private root: Object3D;
+    private _nearestAnimatedRoot?: Object3D = undefined;
     private clip: AnimationClip | null;
     // Playback speed. Does not affect how the animation is written, just how fast actions play it back.
@@ -38,7 +41,52 @@
         this.ext = ext;
         this.root = root;
         this.clip = clip;
+        this._nearestAnimatedRoot = this.getNearestAnimatedRoot();
     }
+    private static isDescendantOf(parent: Object3D, child: Object3D) {
+        let current: Object3D | null = child;
+        if (!current || !parent) return false;
+        while (current) {
+            if (!current) return false;
+            if (current === parent) return true;
+            current = current.parent;
+        }
+        return false;
+    }
+    /** Finds the nearest actually animated object under root based on the tracks in the AnimationClip. */
+    getNearestAnimatedRoot() {
+        let highestRoot: Object3D | undefined = undefined;
+        try {
+            for (const track of this.clip?.tracks ?? []) {
+                const parsedPath = PropertyBinding.parseTrackName(track.name);
+                let animationTarget = PropertyBinding.findNode(this.root, parsedPath.nodeName);
+                if (animationTarget) {
+                    if (!highestRoot) highestRoot = animationTarget;
+                    else {
+                        if (animationTarget === highestRoot) continue;
+                        if (RegisteredAnimationInfo.isDescendantOf(highestRoot, animationTarget)) continue;
+                        if (!RegisteredAnimationInfo.isDescendantOf(animationTarget, highestRoot)) {
+                            // TODO test this, should find the nearest common ancestor
+                            while (!RegisteredAnimationInfo.isDescendantOf(animationTarget, highestRoot) && animationTarget.parent) {
+                                animationTarget = animationTarget.parent;
+                            }
+                            if (!RegisteredAnimationInfo.isDescendantOf(animationTarget, highestRoot)) {
+                                console.error("USDZExporter: Animation clip targets multiple roots that are not parent/child. Please report a bug", this.root, this.clip, highestRoot, animationTarget);
+                            }
+                        }
+                        highestRoot = animationTarget;
+                    }
+                }
+            }
+        } catch (e) {
+            console.error("USDZExporter: Exception when trying to find nearest animated root. Please report a bug", e);
+            highestRoot = undefined;
+        }
+        return highestRoot;
+    }
 }
 export class TransformData {
@@ -46,9 +94,8 @@
     pos?: KeyframeTrack;
     rot?: KeyframeTrack;
     scale?: KeyframeTrack;
-    get frameRate(): number { return 60; }
-    private root: Object3D | null;
+    private root: Object3D | null;
     private target: Object3D;
     private duration = 0;
     private useRootMotion = false;
@@ -56,8 +103,9 @@
     /** This value can theoretically be anything – a value of 1 is good to clearly see animation gaps.
      * For production, a value of 1/60 is enough, since the files can then still properly play back at 60fps.
      */
-    static animationDurationPadding = 1 / 60;
-    static restPoseClipDuration = 1 / 60;
+    static frameRate = 60;
+    static animationDurationPadding = 6 / 60;
+    static restPoseClipDuration = 6 / 60;
     constructor(root: Object3D | null, target: Object3D, clip: AnimationClip | null) {
         this.root = root;
@@ -73,6 +121,17 @@
             this.duration = clip.duration;
         }
+        // warn if the duration does not equal the maximum time value in the tracks
+        if (clip && clip.tracks) {
+            const maxTime = Math.max(...clip.tracks.map((t) => t.times[t.times.length - 1]));
+            if (maxTime !== this.duration) {
+                console.warn("USDZExporter: Animation clip duration does not match the maximum time value in the tracks.", clip, maxTime, this.duration);
+                // We need to override the duration, otherwise we end up with gaps in the exported animation
+                // where there are actually no written animation values
+                this.duration = maxTime;
+            }
+        }
         const animator = GameObject.getComponent(root, Animator);
         if (animator) this.useRootMotion = animator.applyRootMotion;
     }
@@ -132,8 +191,9 @@
         if (generateRot && rotTimesArray) for (const t of rotTimesArray) timesArray.push(t);
         if (generateScale && scaleTimesArray) for (const t of scaleTimesArray) timesArray.push(t);
-        // we also need to make sure we have start and end times for these tracks
-        // TODO seems we can't get track duration from the KeyframeTracks
+        // We also need to make sure we have start and end times for these tracks
+        // We already ensure this is the case for duration – it's always the maximum time value in the tracks
+        // (see constructor)
         if (!timesArray.includes(0)) timesArray.push(0);
         // sort times so it's increasing
@@ -166,9 +226,34 @@
         if (!rotationInterpolant) rotation.set(object.quaternion.x, object.quaternion.y, object.quaternion.z, object.quaternion.w);
         if (!scaleInterpolant) scale.set(object.scale.x, object.scale.y, object.scale.z);
-        for (let index = 0; index < timesArray.length; index++) {
-            const time = timesArray[index];
+        // WORKAROUND because it seems that sometimes we get a quaternion interpolant with a valueSize
+        // of its own length... which then goes into an endless loop
+        if (positionInterpolant && positionInterpolant.valueSize !== 3)
+            positionInterpolant.valueSize = 3;
+        if (rotationInterpolant && rotationInterpolant.valueSize !== 4)
+            rotationInterpolant.valueSize = 4;
+        if (scaleInterpolant && scaleInterpolant.valueSize !== 3)
+            scaleInterpolant.valueSize = 3;
+        // We're optionally padding with one extra time step at the beginning and the end
+        // So that we don't get unwanted interpolations between clips (during the padding time)
+        const extraFrame = 0; // TransformData.animationDurationPadding > 1 / 60 ? 1 : 0;
+        for (let index = 0 - extraFrame; index < timesArray.length + extraFrame; index++) {
+            let time = 0;
+            let returnTime = 0;
+            if (index < 0) {
+                time = timesArray[0];
+                returnTime = time - (TransformData.animationDurationPadding / 2) + 1 / 60;
+            }
+            else if (index >= timesArray.length) {
+                time = timesArray[timesArray.length - 1];
+                returnTime = time + (TransformData.animationDurationPadding / 2) - 1 / 60;
+            }
+            else {
+                time = timesArray[index];
+                returnTime = time;
+            }
             if (positionInterpolant) {
                 const pos = positionInterpolant.evaluate(time);
                 translation.set(pos[0], pos[1], pos[2]);
@@ -192,21 +277,23 @@
                 rootMatrix.decompose(translation, rotation, scale);
             }
-            yield { time, translation, rotation, scale, index };
+            yield { time: returnTime, translation, rotation, scale, index };
         }
     }
 }
-declare type AnimationDict = Map<Object3D, Array<TransformData>>;
 export class AnimationExtension implements IUSDExporterExtension {
     get extensionName(): string { return "animation" }
+    get animationData() { return this.dict; }
+    get registeredClips() { return this.clipToStartTime.keys(); }
+    get animatedRoots() { return this.rootTargetMap.keys(); }
+    get holdClipMap() { return this.clipToHoldClip; }
     /** For each animated object, contains time/pos/rot/scale samples in the format that USD needs,
      *  ready to be written to the .usda file.
      */
-    private dict: AnimationDict = new Map();
+    private dict: Map<Object3D, Array<TransformData>> = new Map();
     /** Map of all roots (Animation/Animator or scene) and all targets that they animate.
      *  We need that info so that we can ensure that each target has the same number of TransformData entries
      *  so that switching between animations doesn't result in data "leaking" to another clip.
@@ -215,6 +302,9 @@
     private rootAndClipToRegisteredAnimationMap = new Map<string, RegisteredAnimationInfo>();
     /** Clips registered for each root */
     private rootToRegisteredClip = new Map<Object3D, Array<AnimationClip>>();
+    private lastClipEndTime = 0;
+    private clipToStartTime: Map<AnimationClip, number> = new Map();
+    private clipToHoldClip: Map<AnimationClip, AnimationClip> = new Map();
     private serializers: SerializeAnimation[] = [];
@@ -232,6 +322,7 @@
         // return the end time + padding of the rest clip, if any exists
         if (!this.injectRestPoses) return 0;
         if (this.rootAndClipToRegisteredAnimationMap.size === 0) return 0;
+        // return 0;
         return (TransformData.restPoseClipDuration + TransformData.animationDurationPadding) * 60;
     }
@@ -253,6 +344,7 @@
         return currentCount ?? 0;
     }
+    /*
     // TODO why do we have this here and on TransformData? Can RegisteredAnimationInfo not cache this value?
     // TODO we probably want to assert here that this is the same value on all nodes
     getStartTime01(root: Object3D, clip: AnimationClip | null) {
@@ -275,7 +367,18 @@
         return currentStartTime;
     }
+    */
+    getStartTimeByClip(clip: AnimationClip | null) {
+        if (!clip) return 0;
+        if (!this.clipToStartTime.has(clip)) {
+            console.error("USDZExporter: Missing start time for clip – please report a bug.", clip);
+            return 0;
+        }
+        const time = this.clipToStartTime.get(clip)!;
+        return time;
+    }
     // The same clip could be registered for different roots. All of them need written animation data then.
     // The same root could have multiple clips registered to it. If it does, the clips need to write
     // independent time data, so that playing back an animation on that root doesn't result in data "leaking"/"overlapping".
@@ -364,6 +467,7 @@
             // Inject a rest pose if we don't have it already
             if (this.injectRestPoses && !transformDataForTarget[0]) {
+                console.log("Injecting rest pose", animationTarget, clip, "at slot", currentCount);
                 transformDataForTarget[0] = new TransformData(null, animationTarget, null);
             }
             // These all need to be at the same index, otherwise our padding went wrong
@@ -412,6 +516,26 @@
             const registered = this.rootToRegisteredClip.get(root);
             if (!registered) this.rootToRegisteredClip.set(root, [clip]);
             else registered.push(clip);
+            const lastClip = this.clipToStartTime.get(clip);
+            if (!lastClip) {
+                if (this.lastClipEndTime == null) this.lastClipEndTime = TransformData.restPoseClipDuration;
+                let newStartTime = this.lastClipEndTime + TransformData.animationDurationPadding;
+                let newEndTime = newStartTime + clip.duration;
+                // Round these times, makes it easier to understand what happens in the file
+                const roundedStartTime = Math.round(newStartTime * 60) / 60;
+                const roundedEndTime = Math.round(newEndTime * 60) / 60;
+                if (Math.abs(roundedStartTime - newStartTime) < 0.01) newStartTime = roundedStartTime;
+                if (Math.abs(roundedEndTime - newEndTime) < 0.01) newEndTime = roundedEndTime;
+                // Round newStartTime up to the next frame
+                newStartTime = Math.ceil(newStartTime);
+                newEndTime = newStartTime + clip.duration;
+                this.clipToStartTime.set(clip, newStartTime);
+                this.lastClipEndTime = newEndTime;
+            }
         }
         return info;
     }
@@ -486,7 +610,7 @@
         }, false);
         // we need to be able to retarget serialization to empty parents before actually serializing (we do that in another callback)
-        const ser = new SerializeAnimation(object, this.dict);
+        const ser = new SerializeAnimation(object, this);
         this.serializers.push(ser);
         ser.registerCallback(model);
     }
@@ -498,15 +622,18 @@
 class SerializeAnimation {
-    object: Object3D;
-    dict: AnimationDict;
     model: USDObject | undefined = undefined;
+    private object: Object3D;
+    private animationData: Map<Object3D, Array<TransformData>>;
+    private ext: AnimationExtension;
     private callback?: (writer: USDWriter, context: USDZExporterContext) => void;
-    constructor(object: Object3D, dict: AnimationDict) {
+    constructor(object: Object3D, ext: AnimationExtension) {
         this.object = object;
-        this.dict = dict;
+        this.animationData = ext.animationData;
+        this.ext = ext;
     }
     registerCallback(model: USDObject) {
@@ -522,9 +649,9 @@
             this.model.addEventListener("serialize", this.callback);
     }
-    skinnedMeshExport(writer: USDWriter, _context: USDZExporterContext) {
+    skinnedMeshExport(writer: USDWriter, _context: USDZExporterContext, ext: AnimationExtension) {
         const model = this.model;
-        const dict = this.dict;
+        const dict = this.animationData;
         if (!model) return;
         if ( model.skinnedMesh ) {
@@ -636,6 +763,7 @@
                 for (const [bone, transformDatas] of boneToTransformData) {
+                    /*
                     // calculate start times from the transformDatas
                     const startTimes = new Array<number>();
                     let currentStartTime = 0;
@@ -643,11 +771,12 @@
                         startTimes.push(currentStartTime);
                         currentStartTime += transformDatas[i].getDuration() + TransformData.animationDurationPadding;
                     }
+                    */
                     for (let i = 0; i < transformDatas.length; i++) {
                         const transformData = transformDatas[i];
                         // const timeOffset = transformData.getStartTime(dict);
-                        const timeOffset = startTimes[i];
+                        const timeOffset = ext.getStartTimeByClip(transformData.clip);
                         if (times.length <= i) {
                             times.push({pos: [], rot: [], scale: [], timeOffset});
                         }
@@ -883,15 +1012,13 @@
         }
     }
     onSerialize(writer: USDWriter, _context: USDZExporterContext) {
         if (!this.model) return;
         // Workaround: Sanitize TransformData for this object.
         // This works around an issue with wrongly detected animation roots, where some of the indices
         // in the TransformData array are not property set. Reproduces with golem_yokai.glb
-        const arr0 = this.dict.get(this.object);
+        const arr0 = this.animationData.get(this.object);
         if (arr0) {
             for (let i = 0; i < arr0.length; i++) {
                 if (arr0[i] !== undefined) continue;
@@ -899,15 +1026,17 @@
             }
         }
-        this.skinnedMeshExport(writer, _context);
+        const ext = this.ext;
+        this.skinnedMeshExport(writer, _context, ext);
         const object = this.object;
         const model = this.model;
         // do we have animation data for this node? if not, return
-        const arr = this.dict.get(object);
-        if (!arr) return;
+        const animationData = this.animationData.get(object);
+        if (!animationData) return;
         // Skinned meshes are handled separately by the method above.
         // They need to be handled first (before checking for animation data) because animation needs to be exported
         // as part of the skinned mesh and that may not be animated at all – if any bone is animated we need to export.
@@ -922,7 +1051,7 @@
         //@ts-ignore
         // if (object.isBone) return;
-        if (debugSerialization) console.log("SERIALIZE", this.model.name, this.object.type, arr);
+        if (debugSerialization) console.log("SERIALIZE", this.model.name, this.object.type, animationData);
         // const composedTransform = new Matrix4();
         // writer.appendLine("matrix4d xformOp:transform.timeSamples = {");
@@ -935,16 +1064,18 @@
         // calculate start times
         // calculate start times from the transformDatas
+        /*
         const startTimes = new Array<number>();
         let currentStartTime = 0;
-        for (let i = 0; i < arr.length; i++) {
+        for (let i = 0; i < animationData.length; i++) {
             startTimes.push(currentStartTime);
-            if (arr[i] === undefined) {
-                console.error("Got an undefined transform data, this is likely a bug.", object, arr);
+            if (animationData[i] === undefined) {
+                console.error("Got an undefined transform data, this is likely a bug.", object, animationData);
                 continue;
             }
-            currentStartTime += arr[i].getDuration() + TransformData.animationDurationPadding;
+            currentStartTime += animationData[i].getDuration() + TransformData.animationDurationPadding;
         }
+        */
         const formatter = Intl.NumberFormat("en-US", {
             maximumFractionDigits: 3,
@@ -981,7 +1112,7 @@
             for (let i = 0; i < arr.length; i++) {
                 const transformData = arr[i];
                 if (!transformData) continue;
-                const startTime = startTimes[i];
+                const startTime = ext.getStartTimeByClip(transformData.clip);
                 const timesArray = transformData.getSortedTimesArray(type === "position", type === "rotation", type === "scale");
                 if (!timesArray || timesArray.length === 0) {
@@ -989,31 +1120,38 @@
                     continue;
                 }
-                // if (debug) // writing out the clip name and duration is useful even when not debugging
+                const isRestPose = !transformData.clip;
+                const hasPos = type === "position" && (transformData.pos || isRestPose);
+                const hasRot = type === "rotation" && (transformData.rot || isRestPose);
+                const hasScale = type === "scale" && (transformData.scale || isRestPose);
+                // Writing out the clip name and duration.
+                // If this is a rest pose clip, we still want to write it out, as it's a valid clip.
+                if (hasPos || hasRot || hasScale)
                 {
                     const clipName = transformData.clip?.name ?? "rest";
                     const duration = transformData.getDuration();
                     if (debug) console.log("Write .timeSamples:", clipName, startTime, duration, arr);
-                    writer.appendLine("# " + clipName + ": start=" + formatter.format(startTime * transformData.frameRate) + ", length=" + formatter.format(duration * transformData.frameRate) + ", frames=" + transformData.getFrames());
+                    writer.appendLine("# " + clipName + ": start=" + formatter.format(startTime * TransformData.frameRate) + ", length=" + formatter.format(duration * TransformData.frameRate) + ", frames=" + transformData.getFrames());
                 }
-                if (type === "position" && transformData.pos) {
+                if (hasPos) {
                     for (const { time, translation } of transformData.getValues(timesArray, true, false, false)) {
-                        const timeStr = formatter.format((startTime + time) * transformData.frameRate);
+                        const timeStr = formatter.format((startTime + time) * TransformData.frameRate);
                         const line = `${timeStr}: (${fn(translation.x)}, ${fn(translation.y)}, ${fn(translation.z)}),`;
                         writer.appendLine(line);
                     }
                 }
-                if (type === "rotation" && transformData.rot) {
+                if (hasRot) {
                     for (const { time, rotation } of transformData.getValues(timesArray, false, true, false)) {
-                        const timeStr = formatter.format((startTime + time) * transformData.frameRate);
+                        const timeStr = formatter.format((startTime + time) * TransformData.frameRate);
                         const line = `${timeStr}: (${fn(rotation.w)}, ${fn(rotation.x)}, ${fn(rotation.y)}, ${fn(rotation.z)}),`;
                         writer.appendLine(line);
                     }
                 }
-                if (type === "scale" && transformData.scale) {
+                if (hasScale) {
                     for (const { time, scale } of transformData.getValues(timesArray, false, false, true)) {
-                        const timeStr = formatter.format((startTime + time) * transformData.frameRate);
+                        const timeStr = formatter.format((startTime + time) * TransformData.frameRate);
                         const line = `${timeStr}: (${fn(scale.x)}, ${fn(scale.y)}, ${fn(scale.z)}),`;
                         writer.appendLine(line);
                     }
@@ -1021,9 +1159,9 @@
             }
             writer.closeBlock();
         }
-        writeAnimationTimesamples(arr, "position");
-        writeAnimationTimesamples(arr, "rotation");
-        writeAnimationTimesamples(arr, "scale");
+        writeAnimationTimesamples(animationData, "position");
+        writeAnimationTimesamples(animationData, "rotation");
+        writeAnimationTimesamples(animationData, "scale");
         // We must be careful here that we don't overwrite the xformOpOrder that the object already had.
         // it _might_ not even have any (if the position/quaternion/scale are all indentity values)

src/engine-components/export/usdz/extensions/behavior/Behaviour.ts CHANGED Viewed

@@ -79,7 +79,7 @@
     onExportObject(_object, model: USDObject, context) {
         for (const beh of this.behaviourComponents) {
-            if (debug) console.log("onExportObject: createBehaviours", beh);
+            // if (debug) console.log("onExportObject: createBehaviours", beh);
             beh.createBehaviours?.call(beh, this, model, context);
         }
     }
@@ -100,39 +100,160 @@
         const triggerSources = new Set<Target>();
         const actionTargets = new Set<Target>();
         const targetUuids = new Set<string>();
+        const playAnimationActions = new Set<ActionModel>();
-        function collect (actionModel: IBehaviorElement) {
+        // We're assembling a mermaid graph on the go, for easier debugging
+        const createMermaidGraphForDebugging = debug;
+        let mermaidGraph = "graph LR\n";
+        let mermaidGraphTopLevel = "";
+        function collectAction (actionModel: IBehaviorElement) {
             if (actionModel instanceof GroupActionModel) {
+                if (createMermaidGraphForDebugging) mermaidGraph += `subgraph Group_${actionModel.id}\n`;
                 for (const action of actionModel.actions) {
-                    collect(action);
+                    if (createMermaidGraphForDebugging) mermaidGraph += `${actionModel.id}[${actionModel.id}] -- ${actionModel.type} --> ${action.id}[${action.id}]\n`;
+                    collectAction(action);
                 }
+                if (createMermaidGraphForDebugging) mermaidGraph += `end\n`;
             }
             else if (actionModel instanceof ActionModel) {
+                if (actionModel.tokenId === "StartAnimation") {
+                    playAnimationActions.add(actionModel);
+                }
                 const affected = actionModel.affectedObjects;
                 if (affected) {
-                    if (typeof affected === "object")
+                    if (Array.isArray(affected)) {
+                        for (const a of affected) {
+                            actionTargets.add(a as Target);
+                            //@ts-ignore
+                            if (createMermaidGraphForDebugging) mermaidGraphTopLevel += `${actionModel.id}[${actionModel.id}] -- ${actionModel.tokenId} --> ${a.uuid}(("${a.displayName || a.name || a.uuid}"))\n`;
+                        }
+                    }
+                    else if (typeof affected === "object") {
                         actionTargets.add(affected as Target);
-                    else if (typeof affected === "string")
+                        //@ts-ignore
+                        if (createMermaidGraphForDebugging) mermaidGraphTopLevel += `${actionModel.id}[${actionModel.id}] -- ${actionModel.tokenId} --> ${affected.uuid}(("${affected.displayName || affected.name || affected.uuid}"))\n`;
+                    }
+                    else if (typeof affected === "string") {
                         actionTargets.add({uuid: affected} as any as Target);
+                    }
                 }
                 const xform = actionModel.xFormTarget;
                 if (xform) {
-                    if (typeof xform === "object")
+                    if (typeof xform === "object") {
                         actionTargets.add(xform as Target);
-                    else if (typeof xform === "string")
+                        //@ts-ignore
+                        if (createMermaidGraphForDebugging) mermaidGraphTopLevel += `${actionModel.id}[${actionModel.id}] -- ${actionModel.tokenId} --> ${xform.uuid}(("${xform.displayName || xform.name || xform.uuid}"))\n`;
+                    }
+                    else if (typeof xform === "string") {
                         actionTargets.add({uuid: xform} as any as Target);
+                    }
                 }
             }
         }
+        function collectTrigger(trigger: IBehaviorElement | IBehaviorElement[], action: IBehaviorElement) {
+            if (Array.isArray(trigger)) {
+                for (const t of trigger)
+                    collectTrigger(t, action);
+            }
+            else if (trigger instanceof TriggerModel) {
+                if (typeof trigger.targetId === "object") {
+                    triggerSources.add(trigger.targetId as Target);
+                    //@ts-ignore
+                    if (createMermaidGraphForDebugging) mermaidGraphTopLevel += `${trigger.targetId.uuid}(("${trigger.targetId.displayName}")) --> ${trigger.id}[${trigger.id}]\n`;
+                }
+                //@ts-ignore
+                if (createMermaidGraphForDebugging) mermaidGraph += `${trigger.id}((${trigger.id})) -- ${trigger.tokenId}${trigger.type ? ":" + trigger.type : ""} --> ${action.id}[${action.tokenId || action.id}]\n`;
+            }
+        }
         // collect all targets of all triggers and actions
         for (const beh of this.behaviours) {
-            if (beh.trigger instanceof TriggerModel && typeof beh.trigger.targetId === "object" )
-                triggerSources.add(beh.trigger.targetId as Target);
-            collect(beh.action);
+            if (createMermaidGraphForDebugging) mermaidGraph += `subgraph Behavior_${beh.id}\n`;
+            collectAction(beh.action);
+            collectTrigger(beh.trigger, beh.action);
+            if (createMermaidGraphForDebugging) mermaidGraph += `end\n`;
         }
+        if (createMermaidGraphForDebugging) mermaidGraph += "\n" + mermaidGraphTopLevel;
+        if (createMermaidGraphForDebugging) {
+            console.log("All USDZ behaviours", this.behaviours);
+            console.warn("The Mermaid graph can be pasted into https://massive-mermaid.glitch.me/ or https://mermaid.live/edit. It should be in your clipboard already!");
+            console.log(mermaidGraph);
+            // copy to clipboard, can be pasted into https://massive-mermaid.glitch.me/ or https://mermaid.live/edit
+            navigator.clipboard.writeText(mermaidGraph);
+        }
+        {
+            // Validation: Check if any PlayAnimation actions are overlapping.
+            // That means: the target of one of the actions is a child of any other target of another action.
+            // This leads to undefined behaviour in the runtime.
+            // See FB15122057 for more information.
+            let animationsGraph = "gantt\ntitle Animations\ndateFormat X\naxisFormat %s\n";
+            const arr = Array.from(playAnimationActions);
+            const animationTargetObjects = new Set<USDObject>();
+            for (const a of arr) {
+                if (!a.affectedObjects) continue;
+                if (typeof a.affectedObjects === "string") continue;
+                if (Array.isArray(a.affectedObjects)) {
+                    for (const o of a.affectedObjects) {
+                        animationTargetObjects.add(o as USDObject);
+                    }
+                }
+                else {
+                    animationTargetObjects.add(a.affectedObjects as USDObject);
+                }
+                if (createMermaidGraphForDebugging) {
+                    animationsGraph += `section ${a.animationName} (${a.id})\n`;
+                    animationsGraph += `${a.id} : ${a.start}, ${a.duration}s\n`;
+                }
+            }
+            if (createMermaidGraphForDebugging) {
+                console.log(animationsGraph);
+            }
+            const animationTargetPaths = new Set<{path: string, obj: USDObject}>();
+            for (const o of animationTargetObjects) {
+                if (!o.getPath) {
+                    console.error("USDZExporter: Animation target object has no getPath method. This is likely a bug", o);
+                }
+                let path = o.getPath();
+                // remove < and >, these are part of USD paths
+                if (path.startsWith("<")) path = path.substring(1);
+                if (path.endsWith(">")) path = path.substring(0, path.length - 1);
+                animationTargetPaths.add({path, obj: o});
+            }
+            // order by length
+            const sortedPaths = Array.from(animationTargetPaths).sort((a, b) => a.path.length - b.path.length);
+            const overlappingTargets = new Array<{child: string, parent: string}>();
+            for (let i = 0; i < sortedPaths.length; i++) {
+                for (let j = i + 1; j < sortedPaths.length; j++) {
+                    if (sortedPaths[j].path.startsWith(sortedPaths[i].path)) {
+                        const c = sortedPaths[j];
+                        const p = sortedPaths[i];
+                        overlappingTargets.push({child: c.obj.displayName + " (" + c.path + ")", parent: p.obj.displayName + " (" + p.path + ")"});
+                    }
+                }
+            }
+            // There's some overlapping animation targets – we should warn here, so that this
+            // can be resolved in the scene.
+            if (overlappingTargets.length) {
+                console.warn("USDZExporter: There are overlapping PlayAnimation actions. This can lead to undefined runtime behaviour when playing multiple animations. Please restructure the hierarchy so that animations don't overlap.",
+                    {
+                        overlappingTargets,
+                        playAnimationActions,
+                    }
+                );
+            }
+        }
         for (const source of new Set([...triggerSources, ...actionTargets])) {
             // shouldn't happen but strictly speaking a trigger source could be set to an array
             if (Array.isArray(source)) {
@@ -191,103 +312,4 @@
         this.behaviourComponentsCopy.length = 0;
         this.audioClipsCopy.length = 0;
     }
-    // combine behaviours that have tap triggers on the same object
-    // private combineBehavioursWithSameTapActions() {
-    //     // TODO: if behaviours have different settings (e.g. one is exclusive and another one is not) this wont work - we need more logic for that
-    //     const combined: { [key: string]: { behaviorId: string, trigger: TriggerModel, actions: IBehaviorElement[] } } = {};
-    //     for (let i = this.behaviours.length - 1; i >= 0; i--) {
-    //         const beh = this.behaviours[i];
-    //         const trigger = beh.trigger as TriggerModel;
-    //         if (!Array.isArray(trigger) && TriggerBuilder.isTapTrigger(trigger)) {
-    //             const targetObject = trigger.targetId;
-    //             if (!targetObject) continue;
-    //             if (!combined[targetObject]) {
-    //                 combined[targetObject] = { behaviorId: beh.id, trigger: trigger, actions: [] };
-    //             }
-    //             const action = beh.action;
-    //             combined[targetObject].actions.push(action);
-    //             this.behaviours.splice(i, 1);
-    //         }
-    //     }
-    //     for (const key in combined) {
-    //         const val = combined[key];
-    //         console.log("Combine " + val.actions.length + " actions on " + val.trigger.id, val.actions);
-    //         const beh = new BehaviorModel(val.behaviorId, val.trigger, ActionBuilder.sequence(...val.actions));
-    //         this.behaviours.push(beh);
-    //     }
-    // }
-}
-        // const playAnimationOnTap = new BehaviorModel("b_" + model.name + "_playanim", TriggerBuilder.tapTrigger(model),
-        //     ActionBuilder.parallel(
-        //         ActionBuilder.lookAtCameraAction(model),
-        //         ActionBuilder.sequence(
-        //             //ActionBuilder.startAnimationAction(model, 0, 0, 1, false, true),
-        //             ActionBuilder.emphasize(model, 1, MotionType.Float),
-        //             ActionBuilder.waitAction(1),
-        //             ActionBuilder.emphasize(model, 1, MotionType.Blink),
-        //             ActionBuilder.waitAction(1),
-        //             ActionBuilder.emphasize(model, 1, MotionType.Jiggle),
-        //             ActionBuilder.waitAction(1),
-        //             ActionBuilder.emphasize(model, 1, MotionType.Pulse),
-        //             ActionBuilder.waitAction(1),
-        //             ActionBuilder.emphasize(model, 1, MotionType.Spin),
-        //             ActionBuilder.waitAction(1),
-        //             ActionBuilder.emphasize(model, 1, MotionType.Bounce),
-        //             ActionBuilder.waitAction(1),
-        //             ActionBuilder.emphasize(model, 1, MotionType.Flip),
-        //             ActionBuilder.waitAction(1),
-        //         ).makeLooping()
-        //     ).makeLooping()
-        // );
-        // this.behaviours.push(playAnimationOnTap);
-        // return;
-        // const identityMatrix = new Matrix4().identity();
-        // const emptyParent = new USDZObject(model.name + "_empty", model.matrix);
-        // const parent = model.parent;
-        // parent.add(emptyParent);
-        // model.matrix = identityMatrix;
-        // emptyParent.add(model);
-        // const geometry = new SphereGeometry(.6, 32, 16);
-        // const modelVariant = new USDZObject(model.name + "_variant", identityMatrix, geometry, new MeshStandardMaterial({ color: 0xff0000 }));
-        // emptyParent.add(modelVariant);
-        // const matrix2 = new Matrix4();
-        // matrix2.makeTranslation(.5, 0, 0);
-        // const modelVariant2 = new USDZObject(model.name + "_variant2", matrix2, geometry, new MeshStandardMaterial({ color: 0xffff00 }));
-        // emptyParent.add(modelVariant2);
-        // const hideVariantOnStart = new BehaviorModel("b_" + model.name + "_start", TriggerBuilder.sceneStartTrigger(), ActionBuilder.fadeAction(modelVariant, 0, false));
-        // this.behaviours.push(hideVariantOnStart);
-        // const showVariant = new BehaviorModel("b_" + model.name + "_show_variant", [TriggerBuilder.tapTrigger(model)], new GroupActionModel("group", [
-        //     ActionBuilder.fadeAction(model, 0, false),
-        //     ActionBuilder.fadeAction(modelVariant, 0, true),
-        // ]));
-        // this.behaviours.push(showVariant);
-        // const showOriginal = new BehaviorModel("b_" + model.name + "_show_original", [
-        //     TriggerBuilder.tapTrigger(modelVariant),
-        //     TriggerBuilder.tapTrigger(modelVariant2)
-        // ],
-        //     new GroupActionModel("group", [
-        //         ActionBuilder.fadeAction([modelVariant, modelVariant2], 0, false),
-        //         //ActionBuilder.waitAction(1),
-        //         ActionBuilder.fadeAction(model, 0, true),
-        //         //ActionBuilder.waitAction(.2),
-        //         ActionBuilder.startAnimationAction(model, 0, 1000, 1, false, true),
-        //         //ActionBuilder.lookAtCameraAction(model, 2, Vec3.forward, Vec3.up),
-        //         //ActionBuilder.waitAction(1),
-        //         //ActionBuilder.fadeAction(modelVariant2, 0, true),
-        //     ]).makeSequence());
-        // this.behaviours.push(showOriginal);
+}

src/engine-components/export/usdz/extensions/behavior/BehaviourComponents.ts CHANGED Viewed

@@ -1,4 +1,4 @@
-import { Material, Matrix4, Mesh, Object3D, Quaternion, Vector3 } from "three";
+import { AnimationClip, Material, Matrix4, Mesh, Object3D, Quaternion, Vector3 } from "three";
 import { isDevEnvironment, showBalloonWarning } from "../../../../../engine/debug/index.js";
 import { serializable } from "../../../../../engine/engine_serialization_decorator.js";
@@ -13,18 +13,19 @@
 import { Behaviour, GameObject } from "../../../../Component.js";
 import type { IPointerClickHandler, PointerEventData } from "../../../../ui/PointerEvents.js";
 import { ObjectRaycaster,Raycaster } from "../../../../ui/Raycaster.js";
-import { makeNameSafeForUSD,USDDocument, USDObject, USDWriter, USDZExporterContext } from "../../ThreeUSDZExporter.js";
+import { makeNameSafeForUSD,USDDocument, USDObject, USDZExporterContext } from "../../ThreeUSDZExporter.js";
 import { AnimationExtension, RegisteredAnimationInfo, type UsdzAnimation } from "../Animation.js";
 import { AudioExtension } from "./AudioExtension.js";
 import type { BehaviorExtension, UsdzBehaviour } from "./Behaviour.js";
-import { ActionBuilder, ActionModel, BehaviorModel, EmphasizeActionMotionType,type IBehaviorElement, TriggerBuilder } from "./BehavioursBuilder.js";
+import { ActionBuilder, ActionModel, BehaviorModel, EmphasizeActionMotionType,type IBehaviorElement, Target, TriggerBuilder } from "./BehavioursBuilder.js";
 const debug = getParam("debugusdzbehaviours");
 function ensureRaycaster(obj: GameObject) {
     if (!obj) return;
     if (!obj.getComponentInParent(Raycaster)) {
-        if (isDevEnvironment()) console.warn("Create Raycaster on " + obj.name + " because no raycaster was found in the hierarchy")
+        if (isDevEnvironment())
+            console.debug("Raycaster on \"" + obj.name + "\" was automatically added, because no raycaster was found in the parent hierarchy.")
         obj.addComponent(ObjectRaycaster);
     }
 }
@@ -851,6 +852,7 @@
         document.traverse(model => {
             if (model.uuid === this.target?.uuid) {
                 const sequence = PlayAnimationOnClick.getActionForSequences(
+                    document,
                     model,
                     this.animationSequence,
                     this.animationLoopAfterSequence,
@@ -870,12 +872,12 @@
         });
     }
-    static getActionForSequences(model: USDObject, animationSequence?: Array<RegisteredAnimationInfo>, animationLoopAfterSequence?: Array<RegisteredAnimationInfo>, randomOffsetNormalized?: number) {
+    static getActionForSequences(_document: USDDocument, model: Target, animationSequence?: Array<RegisteredAnimationInfo>, animationLoopAfterSequence?: Array<RegisteredAnimationInfo>, randomOffsetNormalized?: number) {
-        const getOrCacheAction = (model: USDObject, anim: RegisteredAnimationInfo) => {
+        const getOrCacheAction = (model: Target, anim: RegisteredAnimationInfo) => {
             let action = PlayAnimationOnClick.animationActions.find(a => a.affectedObjects == model && a.start == anim.start && a.duration == anim.duration && a.animationSpeed == anim.speed);
             if (!action) {
-                action = ActionBuilder.startAnimationAction(model, anim.start, anim.duration, anim.speed) as ActionModel;
+                action = ActionBuilder.startAnimationAction(model, anim) as ActionModel;
                 PlayAnimationOnClick.animationActions.push(action);
             }
             return action;
@@ -982,7 +984,7 @@
                 // find the first transition without parameters
                 // TODO we could also find the first _valid_ transition here instead based on the current parameters.
                 const transition = currentState.transitions.find(t => t.conditions.length === 0);
-                const nextState = transition ? runtimeController["getState"](transition.destinationState, 0) : null;
+                const nextState: State | null = transition ? runtimeController["getState"](transition.destinationState, 0) : null;
                 // abort: we found a state loop
                 if (nextState && visitedStates.includes(nextState)) {
                     currentState = nextState;
@@ -1008,6 +1010,14 @@
                 const firstStateInLoop = visitedStates.indexOf(currentState);
                 statesUntilLoop = visitedStates.slice(0, firstStateInLoop); // can be empty, which means we're looping all
                 statesLooping = visitedStates.slice(firstStateInLoop); // can be empty, which means nothing is looping
+                // Potentially we need to prevent self-transitions into a non-looping state, these do not result in a loop in the runtime
+                /*
+                if (statesLooping.length === 1 && !statesLooping[0].motion?.isLooping) {
+                    statesUntilLoop.push(statesLooping[0]);
+                    statesLooping = [];
+                }
+                */
                 if (debug) console.log("found loop from " + stateName, "states until loop", statesUntilLoop, "states looping", statesLooping);
             }
             else {
@@ -1015,6 +1025,52 @@
                 statesLooping = [];
                 if (debug) console.log("found no loop from " + stateName, "states", statesUntilLoop);
             }
+            // If we do NOT loop, we need to explicitly add a single-keyframe clip with a "hold" pose
+            // to prevent the animation from resetting to the start.
+            if (!statesLooping.length) {
+                const lastState = statesUntilLoop[statesUntilLoop.length - 1];
+                const lastClip = lastState.motion?.clip;
+                if (lastClip) {
+                    let clipCopy: AnimationClip | undefined;
+                    if (ext.holdClipMap.has(lastClip)) {
+                        clipCopy = ext.holdClipMap.get(lastClip);
+                    }
+                    else {
+                        // We're creating a "hold" clip here; exactly 1 second long, and inteprolates exactly on the duration of the clip
+                        const holdStateName = lastState.name + "_hold";
+                        clipCopy = lastClip.clone();
+                        clipCopy.duration = 1;
+                        clipCopy.name = holdStateName;
+                        const lastFrame = lastClip.duration;
+                        clipCopy.tracks = lastClip.tracks.map(t => {
+                            const trackCopy = t.clone();
+                            trackCopy.times = new Float32Array([0, lastFrame]);
+                            const len = t.values.length;
+                            const size = t.getValueSize();;
+                            const lastValue = t.values.slice(len - size, len);
+                            // we need this twice
+                            trackCopy.values = new Float32Array(2 * size);
+                            trackCopy.values.set(lastValue, 0);
+                            trackCopy.values.set(lastValue, size);
+                            return trackCopy;
+                        });
+                        clipCopy.name = holdStateName;
+                        ext.holdClipMap.set(lastClip, clipCopy);
+                    }
+                    if (clipCopy) {
+                        const holdState = {
+                            name: clipCopy.name,
+                            motion: { clip: clipCopy, isLooping: false, name: clipCopy.name },
+                            speed: 1,
+                            transitions: [],
+                            behaviours: [],
+                            hash: lastState.hash + 1,
+                        }
+                        statesLooping.push(holdState);
+                    }
+                }
+            }
         }
         // Special case: someone's trying to play an empty clip without any motion data, no loops or anything.

src/engine-components/export/usdz/extensions/behavior/BehavioursBuilder.ts CHANGED Viewed

@@ -2,6 +2,7 @@
 import { getParam } from "../../../../../engine/engine_utils.js";
 import { makeNameSafeForUSD,USDDocument, USDObject, USDWriter } from "../../ThreeUSDZExporter.js";
+import type { RegisteredAnimationInfo } from "./../Animation.js";
 import { BehaviorExtension } from "./Behaviour.js";
 const debug = getParam("debugusdz");
@@ -347,7 +348,7 @@
     private static global_id: number = 0;
     id: string;
-    tokenId?: string;
+    tokenId?: "ChangeScene" | "Visibility" | "StartAnimation" | "Wait" | "LookAtCamera" | "Emphasize" | "Transform" | "Audio" | "Impulse";
     affectedObjects?: string | Target;
     easeType?: EaseType;;
     motionType: EmphasizeActionMotionType | VisibilityActionMotionType | undefined = undefined;
@@ -368,6 +369,10 @@
     multiplePerformOperation?: MultiplePerformOperation;
     velocity?: Vec3;
+    // extra info written as comment at the beginning of the action
+    comment?: string;
+    animationName?: string;
     clone(): ActionModel {
         const copy = new ActionModel();
         const id = copy.id;
@@ -379,17 +384,14 @@
     constructor(affectedObjects?: string | Target, id?: string) {
         if (affectedObjects) this.affectedObjects = affectedObjects;
         if (id) this.id = id;
-        /*else if (affectedObjects) {
-            this.id = "Action_" + sanitizeId(affectedObjects);
-        }
-        else */
-        else
-            this.id = "Action";
+        else this.id = "Action";
         this.id += "_" + ActionModel.global_id++;
     }
     writeTo(document: USDDocument, writer: USDWriter) {
         writer.beginBlock(`def Preliminary_Action "${this.id}"`);
+        if (this.comment)
+            writer.appendLine(`# ${this.comment}`);
         if (this.affectedObjects) {
             if (typeof this.affectedObjects !== "string") this.affectedObjects = resolve(this.affectedObjects, document);
             writer.appendLine('rel affectedObjects = ' + this.affectedObjects);
@@ -518,9 +520,26 @@
      * @param start offset in seconds!
      * @param duration in seconds! 0 means play to end
      */
-    static startAnimationAction(targetObject: Target, start: number, duration: number = 0, animationSpeed: number = 1, reversed: boolean = false, pingPong: boolean = false): IBehaviorElement {
+    static startAnimationAction(targetObject: Target, anim: RegisteredAnimationInfo, reversed: boolean = false, pingPong: boolean = false): IBehaviorElement {
         const act = new ActionModel(targetObject);
         act.tokenId = "StartAnimation";
+        /*
+        if (targetObject instanceof USDObject) {
+            act.cachedTargetObject = targetObject;
+            // try to retarget the animation – this improves animation playback with overlapping roots.
+            act.affectedObjects = anim.nearestAnimatedRoot;
+        }
+        */
+        const start = anim.start;
+        const duration = anim.duration;
+        const animationSpeed = anim.speed;
+        const animationName = anim.clipName;
+        act.comment = `Animation: ${animationName}, start=${start * 60}, length=${duration * 60}, end=${(start + duration) * 60}`;
+        act.animationName = animationName;
         // start is time in seconds, the documentation is not right here
         act.start = start;
         // duration of 0 is play to end

src/engine/debug/debug_console.ts CHANGED Viewed

@@ -3,7 +3,7 @@
 import { isDevEnvironment } from "./debug.js";
 import { getErrorCount, makeErrorsVisibleForDevelopment } from "./debug_overlay.js";
-let consoleInstance: VConsole | null = null;
+let consoleInstance: VConsole | null | undefined = undefined;
 let consoleHtmlElement: HTMLElement | null = null;
 let consoleSwitchButton: HTMLElement | null = null;
 let isLoading = false;
@@ -48,7 +48,7 @@
 export function showDebugConsole() {
-    if (consoleInstance !== null) {
+    if (consoleInstance) {
         isVisible = true;
         consoleInstance.showSwitch();
         return;
@@ -57,7 +57,7 @@
 }
 export function hideDebugConsole() {
-    if (consoleInstance === null) return;
+    if (!consoleInstance) return;
     isVisible = false;
     consoleInstance.hide();
     consoleInstance.hideSwitch();
@@ -121,12 +121,19 @@
 }
 function createConsole(startHidden: boolean = false) {
-    if (consoleInstance) return;
+    if (consoleInstance !== undefined) return;
     if (isLoading) return;
     isLoading = true;
     const script = document.createElement("script");
     script.onload = () => {
+        // check if VConsole is now defined on globalThis
+        if (!globalThis.VConsole) {
+            console.warn("🌵 Debug console failed to load.");
+            isLoading = false;
+            consoleInstance = null;
+            return;
+        }
         isLoading = false;
         isVisible = true;
         beginWatchingLogs();
@@ -235,11 +242,17 @@
         }
     };
+    script.onerror = () => {
+        console.warn("🌵 Debug console failed to load.");
+        isLoading = false;
+        consoleInstance = null;
+    };
     script.src = "https://unpkg.com/vconsole@latest/dist/vconsole.min.js";
     document.body.appendChild(script);
 }
 function createInspectPlugin() {
+    if (!globalThis.VConsole) return;
     const plugin = new VConsole.VConsolePlugin("needle-console", "🌵 Inspect glTF");
     const getIframe = () => {
         return document.querySelector("#__vc_plug_" + plugin._id + " iframe") as HTMLIFrameElement;

src/engine/engine_element.ts CHANGED Viewed

@@ -1,7 +1,6 @@
 import { AgXToneMapping, Color, LinearToneMapping, NeutralToneMapping, NoToneMapping } from "three";
-import { getLoader, registerLoader } from "../engine/engine_gltf.js";
-import { GameObject } from "../engine-components/Component.js";
+import { registerLoader } from "../engine/engine_gltf.js";
 import { isDevEnvironment, showBalloonWarning } from "./debug/index.js";
 import { VERSION } from "./engine_constants.js";
 import { TonemappingAttributeOptions } from "./engine_element_attributes.js";
@@ -10,7 +9,7 @@
 import { hasCommercialLicense } from "./engine_license.js";
 import { setDracoDecoderPath, setDracoDecoderType, setKtx2TranscoderPath } from "./engine_loaders.js";
 import { NeedleGltfLoader } from "./engine_scenetools.js";
-import { Context, ContextCreateArgs, LoadingProgressArgs } from "./engine_setup.js";
+import { Context, ContextCreateArgs } from "./engine_setup.js";
 import { type INeedleEngineComponent, type LoadedGLTF } from "./engine_types.js";
 import { getParam } from "./engine_utils.js";
 import { ensureFonts } from "./webcomponents/fonts.js";
@@ -156,6 +155,12 @@
             height: 100%;
         }
     }
+    :host > div.canvas-wrapper {
+        width: 100%;
+        height: 100%;
+    }
     :host canvas {
         position: absolute;
         user-select: none;
@@ -172,6 +177,7 @@
     }
     :host .content {
         position: absolute;
+        top: 0;
         width: 100%;
         height: 100%;
         visibility: visible;
@@ -194,7 +200,7 @@
         z-index: 9999;
     }
 </style>
-<div> <!-- this wrapper is necessary for WebXR https://github.com/meta-quest/immersive-web-emulator/issues/55 -->
+<div class="canvas-wrapper"> <!-- this wrapper is necessary for WebXR https://github.com/meta-quest/immersive-web-emulator/issues/55 -->
     <canvas></canvas>
 </div>
 <div class="content">

src/engine/engine_gameobject.ts CHANGED Viewed

@@ -50,6 +50,9 @@
     components?: boolean;
 }
+/**
+ * Instantiation options for {@link syncInstantiate}
+ */
 export class InstantiateOptions implements IInstantiateOptions {
     idProvider?: UIDProvider | undefined;
     parent?: string | undefined | Object3D;

src/engine/engine_gltf_builtin_components.ts CHANGED Viewed

@@ -89,7 +89,7 @@
             queue.length = 0;
         }
         // when dropping the same file multiple times we need to generate new guids
-        // e.g. SyncTransform sends its own guid to the server to know about ownership
+        // e.g. SyncedTransform sends its own guid to the server to know about ownership
         // so it requires a unique guid for a new instance
         // doing it here at the end of resolving of references should ensure that
         // and this should run before awake and onenable of newly created components

src/engine/engine_networking_instantiate.ts CHANGED Viewed

@@ -111,9 +111,12 @@
 }
 /**
- * @param obj - The object or component to be destroyed
- * @param con - The network connection to send the destroy event to
- * @param recursive - If true, all children will be destroyed as well. Default is true
+ * Destroy an object across the network. See also {@link syncInstantiate}.
+ * @param obj The object or component to be destroyed
+ * @param con The network connection to send the destroy event to
+ * @param recursive If true, all children will be destroyed as well. Default is true
+ * @param opts Options for the destroy operation
+ * @category Networking
  */
 export function syncDestroy(obj: GameObject | Component, con: INetworkConnection, recursive: boolean = true, opts?: SyncDestroyOptions) {
     if (!obj) return;
@@ -162,13 +165,18 @@
 }
-// when a file is instantiated via some server (e.g. via file drop) we also want to send the info where the file can be downloaded
-// doing it this route will ensure we have
+//
-/** @internal */
+/**
+ * When a file is instantiated via some server (e.g. via file drop) we also want to send the info where the file can be downloaded.
+ * @internal
+ */
 export class HostData {
+    /** File to download */
     filename: string;
+    /** Checksum to verify its the correct file */
     hash: string;
+    /** Expected size of the referenced file and its dependencies */
     size: number;
     constructor(filename: string, hash: string, size: number) {
@@ -205,8 +213,15 @@
     }
 }
+/**
+ * Instantiation options for {@link syncInstantiate}
+ */
 export type SyncInstantiateOptions = IInstantiateOptions & Pick<IModel, "deleteOnDisconnect">;
+/**
+ * Instantiate an object across the network. See also {@link syncDestroy}.
+ * @category Networking
+ */
 export function syncInstantiate(object: GameObject | Object3D, opts: SyncInstantiateOptions, hostData?: HostData, save?: boolean): GameObject | null {
     const obj: GameObject = object as GameObject;

src/engine/engine_networking.ts CHANGED Viewed

@@ -40,9 +40,9 @@
  * @link https://engine.needle.tools/docs/networking.html#manual-networking
  * */
 export enum RoomEvents {
-    /** Internal: sent to the server when attempting to join a room */
+    /** Internal: Sent to the server when attempting to join a room */
     Join = "join-room",
-    /** Internal: sent to the server when attempting to leave a room */
+    /** Internal: Sent to the server when attempting to leave a room */
     Leave = "leave-room",
     /** Incoming: When the local user has joined a room */
     JoinedRoom = "joined-room",
@@ -52,6 +52,7 @@
     UserJoinedRoom = "user-joined-room",
     /** Incoming: When a other user has left the room */
     UserLeftRoom = "user-left-room",
+    /** When a user joins a room, the server sends the entire room state. Afterwards, the server sends the room-state-sent event. */
     RoomStateSent = "room-state-sent",
 }

src/engine/engine_time_utils.ts CHANGED Viewed

@@ -83,7 +83,7 @@
      * // Enable console logging:
      * Progress.start("export-usdz", { logTimings: true });
      */
-    static start(scope: string, options?: ProgressStartOptions | string ) {
+    static start(scope: string, options?: ProgressStartOptions | string) {
         if (typeof options === "string") options = { parentScope: options };
         const p = new ProgressEntry(scope, options);
         progressCache.set(scope, p);

src/engine-components/export/usdz/index.ts CHANGED Viewed

@@ -1,3 +1,3 @@
 export { type UsdzBehaviour } from "./extensions/behavior/Behaviour.js";
-export { imageToCanvas, makeNameSafeForUSD, USDZExporter as NeedleUSDZExporter, USDObject, USDWriter,type USDZExporterContext } from "./ThreeUSDZExporter.js";
+export { imageToCanvas, makeNameSafeForUSD, USDZExporter as NeedleUSDZExporter, USDDocument, USDObject, USDWriter,type USDZExporterContext } from "./ThreeUSDZExporter.js";
 export { USDZExporter } from "./USDZExporter.js";

src/engine-components/export/usdz/extensions/behavior/PhysicsExtension.ts CHANGED Viewed

@@ -9,9 +9,7 @@
 export class PhysicsExtension implements IUSDExporterExtension {
-    get extensionName(): string {
-        return "Physics";
-    }
+    get extensionName(): string { return "Physics"; }
     onExportObject?(object: Object3D, model : USDObject, _context: USDZExporterContext) {

src/engine-components-experimental/networking/PlayerSync.ts CHANGED Viewed

@@ -14,6 +14,11 @@
 declare type PlayerSyncWithAsset = PlayerSync & Required<Pick<PlayerSync, "asset">>;
+/**
+ * This component instantiates an asset for each player that joins a networked room. The asset will be destroyed when the player leaves the room.
+ * The asset should have a PlayerState component, and can have other components like SyncedTransform, custom components, etc.
+ * @category Networking
+ */
 export class PlayerSync extends Behaviour {
     /**

src/engine-components/RigidBody.ts CHANGED Viewed

@@ -316,7 +316,10 @@
         this._watch.start(true, true);
         this.startCoroutine(this.beforePhysics(), FrameEvent.LateUpdate);
         if (isDevEnvironment() && !this.context.physics.engine?.getBody(this)) {
-            console.warn(`Rigidbody was not created: Does your object (${this.name}) have a collider?`);
+            if (!globalThis["NEEDLE_USE_RAPIER"])
+                console.warn(`Rigidbody could not be created: Rapier physics are explicitly disabled.`);
+            else
+                console.warn(`Rigidbody could not be created. Ensure \"(${this.name}\" has a Collider component.`);
         }
     }

src/engine-components/ShadowCatcher.ts CHANGED Viewed

@@ -122,6 +122,7 @@
             return;
             `);
         }
+        material.userData.isLightBlendMaterial = true;
     }
     // ShadowMaterial: only does a mask; shadowed areas are fully black.
@@ -135,12 +136,14 @@
                 material.opacity = this.shadowColor.alpha;
                 this.applyMaterialOptions(material);
                 this.targetMesh.material = material;
+                material.userData.isShadowCatcherMaterial = true;
             }
             else {
                 const material = this.targetMesh.material as ShadowMaterial;
                 material.color = this.shadowColor;
                 material.opacity = this.shadowColor.alpha;
                 this.applyMaterialOptions(material);
+                material.userData.isShadowCatcherMaterial = true;
             }
         }
     }

src/engine-components/export/usdz/ThreeUSDZExporter.ts CHANGED Viewed

@@ -41,6 +41,7 @@
 import { BehaviorExtension } from '../../api.js';
 import type { IUSDExporterExtension } from './Extension.js';
 import type { AnimationExtension } from './extensions/Animation.js';
+import type { PhysicsExtension } from './extensions/behavior/PhysicsExtension.js';
 function makeNameSafe( str ) {
 	// Remove characters that are not allowed in USD ASCII identifiers
@@ -395,12 +396,21 @@
 	}
 	buildHeader( _context: USDZExporterContext ) {
 		const animationExtension = _context.extensions?.find( ext => ext?.extensionName === 'animation' ) as AnimationExtension | undefined;
 		const behaviorExtension = _context.extensions?.find( ext => ext?.extensionName === 'Behaviour' ) as BehaviorExtension | undefined;
+		const physicsExtension = _context.extensions?.find( ext => ext?.extensionName === 'Physics' ) as PhysicsExtension | undefined;
 		const startTimeCode = animationExtension?.getStartTimeCode() ?? 0;
 		const endTimeCode = animationExtension?.getEndTimeCode() ?? 0;
+		let comment = "";
+		const registeredClips = animationExtension?.registeredClips;
+		if (registeredClips) {
+			for ( const clip of registeredClips ) {
+				comment += `\t# Animation: ${clip.name}, start=${animationExtension.getStartTimeByClip(clip) * 60}, length=${clip.duration * 60}\n`;
+			}
+		}
+		const comments = comment;
 		return `#usda 1.0
 (
@@ -409,6 +419,7 @@
 		dictionary Needle = {
 			bool animations = ${animationExtension ? 1 : 0}
 			bool interactive = ${behaviorExtension ? 1 : 0}
+			bool physics = ${physicsExtension ? 1 : 0}
 			bool quickLookCompatible = ${_context.quickLookCompatible ? 1 : 0}
 		}
 	}
@@ -420,6 +431,7 @@
 	timeCodesPerSecond = 60
 	framesPerSecond = 60
 	doc = """Generated by Needle Engine USDZ Exporter ${VERSION}"""
+${comments}
 )
 `;
@@ -582,6 +594,7 @@
 	sceneAnchoringOptions: USDZExporterOptions = new USDZExporterOptions();
 	extensions: Array<IUSDExporterExtension> = [];
 	keepObject?: (object: Object3D) => boolean;
+	beforeWritingDocument?: () => void;
 	constructor() {
@@ -697,6 +710,9 @@
 		// Moved into parseDocument callback for proper defaultPrim encapsulation
 		// context.output += buildMaterials( materials, textures, options.quickLookCompatible );
+		// callback for validating after all export has been done
+		context.exporter?.beforeWritingDocument?.();
 		const header = context.document.buildHeader( context );
 		const final = header + '\n' + context.output;
@@ -707,7 +723,7 @@
 		context.output = '';
 		Progress.report("export-usdz", { message: "Exporting textures", autoStep: 10 });
-		Progress.start("export-usdz-textures", "export-usdz");
+		Progress.start("export-usdz-textures", { parentScope: "export-usdz", logTimings: false });
 		const decompressionRenderer = new WebGLRenderer( {
 			antialias: false,
 			alpha: true,
@@ -1823,7 +1839,7 @@
 	if ( attribute === undefined ) {
-		console.warn( 'USDZExporter: Attribute is missing. Results may be undefined.' );
+		console.warn( 'USDZExporter: A mesh attribute is missing and will be set with placeholder data. The result may look incorrect.' );
 		return Array( count ).fill( '(0, 0, 1)' ).join( ', ' );
 	}
@@ -1943,12 +1959,45 @@
 function buildMaterial( material: MeshBasicMaterial, textures: TextureMap, quickLookCompatible = false ) {
+	const materialName = getMaterialName(material);
+	console.log(material);
+	// Special case: occluder material
+	// Supported on iOS 18+ and visionOS 1+
+	const isShadowCatcherMaterial =
+		material.colorWrite === false ||
+		(material.userData?.isShadowCatcherMaterial || material.userData?.isLightBlendMaterial);
+	if (isShadowCatcherMaterial) {
+		// Two options here:
+		// - ND_realitykit_occlusion_surfaceshader (non-shadow receiving)
+		// - ND_realitykit_shadowreceiver_surfaceshader (shadow receiving)
+		const mode = (material.userData.isLightBlendMaterial || material.userData.isShadowCatcherMaterial)
+			? "ND_realitykit_shadowreceiver_surfaceshader"
+			: "ND_realitykit_occlusion_surfaceshader";
+		return `
+		def Material "${materialName}" ${material.name ?`(
+			displayName = "${material.name}"
+		)` : ''}
+		{
+			token outputs:mtlx:surface.connect = ${materialRoot}/${materialName}/Occlusion.outputs:out>
+			def Shader "Occlusion"
+			{
+				uniform token info:id = "${mode}"
+				token outputs:out
+			}
+		}`;
+	}
 	// https://graphics.pixar.com/usd/docs/UsdPreviewSurface-Proposal.html
 	const pad = '                ';
 	const inputs: Array<string> = [];
 	const samplers: Array<string> = [];
-	const materialName = getMaterialName(material);
 	const usedUVChannels: Set<number> = new Set();
 	function texName(tex: Texture) {

src/engine-components/export/usdz/USDZExporter.ts CHANGED Viewed

@@ -1,7 +1,7 @@
 import { NEEDLE_progressive } from "@needle-tools/gltf-progressive";
 import { Euler, Matrix4, Mesh, Object3D, Quaternion, Vector3 } from "three";
-import { showBalloonMessage, showBalloonWarning } from "../../../engine/debug/index.js";
+import { isDevEnvironment, showBalloonMessage, showBalloonWarning } from "../../../engine/debug/index.js";
 import { hasProLicense } from "../../../engine/engine_license.js";
 import { serializable } from "../../../engine/engine_serialization.js";
 import { getFormattedDate, Progress } from "../../../engine/engine_time_utils.js";
@@ -9,10 +9,12 @@
 import { WebXRButtonFactory } from "../../../engine/webcomponents/WebXRButtons.js";
 import { InternalUSDZRegistry } from "../../../engine/xr/usdz.js"
 import { InstancingHandler } from "../../../engine-components/RendererInstancing.js";
+import { Collider } from "../../Collider.js";
 import { Behaviour, GameObject } from "../../Component.js";
 import { ContactShadows } from "../../ContactShadows.js";
 import { GroundProjectedEnv } from "../../GroundProjection.js";
 import { Renderer } from "../../Renderer.js"
+import { Rigidbody } from "../../RigidBody.js";
 import { SpriteRenderer } from "../../SpriteRenderer.js";
 import { WebARSessionRoot } from "../../webxr/WebARSessionRoot.js";
 import { WebXR } from "../../webxr/WebXR.js";
@@ -115,7 +117,15 @@
     @serializable()
     interactive: boolean = true;
+    /** Enabling this option will export the USDZ file with RealityKit physics components.
+     * Rigidbody and Collider components will be converted to their RealityKit counterparts.
+     * Physics are supported on QuickLook in iOS 18+ and VisionOS 1+.
+     * Physics export is automatically turned off when there are no Rigidbody components anywhere on the exported object.
+     */
     @serializable()
+    physics: boolean = true;
+    @serializable()
     allowCreateQuicklookButton: boolean = true;
     @serializable()
@@ -149,14 +159,6 @@
             this.objectToExport = this.gameObject;
         if (!this.objectToExport?.children?.length && !(this.objectToExport as Mesh)?.isMesh)
             this.objectToExport = this.context.scene;
-        if (this.interactive) {
-            this.extensions.push(new BehaviorExtension());
-            this.extensions.push(new AudioExtension());
-            this.extensions.push(new PhysicsExtension());
-            this.extensions.push(new TextExtension());
-            this.extensions.push(new USDZUIExtension());
-        }
     }
     /** @internal */
@@ -356,7 +358,30 @@
         // TODO we probably want to do that with all the extensions...
         // Ordering of extensions is important
         const animExt = new AnimationExtension(this.quickLookCompatible);
-        const extensions: any = [animExt, ...this.extensions]
+        let physicsExt: PhysicsExtension | undefined = undefined;
+        const defaultExtensions: IUSDExporterExtension[] = [];
+        if (this.interactive) {
+            defaultExtensions.push(new BehaviorExtension());
+            defaultExtensions.push(new AudioExtension());
+            // If physics are enabled, and there are any Rigidbody components in the scene,
+            // add the PhysicsExtension to the default extensions.
+            if (globalThis["NEEDLE_USE_RAPIER"]) {
+                const rigidbodies = GameObject.getComponentsInChildren(objectToExport, Rigidbody);
+                if (rigidbodies.length > 0) {
+                    if (this.physics) {
+                        physicsExt = new PhysicsExtension();
+                        defaultExtensions.push(physicsExt);
+                    }
+                    else if (isDevEnvironment()) {
+                        console.warn("USDZExporter: Physics export is disabled, but there are active Rigidbody components in the scene. They will not be exported.");
+                    }
+                }
+            }
+            defaultExtensions.push(new TextExtension());
+            defaultExtensions.push(new USDZUIExtension());
+        }
+        const extensions: any = [animExt, ...defaultExtensions, ...this.extensions];
         const eventArgs = { self: this, exporter: exporter, extensions: extensions, object: objectToExport };
         Progress.report("export-usdz", "Invoking before-export");
@@ -401,6 +426,20 @@
             return keep;
         }
+        exporter.beforeWritingDocument = () => {
+            // Warn if there are any physics components on animated objects or their children
+            if (isDevEnvironment() && animExt && physicsExt) {
+                const animatedObjects = animExt.animatedRoots;
+                for (const object of animatedObjects) {
+                    const rigidBodySources = GameObject.getComponentsInChildren(object, Rigidbody).filter(c => c.enabled);
+                    const colliderSources = GameObject.getComponents(object, Collider).filter(c => c.enabled && !c.isTrigger);
+                    if (rigidBodySources.length > 0 || colliderSources.length > 0) {
+                        console.error("An animated object has physics components in its child hierarchy. This can lead to undefined behaviour due to a bug in Apple's QuickLook (FB15925487). Remove the physics components from child objects or verify that you get the expected results.", object);
+                    }
+                }
+            }
+        };
         // Collect invisible objects so that we can disable them if
         // - we're exporting for QuickLook
         // - and interactive behaviors are allowed.