Needle Engine 3.37.7-beta

Files changed (11) hide show

src/engine-components/export/usdz/extensions/behavior/Behaviour.ts +2 -0
src/engine-components/export/usdz/extensions/behavior/BehaviourComponents.ts +18 -1
src/engine-components/export/usdz/extensions/behavior/BehavioursBuilder.ts +34 -7
src/engine/engine_networking_auto.ts +13 -17
src/engine/engine_networking_types.ts +1 -2
src/engine/engine_physics_rapier.ts +5 -1
src/engine-components/export/usdz/index.ts +1 -1
src/engine-components/export/usdz/ThreeUSDZExporter.ts +11 -5
src/engine-components/timeline/TimelineTracks.ts +6 -0
src/engine-components/export/usdz/USDZExporter.ts +60 -13
src/engine-components/webxr/controllers/XRControllerModel.ts +1 -1

src/engine-components/export/usdz/extensions/behavior/Behaviour.ts CHANGED Viewed

@@ -52,11 +52,13 @@
                 }
             }, false);
         });
+        if (debug) console.log("onBeforeBuildDocument: all components", this.behaviourComponents);
         return Promise.all(beforeCreateDocumentPromises);
     }
     onExportObject(_object, model: USDObject, context) {
         for (const beh of this.behaviourComponents) {
+            if (debug) console.log("onExportObject: createBehaviours", beh);
             beh.createBehaviours?.call(beh, this, model, context);
         }
     }

src/engine-components/export/usdz/extensions/behavior/BehaviourComponents.ts CHANGED Viewed

@@ -12,7 +12,7 @@
 import { Behaviour, GameObject } from "../../../../Component.js";
 import type { IPointerClickHandler, PointerEventData } from "../../../../ui/PointerEvents.js";
 import { ObjectRaycaster,Raycaster } from "../../../../ui/Raycaster.js";
-import { makeNameSafeForUSD,USDDocument, USDObject, USDZExporterContext } from "../../ThreeUSDZExporter.js";
+import { makeNameSafeForUSD,USDDocument, USDObject, USDWriter, USDZExporterContext } from "../../ThreeUSDZExporter.js";
 import { AnimationExtension, RegisteredAnimationInfo, type UsdzAnimation } from "../Animation.js";
 import { AudioExtension } from "./AudioExtension.js";
 import type { BehaviorExtension, UsdzBehaviour } from "./Behaviour.js";
@@ -521,6 +521,23 @@
                 ));
             }
+            // add InputTargetComponent for VisionOS direct/indirect interactions
+            const addInputTargetComponent = (model: USDObject) => {
+                const empty = USDObject.createEmpty();
+                empty.name = "InputTarget";
+                empty.displayName = undefined;
+                empty.type = "RealityKitComponent";
+                empty.onSerialize = (writer: USDWriter) => {
+                    writer.appendLine("bool allowsDirectInput = 1");
+                    writer.appendLine("bool allowsIndirectInput = 1");
+                    writer.appendLine('uniform token info:id = "RealityKit.InputTarget"');
+                };
+                model.add(empty);
+            }
+            addInputTargetComponent(selfModel);
+            if (this.toggleModel) addInputTargetComponent(this.toggleModel);
             // Ensure initial states are set correctly so that we get the same result as was currently active in the runtime
             const objectsToHide = new Array<USDObject>();
             if (!this.targetStateBeforeCreatingDocument)

src/engine-components/export/usdz/extensions/behavior/BehavioursBuilder.ts CHANGED Viewed

@@ -1,4 +1,4 @@
-import { Object3D } from "three";
+import { Object3D, Vector3 } from "three";
 import { EnumToPrimitiveUnion, getParam } from "../../../../../engine/engine_utils.js";
 import { makeNameSafeForUSD,USDDocument, USDObject, USDWriter } from "../../ThreeUSDZExporter.js";
@@ -47,9 +47,9 @@
         else
             triggerString = `<${this.trigger.id}>`;
-        writer.appendLine(`rel triggers = ${triggerString} `);
+        writer.appendLine(`rel triggers = ${triggerString}`);
         writer.appendLine(`rel actions = <${this.action.id}>`);
-        writer.appendLine(`uniform bool exclusive = ${this.exclusive}`);
+        writer.appendLine(`uniform bool exclusive = ${this.exclusive ? 1 : 0}`); // Apple uses 0 and 1 for bools
         writer.appendLine();
         if (Array.isArray(this.trigger)) {
             for (const trigger of this.trigger) {
@@ -218,8 +218,8 @@
         writer.appendLine();
         writer.appendLine(`token info:id = "Group"`);
-        writer.appendLine(`bool loops = ${this.loops > 0 ? "true" : "false" } `);
-        writer.appendLine(`int performCount = ${Math.max(0, this.performCount)} `);
+        writer.appendLine(`bool loops = ${this.loops}`);
+        writer.appendLine(`int performCount = ${this.loops > 0 ? 0 : Math.max(0, this.performCount)}`);
         writer.appendLine(`token type = "${this.type}"`);
         writer.appendLine();
@@ -260,7 +260,7 @@
     tokenId?: string;
     affectedObjects?: string | Target;
     easeType?: EaseType;;
-    motionType: EmphasizeActionMotionType | VisibilityActionMotionType = "none";
+    motionType: EmphasizeActionMotionType | VisibilityActionMotionType | undefined = undefined;
     duration?: number;
     moveDistance?: number;
     style?: MotionStyle;
@@ -276,6 +276,7 @@
     gain?: number;
     auralMode?: AuralMode;
     multiplePerformOperation?: MultiplePerformOperation;
+    velocity?: Vec3;
     clone(): ActionModel {
         const copy = new ActionModel();
@@ -309,7 +310,10 @@
             writer.appendLine(`token easeType = "${this.easeType}"`);
         if (this.tokenId)
             writer.appendLine(`token info:id = "${this.tokenId}"`);
-        writer.appendLine(`token motionType = "${this.motionType}"`);
+        if (this.tokenId === "ChangeScene")
+            writer.appendLine(`rel scene = </StageRoot/Scenes/Scene>`);
+        if (this.motionType !== undefined)
+            writer.appendLine(`token motionType = "${this.motionType}"`);
         if (typeof this.moveDistance === "number")
             writer.appendLine(`double moveDistance = ${this.moveDistance} `);
         if (this.style)
@@ -349,6 +353,9 @@
         if (typeof this.multiplePerformOperation === "string") {
             writer.appendLine(`token multiplePerformOperation = "${this.multiplePerformOperation}"`);
         }
+        if (typeof this.velocity === "object") {
+            writer.appendLine(`vector3d velocity = (${this.velocity.x}, ${this.velocity.y}, ${this.velocity.z})`);
+        }
         writer.closeBlock();
     }
 }
@@ -452,6 +459,7 @@
         const act = new ActionModel();
         act.tokenId = "Wait";
         act.duration = duration;
+        act.motionType = undefined;
         return act;
     }
@@ -480,6 +488,8 @@
         const act = new ActionModel(targets);
         act.tokenId = "Transform";
         act.duration = duration;
+        // Workaround for a bug in QuickLook: if duration is 0, loops stop somewhat randomly. FB13759712
+        act.duration = Math.max(0.000001, duration);
         act.type = transformType;
         act.easeType = duration > 0 ? easeType : "none";
         if (Array.isArray(transformTarget)) {
@@ -500,6 +510,23 @@
         return act;
     }
+    // Supported only on VisionOS, Preliminary Behaviours can affect RealityKit physics as well
+    static impulseAction(targets: Target, velocity: Vec3) {
+        const act = new ActionModel(targets);
+        act.tokenId = "Impulse";
+        act.velocity = velocity;
+        return act;
+    }
+    // Currently doesn't work on VisionOS, see FB13761990
+    /*
+    static reloadSceneAction() {
+        const act = new ActionModel();
+        act.tokenId = "ChangeScene";
+        // rel scene = ... is implicit since we only allow one scene right now
+        return act;
+    }
+    */
 }
 export { Vec3 as USDVec3 }

src/engine/engine_networking_auto.ts CHANGED Viewed

@@ -1,4 +1,6 @@
 import { isDevEnvironment } from "./debug/index.js";
+import { RoomEvents } from "./engine_networking.js";
+import { INetworkConnection, SendQueue } from "./engine_networking_types.js";
 import type { IComponent } from "./engine_types.js";
 import { getParam } from "./engine_utils.js";
@@ -40,8 +42,6 @@
     private changedProperties: { [key: string]: any } = {};
     private data = {};
-    private _boundEvent?: Function;
-    private _handleReceivingMethod?: Function;
     get networkingKey(): string {
         return this.comp.guid;
@@ -57,10 +57,8 @@
         this._isInit = true;
         this.comp = comp;
         // console.log("INIT", this.comp.name, this.networkingKey);
-        this._boundEvent = this.onHandleSending.bind(this);
-        this.comp.context.post_render_callbacks.push(this._boundEvent);
-        this._handleReceivingMethod = this.onHandleReceiving.bind(this);
-        this.comp.context.connection.beginListen(this.networkingKey, this._handleReceivingMethod);
+        this.comp.context.post_render_callbacks.push(this.onHandleSending);
+        this.comp.context.connection.beginListen(this.networkingKey, this.onHandleReceiving);
         const state = this.comp.context.connection.tryGetState(this.comp.guid);
         if (state) this.onHandleReceiving(state);
@@ -68,10 +66,8 @@
     destroy() {
         if (!this._isInit) return;
-        if (this._boundEvent)
-            this.comp.context.post_render_callbacks.splice(this.comp.context.post_render_callbacks.indexOf(this._boundEvent), 1);
-        if (this._handleReceivingMethod)
-            this.comp.context.connection.stopListen(this.networkingKey, this._handleReceivingMethod);
+        this.comp.context.post_render_callbacks.splice(this.comp.context.post_render_callbacks.indexOf(this.onHandleSending), 1);
+        this.comp.context.connection.stopListen(this.networkingKey, this.onHandleReceiving);
         //@ts-ignore
         this.comp = null;
         this._isInit = false;
@@ -84,12 +80,11 @@
         this.changedProperties[propertyName] = value;
     }
-    private onHandleSending() {
+    private onHandleSending = () => {
         if (!this.hasChanges) return;
         this.hasChanges = false;
-        // console.log(this.changedProperties);
-        const net = this.comp.context.connection;
-        if (!net || !net.isConnected) {
+        const net = this.comp.context.connection as INetworkConnection
+        if (!net || !net.isConnected || !net.isInRoom) {
             for (const key in this.changedProperties)
                 delete this.changedProperties[key];
             return;
@@ -105,11 +100,12 @@
             this.data[name] = value;
         }
         // console.log("SEND", this.comp.name, this.data, this.networkingKey);
-        net.send(this.networkingKey, this.data);
+        net.send(this.networkingKey, this.data, SendQueue.Queued);
     }
-    private onHandleReceiving(val) {
-        if(debug) console.log("RECEIVE", this.comp.name, this.comp.guid, val);
+    private onHandleReceiving = (val) => {
+        if (debug)
+            console.log("RECEIVE", this.comp.name, this.comp.guid, val);
         if (!this._isInit) return;
         if (!this.comp) return;
         const guid = val.guid;

src/engine/engine_networking_types.ts CHANGED Viewed

@@ -13,7 +13,6 @@
 export declare interface INetworkConnection {
     get isConnected(): boolean;
+    get isInRoom(): boolean;
     send(key: string, data: IModel | object | boolean | null | string | number, queue: SendQueue): unknown;
 }

src/engine/engine_physics_rapier.ts CHANGED Viewed

@@ -1023,7 +1023,11 @@
         if (!this.eventQueue) {
             this.eventQueue = new EventQueue(false);
         }
-        if (dt) {
+        if (dt === undefined || dt <= 0) {
+            this._isUpdatingPhysicsWorld = false;
+            return;
+        }
+        else if (dt !== undefined) {
             // if we make to sudden changes to the timestep the physics can get unstable
             // https://rapier.rs/docs/user_guides/javascript/integration_parameters/#dt
             this.world.timestep = Mathf.lerp(this.world.timestep, dt, 0.8);

src/engine-components/export/usdz/index.ts CHANGED Viewed

@@ -1,3 +1,3 @@
 export { type UsdzBehaviour } from "./extensions/behavior/Behaviour.js";
-export { imageToCanvas,USDObject } from "./ThreeUSDZExporter.js";
+export { imageToCanvas, makeNameSafeForUSD, USDZExporter as NeedleUSDZExporter, USDObject, USDWriter,type USDZExporterContext } from "./ThreeUSDZExporter.js";
 export { USDZExporter } from "./USDZExporter.js";

src/engine-components/export/usdz/ThreeUSDZExporter.ts CHANGED Viewed

@@ -93,7 +93,8 @@
 	uuid: string;
 	name: string;
-	displayName: string;
+	type?: string; // by default, Xform is used
+	displayName?: string;
 	matrix: Matrix4;
 	private _isDynamic: boolean;
 	get isDynamic() { return this._isDynamic; }
@@ -1218,13 +1219,17 @@
 	}
 	else if ( camera )
 		writer.beginBlock( `def Camera "${name}"`, "(", false );
+	else if ( model.type !== undefined)
+		writer.beginBlock( `def ${model.type} "${name}"` );
 	else
 		writer.beginBlock( `def Xform "${name}"`, "(", false);
 	if (model.displayName)
 		writer.appendLine(`displayName = "${model.displayName}"`);
-	writer.closeBlock( ")" );
-	writer.beginBlock();
+	if (model.type === undefined) {
+		writer.closeBlock( ")" );
+		writer.beginBlock();
+	}
 	if ( geometry && material ) {
 		const materialName = getMaterialName(material);
@@ -1245,10 +1250,11 @@
 		writer.appendLine( `rel skel:animationSource = <Rig/_anim>`);
 		writer.appendLine( `matrix4d xformOp:transform = ${buildMatrix(new Matrix4())}` ); // always identity / in world space
 	}
-	else {
+	else if (model.type === undefined) {
 		writer.appendLine( `matrix4d xformOp:transform = ${transform}` );
 	}
-	writer.appendLine( 'uniform token[] xformOpOrder = ["xformOp:transform"]' );
+	if (model.type === undefined)
+		writer.appendLine( 'uniform token[] xformOpOrder = ["xformOp:transform"]' );
 	if ( camera ) {

src/engine-components/timeline/TimelineTracks.ts CHANGED Viewed

@@ -150,6 +150,12 @@
     private _didBind: boolean = false;
     private _animator: Animator | null = null;
+    onDisable() {
+        // if this track is disabled we need to stop the currently active actions
+        this.mixer?.stopAllAction();
+    }
     // Using this callback instead of onEnable etc
     // because we want to re-enable the animator when the director is at the end and wrap mode is set to none
     // in which case the director is stopped (but not disabled)

src/engine-components/export/usdz/USDZExporter.ts CHANGED Viewed

@@ -1,4 +1,4 @@
-import { Matrix4, Mesh, Object3D } from "three";
+import { Matrix4, Mesh, Object3D, Quaternion, Vector3 } from "three";
 import { showBalloonMessage, showBalloonWarning } from "../../../engine/debug/index.js";
 import { hasProLicense } from "../../../engine/engine_license.js";
@@ -8,6 +8,7 @@
 import { Behaviour, GameObject } from "../../Component.js";
 import { Renderer } from "../../Renderer.js"
 import { WebARSessionRoot } from "../../webxr/WebARSessionRoot.js";
+import { WebXR } from "../../webxr/WebXR.js";
 import { WebXRButtonFactory } from "../../webxr/WebXRButtons.js";
 import { XRFlag, XRState, XRStateFlag } from "../../webxr/XRFlag.js";
 import type { IUSDExporterExtension } from "./Extension.js";
@@ -280,6 +281,7 @@
     }
     private readonly _currentExportTasks = new Map<Object3D, Promise<Blob | null>>();
+    private _previousTimeScale: number = 1;
     private async internalExport(objectToExport: Object3D): Promise<Blob | null> {
@@ -335,9 +337,6 @@
         const currentXRState = XRState.Global.Mask;
         XRState.Global.Set(XRStateFlag.AR);
-        // make sure we apply the AR scale
-        this.applyWebARSessionRoot();
         const exporter = new ThreeUSDZExporter();
         const extensions: any = [...this.extensions]
@@ -349,6 +348,13 @@
         Progress.report("export-usdz", "Invoking before-export");
         this.dispatchEvent(new CustomEvent("before-export", { detail: eventArgs }))
+        // make sure we apply the AR scale
+        this.applyWebARSessionRoot();
+        // freeze time
+        this._previousTimeScale = this.context.time.timeScale;
+        this.context.time.timeScale = 0;
         // Implicit registration and actions for Animators and Animation components
         // Currently, Animators properly build PlayAnimation actions, but Animation components don't.
@@ -366,7 +372,7 @@
         exporter.keepObject = (object) => {
             // TODO We need to take more care with disabled renderers. This currently breaks when any renderer is disabled
             // and then enabled at runtime by e.g. SetActiveOnClick, requiring extra work to enable them before export,
-            // cache their state, and then reset their state after export. See
+            // cache their state, and then reset their state after export.
             const renderer = GameObject.getComponent(object, Renderer)
             if (renderer && !renderer.enabled) return false;
             return true;
@@ -396,6 +402,11 @@
         const blob = new Blob([arraybuffer], { type: 'model/vnd.usdz+zip' });
+        this.revertWebARSessionRoot();
+        // unfreeze time
+        this.context.time.timeScale = this._previousTimeScale;
         Progress.report("export-usdz", "Invoking after-export");
         this.dispatchEvent(new CustomEvent("after-export", { detail: eventArgs }))
@@ -531,6 +542,12 @@
     }
     private static invertForwardMatrix = new Matrix4().makeRotationY(Math.PI);
+    private _rootSessionRootWasAppliedTo: Object3D | null = null;
+    private _rootPositionBeforeExport: Vector3 = new Vector3();
+    private _rootRotationBeforeExport: Quaternion = new Quaternion();
+    private _rootScaleBeforeExport: Vector3 = new Vector3();
     private applyWebARSessionRoot() {
         if (!this.objectToExport) return;
@@ -539,23 +556,39 @@
         let sessionRoot = GameObject.getComponentInParent(this.objectToExport, WebARSessionRoot);
         const hasSessionRootInParentHierarchy = sessionRoot !== null && sessionRoot !== undefined;
         // if it's not in the parent hierarchy BUT in the child hierarchy we apply it to the sessionRoot object itself
-        // that#s the case when no objectToExport is explictly assigned and the whole scene is being exported
+        // that's the case when no objectToExport is explictly assigned and the whole scene is being exported
         if(!sessionRoot) sessionRoot = GameObject.getComponentInChildren(this.objectToExport, WebARSessionRoot);
-        if (debug) console.log("applyWebARSessionRoot", sessionRoot);
+        if (debug) console.log("applyWebARSessionRoot", sessionRoot, sessionRoot?.arScale);
+        let arScale = 1;
+        let invertForward = false;
+        const target = hasSessionRootInParentHierarchy || !sessionRoot ? this.objectToExport : sessionRoot.gameObject;
         if (!sessionRoot) {
-            if(debug) console.warn("No WebARSessionRoot found in parent hierarchy", this.objectToExport);
-            return;
+            const xr = GameObject.findObjectOfType(WebXR);
+            if (xr) arScale = xr.arSceneScale;
         }
+        else {
+            arScale = sessionRoot.arScale;
+            invertForward = sessionRoot.invertForward;
+        }
         // either apply the scale to the object being exported or to the sessionRoot object itself
-        const target = hasSessionRootInParentHierarchy ? this.objectToExport : sessionRoot.gameObject;
-        const scale = 1 / sessionRoot!.arScale;
-        target.matrix.makeScale(scale, scale, scale);
-        if (sessionRoot.invertForward) {
+        const scale = 1 / arScale;
+        this._rootSessionRootWasAppliedTo = target;
+        this._rootPositionBeforeExport.copy(target.position);
+        this._rootRotationBeforeExport.copy(target.quaternion);
+        this._rootScaleBeforeExport.copy(target.scale);
+        target.scale.multiplyScalar(scale);
+        // legacy, should likely be deleted
+        if (invertForward) {
             target.matrix.multiply(USDZExporter.invertForwardMatrix);
         }
+        // udate childs as well
+        target.updateMatrixWorld(true);
         // TODO we should refactor this and use one common method in WebARSessionRoot to place an object –
         // basically the inverted effect of WebARSessionRoot.onApplyPose()
@@ -563,7 +596,21 @@
         // TODO why are we not reverting this transformation after the export?
     }
+    private revertWebARSessionRoot() {
+        if (!this.objectToExport) return;
+        if (!this._rootSessionRootWasAppliedTo) return;
+        const target = this._rootSessionRootWasAppliedTo;
+        target.position.copy(this._rootPositionBeforeExport);
+        target.quaternion.copy(this._rootRotationBeforeExport);
+        target.scale.copy(this._rootScaleBeforeExport);
+        // udate childs as well
+        target.updateMatrixWorld(true);
+        this._rootSessionRootWasAppliedTo = null;
+    }
     private createQuicklookButton() {
         const buttoncontainer = WebXRButtonFactory.getOrCreate();
         const button = buttoncontainer.createQuicklookButton();

src/engine-components/webxr/controllers/XRControllerModel.ts CHANGED Viewed

@@ -243,7 +243,7 @@
         const loader = new GLTFLoader();
         addDracoAndKTX2Loaders(loader, context);
-        await registerExtensions(loader, context, this.sourceId!);
+        await registerExtensions(loader, context, this.sourceId ?? "");
         loader.setPath('https://cdn.jsdelivr.net/npm/@webxr-input-profiles/assets@1.0/dist/profiles/generic-hand/');
         // TODO: we should handle the loading here ourselves to not have this requirement of a specific model name