Needle Engine 3.4.0-alpha

Files changed (27) hide show

src/engine/codegen/register_types.js +52 -4
plugins/vite/reload.js +13 -2
src/engine-components/Animation.ts +4 -0
src/engine-components/export/usdz/extensions/Animation.ts +37 -45
src/engine-components/ui/BaseUIComponent.ts +7 -1
src/engine-components/ui/Canvas.ts +80 -5
src/engine-components/codegen/components.ts +25 -1
src/engine/engine_gameobject.ts +3 -2
src/engine/engine_three_utils.ts +2 -2
src/engine-components/export/usdz/Extension.ts +4 -5
src/engine-components/ui/Graphic.ts +2 -0
src/engine-components/ui/Image.ts +3 -3
src/engine-components/ui/Interfaces.ts +30 -6
src/engine-components/ui/Layout.ts +303 -4
src/engine-components/utils/LookAt.ts +60 -7
src/engine-components/postprocessing/PostProcessingHandler.ts +1 -1
src/engine-components/ui/RectTransform.ts +65 -40
src/engine-components/export/usdz/types.ts +0 -39
src/engine-components/export/usdz/USDZExporter.ts +39 -17
src/engine-components/webxr/WebXRImageTracking.ts +100 -27
src/engine-components/export/usdz/extensions/behavior/Actions.ts +99 -0
src/engine-components/export/usdz/extensions/behavior/Behaviour.ts +181 -0
src/engine-components/export/usdz/extensions/behavior/BehaviourComponents.ts +503 -0
src/engine-components/export/usdz/extensions/behavior/BehavioursBuilder.ts +459 -0
src/engine-components/export/usdz/extensions/DocumentExtension.ts +10 -0
src/engine-components/export/usdz/ThreeUSDZExporter.ts +1280 -0
src/engine-components/export/usdz/extensions/USDZText.ts +142 -0

src/engine/codegen/register_types.js CHANGED Viewed

@@ -1,7 +1,10 @@
 import { TypeStore } from "./../engine_typestore"
 // Import types
 import { __Ignore } from "../../engine-components/codegen/components";
+import { ActionBuilder } from "../../engine-components/export/usdz/extensions/behavior/BehavioursBuilder";
+import { ActionCollection } from "../../engine-components/export/usdz/extensions/behavior/Actions";
+import { ActionModel } from "../../engine-components/export/usdz/extensions/behavior/BehavioursBuilder";
 import { AlignmentConstraint } from "../../engine-components/AlignmentConstraint";
 import { Animation } from "../../engine-components/Animation";
 import { AnimationCurve } from "../../engine-components/AnimationCurve";
@@ -26,6 +29,8 @@
 import { AxesHelper } from "../../engine-components/AxesHelper";
 import { BaseUIComponent } from "../../engine-components/ui/BaseUIComponent";
 import { BasicIKConstraint } from "../../engine-components/BasicIKConstraint";
+import { BehaviorExtension } from "../../engine-components/export/usdz/extensions/behavior/Behaviour";
+import { BehaviorModel } from "../../engine-components/export/usdz/extensions/behavior/BehavioursBuilder";
 import { Behaviour } from "../../engine-components/Component";
 import { Bloom } from "../../engine-components/postprocessing/Effects/Bloom";
 import { BoxCollider } from "../../engine-components/Collider";
@@ -37,6 +42,8 @@
 import { Canvas } from "../../engine-components/ui/Canvas";
 import { CanvasGroup } from "../../engine-components/ui/CanvasGroup";
 import { CapsuleCollider } from "../../engine-components/Collider";
+import { ChangeMaterialOnClick } from "../../engine-components/export/usdz/extensions/behavior/BehaviourComponents";
+import { ChangeTransformOnClick } from "../../engine-components/export/usdz/extensions/behavior/BehaviourComponents";
 import { CharacterController } from "../../engine-components/CharacterController";
 import { CharacterControllerInput } from "../../engine-components/CharacterController";
 import { ChromaticAberration } from "../../engine-components/postprocessing/Effects/ChromaticAberration";
@@ -50,10 +57,12 @@
 import { DeleteBox } from "../../engine-components/DeleteBox";
 import { DepthOfField } from "../../engine-components/postprocessing/Effects/DepthOfField";
 import { DeviceFlag } from "../../engine-components/DeviceFlag";
+import { DocumentExtension } from "../../engine-components/export/usdz/extensions/DocumentExtension";
 import { DragControls } from "../../engine-components/DragControls";
 import { DropListener } from "../../engine-components/DropListener";
 import { Duplicatable } from "../../engine-components/Duplicatable";
 import { EmissionModule } from "../../engine-components/ParticleSystemModules";
+import { EmphasizeOnClick } from "../../engine-components/export/usdz/extensions/behavior/BehaviourComponents";
 import { EventList } from "../../engine-components/EventList";
 import { EventListEvent } from "../../engine-components/EventList";
 import { EventSystem } from "../../engine-components/ui/EventSystem";
@@ -70,13 +79,14 @@
 import { GridHelper } from "../../engine-components/GridHelper";
 import { GridLayoutGroup } from "../../engine-components/ui/Layout";
 import { GroundProjectedEnv } from "../../engine-components/GroundProjection";
+import { GroupActionModel } from "../../engine-components/export/usdz/extensions/behavior/BehavioursBuilder";
+import { HideOnStart } from "../../engine-components/export/usdz/extensions/behavior/BehaviourComponents";
 import { HingeJoint } from "../../engine-components/Joints";
 import { HorizontalLayoutGroup } from "../../engine-components/ui/Layout";
 import { Image } from "../../engine-components/ui/Image";
 import { InheritVelocityModule } from "../../engine-components/ParticleSystemModules";
 import { InputField } from "../../engine-components/ui/InputField";
 import { Interactable } from "../../engine-components/Interactable";
-import { LayoutGroup } from "../../engine-components/ui/Layout";
 import { Light } from "../../engine-components/Light";
 import { LimitVelocityOverLifetimeModule } from "../../engine-components/ParticleSystemModules";
 import { LODGroup } from "../../engine-components/LODGroup";
@@ -98,17 +108,21 @@
 import { OpenURL } from "../../engine-components/utils/OpenURL";
 import { OrbitControls } from "../../engine-components/OrbitControls";
 import { Outline } from "../../engine-components/ui/Outline";
+import { Padding } from "../../engine-components/ui/Layout";
 import { ParticleBurst } from "../../engine-components/ParticleSystemModules";
 import { ParticleSubEmitter } from "../../engine-components/ParticleSystemSubEmitter";
 import { ParticleSystem } from "../../engine-components/ParticleSystem";
 import { ParticleSystemRenderer } from "../../engine-components/ParticleSystem";
 import { PixelationEffect } from "../../engine-components/postprocessing/Effects/Pixelation";
 import { PlayableDirector } from "../../engine-components/timeline/PlayableDirector";
+import { PlayAnimationOnClick } from "../../engine-components/export/usdz/extensions/behavior/BehaviourComponents";
 import { PlayerColor } from "../../engine-components/PlayerColor";
 import { PlayerState } from "../../engine-components-experimental/networking/PlayerSync";
 import { PlayerSync } from "../../engine-components-experimental/networking/PlayerSync";
 import { PointerEventData } from "../../engine-components/ui/PointerEvents";
 import { PostProcessingHandler } from "../../engine-components/postprocessing/PostProcessingHandler";
+import { PreliminaryAction } from "../../engine-components/export/usdz/extensions/behavior/BehaviourComponents";
+import { PreliminaryTrigger } from "../../engine-components/export/usdz/extensions/behavior/BehaviourComponents";
 import { PresentationMode } from "../../engine-components-experimental/Presentation";
 import { QuickLookOverlay } from "../../engine-components/export/usdz/USDZExporter";
 import { RawImage } from "../../engine-components/ui/Image";
@@ -127,6 +141,7 @@
 import { SceneSwitcher } from "../../engine-components/SceneSwitcher";
 import { ScreenCapture } from "../../engine-components/ScreenCapture";
 import { ScreenSpaceAmbientOcclusion } from "../../engine-components/postprocessing/Effects/ScreenspaceAmbientOcclusion";
+import { SetActiveOnClick } from "../../engine-components/export/usdz/extensions/behavior/BehaviourComponents";
 import { ShadowCatcher } from "../../engine-components/ShadowCatcher";
 import { ShapeModule } from "../../engine-components/ParticleSystemModules";
 import { SignalAsset } from "../../engine-components/timeline/SignalAsset";
@@ -151,24 +166,33 @@
 import { SyncedCamera } from "../../engine-components/SyncedCamera";
 import { SyncedRoom } from "../../engine-components/SyncedRoom";
 import { SyncedTransform } from "../../engine-components/SyncedTransform";
+import { TapGestureTrigger } from "../../engine-components/export/usdz/extensions/behavior/BehaviourComponents";
 import { TeleportTarget } from "../../engine-components/webxr/WebXRController";
 import { TestRunner } from "../../engine-components/TestRunner";
 import { TestSimulateUserData } from "../../engine-components/TestRunner";
 import { Text } from "../../engine-components/ui/Text";
+import { TextBuilder } from "../../engine-components/export/usdz/extensions/USDZText";
+import { TextExtension } from "../../engine-components/export/usdz/extensions/USDZText";
 import { TextureSheetAnimationModule } from "../../engine-components/ParticleSystemModules";
 import { TiltShiftEffect } from "../../engine-components/postprocessing/Effects/TiltShiftEffect";
 import { ToneMapping } from "../../engine-components/postprocessing/Effects/Tonemapping";
 import { TrailModule } from "../../engine-components/ParticleSystemModules";
 import { TransformData } from "../../engine-components/export/usdz/extensions/Animation";
 import { TransformGizmo } from "../../engine-components/TransformGizmo";
+import { TriggerBuilder } from "../../engine-components/export/usdz/extensions/behavior/BehavioursBuilder";
+import { TriggerModel } from "../../engine-components/export/usdz/extensions/behavior/BehavioursBuilder";
 import { UIRaycastUtils } from "../../engine-components/ui/RaycastUtils";
 import { UIRootComponent } from "../../engine-components/ui/BaseUIComponent";
 import { UsageMarker } from "../../engine-components/Interactable";
+import { USDZBehaviours } from "../../engine-components/export/usdz/extensions/behavior/Behaviour";
 import { USDZExporter } from "../../engine-components/export/usdz/USDZExporter";
+import { USDZText } from "../../engine-components/export/usdz/extensions/USDZText";
+import { VariantAction } from "../../engine-components/export/usdz/extensions/behavior/Actions";
 import { VelocityOverLifetimeModule } from "../../engine-components/ParticleSystemModules";
 import { VerticalLayoutGroup } from "../../engine-components/ui/Layout";
 import { VideoPlayer } from "../../engine-components/VideoPlayer";
 import { Vignette } from "../../engine-components/postprocessing/Effects/Vignette";
+import { VisibilityAction } from "../../engine-components/export/usdz/extensions/behavior/BehaviourComponents";
 import { Voip } from "../../engine-components/Voip";
 import { Volume } from "../../engine-components/postprocessing/Volume";
 import { VolumeParameter } from "../../engine-components/postprocessing/VolumeParameter";
@@ -190,9 +214,12 @@
 import { XRGrabRendering } from "../../engine-components/webxr/WebXRGrabRendering";
 import { XRRig } from "../../engine-components/webxr/WebXRRig";
 import { XRState } from "../../engine-components/XRFlag";
 // Register types
 TypeStore.add("__Ignore", __Ignore);
+TypeStore.add("ActionBuilder", ActionBuilder);
+TypeStore.add("ActionCollection", ActionCollection);
+TypeStore.add("ActionModel", ActionModel);
 TypeStore.add("AlignmentConstraint", AlignmentConstraint);
 TypeStore.add("Animation", Animation);
 TypeStore.add("AnimationCurve", AnimationCurve);
@@ -217,6 +244,8 @@
 TypeStore.add("AxesHelper", AxesHelper);
 TypeStore.add("BaseUIComponent", BaseUIComponent);
 TypeStore.add("BasicIKConstraint", BasicIKConstraint);
+TypeStore.add("BehaviorExtension", BehaviorExtension);
+TypeStore.add("BehaviorModel", BehaviorModel);
 TypeStore.add("Behaviour", Behaviour);
 TypeStore.add("Bloom", Bloom);
 TypeStore.add("BoxCollider", BoxCollider);
@@ -228,6 +257,8 @@
 TypeStore.add("Canvas", Canvas);
 TypeStore.add("CanvasGroup", CanvasGroup);
 TypeStore.add("CapsuleCollider", CapsuleCollider);
+TypeStore.add("ChangeMaterialOnClick", ChangeMaterialOnClick);
+TypeStore.add("ChangeTransformOnClick", ChangeTransformOnClick);
 TypeStore.add("CharacterController", CharacterController);
 TypeStore.add("CharacterControllerInput", CharacterControllerInput);
 TypeStore.add("ChromaticAberration", ChromaticAberration);
@@ -241,10 +272,12 @@
 TypeStore.add("DeleteBox", DeleteBox);
 TypeStore.add("DepthOfField", DepthOfField);
 TypeStore.add("DeviceFlag", DeviceFlag);
+TypeStore.add("DocumentExtension", DocumentExtension);
 TypeStore.add("DragControls", DragControls);
 TypeStore.add("DropListener", DropListener);
 TypeStore.add("Duplicatable", Duplicatable);
 TypeStore.add("EmissionModule", EmissionModule);
+TypeStore.add("EmphasizeOnClick", EmphasizeOnClick);
 TypeStore.add("EventList", EventList);
 TypeStore.add("EventListEvent", EventListEvent);
 TypeStore.add("EventSystem", EventSystem);
@@ -261,13 +294,14 @@
 TypeStore.add("GridHelper", GridHelper);
 TypeStore.add("GridLayoutGroup", GridLayoutGroup);
 TypeStore.add("GroundProjectedEnv", GroundProjectedEnv);
+TypeStore.add("GroupActionModel", GroupActionModel);
+TypeStore.add("HideOnStart", HideOnStart);
 TypeStore.add("HingeJoint", HingeJoint);
 TypeStore.add("HorizontalLayoutGroup", HorizontalLayoutGroup);
 TypeStore.add("Image", Image);
 TypeStore.add("InheritVelocityModule", InheritVelocityModule);
 TypeStore.add("InputField", InputField);
 TypeStore.add("Interactable", Interactable);
-TypeStore.add("LayoutGroup", LayoutGroup);
 TypeStore.add("Light", Light);
 TypeStore.add("LimitVelocityOverLifetimeModule", LimitVelocityOverLifetimeModule);
 TypeStore.add("LODGroup", LODGroup);
@@ -289,17 +323,21 @@
 TypeStore.add("OpenURL", OpenURL);
 TypeStore.add("OrbitControls", OrbitControls);
 TypeStore.add("Outline", Outline);
+TypeStore.add("Padding", Padding);
 TypeStore.add("ParticleBurst", ParticleBurst);
 TypeStore.add("ParticleSubEmitter", ParticleSubEmitter);
 TypeStore.add("ParticleSystem", ParticleSystem);
 TypeStore.add("ParticleSystemRenderer", ParticleSystemRenderer);
 TypeStore.add("PixelationEffect", PixelationEffect);
 TypeStore.add("PlayableDirector", PlayableDirector);
+TypeStore.add("PlayAnimationOnClick", PlayAnimationOnClick);
 TypeStore.add("PlayerColor", PlayerColor);
 TypeStore.add("PlayerState", PlayerState);
 TypeStore.add("PlayerSync", PlayerSync);
 TypeStore.add("PointerEventData", PointerEventData);
 TypeStore.add("PostProcessingHandler", PostProcessingHandler);
+TypeStore.add("PreliminaryAction", PreliminaryAction);
+TypeStore.add("PreliminaryTrigger", PreliminaryTrigger);
 TypeStore.add("PresentationMode", PresentationMode);
 TypeStore.add("QuickLookOverlay", QuickLookOverlay);
 TypeStore.add("RawImage", RawImage);
@@ -318,6 +356,7 @@
 TypeStore.add("SceneSwitcher", SceneSwitcher);
 TypeStore.add("ScreenCapture", ScreenCapture);
 TypeStore.add("ScreenSpaceAmbientOcclusion", ScreenSpaceAmbientOcclusion);
+TypeStore.add("SetActiveOnClick", SetActiveOnClick);
 TypeStore.add("ShadowCatcher", ShadowCatcher);
 TypeStore.add("ShapeModule", ShapeModule);
 TypeStore.add("SignalAsset", SignalAsset);
@@ -342,24 +381,33 @@
 TypeStore.add("SyncedCamera", SyncedCamera);
 TypeStore.add("SyncedRoom", SyncedRoom);
 TypeStore.add("SyncedTransform", SyncedTransform);
+TypeStore.add("TapGestureTrigger", TapGestureTrigger);
 TypeStore.add("TeleportTarget", TeleportTarget);
 TypeStore.add("TestRunner", TestRunner);
 TypeStore.add("TestSimulateUserData", TestSimulateUserData);
 TypeStore.add("Text", Text);
+TypeStore.add("TextBuilder", TextBuilder);
+TypeStore.add("TextExtension", TextExtension);
 TypeStore.add("TextureSheetAnimationModule", TextureSheetAnimationModule);
 TypeStore.add("TiltShiftEffect", TiltShiftEffect);
 TypeStore.add("ToneMapping", ToneMapping);
 TypeStore.add("TrailModule", TrailModule);
 TypeStore.add("TransformData", TransformData);
 TypeStore.add("TransformGizmo", TransformGizmo);
+TypeStore.add("TriggerBuilder", TriggerBuilder);
+TypeStore.add("TriggerModel", TriggerModel);
 TypeStore.add("UIRaycastUtils", UIRaycastUtils);
 TypeStore.add("UIRootComponent", UIRootComponent);
 TypeStore.add("UsageMarker", UsageMarker);
+TypeStore.add("USDZBehaviours", USDZBehaviours);
 TypeStore.add("USDZExporter", USDZExporter);
+TypeStore.add("USDZText", USDZText);
+TypeStore.add("VariantAction", VariantAction);
 TypeStore.add("VelocityOverLifetimeModule", VelocityOverLifetimeModule);
 TypeStore.add("VerticalLayoutGroup", VerticalLayoutGroup);
 TypeStore.add("VideoPlayer", VideoPlayer);
 TypeStore.add("Vignette", Vignette);
+TypeStore.add("VisibilityAction", VisibilityAction);
 TypeStore.add("Voip", Voip);
 TypeStore.add("Volume", Volume);
 TypeStore.add("VolumeParameter", VolumeParameter);

plugins/vite/reload.js CHANGED Viewed

@@ -9,6 +9,7 @@
 const __dirname = path.dirname(__filename);
 const filesUsingHotReload = new Set();
+let assetsDirectory = "";
 export const needleReload = (command, config, userSettings) => {
     if (command === "build") return;
@@ -23,8 +24,9 @@
         if (res) config = res;
     }
+    const projectConfig = tryLoadProjectConfig();
+    assetsDirectory = path.resolve(projectConfig?.assetsDirectory || "assets");
-    const projectConfig = tryLoadProjectConfig();
     const buildDirectory = projectConfig?.buildDirectory?.length ? process.cwd().replaceAll("\\", "/") + "/" + projectConfig?.buildDirectory : "";
     if (buildDirectory?.length) {
         setTimeout(() => console.log("Build directory: ", buildDirectory), 100);
@@ -153,9 +155,18 @@
         }
     }
-    if (file.endsWith(".vue") || file.endsWith(".ts") || file.endsWith(".js") || file.endsWith(".jsx") || file.endsWith(".tsx"))
+    // these are known file types we export from integrations
+    const knownExportFileTypes = [ ".glb", ".gltf", ".bin", "exr", ".ktx2", ".mp3", ".ogg", ".mp4", ".webm" ];
+    if (!knownExportFileTypes.some((type) => file.endsWith(type)))
         return;
+    // we only care about exports into "assets"
+    if (!path.resolve(file).startsWith(assetsDirectory))
+        return;
+    if (file.endsWith(".svelte") || file.endsWith(".vue") || file.endsWith(".ts") || file.endsWith(".js") || file.endsWith(".jsx") || file.endsWith(".tsx"))
+        return;
     if (file.endsWith(lockFileName)) return;
     let fileSize = "";
     const isGlbOrGltfFile = file.endsWith(".glb") || file.endsWith(".bin");

src/engine-components/Animation.ts CHANGED Viewed

@@ -176,6 +176,10 @@
                 return this.internalOnPlay(act, options);
             }
         }
+        if (!(clip instanceof AnimationClip)) {
+            console.warn("Clip is no AnimationClip", clip, "on object: " + this.name)
+            return;
+        }
         const act = this.mixer.clipAction(clip);
         this.actions.push(act);
         return this.internalOnPlay(act, options);

src/engine-components/export/usdz/extensions/Animation.ts CHANGED Viewed

@@ -1,17 +1,18 @@
 import { GameObject } from "../../../Component";
 import { getParam } from "../../../../engine/engine_utils";
-import { Object3D, Color, Matrix4, MeshStandardMaterial, Vector3, Quaternion, Interpolant } from "three";
-//@ts-ignore
-import { USDZObject, buildMatrix } from "three/examples/jsm/exporters/USDZExporter"
-import { IUSDZExporterExtension } from "../Extension";
+import { USDObject, buildMatrix } from "../ThreeUSDZExporter";
+import { IUSDExporterExtension } from "../Extension";
+import { Object3D, Matrix4, Vector3, Quaternion, Interpolant, AnimationClip, KeyframeTrack } from "three";
 const debug = getParam("debugusdzanimation");
 export interface UsdzAnimation {
-    createAnimation(ext: AnimationExtension, model: USDZObject, context);
+    createAnimation(ext: AnimationExtension, model: USDObject, context);
 }
-export type AnimationClipCollection = Array<{ root: Object3D, clips: Array<THREE.AnimationClip> }>;
+export type AnimationClipCollection = Array<{ root: Object3D, clips: Array<AnimationClip> }>;
 export class RegisteredAnimationInfo {
@@ -20,9 +21,9 @@
     private ext: AnimationExtension;
     private root: Object3D;
-    private clip: THREE.AnimationClip;
+    private clip: AnimationClip;
-    constructor(ext: AnimationExtension, root: THREE.Object3D, clip: THREE.AnimationClip) {
+    constructor(ext: AnimationExtension, root: Object3D, clip: AnimationClip) {
         this.ext = ext;
         this.root = root;
         this.clip = clip;
@@ -30,17 +31,17 @@
 }
 export class TransformData {
-    clip: THREE.AnimationClip;
-    pos?: THREE.KeyframeTrack;
-    rot?: THREE.KeyframeTrack;
-    scale?: THREE.KeyframeTrack;
+    clip: AnimationClip;
+    pos?: KeyframeTrack;
+    rot?: KeyframeTrack;
+    scale?: KeyframeTrack;
     get frameRate(): number { return 60; }
     private ext: AnimationExtension;
     private root: Object3D;
     private target: Object3D;
-    constructor(ext: AnimationExtension, root: Object3D, target: Object3D, clip: THREE.AnimationClip) {
+    constructor(ext: AnimationExtension, root: Object3D, target: Object3D, clip: AnimationClip) {
         this.ext = ext;
         this.root = root;
         this.target = target;
@@ -78,14 +79,14 @@
 declare type AnimationDict = Map<Object3D, Array<TransformData>>;
-export class AnimationExtension implements IUSDZExporterExtension {
+export class AnimationExtension implements IUSDExporterExtension {
     get extensionName(): string { return "animation" }
     private dict: AnimationDict = new Map();
     // private rootTargetMap: Map<Object3D, Object3D[]> = new Map();
     private rootTargetMap: Map<Object3D, Object3D[]> = new Map();
-    getStartTime01(root: Object3D, clip: THREE.AnimationClip) {
+    getStartTime01(root: Object3D, clip: AnimationClip) {
         const targets = this.rootTargetMap.get(root);
         if (!targets) return Infinity;
         let longestStartTime: number = -1;
@@ -108,7 +109,7 @@
         return longestStartTime;
     }
-    registerAnimation(root: Object3D, clip: THREE.AnimationClip): RegisteredAnimationInfo | null {
+    registerAnimation(root: Object3D, clip: AnimationClip): RegisteredAnimationInfo | null {
         if (!clip || !root) return null;
         if (!this.rootTargetMap.has(root)) this.rootTargetMap.set(root, []);
         // this.rootTargetMap.get(root)?.push(clip);
@@ -165,7 +166,7 @@
         }
     }
-    onExportObject(object, model: USDZObject, _context) {
+    onExportObject(object, model: USDObject, _context) {
         GameObject.foreachComponent(object, (comp) => {
             const c = comp as unknown as UsdzAnimation;
@@ -187,7 +188,7 @@
     object: Object3D;
     dict: AnimationDict;
-    model: USDZObject;
+    model: USDObject | undefined = undefined;
     private callback?: Function;
@@ -196,7 +197,7 @@
         this.dict = dict;
     }
-    registerCallback(model: USDZObject) {
+    registerCallback(model: USDObject) {
         if (this.model && this.callback) {
             this.model.removeEventListener("serialize", this.callback);
         }
@@ -209,6 +210,7 @@
     }
     onSerialize(writer, _context) {
+        if (!this.model) return;
         if (debug)
             console.log("SERIALIZE", this.model.name, this.object.type);
         // do we have a track for this?
@@ -228,22 +230,24 @@
         const rotation = new Quaternion();
         const scale = new Vector3(1, 1, 1);
-        // TODO doesn't support individual time arrays right now
-        // could use these in case we don't have time values that are identical
-        /*
-        const translationInterpolant = o.pos?.createInterpolant() as THREE.Interpolant;
-        const rotationInterpolant = o.rot?.createInterpolant() as THREE.Interpolant;
-        const scaleInterpolant = o.scale?.createInterpolant() as THREE.Interpolant;
-        */
         writer.appendLine("matrix4d xformOp:transform.timeSamples = {");
         writer.indent++;
         for (const transformData of arr) {
-            let timesArray = transformData.pos?.times;
-            if (!timesArray || transformData.rot && transformData.rot.times?.length > timesArray?.length) timesArray = transformData.rot?.times;
-            if (!timesArray || transformData.scale && transformData.scale.times?.length > timesArray?.length) timesArray = transformData.scale?.times;
-            if (!timesArray) {
+            let posTimesArray = transformData.pos?.times;
+            let rotTimesArray = transformData.rot?.times;
+            let scaleTimesArray = transformData.scale?.times;
+            // timesArray is the sorted union of all time values
+            let timesArray: number[] = [];
+            if (posTimesArray) for (const t of posTimesArray) timesArray.push(t);
+            if (rotTimesArray) for (const t of rotTimesArray) timesArray.push(t);
+            if (scaleTimesArray) for (const t of scaleTimesArray) timesArray.push(t);
+            // sort
+            timesArray.sort((a, b) => a - b);
+            timesArray = [...new Set(timesArray)];
+            if (!timesArray || timesArray.length === 0) {
                 console.error("got an animated object but no time values??", object, transformData);
                 continue;
             }
@@ -276,8 +280,8 @@
                     rotation.set(quat[0], quat[1], quat[2], quat[3]);
                 }
                 if (scaleInterpolant) {
-                    const scale = scaleInterpolant.evaluate(time);
-                    scale.set(scale[0], scale[1], scale[2]);
+                    const scaleVal = scaleInterpolant.evaluate(time);
+                    scale.set(scaleVal[0], scaleVal[1], scaleVal[2]);
                 }
                 composedTransform.compose(translation, rotation, scale);
@@ -290,17 +294,5 @@
         }
         writer.indent--;
         writer.appendLine("}");
-        /*
-        let transform3 = new Matrix4();
-        transform3.compose(0.2,0,0);
-        const transform = buildMatrix(model.matrix);
-        const transform2 = buildMatrix(transform3.multiply(model.matrix));
-        writer.appendLine(`matrix4d xformOp:transform.timeSamples = {
-            0: ${transform},
-            30: ${transform2}
-        }`);
-        */
     }
 }

src/engine-components/ui/BaseUIComponent.ts CHANGED Viewed

@@ -4,7 +4,7 @@
 import { EventSystem } from "./EventSystem";
 import { showGizmos } from '../../engine/engine_default_parameters';
 import { AxesHelper, Object3D } from 'three';
-import { IGraphic } from './Interfaces';
+import { ICanvas, IGraphic } from './Interfaces';
 import { ShadowCastingMode } from '../Renderer';
 export const includesDir = "./include";
@@ -50,6 +50,12 @@
         return this._root;
     }
+    protected get Canvas() {
+        const cv = this.Root as any as ICanvas;
+        if (cv?.isCanvas) return cv;
+        return null;
+    }
     // private _intermediate?: Object3D;
     protected _parentComponent?: BaseUIComponent | null = undefined;

src/engine-components/ui/Canvas.ts CHANGED Viewed

@@ -3,12 +3,14 @@
 import { FrameEvent } from "../../engine/engine_setup";
 import { BaseUIComponent, UIRootComponent } from "./BaseUIComponent";
 import { GameObject } from "../Component";
-import { Object3D } from "three";
+import { Matrix4, Object3D } from "three";
 import { RectTransform } from "./RectTransform";
-import { ICanvas } from "./Interfaces";
+import { ICanvas, ILayoutGroup, IRectTransform } from "./Interfaces";
 import { Camera } from "../Camera";
 import { EventSystem } from "./EventSystem";
 import * as ThreeMeshUI from 'three-mesh-ui'
+import { getParam } from "../../engine/engine_utils";
+import { LayoutGroup } from "./Layout";
 export enum RenderMode {
     ScreenSpaceOverlay = 0,
@@ -17,8 +19,14 @@
     Undefined = -1,
 }
+const debugLayout = getParam("debuguilayout");
 export class Canvas extends UIRootComponent implements ICanvas {
+    get isCanvas() {
+        return true;
+    }
     get screenspace(): any {
         return this.renderMode !== RenderMode.WorldSpace;
     }
@@ -122,6 +130,10 @@
         super.awake();
     }
+    start() {
+        this.onUpdateRenderMode();
+    }
     onEnable() {
         super.onEnable();
         this._updateRenderSettingsRoutine = undefined;
@@ -149,7 +161,29 @@
     private _boundRenderSettingsChanged = this.onRenderSettingsChanged.bind(this);
     private previousParent: Object3D | null = null;
+    private _lastMatrixWorld: Matrix4 | null = null;
+    private _rectTransforms: IRectTransform[] = [];
+    registerTransform(rt: IRectTransform) {
+        this._rectTransforms.push(rt);
+    }
+    unregisterTransform(rt: IRectTransform) {
+        const index = this._rectTransforms.indexOf(rt);
+        if (index !== -1) {
+            this._rectTransforms.splice(index, 1);
+        }
+    }
+    private _layoutGroups: Map<Object3D, ILayoutGroup> = new Map();
+    registerLayoutGroup(group: ILayoutGroup) {
+        const obj = group.gameObject;
+        this._layoutGroups.set(obj, group)
+    }
+    unregisterLayoutGroup(group: ILayoutGroup) {
+        const obj = group.gameObject;
+        this._layoutGroups.delete(obj);
+    }
     onBeforeRenderRoutine = () => {
         if (this.renderOnTop) {
             // This is just a test but in reality it should be combined with all world canvases with render on top in one render pass
@@ -158,6 +192,7 @@
         }
         else {
             this.onUpdateRenderMode();
+            this.handleLayoutUpdates();
             // TODO: we might need to optimize this. This is here to make sure the TMUI text clipping matrices are correct. Ideally the text does use onBeforeRender and apply the clipping matrix there so we dont have to force update all the matrices here
             this.shadowComponent?.updateMatrixWorld(true);
             this.shadowComponent?.updateWorldMatrix(true, true);
@@ -171,13 +206,42 @@
             this.context.renderer.autoClear = false;
             this.context.renderer.clearDepth();
             this.onUpdateRenderMode(true);
+            this.handleLayoutUpdates();
             this.shadowComponent?.updateMatrixWorld(true);
+            // this.handleLayoutUpdates();
             EventSystem.ensureUpdateMeshUI(ThreeMeshUI, this.context);
             this.context.renderer.render(this.gameObject, this.context.mainCamera);
             this.context.renderer.autoClear = true;
         }
+        this._lastMatrixWorld?.copy(this.gameObject.matrixWorld);
     }
+    private handleLayoutUpdates() {
+        if (this._lastMatrixWorld === null) {
+            this._lastMatrixWorld = new Matrix4();
+        }
+        const matrixWorldChanged = !this._lastMatrixWorld.equals(this.gameObject.matrixWorld);
+        if (debugLayout && matrixWorldChanged) console.log("Canvas Layout changed", this.context.time.frameCount, this.name);
+        // TODO: optimize this, we should only need to update a subhierarchy of the parts where layout has changed
+        let didLog = false;
+        for (const ch of this._rectTransforms) {
+            if (matrixWorldChanged) ch.markDirty();
+            let layout = this._layoutGroups.get(ch.gameObject);
+            if(ch.isDirty && !layout){
+                layout = ch.gameObject.getComponentInParent(LayoutGroup) as LayoutGroup;
+            }
+            if (ch.isDirty || layout?.isDirty) {
+                if (debugLayout && !didLog) {
+                    console.log("CANVAS UPDATE ### " + ch.name + " ##################################### " + this.context.time.frame);
+                    // didLog = true;
+                }
+                layout?.updateLayout();
+                ch.updateTransform();
+            }
+        }
+    }
     applyRenderSettings() {
         this.onRenderSettingsChanged();
     }
@@ -205,6 +269,7 @@
     private _lastWidth: number = -1;
     private _lastHeight: number = -1;
     private onUpdateRenderMode(force: boolean = false) {
         if (!force) {
             if (this._renderMode === this._activeRenderMode && this._lastWidth === this.context.domWidth && this._lastHeight === this.context.domHeight) {
@@ -241,6 +306,13 @@
                 canvas.quaternion.identity();
                 const rect = this.gameObject.getComponent(RectTransform)!;
+                let hasChanged = false;
+                if (rect.sizeDelta.x !== this.context.domWidth) {
+                    hasChanged = true;
+                }
+                if (rect.sizeDelta.y !== this.context.domHeight) {
+                    hasChanged = true;
+                }
                 const vFOV = camera.fieldOfView! * Math.PI / 180;
                 const h = 2 * Math.tan(vFOV / 2) * Math.abs(plane);
@@ -248,11 +320,14 @@
                 canvas.scale.y = h / this.context.domHeight;
                 // Set scale.z, otherwise small offsets in screenspace mode have different visual results based on export scale and other settings
                 canvas.scale.z = .01;
-                rect.sizeDelta.x = this.context.domWidth;
-                rect.sizeDelta.y = this.context.domHeight;
-                rect?.markDirty();
+                if (hasChanged) {
+                    rect.sizeDelta.x = this.context.domWidth;
+                    rect.sizeDelta.y = this.context.domHeight;
+                    rect?.markDirty();
+                }
                 // this.context.scene.add(this.gameObject)
                 // this.gameObject.scale.multiplyScalar(.01);
                 // this.gameObject.position.set(0,0,0);

src/engine-components/codegen/components.ts CHANGED Viewed

@@ -1,5 +1,8 @@
 // Export types
 export class __Ignore {}
+export { ActionBuilder } from "../export/usdz/extensions/behavior/BehavioursBuilder";
+export { ActionCollection } from "../export/usdz/extensions/behavior/Actions";
+export { ActionModel } from "../export/usdz/extensions/behavior/BehavioursBuilder";
 export { AlignmentConstraint } from "../AlignmentConstraint";
 export { Animation } from "../Animation";
 export { AnimationCurve } from "../AnimationCurve";
@@ -24,6 +27,8 @@
 export { AxesHelper } from "../AxesHelper";
 export { BaseUIComponent } from "../ui/BaseUIComponent";
 export { BasicIKConstraint } from "../BasicIKConstraint";
+export { BehaviorExtension } from "../export/usdz/extensions/behavior/Behaviour";
+export { BehaviorModel } from "../export/usdz/extensions/behavior/BehavioursBuilder";
 export { Behaviour } from "../Component";
 export { Bloom } from "../postprocessing/Effects/Bloom";
 export { BoxCollider } from "../Collider";
@@ -35,6 +40,8 @@
 export { Canvas } from "../ui/Canvas";
 export { CanvasGroup } from "../ui/CanvasGroup";
 export { CapsuleCollider } from "../Collider";
+export { ChangeMaterialOnClick } from "../export/usdz/extensions/behavior/BehaviourComponents";
+export { ChangeTransformOnClick } from "../export/usdz/extensions/behavior/BehaviourComponents";
 export { CharacterController } from "../CharacterController";
 export { CharacterControllerInput } from "../CharacterController";
 export { ChromaticAberration } from "../postprocessing/Effects/ChromaticAberration";
@@ -48,10 +55,12 @@
 export { DeleteBox } from "../DeleteBox";
 export { DepthOfField } from "../postprocessing/Effects/DepthOfField";
 export { DeviceFlag } from "../DeviceFlag";
+export { DocumentExtension } from "../export/usdz/extensions/DocumentExtension";
 export { DragControls } from "../DragControls";
 export { DropListener } from "../DropListener";
 export { Duplicatable } from "../Duplicatable";
 export { EmissionModule } from "../ParticleSystemModules";
+export { EmphasizeOnClick } from "../export/usdz/extensions/behavior/BehaviourComponents";
 export { EventList } from "../EventList";
 export { EventListEvent } from "../EventList";
 export { EventSystem } from "../ui/EventSystem";
@@ -68,13 +77,14 @@
 export { GridHelper } from "../GridHelper";
 export { GridLayoutGroup } from "../ui/Layout";
 export { GroundProjectedEnv } from "../GroundProjection";
+export { GroupActionModel } from "../export/usdz/extensions/behavior/BehavioursBuilder";
+export { HideOnStart } from "../export/usdz/extensions/behavior/BehaviourComponents";
 export { HingeJoint } from "../Joints";
 export { HorizontalLayoutGroup } from "../ui/Layout";
 export { Image } from "../ui/Image";
 export { InheritVelocityModule } from "../ParticleSystemModules";
 export { InputField } from "../ui/InputField";
 export { Interactable } from "../Interactable";
-export { LayoutGroup } from "../ui/Layout";
 export { Light } from "../Light";
 export { LimitVelocityOverLifetimeModule } from "../ParticleSystemModules";
 export { LODGroup } from "../LODGroup";
@@ -96,15 +106,19 @@
 export { OpenURL } from "../utils/OpenURL";
 export { OrbitControls } from "../OrbitControls";
 export { Outline } from "../ui/Outline";
+export { Padding } from "../ui/Layout";
 export { ParticleBurst } from "../ParticleSystemModules";
 export { ParticleSubEmitter } from "../ParticleSystemSubEmitter";
 export { ParticleSystem } from "../ParticleSystem";
 export { ParticleSystemRenderer } from "../ParticleSystem";
 export { PixelationEffect } from "../postprocessing/Effects/Pixelation";
 export { PlayableDirector } from "../timeline/PlayableDirector";
+export { PlayAnimationOnClick } from "../export/usdz/extensions/behavior/BehaviourComponents";
 export { PlayerColor } from "../PlayerColor";
 export { PointerEventData } from "../ui/PointerEvents";
 export { PostProcessingHandler } from "../postprocessing/PostProcessingHandler";
+export { PreliminaryAction } from "../export/usdz/extensions/behavior/BehaviourComponents";
+export { PreliminaryTrigger } from "../export/usdz/extensions/behavior/BehaviourComponents";
 export { QuickLookOverlay } from "../export/usdz/USDZExporter";
 export { RawImage } from "../ui/Image";
 export { Raycaster } from "../ui/Raycaster";
@@ -122,6 +136,7 @@
 export { SceneSwitcher } from "../SceneSwitcher";
 export { ScreenCapture } from "../ScreenCapture";
 export { ScreenSpaceAmbientOcclusion } from "../postprocessing/Effects/ScreenspaceAmbientOcclusion";
+export { SetActiveOnClick } from "../export/usdz/extensions/behavior/BehaviourComponents";
 export { ShadowCatcher } from "../ShadowCatcher";
 export { ShapeModule } from "../ParticleSystemModules";
 export { SignalAsset } from "../timeline/SignalAsset";
@@ -146,24 +161,33 @@
 export { SyncedCamera } from "../SyncedCamera";
 export { SyncedRoom } from "../SyncedRoom";
 export { SyncedTransform } from "../SyncedTransform";
+export { TapGestureTrigger } from "../export/usdz/extensions/behavior/BehaviourComponents";
 export { TeleportTarget } from "../webxr/WebXRController";
 export { TestRunner } from "../TestRunner";
 export { TestSimulateUserData } from "../TestRunner";
 export { Text } from "../ui/Text";
+export { TextBuilder } from "../export/usdz/extensions/USDZText";
+export { TextExtension } from "../export/usdz/extensions/USDZText";
 export { TextureSheetAnimationModule } from "../ParticleSystemModules";
 export { TiltShiftEffect } from "../postprocessing/Effects/TiltShiftEffect";
 export { ToneMapping } from "../postprocessing/Effects/Tonemapping";
 export { TrailModule } from "../ParticleSystemModules";
 export { TransformData } from "../export/usdz/extensions/Animation";
 export { TransformGizmo } from "../TransformGizmo";
+export { TriggerBuilder } from "../export/usdz/extensions/behavior/BehavioursBuilder";
+export { TriggerModel } from "../export/usdz/extensions/behavior/BehavioursBuilder";
 export { UIRaycastUtils } from "../ui/RaycastUtils";
 export { UIRootComponent } from "../ui/BaseUIComponent";
 export { UsageMarker } from "../Interactable";
+export { USDZBehaviours } from "../export/usdz/extensions/behavior/Behaviour";
 export { USDZExporter } from "../export/usdz/USDZExporter";
+export { USDZText } from "../export/usdz/extensions/USDZText";
+export { VariantAction } from "../export/usdz/extensions/behavior/Actions";
 export { VelocityOverLifetimeModule } from "../ParticleSystemModules";
 export { VerticalLayoutGroup } from "../ui/Layout";
 export { VideoPlayer } from "../VideoPlayer";
 export { Vignette } from "../postprocessing/Effects/Vignette";
+export { VisibilityAction } from "../export/usdz/extensions/behavior/BehaviourComponents";
 export { Voip } from "../Voip";
 export { Volume } from "../postprocessing/Volume";
 export { VolumeParameter } from "../postprocessing/VolumeParameter";

src/engine/engine_gameobject.ts CHANGED Viewed

@@ -163,8 +163,9 @@
     return internalForEachComponent(instance, cb, recursive);
 }
-export function* foreachComponentEnumerator<T extends IComponent>(instance: Object3D, type?: Constructor<T>, includeChildren: boolean = false): Generator<T> {
+export function* foreachComponentEnumerator<T extends IComponent>(instance: Object3D, type?: Constructor<T>, includeChildren: boolean = false, maxLevel: number = 999, _currentLevel: number = 0): Generator<T> {
     if (!instance?.userData.components) return;
+    if (_currentLevel > maxLevel) return;
     for (const comp of instance.userData.components) {
         if (type && comp?.isComponent === true && comp instanceof type) {
             yield comp;
@@ -175,7 +176,7 @@
     }
     if (includeChildren === true) {
         for (const ch of instance.children) {
-            yield* foreachComponentEnumerator(ch, type, true);
+            yield* foreachComponentEnumerator(ch, type, true, maxLevel, _currentLevel + 1);
         }
     }
 }

src/engine/engine_three_utils.ts CHANGED Viewed

@@ -98,8 +98,8 @@
     const tempVec = _worldScale2;
     const obj2 = obj.parent;
     obj2.getWorldScale(tempVec);
-    tempVec.divide(vec);
-    obj.scale.copy(tempVec);
+    obj.scale.copy(vec);
+    obj.scale.divide(tempVec);
 }
 const _forward = new Vector3();

src/engine-components/export/usdz/Extension.ts CHANGED Viewed

@@ -1,12 +1,11 @@
-import { USDZObject } from "./types";
+import { USDObject } from "./ThreeUSDZExporter";
+export interface IUSDExporterExtension {
-export interface IUSDZExporterExtension {
     get extensionName(): string;
     onBeforeBuildDocument?(context);
     onAfterBuildDocument?(context);
-    onExportObject?(object, model : USDZObject, context);
+    onExportObject?(object, model : USDObject, context);
     onAfterSerialize?(context);
-    onAfterHierarchy?(context);
+    onAfterHierarchy?(context, writer : any);
 }

src/engine-components/ui/Graphic.ts CHANGED Viewed

@@ -9,6 +9,8 @@
 import { GameObject } from '../Component';
 import SimpleStateBehavior from "three-mesh-ui/examples/behaviors/states/SimpleStateBehavior"
 import { Outline } from './Outline';
+import { BehaviorExtension, UsdzBehaviour } from '../../engine-components/export/usdz/extensions/behavior/Behaviour';
+import { USDObject } from '../../engine-components/export/usdz/ThreeUSDZExporter';
 const _colorStateObject: { backgroundColor: Color, backgroundOpacity: number } = {
     backgroundColor: new Color(1, 1, 1),

src/engine-components/ui/Image.ts CHANGED Viewed

@@ -49,9 +49,9 @@
             if(this.sprite?.texture?.name === "Knob") {
                 opts.borderRadius = 999;
             }
-            opts.borderColor = new Color(.4, .4, .4);
-            opts.borderOpacity = this.color.alpha;
-            opts.borderWidth = .3;
+            // opts.borderColor = new Color(.4, .4, .4);
+            // opts.borderOpacity = this.color.alpha;
+            // opts.borderWidth = .3;
         }
     }

src/engine-components/ui/Interfaces.ts CHANGED Viewed

@@ -1,24 +1,48 @@
+import { Behaviour } from "../Component";
 import { IComponent } from "../../engine/engine_types";
 export interface ICanvas {
-    get screenspace() : boolean;
+    get isCanvas(): boolean;
+    get screenspace(): boolean;
+    registerTransform(rt: IRectTransform);
+    unregisterTransform(rt: IRectTransform);
 }
 export interface ICanvasGroup {
-    get isCanvasGroup() : boolean;
+    get isCanvasGroup(): boolean;
     blocksRaycasts: boolean;
     interactable: boolean;
 }
 export interface IGraphic extends IComponent {
-    get isGraphic() : boolean;
+    get isGraphic(): boolean;
     raycastTarget: boolean;
 }
 export interface IRectTransform extends IComponent {
+    get isDirty(): boolean;
+    markDirty();
+    updateTransform();
+}
+export interface IRectTransformChangedReceiver {
+    onParentRectTransformChanged(comp: IRectTransform): void;
 }
-export interface IRectTransformChangedReceiver {
-    onParentRectTransformChanged(comp : IRectTransform) : void;
-}
+export interface ILayoutGroup extends IComponent {
+    get isLayoutGroup(): boolean;
+    get isDirty(): boolean;
+    updateLayout();
+}
+// export abstract class LayoutGroup extends Behaviour implements IRectTransformChangedReceiver, ILayoutGroup {
+//     get isLayoutGroup(): boolean {
+//         return true;
+//     }
+//     updateLayout() {
+//         throw new Error("Method not implemented.");
+//     }
+//     onParentRectTransformChanged(comp: IRectTransform): void {
+//         throw new Error("Method not implemented.");
+//     }
+// }

src/engine-components/ui/Layout.ts CHANGED Viewed

@@ -1,17 +1,316 @@
-import { Behaviour } from "../Component";
+import { ILayoutGroup, IRectTransform, IRectTransformChangedReceiver } from "./Interfaces";
+import { Behaviour, GameObject } from "../Component";
+import { serializable } from "../../engine/engine_serialization";
+import { Canvas } from "./Canvas";
+import { RectTransform } from "./RectTransform";
+import { getParam } from "../../engine/engine_utils";
-export class LayoutGroup extends Behaviour {
+const debug = getParam("debuguilayout");
+export class Padding {
+    @serializable()
+    left: number = 0;
+    @serializable()
+    right: number = 0;
+    @serializable()
+    top: number = 0;
+    @serializable()
+    bottom: number = 0;
+    get vertical() {
+        return this.top + this.bottom;
+    }
+    get horizontal() {
+        return this.left + this.right;
+    }
+}
+export enum TextAnchor {
+    UpperLeft = 0,
+    UpperCenter = 1,
+    UpperRight = 2,
+    MiddleLeft = 3,
+    MiddleCenter = 4,
+    MiddleRight = 5,
+    LowerLeft = 6,
+    LowerCenter = 7,
+    LowerRight = 8,
+    Custom = 9
+}
+enum Axis {
+    Horizontal = "x",
+    Vertical = "y"
+}
+export abstract class LayoutGroup extends Behaviour implements ILayoutGroup {
+    private _rectTransform: RectTransform | null = null;
+    private get rectTransform() {
+        return this._rectTransform;
+    }
+    onParentRectTransformChanged(_comp: IRectTransform): void {
+        this._needsUpdate = true;
+    }
+    private _needsUpdate: boolean = false;
+    get isDirty(): boolean {
+        return this._needsUpdate;
+    }
+    get isLayoutGroup(): boolean {
+        return true;
+    }
+    updateLayout() {
+        if (!this._rectTransform) return;
+        if (debug)
+            console.warn("Layout Update", this.context.time.frame, this.name);
+        this._needsUpdate = false;
+        this.onCalculateLayout(this._rectTransform);
+    }
+    // onBeforeRender(): void {
+    //     this.updateLayout();
+    // }
+    @serializable()
+    childAlignment: TextAnchor = TextAnchor.UpperLeft;
+    @serializable()
     reverseArrangement: boolean = false;
+    @serializable()
+    spacing: number = 0;
+    @serializable(Padding)
+    padding!: Padding;
+    @serializable()
+    minWidth: number = 0;
+    @serializable()
+    minHeight: number = 0;
+    @serializable()
+    flexibleHeight: number = 0;
+    @serializable()
+    flexibleWidth: number = 0;
+    @serializable()
+    preferredHeight: number = 0;
+    @serializable()
+    preferredWidth: number = 0;
+    start() {
+        this._needsUpdate = true;
+    }
+    onEnable(): void {
+        if(debug) console.log(this.name, this);
+        this._rectTransform = this.gameObject.getComponent(RectTransform);
+        const canvas = this.gameObject.getComponentInParent(Canvas);
+        if (canvas) {
+            canvas.registerLayoutGroup(this);
+        }
+        this._needsUpdate = true;
+    }
+    onDisable(): void {
+        const canvas = this.gameObject.getComponentInParent(Canvas);
+        if (canvas) {
+            canvas.unregisterLayoutGroup(this);
+        }
+    }
+    protected abstract onCalculateLayout(rt: RectTransform);
+    // for animation:
+    private set m_Spacing(val) {
+        if (val === this.spacing) return;
+        this._needsUpdate = true;
+        this.spacing = val;
+    }
+    get m_Spacing() {
+        return this.spacing;
+    }
 }
-export class VerticalLayoutGroup extends LayoutGroup {
+export abstract class HorizontalOrVerticalLayoutGroup extends LayoutGroup {
+    @serializable()
+    childControlHeight: boolean = true;
+    @serializable()
+    childControlWidth: boolean = true;
+    @serializable()
+    childForceExpandHeight: boolean = false;
+    @serializable()
+    childForceExpandWidth: boolean = false;
+    @serializable()
+    childScaleHeight: boolean = false;
+    @serializable()
+    childScaleWidth: boolean = false;
+    protected abstract get primaryAxis(): Axis;
+    protected onCalculateLayout(rect: RectTransform) {
+        const axis = this.primaryAxis;
+        const totalWidth = rect.width;
+        let actualWidth = totalWidth;
+        const totalHeight = rect.height;
+        let actualHeight = totalHeight;
+        actualWidth -= this.padding.horizontal;
+        actualHeight -= this.padding.vertical;
+        // console.log(rt.name, "width=" + totalWidth + ", height=" + totalHeight)
+        const paddingAxis = axis === Axis.Horizontal ? this.padding.horizontal : this.padding.vertical;
+        const isHorizontal = axis === Axis.Horizontal;
+        const isVertical = !isHorizontal;
+        const otherAxis = isHorizontal ? "y" : "x";
+        const controlSize = isHorizontal ? this.childControlWidth : this.childControlHeight;
+        const controlSizeOtherAxis = isHorizontal ? this.childControlHeight : this.childControlWidth;
+        const forceExpandSize = isHorizontal ? this.childForceExpandWidth : this.childForceExpandHeight;
+        const forceExpandSizeOtherAxis = isHorizontal ? this.childForceExpandHeight : this.childForceExpandWidth;
+        const actualExpandSize = isHorizontal ? actualHeight : actualWidth;
+        const totalSpace = isHorizontal ? totalWidth : totalHeight;
+        // 0 is left/top, 0.5 is middle, 1 is right/bottom
+        const alignmentOnAxis = 0.5 * (isHorizontal ? this.childAlignment % 3 : Math.floor(this.childAlignment / 3));
+        let start = 0;
+        if (isHorizontal) {
+            start += this.padding.left;
+        }
+        else
+            start += this.padding.top;
+        // Calculate total size of the elements
+        let totalChildSize = 0;
+        let actualRectTransformChildCount = 0;
+        for (let i = 0; i < this.gameObject.children.length; i++) {
+            const ch = this.gameObject.children[i];
+            const rt = GameObject.getComponent(ch, RectTransform);
+            if (rt?.activeAndEnabled) {
+                actualRectTransformChildCount += 1;
+                if (isHorizontal) {
+                    totalChildSize += rt.width;
+                }
+                else {
+                    totalChildSize += rt.height;
+                }
+            }
+        }
+        let sizePerChild = 0;
+        const totalSpacing = this.spacing * (actualRectTransformChildCount - 1)
+        if (forceExpandSize || controlSize) {
+            let size = 0;
+            if (isHorizontal) {
+                size = actualWidth -= totalSpacing;
+            }
+            else {
+                size = actualHeight -= totalSpacing;
+            }
+            if (actualRectTransformChildCount > 0)
+                sizePerChild = size / actualRectTransformChildCount;
+        }
+        let leftOffset = 0;
+        leftOffset += this.padding.left;
+        leftOffset -= this.padding.right;
+        if (alignmentOnAxis !== 0) {
+            start = totalSpace - totalChildSize;
+            start *= alignmentOnAxis;
+            start -= totalSpacing * alignmentOnAxis;
+            if (isHorizontal) {
+                start -= this.padding.right * alignmentOnAxis;
+                start += this.padding.left * (1 - alignmentOnAxis);
+                if (start < this.padding.left) {
+                    start = this.padding.left;
+                }
+            }
+            else {
+                start -= this.padding.bottom * alignmentOnAxis;
+                start += this.padding.top * (1 - alignmentOnAxis);
+                if (start < this.padding.top) {
+                    start = this.padding.top;
+                }
+            }
+        }
+        // Apply layout
+        let k = 0;
+        for (let i = 0; i < this.gameObject.children.length; i++) {
+            const ch = this.gameObject.children[i];
+            const rt = GameObject.getComponent(ch, RectTransform);
+            if (rt?.activeAndEnabled) {
+                rt.pivot?.set(.5, .5);
+                // Horizontal padding
+                const x = totalWidth * .5 + leftOffset * .5;
+                if (rt.anchoredPosition.x !== x)
+                    rt.anchoredPosition.x = x;
+                const y = totalHeight * -.5
+                if (rt.anchoredPosition.y !== y)
+                    rt.anchoredPosition.y = y;
+                // Set the size for the secondary axis (e.g. height for a horizontal layout group)
+                if (forceExpandSizeOtherAxis && controlSizeOtherAxis && rt.sizeDelta[otherAxis] !== actualExpandSize) {
+                    rt.sizeDelta[otherAxis] = actualExpandSize;
+                }
+                // Set the size for the primary axis (e.g. width for a horizontal layout group)
+                if (forceExpandSize && controlSize && rt.sizeDelta[axis] !== sizePerChild) {
+                    rt.sizeDelta[axis] = sizePerChild
+                }
+                const size = isHorizontal ? rt.width : rt.height;
+                let halfSize = size * .5;
+                start += halfSize;
+                if (forceExpandSize) {
+                    let preferredStart = sizePerChild * (k + 1) - sizePerChild * .5;
+                    if (preferredStart > start)
+                        start = preferredStart;
+                }
+                let value = start;
+                if (axis === Axis.Vertical)
+                    value = -value;
+                // Only set the position if it's not already the correct one to avoid triggering the rectTransform dirty event
+                if (rt.anchoredPosition[axis] !== value) {
+                    rt.anchoredPosition[axis] = value
+                }
+                start += halfSize;
+                start += this.spacing;
+                k += 1;
+            }
+        }
+    }
 }
-export class HorizontalLayoutGroup extends LayoutGroup {
+export class VerticalLayoutGroup extends HorizontalOrVerticalLayoutGroup {
+    protected get primaryAxis() {
+        return Axis.Vertical;
+    }
 }
+export class HorizontalLayoutGroup extends HorizontalOrVerticalLayoutGroup {
+    protected get primaryAxis() {
+        return Axis.Horizontal;
+    }
+}
 export class GridLayoutGroup extends LayoutGroup {
+    protected onCalculateLayout() {
+    }
 }

src/engine-components/utils/LookAt.ts CHANGED Viewed

@@ -1,21 +1,74 @@
 import { serializable } from "../../engine/engine_serialization";
 import { Behaviour } from "../Component";
-import { Object3D } from "three";
-import { getWorldPosition, lookAtInverse } from "../../engine/engine_three_utils";
+import { Matrix4, Object3D, Quaternion, Vector3 } from "three";
+import { getWorldPosition, getWorldQuaternion, setWorldQuaternion } from "../../engine/engine_three_utils";
-export class LookAt extends Behaviour {
+import { USDObject } from "../../engine-components/export/usdz/ThreeUSDZExporter";
+import { UsdzBehaviour } from "../../engine-components/export/usdz/extensions/behavior/Behaviour";
+import { ActionBuilder, BehaviorModel, TriggerBuilder, USDVec3 } from "../../engine-components/export/usdz/extensions/behavior/BehavioursBuilder";
+export class LookAt extends Behaviour implements UsdzBehaviour {
     @serializable(Object3D)
     target?: Object3D;
     @serializable()
     invertForward: boolean = false;
+    @serializable()
+    keepUpDirection: boolean = true;
+    @serializable()
+    copyTargetRotation: boolean = false;
+    private static flipYQuat: Quaternion = new Quaternion().setFromAxisAngle(new Vector3(0, 1, 0), Math.PI);
     onBeforeRender(): void {
-        if (!this.target) return;
-        if (!this.invertForward)
-            this.gameObject.lookAt(getWorldPosition(this.target!));
+        let target: Object3D | null | undefined = this.target;
+        if (!target) target = this.context.mainCamera;
+        if (!target) return;
+        const lookTarget = getWorldPosition(target);
+        const lookFrom = getWorldPosition(this.gameObject);
+        if (this.keepUpDirection)
+            lookTarget.y = lookFrom.y;
+        if (this.copyTargetRotation)
+            setWorldQuaternion(this.gameObject, getWorldQuaternion(target));
         else
-            lookAtInverse(this.gameObject, getWorldPosition(this.target!));
+            this.gameObject.lookAt(lookTarget);
+        if (this.invertForward)
+            this.gameObject.quaternion.multiply(LookAt.flipYQuat);
     }
+    createBehaviours(ext, model: USDObject, _context) {
+        if (model.uuid === this.gameObject.uuid) {
+            let alignmentTarget = model;
+            // not entirely sure why we need to do this - looks like LookAt with up vector doesn't work properly in
+            // QuickLook, so we need to introduce an empty parent and rotate the model by 90° around Y
+            if (this.keepUpDirection) {
+                const parent = USDObject.createEmptyParent(model);
+                alignmentTarget = parent;
+                // rotate by 90° - counter-rotation on the parent makes sure
+                // that without Preliminary Behaviours it still looks right
+                parent.matrix.multiply(new Matrix4().makeRotationZ(Math.PI / 2));
+                model.matrix.multiply(new Matrix4().makeRotationZ(-Math.PI / 2));
+            }
+            const lookAt = new BehaviorModel("lookat " + this.name,
+                TriggerBuilder.sceneStartTrigger(),
+                ActionBuilder.lookAtCameraAction(
+                    alignmentTarget,
+                    undefined,
+                    this.invertForward ? USDVec3.back : USDVec3.forward,
+                    this.keepUpDirection ? USDVec3.up : USDVec3.zero
+                ),
+            );
+            ext.addBehavior(lookAt);
+        }
+    }
 }

src/engine-components/postprocessing/PostProcessingHandler.ts CHANGED Viewed

@@ -1,4 +1,4 @@
-import { HalfFloatType, sRGBEncoding, WebGLRenderTarget } from "three";
+import { HalfFloatType } from "three";
 import { Context } from "../../engine/engine_setup";
 import { getParam, isMobileDevice } from "../../engine/engine_utils";
 import { BloomEffect, BrightnessContrastEffect, ChromaticAberrationEffect, DepthDownsamplingPass, DepthOfFieldEffect, Effect, EffectComposer, EffectPass, HueSaturationEffect, NormalPass, Pass, PixelationEffect, RenderPass, SelectiveBloomEffect, SSAOEffect, VignetteEffect } from "postprocessing";

src/engine-components/ui/RectTransform.ts CHANGED Viewed

@@ -7,10 +7,11 @@
 import { getParam } from "../../engine/engine_utils";
 import { onChange } from "./Utils";
 import { foreachComponentEnumerator } from "../../engine/engine_gameobject";
-import { ICanvas, IRectTransform, IRectTransformChangedReceiver } from "./Interfaces";
+import { ICanvas, IRectTransform, IRectTransformChangedReceiver, ILayoutGroup } from "./Interfaces";
 import { GameObject } from '../Component';
 const debug = getParam("debugui");
+const debugLayout = getParam("debuguilayout");
 export class Size {
     width!: number;
@@ -30,7 +31,7 @@
 export class RectTransform extends BaseUIComponent implements IRectTransform, IRectTransformChangedReceiver {
-    offset: number = 0.05;
+    offset: number = 0.1;
     // @serializable(Object3D)
     // root? : Object3D;
@@ -94,14 +95,13 @@
         return this.sizeDelta.y;
     }
-    private lastMatrixWorld!: Matrix4;
+    // private lastMatrixWorld!: Matrix4;
     private lastMatrix!: Matrix4;
     private rectBlock!: Object3D;
     private _transformNeedsUpdate: boolean = false;
     awake() {
         super.awake();
-        this.lastMatrixWorld = new Matrix4();
         this.lastMatrix = new Matrix4();
         this.rectBlock = new Object3D();;
         this.rectBlock.position.z = .1;
@@ -112,9 +112,10 @@
         // TODO: we need to replace this with the watch that e.g. Rigibody is using (or the one in utils?)
         // perhaps we can also just manually check the few properties in the update loops?
-        onChange(this, "_anchoredPosition", () => { this._transformNeedsUpdate = true; });
-        onChange(this, "sizeDelta", () => { this._transformNeedsUpdate = true; });
-        onChange(this, "pivot", () => { this._transformNeedsUpdate = true; });
+        // TODO: check if value actually changed, this is called on assignment
+        onChange(this, "_anchoredPosition", () => { this.markDirty(); });
+        onChange(this, "sizeDelta", () => { this.markDirty(); });
+        onChange(this, "pivot", () => { this.markDirty(); });
         // When exported with an anchored position offset we remove it here
         // because it would otherwise be applied twice when the anchoring is animated
@@ -135,29 +136,68 @@
         super.onEnable();
         this.addShadowComponent(this.rectBlock);
         this._transformNeedsUpdate = true;
+        this.Canvas?.registerTransform(this);
     }
     onDisable() {
         super.onDisable();
         this.removeShadowComponent();
+        this.Canvas?.unregisterTransform(this);
     }
-    onParentRectTransformChanged(_comp: IRectTransform) {
+    onParentRectTransformChanged(comp: IRectTransform) {
+        if (this._transformNeedsUpdate) return;
         // When the parent rect transform changes we have to to recalculate our transform
+        this.onApplyTransform(debugLayout ? `${comp.name} changed` : undefined);
+    }
+    get isDirty() {
+        if(!this._transformNeedsUpdate) this._transformNeedsUpdate = !this.lastMatrix.equals(this.gameObject.matrix);
+        return this._transformNeedsUpdate;
+    }
+    // private _copyMatrixAfterRender: boolean = false;
+    markDirty() {
+        if (this._transformNeedsUpdate) return;
+        if (debugLayout) console.warn("RectTransform markDirty()", this.name)
         this._transformNeedsUpdate = true;
-        this.applyTransform();
+        // If mark dirty is called explictly we want to allow updating the transform again when updateTransform is called
+        // if we dont reset it here we get delayed layout updates
+        this._lastUpdateFrame = -1;
     }
+    /** Will update the transforms if it changed or is dirty */
+    updateTransform() {
+        // TODO: instead of checking matrix again it would perhaps be better to test if position, rotation or scale have changed individually?
+        const transformChanged = this._transformNeedsUpdate || !this.lastMatrix.equals(this.gameObject.matrix);// || !this.lastMatrixWorld.equals(this.gameObject.matrixWorld);
+        if (transformChanged && this.canUpdate()) {
+            this.onApplyTransform(this._transformNeedsUpdate ? "Marked dirty" : "Matrix changed");
+        }
+    }
     private _parentRectTransform?: RectTransform;
+    private _lastUpdateFrame: number = -1;
-    private applyTransform() {
+    private canUpdate() {
+        return this._transformNeedsUpdate && this.activeAndEnabled && this._lastUpdateFrame !== this.context.time.frame;
+    }
+    private onApplyTransform(reason?: string) {
+        // TODO: need to improve the update logic, with this UI updates have some frame delay but dont happen exponentially per hierarchy
+        if (this.context.time.frameCount === this._lastUpdateFrame) return;
+        this._lastUpdateFrame = this.context.time.frameCount;
         const uiobject = this.shadowComponent;
         if (!uiobject) return;
-        this._transformNeedsUpdate = false;
         this._parentRectTransform = GameObject.getComponentInParent(this.gameObject.parent!, RectTransform) as RectTransform;
-        if (debug) console.log("RectTransform ApplyTransform", this.name, this.isRoot());
+        this._transformNeedsUpdate = false;
+        this.lastMatrix.copy(this.gameObject.matrix);
+        if (debugLayout) console.warn("RectTransform → ApplyTransform", this.name + " because " + reason);
         if (!this.isRoot()) {
             // Reset temp matrix
             uiobject.matrix.identity();
@@ -183,8 +223,7 @@
             tempMatrix.setPosition(tempVec.x, tempVec.y, tempVec.z);
             uiobject.matrix.premultiply(tempMatrix);
             // apply scale if necessary
-            if (this.gameObject.scale.x || this.gameObject.scale.y || this.gameObject.scale.z)
-                uiobject.matrix.scale(this.gameObject.scale);
+            uiobject.matrix.scale(this.gameObject.scale);
         }
         else {
             // We have to rotate the canvas when it's in worldspace
@@ -192,43 +231,29 @@
             if (!canvas.screenspace) uiobject.rotation.y = Math.PI;
         }
-        this._copyMatrixAfterRender = true;
-        this.lastMatrix.copy(this.gameObject.matrix);
         // iterate other components on this object that might need to know about the transform change
         // e.g. Graphic components should update their width and height
         const includeChildren = true;
-        for (const comp of foreachComponentEnumerator(this.gameObject, BaseUIComponent, includeChildren)) {
+        for (const comp of foreachComponentEnumerator(this.gameObject, BaseUIComponent, includeChildren, 1)) {
             if (comp === this) continue;
+            if (!comp.activeAndEnabled) continue;
             const callback = comp as any as IRectTransformChangedReceiver;
-            if (callback.onParentRectTransformChanged)
+            if (callback.onParentRectTransformChanged) {
+                // if (debugLayout) console.log(`RectTransform ${this.name} → call`, comp.name + "/" + comp.constructor.name)
                 callback.onParentRectTransformChanged(this);
+            }
         }
-    }
-    private _copyMatrixAfterRender: boolean = false;
-    markDirty() {
-        this._transformNeedsUpdate = true;
+        // const layout = GameObject.getComponentInParent(this.gameObject, ILayoutGroup);
     }
+    // onAfterRender() {
+    //     if (this._copyMatrixAfterRender) {
+    //         // can we only have this event when the transform changed in this frame? Otherwise all RectTransforms will be iterated. Not sure what is better
+    //         this.lastMatrixWorld.copy(this.gameObject.matrixWorld);
+    //     }
+    // }
-    onBeforeRender() {
-        // TODO: instead of checking matrix again it would perhaps be better to test if position, rotation or scale have changed individually?
-        const transformChanged = this.gameObject.matrixWorldNeedsUpdate || this._transformNeedsUpdate || !this.lastMatrixWorld.equals(this.gameObject.matrixWorld) || !this.lastMatrix.equals(this.gameObject.matrix);
-        if (transformChanged)
-        {
-            this.applyTransform();
-        }
-    }
-    onAfterRender() {
-        if (this._copyMatrixAfterRender) {
-            // can we only have this event when the transform changed in this frame? Otherwise all RectTransforms will be iterated. Not sure what is better
-            this.lastMatrixWorld.copy(this.gameObject.matrixWorld);
-        }
-    }
     /** applies the position offset to the passed in vector */
     private applyAnchoring(pos: Vector3) {
         pos.x += this.anchoredPosition.x;

src/engine-components/export/usdz/types.ts DELETED Viewed

@@ -1,39 +0,0 @@
-import { Object3D, Matrix4, Material, BufferGeometry } from "three";
-// keep in sync with USDZExporter.js
-/** implementation is in three */
-export declare class USDZDocument {
-    name: string;
-    get isDocumentRoot(): boolean;
-    add(obj: USDZObject);
-    remove(obj: USDZObject);
-    traverse(callback: (obj: USDZObject) => void);
-    findById(uuid: string): USDZObject | undefined;
-    get isDynamic(): boolean;
-}
-/** implementation is in three */
-export declare class USDZObject {
-    static createEmptyParent(parent: USDZObject);
-    uuid: string;
-    name: string;
-    matrix: Matrix4;
-    material: Material;
-    geometry: BufferGeometry;
-    parent: USDZObject | USDZDocument | null;
-    children: USDZObject[];
-    _eventListeners: { [event: string]: Function[] };
-    isDynamic: boolean;
-    is(obj: Object3D): boolean;
-    isEmpty(): boolean;
-    clone();
-    getPath();
-    add(child: USDZObject);
-    remove(child: USDZObject);
-    addEventListener(evt: string, listener: Function);
-    removeEventListener(evt: string, listener: Function);
-}

src/engine-components/export/usdz/USDZExporter.ts CHANGED Viewed

@@ -1,15 +1,15 @@
 import { delay, getParam, isiOS, isMobileDevice, isSafari } from "../../../engine/engine_utils";
-import { Object3D, Color, Mesh, Matrix4 } from "three";
-import { USDZExporter as ThreeUSDZExporter } from "three/examples/jsm/exporters/USDZExporter";
+import { Object3D, Mesh, Matrix4 } from "three";
+import { USDZExporter as ThreeUSDZExporter } from "./ThreeUSDZExporter";
 import { AnimationExtension } from "./extensions/Animation"
 import { ensureQuicklookLinkIsCreated } from "./utils/quicklook";
 import { getFormattedDate } from "./utils/timeutils";
 import { registerAnimatorsImplictly } from "./utils/animationutils";
-import { IUSDZExporterExtension } from "./Extension";
+import { IUSDExporterExtension } from "./Extension";
 import { Behaviour, GameObject } from "../../Component";
 import { WebXR } from "../../webxr/WebXR"
 import { serializable } from "../../../engine/engine_serialization";
-import { isDevEnvironment, showBalloonMessage, showBalloonWarning } from "../../../engine/debug/debug";
+import { showBalloonMessage, showBalloonWarning } from "../../../engine/debug/debug";
 import { Context } from "../../../engine/engine_setup";
 import { WebARSessionRoot } from "../../webxr/WebARSessionRoot";
 import { hasProLicense } from "../../../engine/engine_license";
@@ -32,7 +32,7 @@
 export class USDZExporter extends Behaviour {
     @serializable(Object3D)
-    objectToExport?: THREE.Object3D;
+    objectToExport?: Object3D;
     @serializable()
     autoExportAnimations: boolean = false;
@@ -43,13 +43,20 @@
     @serializable(QuickLookOverlay)
     overlay?: QuickLookOverlay;
-    extensions: IUSDZExporterExtension[] = [];
+    // Currently not exposed to integrations - not fully tested. Set from code (e.g. image tracking)
+    @serializable()
+    anchoringType: "plane" | "image" | "face" | "none" = "plane";
+    // Currently not exposed to integrations - not fully tested. Set from code (e.g. image tracking)
+    @serializable()
+    planeAnchoringAlignment: "horizontal" | "vertical" | "any" = "horizontal";
+    extensions: IUSDExporterExtension[] = [];
     private link!: HTMLAnchorElement;
     private webxr?: WebXR;
     private webARSessionRoot: WebARSessionRoot | undefined;
     start() {
         if (debug) {
             console.log(this);
@@ -72,13 +79,11 @@
             this.exportAsync();
         });
-        if (!this.objectToExport) this.objectToExport = this.gameObject;
-        if (isDevEnvironment() && (!this.objectToExport?.children?.length && !(this.objectToExport as Mesh)?.isMesh)) {
-            showBalloonWarning("USDZ Exporter has nothing to export");
-            console.warn("USDZExporter has no objects to export assigned:", this)
-        }
+        // fall back to this object or to the scene if it's empty and doesn't have a mesh
+        if (!this.objectToExport)
+            this.objectToExport = this.gameObject;
+        if (!this.objectToExport?.children?.length && !(this.objectToExport as Mesh)?.isMesh)
+            this.objectToExport = this.context.scene;
     }
@@ -135,13 +140,30 @@
         this.dispatchEvent(new CustomEvent("before-export", { detail: eventArgs }))
         let name = this.exportFileName ?? this.objectToExport?.name ?? this.name;
-        if (debug) name += "-" + getFormattedDate();
-        else if (!hasProLicense()) name = name + " - Made with Needle";
+        if (!hasProLicense()) name += "-MadeWithNeedle";
+        name += "-" + getFormattedDate(); // seems iOS caches the file in some cases, this ensures we always have a fresh file
         //@ts-ignore
         exporter.debug = debug;
+        // sanitize anchoring types
+        if (this.anchoringType !== "plane" && this.anchoringType !== "none" && this.anchoringType !== "image" && this.anchoringType !== "face")
+            this.anchoringType = "plane";
+        if (this.planeAnchoringAlignment !== "horizontal" && this.planeAnchoringAlignment !== "vertical" && this.planeAnchoringAlignment !== "any")
+            this.planeAnchoringAlignment = "horizontal";
         //@ts-ignore
-        const arraybuffer = await exporter.parse(this.objectToExport, extensions);
+        const arraybuffer = await exporter.parse(this.objectToExport, {
+            ar: {
+                anchoring: {
+                    type: this.anchoringType,
+                }
+            },
+            planeAnchoring: {
+                alignment: this.planeAnchoringAlignment,
+            },
+            extensions: extensions
+        });
         const blob = new Blob([arraybuffer], { type: 'application/octet-stream' });
         this.dispatchEvent(new CustomEvent("after-export", { detail: eventArgs }))

src/engine-components/webxr/WebXRImageTracking.ts CHANGED Viewed

@@ -1,10 +1,14 @@
-import { WebXR } from "./WebXR";
+import { WebXR, WebXREvent } from "./WebXR";
 import { serializable } from "../../engine/engine_serialization";
 import { Behaviour, GameObject } from "../Component";
 import { Object3D, Quaternion, Vector3 } from "three";
 import { CircularBuffer, getParam } from "../../engine/engine_utils";
 import { AssetReference } from "../../engine/engine_addressables";
+import { showBalloonWarning } from "../../engine/debug";
+import { USDZExporter } from "../../engine-components/export/usdz/USDZExporter";
+import { USDZExporterContext, USDWriter, imageToCanvas } from "../../engine-components/export/usdz/ThreeUSDZExporter";
 // https://github.com/immersive-web/marker-tracking/blob/main/explainer.md
 const debug = getParam("debugimagetracking");
@@ -18,16 +22,6 @@
     readonly measuredSize: number;
     readonly state: "tracked" | "emulated";
-    // private _matrix: Matrix4 | null = null;
-    // private get matrix(): Matrix4 {
-    //     if (!this._matrix) {
-    //         // this._matrix = WebXRTrackedImage._matrixBuffer.get();
-    //         // const matrix = this._pose.transform.matrix;
-    //         // this._matrix.fromArray(matrix);
-    //     }
-    //     return this._matrix!;
-    // }
     /** Copy the image position to a vector */
     getPosition(vec: Vector3) {
         this.ensureTransformData();
@@ -42,13 +36,23 @@
         return quat;
     }
-    applyToObject(object: Object3D) {
+    private static y180 = new Quaternion().setFromAxisAngle(new Vector3(0, 1, 0), Math.PI);
+    applyToObject(object: Object3D, t01: number | undefined = undefined) {
         this.ensureTransformData();
-        object.position.copy(this._position);
-        object.quaternion.copy(this._rotation);
+        // check if position/_position or rotation/_rotation changed more than just a little bit
+        const haveChanged = object.position.distanceToSquared(this._position) > 0.05 || object.quaternion.angleTo(this._rotation) > 0.05;
+        if (t01 === undefined || t01 >= 1 || haveChanged) {
+            object.position.copy(this._position);
+            object.quaternion.copy(this._rotation);
+        }
+        else {
+            t01 = Math.max(0, Math.min(1, t01));
+            object.position.lerp(this._position, t01);
+            object.quaternion.slerp(this._rotation, t01);
+        }
+        object.quaternion.multiply(WebXRTrackedImage.y180);
     }
-    // private static _matrixBuffer: CircularBuffer<Matrix4> = new CircularBuffer(() => new Matrix4(), 20);
     private static _positionBuffer: CircularBuffer<Vector3> = new CircularBuffer(() => new Vector3(), 20);
     private static _rotationBuffer: CircularBuffer<Quaternion> = new CircularBuffer(() => new Quaternion(), 20);
     private _position!: Vector3;
@@ -58,8 +62,14 @@
             this._position = WebXRTrackedImage._positionBuffer.get();
             this._rotation = WebXRTrackedImage._rotationBuffer.get();
             const t = this._pose.transform;
-            this._position.set(-t.position.x, t.position.y, -t.position.z);
-            this._rotation.set(-t.orientation.x, t.orientation.y, -t.orientation.z, t.orientation.w);
+            // when parented to the world, we need to flip data here
+            //this._position.set(-t.position.x, t.position.y, -t.position.z);
+            // this._rotation.set(-t.orientation.x, t.orientation.y, -t.orientation.z, t.orientation.w);
+            // for some reason when parented to the XRRig, we need the original data
+            this._position.set(t.position.x, t.position.y, t.position.z);
+            this._rotation.set(t.orientation.x, t.orientation.y, t.orientation.z, t.orientation.w);
         }
     }
@@ -99,6 +109,32 @@
     imageDoesNotMove: boolean = false;
 }
+class ImageTrackingExtension {
+    get extensionName() { return "image-tracking"; }
+    private filename: string;
+    private widthInMeters: number;
+    private imageData: Uint8Array;
+    constructor(filename: string, imageData: Uint8Array, widthInMeters: number) {
+        this.filename = filename;
+        this.imageData = imageData;
+        this.widthInMeters = widthInMeters;
+    }
+    onAfterHierarchy(_context: USDZExporterContext, writer: USDWriter) {
+        writer.beginBlock(`def Preliminary_ReferenceImage "AnchoringReferenceImage"`);
+        writer.appendLine(`uniform asset image = @tracker/` + this.filename + `@`);
+        writer.appendLine(`uniform double physicalWidth = ` + (this.widthInMeters * 100).toFixed(8));
+        writer.closeBlock();
+    }
+    onAfterSerialize(context: USDZExporterContext) {
+        context.files['tracker/' + this.filename] = this.imageData;
+    }
+}
 export class WebXRImageTracking extends Behaviour {
     @serializable(WebXRImageTrackingModel)
@@ -121,22 +157,43 @@
                     imageElement.addEventListener("load", async () => {
                         const img = await createImageBitmap(imageElement);
                         WebXRImageTracking._imageElements.set(url, img);
+                        // read back Uint8Array to use in USDZ -
+                        // TODO better would be to do that once we actually need it
+                        const canvas = await imageToCanvas( img );
+                        if (canvas) {
+                            const blob = await new Promise( resolve => canvas.toBlob( resolve, 'image/png', 1 ) ) as any;
+                            const arrayBuffer = await blob.arrayBuffer();
+                            const exporter = GameObject.findObjectOfType(USDZExporter);
+                            if (exporter) {
+                                exporter.extensions.push(
+                                    new ImageTrackingExtension("marker.png", new Uint8Array(arrayBuffer), this.trackedImages[0].widthInMeters)
+                                );
+                                exporter.anchoringType = "image";
+                            }
+                        }
                     });
                 }
             }
         }
     }
+    private xr: WebXR | null = null;
     onEnable(): void {
-        WebXR.addEventListener("modify-ar-options", this.onModifyAROptions);
-        WebXR.addEventListener("xrStarted", this.onXRStarted);
-        this.addEventListener("image-tracking", this.onImageTrackingUpdate)
+        this.xr = GameObject.findObjectOfType(WebXR);
+        WebXR.addEventListener(WebXREvent.ModifyAROptions, this.onModifyAROptions);
+        WebXR.addEventListener(WebXREvent.XRStarted, this.onXRStarted);
+        WebXR.addEventListener(WebXREvent.XRUpdate, this.onXRUpdate);
+        this.addEventListener("image-tracking", this.onImageTrackingUpdate);
     }
     onDisable(): void {
-        WebXR.removeEventListener("modify-ar-options", this.onModifyAROptions);
-        WebXR.removeEventListener("xrStarted", this.onXRStarted);
-        this.removeEventListener("image-tracking", this.onImageTrackingUpdate)
+        WebXR.removeEventListener(WebXREvent.ModifyAROptions, this.onModifyAROptions);
+        WebXR.removeEventListener(WebXREvent.XRStarted, this.onXRStarted);
+        WebXR.removeEventListener(WebXREvent.XRUpdate, this.onXRUpdate);
+        this.removeEventListener("image-tracking", this.onImageTrackingUpdate);
     }
     private onModifyAROptions = (event: any) => {
@@ -197,8 +254,12 @@
                     if (asset) {
                         trackedData!.object = asset;
-                        if (asset !== this.gameObject)
-                            this.gameObject.add(asset);
+                        // make sure to parent to the WebXR.rig
+                        if (this.xr) {
+                            this.xr.Rig.add(asset);
+                        }
                         image.applyToObject(asset);
                         if (!asset.activeSelf)
                             GameObject.setActive(asset, true);
@@ -215,6 +276,10 @@
                 if (!trackedData.object) continue;
+                if (this.xr) {
+                    this.xr.Rig.add(trackedData.object);
+                }
                 image.applyToObject(trackedData.object);
                 if (!trackedData.object.activeSelf)
                     GameObject.setActive(trackedData.object, true);
@@ -229,9 +294,17 @@
         }
     };
-    onBeforeRender(frame: XRFrame | null): void {
+    private onXRUpdate = (evt): void => {
+        const frame = evt.frame;
+        if (!frame) return;
+        if (frame.session && !("getImageTrackingResults" in frame)) {
+            showBalloonWarning("Image tracking is currently not supported on this device. On Chrome for Android, you can enable the <a href=\"chrome://flags/#webxr-incubations\">chrome://flags/#webxr-incubations</a> flag.");
+            return;
+        }
         //@ts-ignore
-        if (frame?.session && typeof frame.getImageTrackingResults === "function") {
+        if (frame.session && typeof frame.getImageTrackingResults === "function") {
             //@ts-ignore
             const results = frame.getImageTrackingResults();
             if (results.length) {

src/engine-components/export/usdz/extensions/behavior/Actions.ts ADDED Viewed

@@ -0,0 +1,99 @@
+import { Object3D, Matrix4, Material, BufferGeometry } from "three";
+import { ActionBuilder, ActionModel } from "./BehavioursBuilder";
+import { USDObject, USDDocument } from "../../ThreeUSDZExporter";
+export abstract class DocumentAction {
+    get id(): string { return this.object.uuid; }
+    protected object: Object3D;
+    protected model?: USDObject;
+    constructor(obj: Object3D) {
+        this.object = obj;
+    }
+    apply(document: USDDocument) {
+        if (!this.model) {
+            this.model = document.findById(this.object.uuid);
+            if (!this.model) {
+                console.error("could not find model with id " + this.object.uuid);
+                return;
+            }
+        }
+        this.onApply(document);
+    }
+    protected abstract onApply(document: USDDocument);
+}
+export class VariantAction extends DocumentAction {
+    constructor(obj: Object3D, matrix?: Matrix4, material?: Material, geometry?: BufferGeometry) {
+        super(obj);
+        this.matrix = matrix;
+        this.material = material;
+        this.geometry = geometry;
+    }
+    private matrix: Matrix4 | undefined;
+    private material: Material | undefined;
+    private geometry: BufferGeometry | undefined;
+    protected onApply(_: USDDocument) {
+        const model = this.model;
+        if (!model) return;
+        if (!model.parent?.isDynamic) {
+            USDObject.createEmptyParent(model);
+        }
+        const clone = model.clone();
+        if (this.matrix) clone.matrix = this.matrix;
+        if (this.material) clone.material = this.material;
+        if (this.geometry) clone.geometry = this.geometry;
+        model.parent?.add(clone);
+    }
+    private _enableAction?: ActionModel;
+    private _disableAction?: ActionModel;
+    enable(): ActionModel {
+        if (this._enableAction) return this._enableAction;
+        this._enableAction = ActionBuilder.fadeAction(this.object, 0, true);;
+        return this._enableAction;
+    }
+    disable(): ActionModel {
+        if (this._disableAction) return this._disableAction;
+        this._disableAction = ActionBuilder.fadeAction(this.object, 0, false);;
+        return this._disableAction;
+    }
+}
+export class ActionCollection {
+    private actions: DocumentAction[];
+    private sortedActions?: { [key: string]: DocumentAction[] };
+    constructor(actions: DocumentAction[]) {
+        this.actions = [...actions]
+    }
+    // organize is called once when getting an action for the first time
+    // the sorted actions are baked then and adding new actions will not be added anymore
+    private organize() {
+        this.sortedActions = {};
+        for (const action of this.actions) {
+            const id = action.id;
+            if (!this.sortedActions[id]) {
+                this.sortedActions[id] = [];
+            }
+            this.sortedActions[id].push(action);
+        }
+    }
+    /** returns all document actions affecting the object passed in */
+    getActions(obj: Object3D): DocumentAction[] | null {
+        if (!this.sortedActions) this.organize();
+        return this.sortedActions![obj.uuid];
+    }
+}

src/engine-components/export/usdz/extensions/behavior/Behaviour.ts ADDED Viewed

@@ -0,0 +1,181 @@
+import { Behaviour, GameObject } from "../../../../Component";
+import { USDZExporter } from "../../USDZExporter";
+import { IUSDExporterExtension } from "../../Extension";
+import { USDObject, USDWriter } from "../../ThreeUSDZExporter";
+import { BehaviorModel } from "./BehavioursBuilder";
+import { IContext } from "../../../../../engine/engine_types";
+export interface UsdzBehaviour {
+    createBehaviours?(ext: BehaviorExtension, model: USDObject, context: IContext): void;
+    beforeCreateDocument?(ext: BehaviorExtension, context: IContext): void;
+    afterCreateDocument?(ext: BehaviorExtension, context: IContext): void;
+}
+export class USDZBehaviours extends Behaviour {
+    start() {
+        const exporter = GameObject.findObjectOfType(USDZExporter);
+        if (exporter) {
+            exporter.extensions.push(new BehaviorExtension());
+        }
+    }
+}
+export class BehaviorExtension implements IUSDExporterExtension {
+    get extensionName(): string {
+        return "Behaviour";
+    }
+    private behaviours: BehaviorModel[] = [];
+    addBehavior(beh: BehaviorModel) {
+        this.behaviours.push(beh);
+    }
+    behaviourComponents: Array<UsdzBehaviour> = [];
+    onBeforeBuildDocument(context) {
+        context.root.traverse(e => {
+            GameObject.foreachComponent(e, (comp) => {
+                const c = comp as unknown as UsdzBehaviour;
+                if (
+                    typeof c.createBehaviours === "function" ||
+                    typeof c.beforeCreateDocument === "function" ||
+                    typeof c.afterCreateDocument === "function"
+                ) {
+                    this.behaviourComponents.push(c);
+                    c.beforeCreateDocument?.call(c, this, context);
+                }
+            }, false);
+        });
+    }
+    onExportObject(_object, model: USDObject, context) {
+        for (const beh of this.behaviourComponents) {
+            beh.createBehaviours?.call(beh, this, model, context);
+        }
+    }
+    onAfterBuildDocument(context) {
+        for (const beh of this.behaviourComponents) {
+            if (typeof beh.afterCreateDocument === "function")
+                beh.afterCreateDocument(this, context);
+        }
+        this.behaviourComponents.length = 0;
+    }
+    onAfterHierarchy(context, writer : USDWriter) {
+        if (this.behaviours?.length) {
+            // this.combineBehavioursWithSameTapActions();
+            writer.beginBlock('def Scope "Behaviors"');
+            for (const beh of this.behaviours)
+                beh.writeTo(this, context.document, writer);
+            writer.closeBlock();
+        }
+    }
+    // combine behaviours that have tap triggers on the same object
+    // private combineBehavioursWithSameTapActions() {
+    //     // TODO: if behaviours have different settings (e.g. one is exclusive and another one is not) this wont work - we need more logic for that
+    //     const combined: { [key: string]: { behaviorId: string, trigger: TriggerModel, actions: IBehaviorElement[] } } = {};
+    //     for (let i = this.behaviours.length - 1; i >= 0; i--) {
+    //         const beh = this.behaviours[i];
+    //         const trigger = beh.trigger as TriggerModel;
+    //         if (!Array.isArray(trigger) && TriggerBuilder.isTapTrigger(trigger)) {
+    //             const targetObject = trigger.targetId;
+    //             if (!targetObject) continue;
+    //             if (!combined[targetObject]) {
+    //                 combined[targetObject] = { behaviorId: beh.id, trigger: trigger, actions: [] };
+    //             }
+    //             const action = beh.action;
+    //             combined[targetObject].actions.push(action);
+    //             this.behaviours.splice(i, 1);
+    //         }
+    //     }
+    //     for (const key in combined) {
+    //         const val = combined[key];
+    //         console.log("Combine " + val.actions.length + " actions on " + val.trigger.id, val.actions);
+    //         const beh = new BehaviorModel(val.behaviorId, val.trigger, ActionBuilder.sequence(...val.actions));
+    //         this.behaviours.push(beh);
+    //     }
+    // }
+}
+        // const playAnimationOnTap = new BehaviorModel("b_" + model.name + "_playanim", TriggerBuilder.tapTrigger(model),
+        //     ActionBuilder.parallel(
+        //         ActionBuilder.lookAtCameraAction(model),
+        //         ActionBuilder.sequence(
+        //             //ActionBuilder.startAnimationAction(model, 0, 0, 1, false, true),
+        //             ActionBuilder.emphasize(model, 1, MotionType.Float),
+        //             ActionBuilder.waitAction(1),
+        //             ActionBuilder.emphasize(model, 1, MotionType.Blink),
+        //             ActionBuilder.waitAction(1),
+        //             ActionBuilder.emphasize(model, 1, MotionType.Jiggle),
+        //             ActionBuilder.waitAction(1),
+        //             ActionBuilder.emphasize(model, 1, MotionType.Pulse),
+        //             ActionBuilder.waitAction(1),
+        //             ActionBuilder.emphasize(model, 1, MotionType.Spin),
+        //             ActionBuilder.waitAction(1),
+        //             ActionBuilder.emphasize(model, 1, MotionType.Bounce),
+        //             ActionBuilder.waitAction(1),
+        //             ActionBuilder.emphasize(model, 1, MotionType.Flip),
+        //             ActionBuilder.waitAction(1),
+        //         ).makeLooping()
+        //     ).makeLooping()
+        // );
+        // this.behaviours.push(playAnimationOnTap);
+        // return;
+        // const identityMatrix = new Matrix4().identity();
+        // const emptyParent = new USDZObject(model.name + "_empty", model.matrix);
+        // const parent = model.parent;
+        // parent.add(emptyParent);
+        // model.matrix = identityMatrix;
+        // emptyParent.add(model);
+        // const geometry = new THREE.SphereGeometry(.6, 32, 16);
+        // const modelVariant = new USDZObject(model.name + "_variant", identityMatrix, geometry, new MeshStandardMaterial({ color: 0xff0000 }));
+        // emptyParent.add(modelVariant);
+        // const matrix2 = new Matrix4();
+        // matrix2.makeTranslation(.5, 0, 0);
+        // const modelVariant2 = new USDZObject(model.name + "_variant2", matrix2, geometry, new MeshStandardMaterial({ color: 0xffff00 }));
+        // emptyParent.add(modelVariant2);
+        // const hideVariantOnStart = new BehaviorModel("b_" + model.name + "_start", TriggerBuilder.sceneStartTrigger(), ActionBuilder.fadeAction(modelVariant, 0, false));
+        // this.behaviours.push(hideVariantOnStart);
+        // const showVariant = new BehaviorModel("b_" + model.name + "_show_variant", [TriggerBuilder.tapTrigger(model)], new GroupActionModel("group", [
+        //     ActionBuilder.fadeAction(model, 0, false),
+        //     ActionBuilder.fadeAction(modelVariant, 0, true),
+        // ]));
+        // this.behaviours.push(showVariant);
+        // const showOriginal = new BehaviorModel("b_" + model.name + "_show_original", [
+        //     TriggerBuilder.tapTrigger(modelVariant),
+        //     TriggerBuilder.tapTrigger(modelVariant2)
+        // ],
+        //     new GroupActionModel("group", [
+        //         ActionBuilder.fadeAction([modelVariant, modelVariant2], 0, false),
+        //         //ActionBuilder.waitAction(1),
+        //         ActionBuilder.fadeAction(model, 0, true),
+        //         //ActionBuilder.waitAction(.2),
+        //         ActionBuilder.startAnimationAction(model, 0, 1000, 1, false, true),
+        //         //ActionBuilder.lookAtCameraAction(model, 2, Vec3.forward, Vec3.up),
+        //         //ActionBuilder.waitAction(1),
+        //         //ActionBuilder.fadeAction(modelVariant2, 0, true),
+        //     ]).makeSequence());
+        // this.behaviours.push(showOriginal);

src/engine-components/export/usdz/extensions/behavior/BehaviourComponents.ts ADDED Viewed

@@ -0,0 +1,503 @@
+import { Behaviour, GameObject } from "../../../../Component";
+import { Animator } from "../../../../Animator";
+import { Renderer } from "../../../../Renderer";
+import { serializable } from "../../../../../engine/engine_serialization_decorator";
+import { IPointerClickHandler } from "../../../../ui/PointerEvents";
+import { RegisteredAnimationInfo, UsdzAnimation } from "../Animation";
+import { getWorldPosition, getWorldQuaternion, getWorldScale, setWorldPosition, setWorldQuaternion, setWorldScale } from "../../../../../engine/engine_three_utils";
+import { Object3D, Material, Vector3, Quaternion } from "three";
+import { USDObject } from "../../ThreeUSDZExporter";
+import { BehaviorExtension, UsdzBehaviour } from "./Behaviour";
+import { ActionBuilder, ActionModel, BehaviorModel, MotionType, Space, TriggerBuilder } from "./BehavioursBuilder";
+export class ChangeTransformOnClick extends Behaviour implements IPointerClickHandler, UsdzBehaviour {
+    @serializable(Object3D)
+    object?: Object3D;
+    @serializable(Object3D)
+    target?: Object3D;
+    @serializable()
+    duration: number = 1;
+    @serializable()
+    relativeMotion: boolean = false;
+    private coroutine: Generator | null = null;
+    private targetPos = new Vector3();
+    private targetRot = new Quaternion();
+    private targetScale = new Vector3();
+    private *moveToTarget() {
+        if (!this.target || !this.object) return;
+        const thisPos = getWorldPosition(this.object).clone();
+        const targetPos = getWorldPosition(this.target).clone();
+        const thisRot = getWorldQuaternion(this.object).clone();
+        const targetRot = getWorldQuaternion(this.target).clone();
+        const thisScale = getWorldScale(this.object).clone();
+        const targetScale = getWorldScale(this.target).clone();
+        const dist = thisPos.distanceTo(targetPos);
+        const rotDist = thisRot.angleTo(targetRot);
+        const scaleDist = thisScale.distanceTo(targetScale);
+        if (dist < 0.01 && rotDist < 0.01 && scaleDist < 0.01) {
+            setWorldPosition(this.object, targetPos);
+            setWorldQuaternion(this.object, targetRot);
+            setWorldScale(this.object, targetScale);
+            this.coroutine = null;
+            return;
+        }
+        let t01 = 0;
+        let eased = 0;
+        while (t01 < 1) {
+            t01 += this.context.time.deltaTime / this.duration;
+            if (t01 > 1) t01 = 1;
+            // apply ease-in-out
+            // https://easings.net/
+            eased =  t01 < 0.5 ? 4 * t01 * t01 * t01 : 1 - Math.pow(-2 * t01 + 2, 3) / 2;
+            this.targetPos.lerpVectors(thisPos, targetPos, eased);
+            this.targetRot.slerpQuaternions(thisRot, targetRot, eased);
+            this.targetScale.lerpVectors(thisScale, targetScale, eased);
+            setWorldPosition(this.object, this.targetPos);
+            setWorldQuaternion(this.object, this.targetRot);
+            setWorldScale(this.object, this.targetScale);
+            yield;
+        }
+        this.coroutine = null;
+    }
+    private *moveRelative() {
+        if (!this.target || !this.object) return;
+        const thisPos = this.object.position.clone();
+        const thisRot = this.object.quaternion.clone();
+        const thisScale = this.object.scale.clone();
+        const posOffset = this.target.position.clone();
+        const rotOffset = this.target.quaternion.clone();
+        const scaleOffset = this.target.scale.clone();
+        // convert into right space
+        posOffset.applyQuaternion(this.object.quaternion);
+        this.targetPos.copy(this.object.position).add(posOffset);
+        this.targetRot.copy(this.object.quaternion).multiply(rotOffset);
+        this.targetScale.copy(this.object.scale).multiply(scaleOffset);
+        let t01 = 0;
+        let eased = 0;
+        while (t01 < 1) {
+            t01 += this.context.time.deltaTime / this.duration;
+            if (t01 > 1) t01 = 1;
+            // apply ease-in-out
+            // https://easings.net/
+            eased =  t01 < 0.5 ? 4 * t01 * t01 * t01 : 1 - Math.pow(-2 * t01 + 2, 3) / 2;
+            this.object.position.lerpVectors(thisPos, this.targetPos, eased);
+            this.object.quaternion.slerpQuaternions(thisRot, this.targetRot, eased);
+            this.object.scale.lerpVectors(thisScale, this.targetScale, eased);
+            yield;
+        }
+        this.coroutine = null;
+    }
+    onPointerClick() {
+        if (this.coroutine) this.stopCoroutine(this.coroutine);
+        if (!this.relativeMotion)
+            this.coroutine = this.startCoroutine(this.moveToTarget());
+        else
+            this.coroutine = this.startCoroutine(this.moveRelative());
+    }
+    beforeCreateDocument(ext) {
+        if (this.target && this.object && this.gameObject) {
+            const moveForward = new BehaviorModel("Move to " + this.target?.name,
+                TriggerBuilder.tapTrigger(this.gameObject),
+                ActionBuilder.transformAction(this.object, this.target, this.duration, this.relativeMotion ? Space.Relative : Space.Absolute),
+            );
+            ext.addBehavior(moveForward);
+        }
+    }
+}
+export class ChangeMaterialOnClick extends Behaviour implements IPointerClickHandler, UsdzBehaviour {
+    @serializable(Material)
+    materialToSwitch?: Material;
+    @serializable(Material)
+    variantMaterial?: Material;
+    private _objectsWithThisMaterial: Renderer[] = [];
+    awake() {
+        if (this.variantMaterial && this.materialToSwitch) {
+            const renderer = GameObject.findObjectsOfType(Renderer);
+            for (const rend of renderer) {
+                if (rend.sharedMaterial === this.materialToSwitch) {
+                    this._objectsWithThisMaterial.push(rend);
+                }
+            }
+        }
+    }
+    onPointerClick() {
+        if (!this.variantMaterial) return;
+        for (const rend of this._objectsWithThisMaterial) {
+            rend.sharedMaterial = this.variantMaterial;
+        }
+    }
+    private selfModel!: USDObject;
+    private targetModels: USDObject[] = [];
+    private static _materialTriggersPerId: { [key: string]: ChangeMaterialOnClick[] } = {}
+    createBehaviours(_ext: BehaviorExtension, model: USDObject, _context) {
+        const shouldExport = this._objectsWithThisMaterial.find(o => o.gameObject.uuid === model.uuid);
+        if (shouldExport) {
+            this.targetModels.push(model);
+        }
+        if (this.gameObject.uuid === model.uuid) {
+            this.selfModel = model;
+            if (this.materialToSwitch) {
+                if (!ChangeMaterialOnClick._materialTriggersPerId[this.materialToSwitch.uuid])
+                    ChangeMaterialOnClick._materialTriggersPerId[this.materialToSwitch.uuid] = [];
+                ChangeMaterialOnClick._materialTriggersPerId[this.materialToSwitch.uuid].push(this);
+            }
+        }
+    }
+    afterCreateDocument(ext: BehaviorExtension, _context) {
+        if (!this.materialToSwitch) return;
+        const handlers = ChangeMaterialOnClick._materialTriggersPerId[this.materialToSwitch.uuid];
+        if (handlers) {
+            const variants: { [key: string]: USDObject[] } = {}
+            for (const handler of handlers) {
+                const createdVariants = handler.createVariants();
+                if (createdVariants && createdVariants.length > 0)
+                    variants[handler.selfModel.uuid] = createdVariants;
+            }
+            const otherVariants: any[] = [];
+            for (const handler of handlers) {
+                for (const key in variants) {
+                    if (key !== handler.selfModel.uuid) {
+                        otherVariants.push(variants[key]);
+                    }
+                }
+                handler.createAndAttachBehaviors(ext, variants[handler.selfModel.uuid], otherVariants);
+            }
+        }
+        delete ChangeMaterialOnClick._materialTriggersPerId[this.materialToSwitch.uuid];
+    }
+    private createAndAttachBehaviors(ext: BehaviorExtension, myVariants, otherVariants) {
+        const start: ActionModel[] = [];
+        const select: ActionModel[] = [];
+        // the order here matters
+        for (const target of this.targetModels) {
+            const hideOriginal = ActionBuilder.fadeAction(target, 0, false);
+            select.push(hideOriginal);
+        }
+        for (const v of otherVariants) {
+            select.push(ActionBuilder.fadeAction(v, 0, false));
+        }
+        for (const v of myVariants) {
+            start.push(ActionBuilder.fadeAction(v, 0, false));
+            select.push(ActionBuilder.fadeAction(v, 0, true));
+        }
+        ext.addBehavior(new BehaviorModel("Select " + this.selfModel.name,
+            TriggerBuilder.tapTrigger(this.selfModel),
+            ActionBuilder.parallel(...select))
+        );
+        ext.addBehavior(new BehaviorModel("Start hidden " + this.selfModel.name,
+            TriggerBuilder.sceneStartTrigger(),
+            ActionBuilder.parallel(...start))
+        );
+    }
+    private createVariants() {
+        if (!this.variantMaterial) return null;
+        const variantModels: USDObject[] = [];
+        for (const target of this.targetModels) {
+            const variant = target.clone();
+            variant.name += " variant_" + this.variantMaterial.name;
+            variant.name = variant.name.replace(/\s/g, "_");
+            variant.material = this.variantMaterial;
+            variant.geometry = target.geometry;
+            variant.matrix = target.matrix;
+            if (!target.parent || !target.parent.isEmpty()) {
+                USDObject.createEmptyParent(target);
+            }
+            if (target.parent) target.parent.add(variant);
+            variantModels.push(variant);
+        }
+        return variantModels;
+    }
+}
+export class SetActiveOnClick extends Behaviour implements IPointerClickHandler, UsdzBehaviour {
+    @serializable(Object3D)
+    target?: Object3D;
+    // TODO bring back, doesn't work yet
+    private toggleOnClick: boolean = false;
+    @serializable()
+    targetState: boolean = true;
+    @serializable()
+    hideSelf: boolean = true;
+    onPointerClick() {
+        if (!this.toggleOnClick && this.hideSelf)
+            this.gameObject.visible = false;
+        if (this.target)
+            this.target.visible = this.toggleOnClick ? !this.target.visible : this.targetState;
+    }
+    private selfModel!: USDObject;
+    private otherModel?: USDObject;
+    private toggleModel?: USDObject;
+    createBehaviours(_, model, _context) {
+        if (model.uuid === this.gameObject.uuid) {
+            this.selfModel = model;
+        }
+    }
+    private stateBeforeCreatingDocument: boolean = false;
+    beforeCreateDocument() {
+        this.stateBeforeCreatingDocument = this.gameObject.visible;
+        this.gameObject.visible = true;
+    }
+    afterCreateDocument(ext, context) {
+        if (!this.target) return;
+        this.otherModel = context.document.findById(this.target.uuid);
+        if (this.selfModel && this.otherModel) {
+            let hideClickedObject = this.hideSelf;
+            let targetState = this.targetState;
+            if (this.toggleOnClick) {
+                hideClickedObject = true;
+                targetState = !this.target.visible;
+                // TODO check where we have to create the clone; here it doesn't show up
+                this.toggleModel = this.selfModel.clone();
+                this.toggleModel.name += "_toggle";
+                if (this.selfModel.parent)
+                    this.selfModel.parent.add(this.toggleModel);
+            }
+            const sequence: ActionModel[] = [];
+            if (hideClickedObject)
+                sequence.push(ActionBuilder.fadeAction(this.selfModel, 0, false));
+            if (this.toggleModel)
+                sequence.push(ActionBuilder.fadeAction(this.toggleModel, 0, true));
+            sequence.push(ActionBuilder.fadeAction(this.otherModel, 0, this.targetState));
+            ext.addBehavior(new BehaviorModel("Toggle_" + this.selfModel.name + "_hideSelf",
+                TriggerBuilder.tapTrigger(this.selfModel),
+                ActionBuilder.sequence(...sequence)
+            ));
+            // TODO this is relatively similar to the material variant switching logic, can we reuse that?
+            if (this.toggleOnClick && this.toggleModel) {
+                const toggleSequence: ActionModel[] = [];
+                toggleSequence.push(ActionBuilder.fadeAction(this.toggleModel, 0, false));
+                toggleSequence.push(ActionBuilder.fadeAction(this.selfModel, 0, true));
+                toggleSequence.push(ActionBuilder.fadeAction(this.otherModel, 0, !this.targetState));
+                ext.addBehavior(new BehaviorModel("Toggle_" + this.selfModel.name + "_toggleSelf",
+                    TriggerBuilder.tapTrigger(this.toggleModel),
+                    ActionBuilder.sequence(...toggleSequence)
+                ));
+                ext.addBehavior(new BehaviorModel("HideOnStart_" + this.gameObject.name,
+                    TriggerBuilder.sceneStartTrigger(),
+                    ActionBuilder.fadeAction(this.toggleModel, 0, false)
+                ));
+            }
+        }
+        this.gameObject.visible = this.stateBeforeCreatingDocument;
+    }
+}
+export class HideOnStart extends Behaviour implements UsdzBehaviour {
+    start() {
+        this.gameObject.visible = false;
+    }
+    createBehaviours(ext, model, _context) {
+        if (model.uuid === this.gameObject.uuid)
+            ext.addBehavior(new BehaviorModel("HideOnStart_" + this.gameObject.name,
+                TriggerBuilder.sceneStartTrigger(),
+                ActionBuilder.fadeAction(model, 0, false)
+            ));
+    }
+    beforeCreateDocument() {
+        this.gameObject.visible = true;
+    }
+    afterCreateDocument() {
+        this.gameObject.visible = false;
+    }
+}
+export class EmphasizeOnClick extends Behaviour implements UsdzBehaviour {
+    @serializable()
+    target?: Object3D;
+    @serializable()
+    duration: number = 0.5;
+    @serializable()
+    motionType: MotionType = MotionType.bounce;
+    beforeCreateDocument() { }
+    createBehaviours(ext, model, _context) {
+        if (!this.target) return;
+        if (model.uuid === this.gameObject.uuid)
+        {
+            const emphasize = new BehaviorModel("emphasize " + this.name,
+                TriggerBuilder.tapTrigger(this.gameObject),
+                ActionBuilder.emphasize(this.target, this.duration, this.motionType, undefined, "basic"),
+            );
+            ext.addBehavior(emphasize);
+        }
+    }
+    afterCreateDocument(_ext, _context) { }
+}
+export class PlayAnimationOnClick extends Behaviour implements IPointerClickHandler, UsdzBehaviour, UsdzAnimation {
+    @serializable(Object3D)
+    target?: Object3D;
+    @serializable(Animator)
+    animator?: Animator;
+    @serializable()
+    stateName?: string;
+    onPointerClick() {
+        if (!this.target) return;
+        if (this.stateName)
+            this.animator?.play(this.stateName, 0, 0, .1);
+    }
+    private selfModel: any;
+    private registeredAnimationModel: any;
+    private registeredAnimation?: RegisteredAnimationInfo;
+    createBehaviours(_ext, model, _context) {
+        if (model.uuid === this.gameObject.uuid)
+            this.selfModel = model;
+    }
+    afterCreateDocument(ext, context) {
+        if (!this.registeredAnimation || !this.registeredAnimationModel) return;
+        const document = context.document;
+        document.traverse(model => {
+            if (model.uuid === this.target?.uuid && this.registeredAnimation) {
+                const playAnimationOnTap = new BehaviorModel("tap " + this.name + " for " + this.stateName + " on " + this.target?.name,
+                    TriggerBuilder.tapTrigger(this.selfModel),
+                    ActionBuilder.startAnimationAction(model, this.registeredAnimation.start, this.registeredAnimation.duration)
+                );
+                ext.addBehavior(playAnimationOnTap);
+            }
+        });
+    }
+    createAnimation(ext, model, _context) {
+        if (this.target && this.animator) {
+            const state = this.animator?.runtimeAnimatorController?.findState(this.stateName);
+            this.registeredAnimationModel = model;
+            this.registeredAnimation = ext.registerAnimation(this.target, state?.motion.clip);
+        }
+    }
+}
+export class PreliminaryAction extends Behaviour {
+    getType(): string | void { }
+    @serializable(Object3D)
+    target?: Object3D;
+    getDuration(): number | void { };
+}
+export class PreliminaryTrigger extends Behaviour {
+    @serializable(PreliminaryAction)
+    target?: PreliminaryAction;
+}
+export class VisibilityAction extends PreliminaryAction {
+    //@type int
+    @serializable()
+    type: VisibilityActionType = VisibilityActionType.Hide;
+    @serializable()
+    duration: number = 1;
+    getType() {
+        switch (this.type) {
+            case VisibilityActionType.Hide: return "hide";
+            case VisibilityActionType.Show: return "show";
+        }
+    }
+    getDuration() {
+        return this.duration;
+    }
+}
+export class TapGestureTrigger extends PreliminaryTrigger {
+}
+export enum VisibilityActionType {
+    Show = 0,
+    Hide = 1,
+}

src/engine-components/export/usdz/extensions/behavior/BehavioursBuilder.ts ADDED Viewed

@@ -0,0 +1,459 @@
+import { Object3D } from "three";
+import { USDDocument, USDObject, USDWriter, makeNameSafeForUSD } from "../../ThreeUSDZExporter";
+import { BehaviorExtension } from "./Behaviour";
+// TODO: rename to usdz element
+export interface IBehaviorElement {
+    id: string;
+    writeTo(document: USDDocument, writer: USDWriter);
+}
+export class BehaviorModel {
+    static global_id: number = 0;
+    id: string;
+    trigger: IBehaviorElement | IBehaviorElement[];
+    action: IBehaviorElement;
+    exclusive: boolean = false;
+    makeExclusive(exclusive: boolean): BehaviorModel {
+        this.exclusive = exclusive;
+        return this;
+    }
+    constructor(id: string, trigger: IBehaviorElement | IBehaviorElement[], action: IBehaviorElement) {
+        this.id = "Behavior_" + makeNameSafeForUSD(id) + "_" + BehaviorModel.global_id++;
+        this.trigger = trigger;
+        this.action = action;
+    }
+    writeTo(_ext: BehaviorExtension, document: USDDocument, writer: USDWriter) {
+        if (!this.trigger || !this.action) return;
+        writer.beginBlock(`def Preliminary_Behavior "${this.id}"`);
+        writer.appendLine(`rel actions = <${this.action.id}>`);
+        writer.appendLine(`uniform bool exclusive = ${this.exclusive}`);
+        let triggerString = "";
+        if (Array.isArray(this.trigger)) {
+            triggerString = "[";
+            for (let i = 0; i < this.trigger.length; i++) {
+            +