Needle Engine 3.4.0-alpha

Files changed (27) hide show

src/engine/codegen/register_types.js +52 -4
plugins/vite/reload.js +13 -2
src/engine-components/Animation.ts +4 -0
src/engine-components/export/usdz/extensions/Animation.ts +37 -45
src/engine-components/ui/BaseUIComponent.ts +7 -1
src/engine-components/ui/Canvas.ts +80 -5
src/engine-components/codegen/components.ts +25 -1
src/engine/engine_gameobject.ts +3 -2
src/engine/engine_three_utils.ts +2 -2
src/engine-components/export/usdz/Extension.ts +4 -5
src/engine-components/ui/Graphic.ts +2 -0
src/engine-components/ui/Image.ts +3 -3
src/engine-components/ui/Interfaces.ts +30 -6
src/engine-components/ui/Layout.ts +303 -4
src/engine-components/utils/LookAt.ts +60 -7
src/engine-components/postprocessing/PostProcessingHandler.ts +1 -1
src/engine-components/ui/RectTransform.ts +65 -40
src/engine-components/export/usdz/types.ts +0 -39
src/engine-components/export/usdz/USDZExporter.ts +39 -17
src/engine-components/webxr/WebXRImageTracking.ts +100 -27
src/engine-components/export/usdz/extensions/behavior/Actions.ts +99 -0
src/engine-components/export/usdz/extensions/behavior/Behaviour.ts +181 -0
src/engine-components/export/usdz/extensions/behavior/BehaviourComponents.ts +503 -0
src/engine-components/export/usdz/extensions/behavior/BehavioursBuilder.ts +459 -0
src/engine-components/export/usdz/extensions/DocumentExtension.ts +10 -0
src/engine-components/export/usdz/ThreeUSDZExporter.ts +1280 -0
src/engine-components/export/usdz/extensions/USDZText.ts +142 -0

src/engine/codegen/register_types.js CHANGED Viewed

@@ -1,7 +1,10 @@
 import { TypeStore } from "./../engine_typestore"
 // Import types
 import { __Ignore } from "../../engine-components/codegen/components";
+import { ActionBuilder } from "../../engine-components/export/usdz/extensions/behavior/BehavioursBuilder";
+import { ActionCollection } from "../../engine-components/export/usdz/extensions/behavior/Actions";
+import { ActionModel } from "../../engine-components/export/usdz/extensions/behavior/BehavioursBuilder";
 import { AlignmentConstraint } from "../../engine-components/AlignmentConstraint";
 import { Animation } from "../../engine-components/Animation";
 import { AnimationCurve } from "../../engine-components/AnimationCurve";
@@ -26,6 +29,8 @@
 import { AxesHelper } from "../../engine-components/AxesHelper";
 import { BaseUIComponent } from "../../engine-components/ui/BaseUIComponent";
 import { BasicIKConstraint } from "../../engine-components/BasicIKConstraint";
+import { BehaviorExtension } from "../../engine-components/export/usdz/extensions/behavior/Behaviour";
+import { BehaviorModel } from "../../engine-components/export/usdz/extensions/behavior/BehavioursBuilder";
 import { Behaviour } from "../../engine-components/Component";
 import { Bloom } from "../../engine-components/postprocessing/Effects/Bloom";
 import { BoxCollider } from "../../engine-components/Collider";
@@ -37,6 +42,8 @@
 import { Canvas } from "../../engine-components/ui/Canvas";
 import { CanvasGroup } from "../../engine-components/ui/CanvasGroup";
 import { CapsuleCollider } from "../../engine-components/Collider";
+import { ChangeMaterialOnClick } from "../../engine-components/export/usdz/extensions/behavior/BehaviourComponents";
+import { ChangeTransformOnClick } from "../../engine-components/export/usdz/extensions/behavior/BehaviourComponents";
 import { CharacterController } from "../../engine-components/CharacterController";
 import { CharacterControllerInput } from "../../engine-components/CharacterController";
 import { ChromaticAberration } from "../../engine-components/postprocessing/Effects/ChromaticAberration";
@@ -50,10 +57,12 @@
 import { DeleteBox } from "../../engine-components/DeleteBox";
 import { DepthOfField } from "../../engine-components/postprocessing/Effects/DepthOfField";
 import { DeviceFlag } from "../../engine-components/DeviceFlag";
+import { DocumentExtension } from "../../engine-components/export/usdz/extensions/DocumentExtension";
 import { DragControls } from "../../engine-components/DragControls";
 import { DropListener } from "../../engine-components/DropListener";
 import { Duplicatable } from "../../engine-components/Duplicatable";
 import { EmissionModule } from "../../engine-components/ParticleSystemModules";
+import { EmphasizeOnClick } from "../../engine-components/export/usdz/extensions/behavior/BehaviourComponents";
 import { EventList } from "../../engine-components/EventList";
 import { EventListEvent } from "../../engine-components/EventList";
 import { EventSystem } from "../../engine-components/ui/EventSystem";
@@ -70,13 +79,14 @@
 import { GridHelper } from "../../engine-components/GridHelper";
 import { GridLayoutGroup } from "../../engine-components/ui/Layout";
 import { GroundProjectedEnv } from "../../engine-components/GroundProjection";
+import { GroupActionModel } from "../../engine-components/export/usdz/extensions/behavior/BehavioursBuilder";
+import { HideOnStart } from "../../engine-components/export/usdz/extensions/behavior/BehaviourComponents";
 import { HingeJoint } from "../../engine-components/Joints";
 import { HorizontalLayoutGroup } from "../../engine-components/ui/Layout";
 import { Image } from "../../engine-components/ui/Image";
 import { InheritVelocityModule } from "../../engine-components/ParticleSystemModules";
 import { InputField } from "../../engine-components/ui/InputField";
 import { Interactable } from "../../engine-components/Interactable";
-import { LayoutGroup } from "../../engine-components/ui/Layout";
 import { Light } from "../../engine-components/Light";
 import { LimitVelocityOverLifetimeModule } from "../../engine-components/ParticleSystemModules";
 import { LODGroup } from "../../engine-components/LODGroup";
@@ -98,17 +108,21 @@
 import { OpenURL } from "../../engine-components/utils/OpenURL";
 import { OrbitControls } from "../../engine-components/OrbitControls";
 import { Outline } from "../../engine-components/ui/Outline";
+import { Padding } from "../../engine-components/ui/Layout";
 import { ParticleBurst } from "../../engine-components/ParticleSystemModules";
 import { ParticleSubEmitter } from "../../engine-components/ParticleSystemSubEmitter";
 import { ParticleSystem } from "../../engine-components/ParticleSystem";
 import { ParticleSystemRenderer } from "../../engine-components/ParticleSystem";
 import { PixelationEffect } from "../../engine-components/postprocessing/Effects/Pixelation";
 import { PlayableDirector } from "../../engine-components/timeline/PlayableDirector";
+import { PlayAnimationOnClick } from "../../engine-components/export/usdz/extensions/behavior/BehaviourComponents";
 import { PlayerColor } from "../../engine-components/PlayerColor";
 import { PlayerState } from "../../engine-components-experimental/networking/PlayerSync";
 import { PlayerSync } from "../../engine-components-experimental/networking/PlayerSync";
 import { PointerEventData } from "../../engine-components/ui/PointerEvents";
 import { PostProcessingHandler } from "../../engine-components/postprocessing/PostProcessingHandler";
+import { PreliminaryAction } from "../../engine-components/export/usdz/extensions/behavior/BehaviourComponents";
+import { PreliminaryTrigger } from "../../engine-components/export/usdz/extensions/behavior/BehaviourComponents";
 import { PresentationMode } from "../../engine-components-experimental/Presentation";
 import { QuickLookOverlay } from "../../engine-components/export/usdz/USDZExporter";
 import { RawImage } from "../../engine-components/ui/Image";
@@ -127,6 +141,7 @@
 import { SceneSwitcher } from "../../engine-components/SceneSwitcher";
 import { ScreenCapture } from "../../engine-components/ScreenCapture";
 import { ScreenSpaceAmbientOcclusion } from "../../engine-components/postprocessing/Effects/ScreenspaceAmbientOcclusion";
+import { SetActiveOnClick } from "../../engine-components/export/usdz/extensions/behavior/BehaviourComponents";
 import { ShadowCatcher } from "../../engine-components/ShadowCatcher";
 import { ShapeModule } from "../../engine-components/ParticleSystemModules";
 import { SignalAsset } from "../../engine-components/timeline/SignalAsset";
@@ -151,24 +166,33 @@
 import { SyncedCamera } from "../../engine-components/SyncedCamera";
 import { SyncedRoom } from "../../engine-components/SyncedRoom";
 import { SyncedTransform } from "../../engine-components/SyncedTransform";
+import { TapGestureTrigger } from "../../engine-components/export/usdz/extensions/behavior/BehaviourComponents";
 import { TeleportTarget } from "../../engine-components/webxr/WebXRController";
 import { TestRunner } from "../../engine-components/TestRunner";
 import { TestSimulateUserData } from "../../engine-components/TestRunner";
 import { Text } from "../../engine-components/ui/Text";
+import { TextBuilder } from "../../engine-components/export/usdz/extensions/USDZText";
+import { TextExtension } from "../../engine-components/export/usdz/extensions/USDZText";
 import { TextureSheetAnimationModule } from "../../engine-components/ParticleSystemModules";
 import { TiltShiftEffect } from "../../engine-components/postprocessing/Effects/TiltShiftEffect";
 import { ToneMapping } from "../../engine-components/postprocessing/Effects/Tonemapping";
 import { TrailModule } from "../../engine-components/ParticleSystemModules";
 import { TransformData } from "../../engine-components/export/usdz/extensions/Animation";
 import { TransformGizmo } from "../../engine-components/TransformGizmo";
+import { TriggerBuilder } from "../../engine-components/export/usdz/extensions/behavior/BehavioursBuilder";
+import { TriggerModel } from "../../engine-components/export/usdz/extensions/behavior/BehavioursBuilder";
 import { UIRaycastUtils } from "../../engine-components/ui/RaycastUtils";
 import { UIRootComponent } from "../../engine-components/ui/BaseUIComponent";
 import { UsageMarker } from "../../engine-components/Interactable";
+import { USDZBehaviours } from "../../engine-components/export/usdz/extensions/behavior/Behaviour";
 import { USDZExporter } from "../../engine-components/export/usdz/USDZExporter";
+import { USDZText } from "../../engine-components/export/usdz/extensions/USDZText";
+import { VariantAction } from "../../engine-components/export/usdz/extensions/behavior/Actions";
 import { VelocityOverLifetimeModule } from "../../engine-components/ParticleSystemModules";
 import { VerticalLayoutGroup } from "../../engine-components/ui/Layout";
 import { VideoPlayer } from "../../engine-components/VideoPlayer";
 import { Vignette } from "../../engine-components/postprocessing/Effects/Vignette";
+import { VisibilityAction } from "../../engine-components/export/usdz/extensions/behavior/BehaviourComponents";
 import { Voip } from "../../engine-components/Voip";
 import { Volume } from "../../engine-components/postprocessing/Volume";
 import { VolumeParameter } from "../../engine-components/postprocessing/VolumeParameter";
@@ -190,9 +214,12 @@
 import { XRGrabRendering } from "../../engine-components/webxr/WebXRGrabRendering";
 import { XRRig } from "../../engine-components/webxr/WebXRRig";
 import { XRState } from "../../engine-components/XRFlag";
 // Register types
 TypeStore.add("__Ignore", __Ignore);
+TypeStore.add("ActionBuilder", ActionBuilder);
+TypeStore.add("ActionCollection", ActionCollection);
+TypeStore.add("ActionModel", ActionModel);
 TypeStore.add("AlignmentConstraint", AlignmentConstraint);
 TypeStore.add("Animation", Animation);
 TypeStore.add("AnimationCurve", AnimationCurve);
@@ -217,6 +244,8 @@
 TypeStore.add("AxesHelper", AxesHelper);
 TypeStore.add("BaseUIComponent", BaseUIComponent);
 TypeStore.add("BasicIKConstraint", BasicIKConstraint);
+TypeStore.add("BehaviorExtension", BehaviorExtension);
+TypeStore.add("BehaviorModel", BehaviorModel);
 TypeStore.add("Behaviour", Behaviour);
 TypeStore.add("Bloom", Bloom);
 TypeStore.add("BoxCollider", BoxCollider);
@@ -228,6 +257,8 @@
 TypeStore.add("Canvas", Canvas);
 TypeStore.add("CanvasGroup", CanvasGroup);
 TypeStore.add("CapsuleCollider", CapsuleCollider);
+TypeStore.add("ChangeMaterialOnClick", ChangeMaterialOnClick);
+TypeStore.add("ChangeTransformOnClick", ChangeTransformOnClick);
 TypeStore.add("CharacterController", CharacterController);
 TypeStore.add("CharacterControllerInput", CharacterControllerInput);
 TypeStore.add("ChromaticAberration", ChromaticAberration);
@@ -241,10 +272,12 @@
 TypeStore.add("DeleteBox", DeleteBox);
 TypeStore.add("DepthOfField", DepthOfField);
 TypeStore.add("DeviceFlag", DeviceFlag);
+TypeStore.add("DocumentExtension", DocumentExtension);
 TypeStore.add("DragControls", DragControls);
 TypeStore.add("DropListener", DropListener);
 TypeStore.add("Duplicatable", Duplicatable);
 TypeStore.add("EmissionModule", EmissionModule);
+TypeStore.add("EmphasizeOnClick", EmphasizeOnClick);
 TypeStore.add("EventList", EventList);
 TypeStore.add("EventListEvent", EventListEvent);
 TypeStore.add("EventSystem", EventSystem);
@@ -261,13 +294,14 @@
 TypeStore.add("GridHelper", GridHelper);
 TypeStore.add("GridLayoutGroup", GridLayoutGroup);
 TypeStore.add("GroundProjectedEnv", GroundProjectedEnv);
+TypeStore.add("GroupActionModel", GroupActionModel);
+TypeStore.add("HideOnStart", HideOnStart);
 TypeStore.add("HingeJoint", HingeJoint);
 TypeStore.add("HorizontalLayoutGroup", HorizontalLayoutGroup);
 TypeStore.add("Image", Image);
 TypeStore.add("InheritVelocityModule", InheritVelocityModule);
 TypeStore.add("InputField", InputField);
 TypeStore.add("Interactable", Interactable);
-TypeStore.add("LayoutGroup", LayoutGroup);
 TypeStore.add("Light", Light);
 TypeStore.add("LimitVelocityOverLifetimeModule", LimitVelocityOverLifetimeModule);
 TypeStore.add("LODGroup", LODGroup);
@@ -289,17 +323,21 @@
 TypeStore.add("OpenURL", OpenURL);
 TypeStore.add("OrbitControls", OrbitControls);
 TypeStore.add("Outline", Outline);
+TypeStore.add("Padding", Padding);
 TypeStore.add("ParticleBurst", ParticleBurst);
 TypeStore.add("ParticleSubEmitter", ParticleSubEmitter);
 TypeStore.add("ParticleSystem", ParticleSystem);
 TypeStore.add("ParticleSystemRenderer", ParticleSystemRenderer);
 TypeStore.add("PixelationEffect", PixelationEffect);
 TypeStore.add("PlayableDirector", PlayableDirector);
+TypeStore.add("PlayAnimationOnClick", PlayAnimationOnClick);
 TypeStore.add("PlayerColor", PlayerColor);
 TypeStore.add("PlayerState", PlayerState);
 TypeStore.add("PlayerSync", PlayerSync);
 TypeStore.add("PointerEventData", PointerEventData);
 TypeStore.add("PostProcessingHandler", PostProcessingHandler);
+TypeStore.add("PreliminaryAction", PreliminaryAction);
+TypeStore.add("PreliminaryTrigger", PreliminaryTrigger);
 TypeStore.add("PresentationMode", PresentationMode);
 TypeStore.add("QuickLookOverlay", QuickLookOverlay);
 TypeStore.add("RawImage", RawImage);
@@ -318,6 +356,7 @@
 TypeStore.add("SceneSwitcher", SceneSwitcher);
 TypeStore.add("ScreenCapture", ScreenCapture);
 TypeStore.add("ScreenSpaceAmbientOcclusion", ScreenSpaceAmbientOcclusion);
+TypeStore.add("SetActiveOnClick", SetActiveOnClick);
 TypeStore.add("ShadowCatcher", ShadowCatcher);
 TypeStore.add("ShapeModule", ShapeModule);
 TypeStore.add("SignalAsset", SignalAsset);
@@ -342,24 +381,33 @@
 TypeStore.add("SyncedCamera", SyncedCamera);
 TypeStore.add("SyncedRoom", SyncedRoom);
 TypeStore.add("SyncedTransform", SyncedTransform);
+TypeStore.add("TapGestureTrigger", TapGestureTrigger);
 TypeStore.add("TeleportTarget", TeleportTarget);
 TypeStore.add("TestRunner", TestRunner);
 TypeStore.add("TestSimulateUserData", TestSimulateUserData);
 TypeStore.add("Text", Text);
+TypeStore.add("TextBuilder", TextBuilder);
+TypeStore.add("TextExtension", TextExtension);
 TypeStore.add("TextureSheetAnimationModule", TextureSheetAnimationModule);
 TypeStore.add("TiltShiftEffect", TiltShiftEffect);
 TypeStore.add("ToneMapping", ToneMapping);
 TypeStore.add("TrailModule", TrailModule);
 TypeStore.add("TransformData", TransformData);
 TypeStore.add("TransformGizmo", TransformGizmo);
+TypeStore.add("TriggerBuilder", TriggerBuilder);
+TypeStore.add("TriggerModel", TriggerModel);
 TypeStore.add("UIRaycastUtils", UIRaycastUtils);
 TypeStore.add("UIRootComponent", UIRootComponent);
 TypeStore.add("UsageMarker", UsageMarker);
+TypeStore.add("USDZBehaviours", USDZBehaviours);
 TypeStore.add("USDZExporter", USDZExporter);
+TypeStore.add("USDZText", USDZText);
+TypeStore.add("VariantAction", VariantAction);
 TypeStore.add("VelocityOverLifetimeModule", VelocityOverLifetimeModule);
 TypeStore.add("VerticalLayoutGroup", VerticalLayoutGroup);
 TypeStore.add("VideoPlayer", VideoPlayer);
 TypeStore.add("Vignette", Vignette);
+TypeStore.add("VisibilityAction", VisibilityAction);
 TypeStore.add("Voip", Voip);
 TypeStore.add("Volume", Volume);
 TypeStore.add("VolumeParameter", VolumeParameter);

plugins/vite/reload.js CHANGED Viewed

@@ -9,6 +9,7 @@
 const __dirname = path.dirname(__filename);
 const filesUsingHotReload = new Set();
+let assetsDirectory = "";
 export const needleReload = (command, config, userSettings) => {
     if (command === "build") return;
@@ -23,8 +24,9 @@
         if (res) config = res;
     }
+    const projectConfig = tryLoadProjectConfig();
+    assetsDirectory = path.resolve(projectConfig?.assetsDirectory || "assets");
-    const projectConfig = tryLoadProjectConfig();
     const buildDirectory = projectConfig?.buildDirectory?.length ? process.cwd().replaceAll("\\", "/") + "/" + projectConfig?.buildDirectory : "";
     if (buildDirectory?.length) {
         setTimeout(() => console.log("Build directory: ", buildDirectory), 100);
@@ -153,9 +155,18 @@
         }
     }
-    if (file.endsWith(".vue") || file.endsWith(".ts") || file.endsWith(".js") || file.endsWith(".jsx") || file.endsWith(".tsx"))
+    // these are known file types we export from integrations
+    const knownExportFileTypes = [ ".glb", ".gltf", ".bin", "exr", ".ktx2", ".mp3", ".ogg", ".mp4", ".webm" ];
+    if (!knownExportFileTypes.some((type) => file.endsWith(type)))
         return;
+    // we only care about exports into "assets"
+    if (!path.resolve(file).startsWith(assetsDirectory))
+        return;
+    if (file.endsWith(".svelte") || file.endsWith(".vue") || file.endsWith(".ts") || file.endsWith(".js") || file.endsWith(".jsx") || file.endsWith(".tsx"))
+        return;
     if (file.endsWith(lockFileName)) return;
     let fileSize = "";
     const isGlbOrGltfFile = file.endsWith(".glb") || file.endsWith(".bin");

src/engine-components/Animation.ts CHANGED Viewed

@@ -176,6 +176,10 @@
                 return this.internalOnPlay(act, options);
             }
         }
+        if (!(clip instanceof AnimationClip)) {
+            console.warn("Clip is no AnimationClip", clip, "on object: " + this.name)
+            return;
+        }
         const act = this.mixer.clipAction(clip);
         this.actions.push(act);
         return this.internalOnPlay(act, options);

src/engine-components/export/usdz/extensions/Animation.ts CHANGED Viewed

@@ -1,17 +1,18 @@
 import { GameObject } from "../../../Component";
 import { getParam } from "../../../../engine/engine_utils";
-import { Object3D, Color, Matrix4, MeshStandardMaterial, Vector3, Quaternion, Interpolant } from "three";
-//@ts-ignore
-import { USDZObject, buildMatrix } from "three/examples/jsm/exporters/USDZExporter"
-import { IUSDZExporterExtension } from "../Extension";
+import { USDObject, buildMatrix } from "../ThreeUSDZExporter";
+import { IUSDExporterExtension } from "../Extension";
+import { Object3D, Matrix4, Vector3, Quaternion, Interpolant, AnimationClip, KeyframeTrack } from "three";
 const debug = getParam("debugusdzanimation");
 export interface UsdzAnimation {
-    createAnimation(ext: AnimationExtension, model: USDZObject, context);
+    createAnimation(ext: AnimationExtension, model: USDObject, context);
 }
-export type AnimationClipCollection = Array<{ root: Object3D, clips: Array<THREE.AnimationClip> }>;
+export type AnimationClipCollection = Array<{ root: Object3D, clips: Array<AnimationClip> }>;
 export class RegisteredAnimationInfo {
@@ -20,9 +21,9 @@
     private ext: AnimationExtension;
     private root: Object3D;
-    private clip: THREE.AnimationClip;
+    private clip: AnimationClip;
-    constructor(ext: AnimationExtension, root: THREE.Object3D, clip: THREE.AnimationClip) {
+    constructor(ext: AnimationExtension, root: Object3D, clip: AnimationClip) {
         this.ext = ext;
         this.root = root;
         this.clip = clip;
@@ -30,17 +31,17 @@
 }
 export class TransformData {
-    clip: THREE.AnimationClip;
-    pos?: THREE.KeyframeTrack;
-    rot?: THREE.KeyframeTrack;
-    scale?: THREE.KeyframeTrack;
+    clip: AnimationClip;
+    pos?: KeyframeTrack;
+    rot?: KeyframeTrack;
+    scale?: KeyframeTrack;
     get frameRate(): number { return 60; }
     private ext: AnimationExtension;
     private root: Object3D;
     private target: Object3D;
-    constructor(ext: AnimationExtension, root: Object3D, target: Object3D, clip: THREE.AnimationClip) {
+    constructor(ext: AnimationExtension, root: Object3D, target: Object3D, clip: AnimationClip) {
         this.ext = ext;
         this.root = root;
         this.target = target;
@@ -78,14 +79,14 @@
 declare type AnimationDict = Map<Object3D, Array<TransformData>>;
-export class AnimationExtension implements IUSDZExporterExtension {
+export class AnimationExtension implements IUSDExporterExtension {
     get extensionName(): string { return "animation" }
     private dict: AnimationDict = new Map();
     // private rootTargetMap: Map<Object3D, Object3D[]> = new Map();
     private rootTargetMap: Map<Object3D, Object3D[]> = new Map();
-    getStartTime01(root: Object3D, clip: THREE.AnimationClip) {
+    getStartTime01(root: Object3D, clip: AnimationClip) {
         const targets = this.rootTargetMap.get(root);
         if (!targets) return Infinity;
         let longestStartTime: number = -1;
@@ -108,7 +109,7 @@
         return longestStartTime;
     }
-    registerAnimation(root: Object3D, clip: THREE.AnimationClip): RegisteredAnimationInfo | null {
+    registerAnimation(root: Object3D, clip: AnimationClip): RegisteredAnimationInfo | null {
         if (!clip || !root) return null;
         if (!this.rootTargetMap.has(root)) this.rootTargetMap.set(root, []);
         // this.rootTargetMap.get(root)?.push(clip);
@@ -165,7 +166,7 @@
         }
     }
-    onExportObject(object, model: USDZObject, _context) {
+    onExportObject(object, model: USDObject, _context) {
         GameObject.foreachComponent(object, (comp) => {
             const c = comp as unknown as UsdzAnimation;
@@ -187,7 +188,7 @@
     object: Object3D;
     dict: AnimationDict;
-    model: USDZObject;
+    model: USDObject | undefined = undefined;
     private callback?: Function;
@@ -196,7 +197,7 @@
         this.dict = dict;
     }
-    registerCallback(model: USDZObject) {
+    registerCallback(model: USDObject) {
         if (this.model && this.callback) {
             this.model.removeEventListener("serialize", this.callback);
         }
@@ -209,6 +210,7 @@
     }
     onSerialize(writer, _context) {
+        if (!this.model) return;
         if (debug)
             console.log("SERIALIZE", this.model.name, this.object.type);
         // do we have a track for this?
@@ -228,22 +230,24 @@
         const rotation = new Quaternion();
         const scale = new Vector3(1, 1, 1);
-        // TODO doesn't support individual time arrays right now
-        // could use these in case we don't have time values that are identical
-        /*
-        const translationInterpolant = o.pos?.createInterpolant() as THREE.Interpolant;
-        const rotationInterpolant = o.rot?.createInterpolant() as THREE.Interpolant;
-        const scaleInterpolant = o.scale?.createInterpolant() as THREE.Interpolant;
-        */
         writer.appendLine("matrix4d xformOp:transform.timeSamples = {");
         writer.indent++;
         for (const transformData of arr) {
-            let timesArray = transformData.pos?.times;
-            if (!timesArray || transformData.rot && transformData.rot.times?.length > timesArray?.length) timesArray = transformData.rot?.times;
-            if (!timesArray || transformData.scale && transformData.scale.times?.length > timesArray?.length) timesArray = transformData.scale?.times;
-            if (!timesArray) {
+            let posTimesArray = transformData.pos?.times;
+            let rotTimesArray = transformData.rot?.times;
+            let scaleTimesArray = transformData.scale?.times;
+            // timesArray is the sorted union of all time values
+            let timesArray: number[] = [];
+            if (posTimesArray) for (const t of posTimesArray) timesArray.push(t);
+            if (rotTimesArray) for (const t of rotTimesArray) timesArray.push(t);
+            if (scaleTimesArray) for (const t of scaleTimesArray) timesArray.push(t);
+            // sort
+            timesArray.sort((a, b) => a - b);
+            timesArray = [...new Set(timesArray)];
+            if (!timesArray || timesArray.length === 0) {
                 console.error("got an animated object but no time values??", object, transformData);
                 continue;
             }
@@ -276,8 +280,8 @@
                     rotation.set(quat[0], quat[1], quat[2], quat[3]);
                 }
                 if (scaleInterpolant) {
-                    const scale = scaleInterpolant.evaluate(time);
-                    scale.set(scale[0], scale[1], scale[2]);
+                    const scaleVal = scaleInterpolant.evaluate(time);
+                    scale.set(scaleVal[0], scaleVal[1], scaleVal[2]);
                 }
                 composedTransform.compose(translation, rotation, scale);
@@ -290,17 +294,5 @@
         }
         writer.indent--;
         writer.appendLine("}");
-        /*
-        let transform3 = new Matrix4();
-        transform3.compose(0.2,0,0);
-        const transform = buildMatrix(model.matrix);
-        const transform2 = buildMatrix(transform3.multiply(model.matrix));
-        writer.appendLine(`matrix4d xformOp:transform.timeSamples = {
-            0: ${transform},
-            30: ${transform2}
-        }`);
-        */
     }
 }

src/engine-components/ui/BaseUIComponent.ts CHANGED Viewed

@@ -4,7 +4,7 @@
 import { EventSystem } from "./EventSystem";
 import { showGizmos } from '../../engine/engine_default_parameters';
 import { AxesHelper, Object3D } from 'three';
-import { IGraphic } from './Interfaces';
+import { ICanvas, IGraphic } from './Interfaces';
 import { ShadowCastingMode } from '../Renderer';
 export const includesDir = "./include";
@@ -50,6 +50,12 @@
         return this._root;
     }
+    protected get Canvas() {
+        const cv = this.Root as any as ICanvas;
+        if (cv?.isCanvas) return cv;
+        return null;
+    }
     // private _intermediate?: Object3D;
     protected _parentComponent?: BaseUIComponent | null = undefined;

src/engine-components/ui/Canvas.ts CHANGED Viewed

@@ -3,12 +3,14 @@
 import { FrameEvent } from "../../engine/engine_setup";
 import { BaseUIComponent, UIRootComponent } from "./BaseUIComponent";
 import { GameObject } from "../Component";
-import { Object3D } from "three";
+import { Matrix4, Object3D } from "three";
 import { RectTransform } from "./RectTransform";
-import { ICanvas } from "./Interfaces";
+import { ICanvas, ILayoutGroup, IRectTransform } from "./Interfaces";
 import { Camera } from "../Camera";
 import { EventSystem } from "./EventSystem";
 import * as ThreeMeshUI from 'three-mesh-ui'
+import { getParam } from "../../engine/engine_utils";
+import { LayoutGroup } from "./Layout";
 export enum RenderMode {
     ScreenSpaceOverlay = 0,
@@ -17,8 +19,14 @@
     Undefined = -1,
 }
+const debugLayout = getParam("debuguilayout");
 export class Canvas extends UIRootComponent implements ICanvas {
+    get isCanvas() {
+        return true;
+    }
     get screenspace(): any {
         return this.renderMode !== RenderMode.WorldSpace;
     }
@@ -122,6 +130,10 @@
         super.awake();
     }
+    start() {
+        this.onUpdateRenderMode();
+    }
     onEnable() {
         super.onEnable();
         this._updateRenderSettingsRoutine = undefined;
@@ -149,7 +161,29 @@
     private _boundRenderSettingsChanged = this.onRenderSettingsChanged.bind(this);
     private previousParent: Object3D | null = null;
+    private _lastMatrixWorld: Matrix4 | null = null;
+    private _rectTransforms: IRectTransform[] = [];
+    registerTransform(rt: IRectTransform) {
+        this._rectTransforms.push(rt);
+    }
+    unregisterTransform(rt: IRectTransform) {
+        const index = this._rectTransforms.indexOf(rt);
+        if (index !== -1) {
+            this._rectTransforms.splice(index, 1);
+        }
+    }
+    private _layoutGroups: Map<Object3D, ILayoutGroup> = new Map();
+    registerLayoutGroup(group: ILayoutGroup) {
+        const obj = group.gameObject;
+        this._layoutGroups.set(obj, group)
+    }
+    unregisterLayoutGroup(group: ILayoutGroup) {
+        const obj = group.gameObject;
+        this._layoutGroups.delete(obj);
+    }
     onBeforeRenderRoutine = () => {
         if (this.renderOnTop) {
             // This is just a test but in reality it should be combined with all world canvases with render on top in one render pass
@@ -158,6 +192,7 @@
         }
         else {
             this.onUpdateRenderMode();
+            this.handleLayoutUpdates();
             // TODO: we might need to optimize this. This is here to make sure the TMUI text clipping matrices are correct. Ideally the text does use onBeforeRender and apply the clipping matrix there so we dont have to force update all the matrices here
             this.shadowComponent?.updateMatrixWorld(true);
             this.shadowComponent?.updateWorldMatrix(true, true);
@@ -171,13 +206,42 @@
             this.context.renderer.autoClear = false;
             this.context.renderer.clearDepth();
             this.onUpdateRenderMode(true);
+            this.handleLayoutUpdates();
             this.shadowComponent?.updateMatrixWorld(true);
+            // this.handleLayoutUpdates();
             EventSystem.ensureUpdateMeshUI(ThreeMeshUI, this.context);
             this.context.renderer.render(this.gameObject, this.context.mainCamera);
             this.context.renderer.autoClear = true;
         }
+        this._lastMatrixWorld?.copy(this.gameObject.matrixWorld);
     }
+    private handleLayoutUpdates() {
+        if (this._lastMatrixWorld === null) {
+            this._lastMatrixWorld = new Matrix4();
+        }
+        const matrixWorldChanged = !this._lastMatrixWorld.equals(this.gameObject.matrixWorld);
+        if (debugLayout && matrixWorldChanged) console.log("Canvas Layout changed", this.context.time.frameCount, this.name);
+        // TODO: optimize this, we should only need to update a subhierarchy of the parts where layout has changed
+        let didLog = false;
+        for (const ch of this._rectTransforms) {
+            if (matrixWorldChanged) ch.markDirty();
+            let layout = this._layoutGroups.get(ch.gameObject);
+            if(ch.isDirty && !layout){
+                layout = ch.gameObject.getComponentInParent(LayoutGroup) as LayoutGroup;
+            }
+            if (ch.isDirty || layout?.isDirty) {
+                if (debugLayout && !didLog) {
+                    console.log("CANVAS UPDATE ### " + ch.name + " ##################################### " + this.context.time.frame);
+                    // didLog = true;
+                }
+                layout?.updateLayout();
+                ch.updateTransform();
+            }
+        }
+    }
     applyRenderSettings() {
         this.onRenderSettingsChanged();
     }
@@ -205,6 +269,7 @@
     private _lastWidth: number = -1;
     private _lastHeight: number = -1;
     private onUpdateRenderMode(force: boolean = false) {
         if (!force) {
             if (this._renderMode === this._activeRenderMode && this._lastWidth === this.context.domWidth && this._lastHeight === this.context.domHeight) {
@@ -241,6 +306,13 @@
                 canvas.quaternion.identity();
                 const rect = this.gameObject.getComponent(RectTransform)!;
+                let hasChanged = false;
+                if (rect.sizeDelta.x !== this.context.domWidth) {
+                    hasChanged = true;
+                }
+                if (rect.sizeDelta.y !== this.context.domHeight) {
+                    hasChanged = true;
+                }
                 const vFOV = camera.fieldOfView! * Math.PI / 180;
                 const h = 2 * Math.tan(vFOV / 2) * Math.abs(plane);
@@ -248,11 +320,14 @@
                 canvas.scale.y = h / this.context.domHeight;
                 // Set scale.z, otherwise small offsets in screenspace mode have different visual results based on export scale and other settings
                 canvas.scale.z = .01;
-                rect.sizeDelta.x = this.context.domWidth;
-                rect.sizeDelta.y = this.context.domHeight;
-                rect?.markDirty();
+                if (hasChanged) {
+                    rect.sizeDelta.x = this.context.domWidth;
+                    rect.sizeDelta.y = this.context.domHeight;
+                    rect?.markDirty();
+                }
                 // this.context.scene.add(this.gameObject)
                 // this.gameObject.scale.multiplyScalar(.01);
                 // this.gameObject.position.set(0,0,0);

src/engine-components/codegen/components.ts CHANGED Viewed

@@ -1,5 +1,8 @@
 // Export types
 export class __Ignore {}
+export { ActionBuilder } from "../export/usdz/extensions/behavior/BehavioursBuilder";
+export { ActionCollection } from "../export/usdz/extensions/behavior/Actions";
+export { ActionModel } from "../export/usdz/extensions/behavior/BehavioursBuilder";
 export { AlignmentConstraint } from "../AlignmentConstraint";
 export { Animation } from "../Animation";
 export { AnimationCurve } from "../AnimationCurve";
@@ -24,6 +27,8 @@
 export { AxesHelper } from "../AxesHelper";
 export { BaseUIComponent } from "../ui/BaseUIComponent";
 export { BasicIKConstraint } from "../BasicIKConstraint";
+export { BehaviorExtension } from "../export/usdz/extensions/behavior/Behaviour";
+export { BehaviorModel } from "../export/usdz/extensions/behavior/BehavioursBuilder";
 export { Behaviour } from "../Component";
 export { Bloom } from "../postprocessing/Effects/Bloom";
 export { BoxCollider } from "../Collider";
@@ -35,6 +40,8 @@
 export { Canvas } from "../ui/Canvas";
 export { CanvasGroup } from "../ui/CanvasGroup";
 export { CapsuleCollider } from "../Collider";
+export { ChangeMaterialOnClick } from "../export/usdz/extensions/behavior/BehaviourComponents";
+export { ChangeTransformOnClick } from "../export/usdz/extensions/behavior/BehaviourComponents";
 export { CharacterController } from "../CharacterController";
 export { CharacterControllerInput } from "../CharacterController";
 export { ChromaticAberration } from "../postprocessing/Effects/ChromaticAberration";
@@ -48,10 +55,12 @@
 export { DeleteBox } from "../DeleteBox";
 export { DepthOfField } from "../postprocessing/Effects/DepthOfField";
 export { DeviceFlag } from "../DeviceFlag";
+export { DocumentExtension } from "../export/usdz/extensions/DocumentExtension";
 export { DragControls } from "../DragControls";
 export { DropListener } from "../DropListener";
 export { Duplicatable } from "../Duplicatable";
 export { EmissionModule } from "../ParticleSystemModules";
+export { EmphasizeOnClick } from "../export/usdz/extensions/behavior/BehaviourComponents";
 export { EventList } from "../EventList";
 export { EventListEvent } from "../EventList";
 export { EventSystem } from "../ui/EventSystem";
@@ -68,13 +77,14 @@
 export { GridHelper } from "../GridHelper";
 export { GridLayoutGroup } from "../ui/Layout";
 export { GroundProjectedEnv } from "../GroundProjection";
+export { GroupActionModel } from "../export/usdz/extensions/behavior/BehavioursBuilder";
+export { HideOnStart } from "../export/usdz/extensions/behavior/BehaviourComponents";
 export { HingeJoint } from "../Joints";
 export { HorizontalLayoutGroup } from "../ui/Layout";
 export { Image } from "../ui/Image";
 export { InheritVelocityModule } from "../ParticleSystemModules";
 export { InputField } from "../ui/InputField";
 export { Interactable } from "../Interactable";
-export { LayoutGroup } from "../ui/Layout";
 export { Light } from "../Light";
 export { LimitVelocityOverLifetimeModule } from "../ParticleSystemModules";
 export { LODGroup } from "../LODGroup";
@@ -96,15 +106,19 @@
 export { OpenURL } from "../utils/OpenURL";
 export { OrbitControls } from "../OrbitControls";
 export { Outline } from "../ui/Outline";
+export { Padding } from "../ui/Layout";
 export { ParticleBurst } from "../ParticleSystemModules";
 export { ParticleSubEmitter } from "../ParticleSystemSubEmitter";
 export { ParticleSystem } from "../ParticleSystem";
 export { ParticleSystemRenderer } from "../ParticleSystem";
 export { PixelationEffect } from "../postprocessing/Effects/Pixelation";
 export { PlayableDirector } from "../timeline/PlayableDirector";
+export { PlayAnimationOnClick } from "../export/usdz/extensions/behavior/BehaviourComponents";
 export { PlayerColor } from "../PlayerColor";
 export { PointerEventData } from "../ui/PointerEvents";
 export { PostProcessingHandler } from "../postprocessing/PostProcessingHandler";
+export { PreliminaryAction } from "../export/usdz/extensions/behavior/BehaviourComponents";
+export { PreliminaryTrigger } from "../export/usdz/extensions/behavior/BehaviourComponents";
 export { QuickLookOverlay } from "../export/usdz/USDZExporter";
 export { RawImage } from "../ui/Image";
 export { Raycaster } from "../ui/Raycaster";
@@ -122,6 +136,7 @@
 export { SceneSwitcher } from "../SceneSwitcher";
 export { ScreenCapture } from "../ScreenCapture";
 export { ScreenSpaceAmbientOcclusion } from "../postprocessing/Effects/ScreenspaceAmbientOcclusion";
+export { SetActiveOnClick } from "../export/usdz/extensions/behavior/BehaviourComponents";
 export { ShadowCatcher } from "../ShadowCatcher";
 export { ShapeModule } from "../ParticleSystemModules";
 export { SignalAsset } from "../timeline/SignalAsset";
@@ -146,24 +161,33 @@
 export { SyncedCamera } from "../SyncedCamera";
 export { SyncedRoom } from "../SyncedRoom";
 export { SyncedTransform } from "../SyncedTransform";
+export { TapGestureTrigger } from "../export/usdz/extensions/behavior/BehaviourComponents";
 export { TeleportTarget } from "../webxr/WebXRController";
 export { TestRunner } from "../TestRunner";
 export { TestSimulateUserData } from "../TestRunner";
 export { Text } from "../ui/Text";
+export { TextBuilder } from "../export/usdz/extensions/USDZText";
+export { TextExtension } from "../export/usdz/extensions/USDZText";
 export { TextureSheetAnimationModule } from "../ParticleSystemModules";
 export { TiltShiftEffect } from "../postprocessing/Effects/TiltShiftEffect";
 export { ToneMapping } from "../postprocessing/Effects/Tonemapping";
 export { TrailModule } from "../ParticleSystemModules";
 export { TransformData } from "../export/usdz/extensions/Animation";
 export { TransformGizmo } from "../TransformGizmo";
+export { TriggerBuilder } from "../export/usdz/extensions/behavior/BehavioursBuilder";
+export { TriggerModel } from "../export/usdz/extensions/behavior/BehavioursBuilder";
 export { UIRaycastUtils } from "../ui/RaycastUtils";
 export { UIRootComponent } from "../ui/BaseUIComponent";
 export { UsageMarker } from "../Interactable";
+export { USDZBehaviours } from "../export/usdz/extensions/behavior/Behaviour";
 export { USDZExporter } from "../export/usdz/USDZExporter";
+export { USDZText } from "../export/usdz/extensions/USDZText";
+export { VariantAction } from "../export/usdz/extensions/behavior/Actions";
 export { VelocityOverLifetimeModule } from "../ParticleSystemModules";
 export { VerticalLayoutGroup } from "../ui/Layout";
 export { VideoPlayer } from "../VideoPlayer";
 export { Vignette } from "../postprocessing/Effects/Vignette";
+export { VisibilityAction } from "../export/usdz/extensions/behavior/BehaviourComponents";
 export { Voip } from "../Voip";
 export { Volume } from "../postprocessing/Volume";
 export { VolumeParameter } from "../postprocessing/VolumeParameter";

src/engine/engine_gameobject.ts CHANGED Viewed

@@ -163,8 +163,9 @@
     return internalForEachComponent(instance, cb, recursive);
 }
-export function* foreachComponentEnumerator<T extends IComponent>(instance: Object3D, type?: Constructor<T>, includeChildren: boolean = false): Generator<T> {
+export function* foreachComponentEnumerator<T extends IComponent>(instance: Object3D, type?: Constructor<T>, includeChildren: boolean = false, maxLevel: number = 999, _currentLevel: number = 0): Generator<T> {
     if (!instance?.userData.components) return;
+    if (_currentLevel > maxLevel) return;
     for (const comp of instance.userData.components) {
         if (type && comp?.isComponent === true && comp instanceof type) {
             yield comp;
@@ -175,7 +176,7 @@
     }
     if (includeChildren === true) {
         for (const ch of instance.children) {
-            yield* foreachComponentEnumerator(ch, type, true);
+            yield* foreachComponentEnumerator(ch, type, true, maxLevel, _currentLevel + 1);
         }
     }
 }

src/engine/engine_three_utils.ts CHANGED Viewed

@@ -98,8 +98,8 @@
     const tempVec = _worldScale2;
     const obj2 = obj.parent;
     obj2.getWorldScale(tempVec);
-    tempVec.divide(vec);
-    obj.scale.copy(tempVec);
+    obj.scale.copy(vec);
+    obj.scale.divide(tempVec);
 }
 const _forward = new Vector3();

src/engine-components/export/usdz/Extension.ts CHANGED Viewed

@@ -1,12 +1,11 @@
-import { USDZObject } from "./types";
+import { USDObject } from "./ThreeUSDZExporter";
+export interface IUSDExporterExtension {
-export interface IUSDZExporterExtension {
     get extensionName(): string;
     onBeforeBuildDocument?(context);
     onAfterBuildDocument?(context);
-    onExportObject?(object, model : USDZObject, context);
+    onExportObject?(object, model : USDObject, context);
     onAfterSerialize?(context);
-    onAfterHierarchy?(context);
+    onAfterHierarchy?(context, writer : any);
 }

src/engine-components/ui/Graphic.ts CHANGED Viewed

@@ -9,6 +9,8 @@
 import { GameObject } from '../Component';
 import SimpleStateBehavior from "three-mesh-ui/examples/behaviors/states/SimpleStateBehavior"
 import { Outline } from './Outline';
+import { BehaviorExtension, UsdzBehaviour } from '../../engine-components/export/usdz/extensions/behavior/Behaviour';
+import { USDObject } from '../../engine-components/export/usdz/ThreeUSDZExporter';
 const _colorStateObject: { backgroundColor: Color, backgroundOpacity: number } = {
     backgroundColor: new Color(1, 1, 1),

src/engine-components/ui/Image.ts CHANGED Viewed

@@ -49,9 +49,9 @@
             if(this.sprite?.texture?.name === "Knob") {
                 opts.borderRadius = 999;
             }
-            opts.borderColor = new Color(.4, .4, .4);
-            opts.borderOpacity = this.color.alpha;
-            opts.borderWidth = .3;
+            // opts.borderColor = new Color(.4, .4, .4);
+            // opts.borderOpacity = this.color.alpha;
+            // opts.borderWidth = .3;
         }
     }

src/engine-components/ui/Interfaces.ts CHANGED Viewed

@@ -1,24 +1,48 @@
+import { Behaviour } from "../Component";
 import { IComponent } from "../../engine/engine_types";
 export interface ICanvas {
-    get screenspace() : boolean;
+    get isCanvas(): boolean;
+    get screenspace(): boolean;
+    registerTransform(rt: IRectTransform);
+    unregisterTransform(rt: IRectTransform);
 }
 export interface ICanvasGroup {
-    get isCanvasGroup() : boolean;
+    get isCanvasGroup(): boolean;
     blocksRaycasts: boolean;
     interactable: boolean;
 }
 export interface IGraphic extends IComponent {
-    get isGraphic() : boolean;
+    get isGraphic(): boolean;
     raycastTarget: boolean;
 }
 export interface IRectTransform extends IComponent {
+    get isDirty(): boolean;
+    markDirty();
+    updateTransform();
+}
+export interface IRectTransformChangedReceiver {
+    onParentRectTransformChanged(comp: IRectTransform): void;
 }
-export interface IRectTransformChangedReceiver {
-    onParentRectTransformChanged(comp : IRectTransform) : void;
-}
+export interface ILayoutGroup extends IComponent {
+    get isLayoutGroup(): boolean;
+    get isDirty(): boolean;
+    updateLayout();
+}
+// export abstract class LayoutGroup extends Behaviour implements IRectTransformChangedReceiver, ILayoutGroup {
+//     get isLayoutGroup(): boolean {
+//         return true;
+//     }
+//     updateLayout() {
+//         throw new Error("Method not implemented.");
+//     }
+//     onParentRectTransformChanged(comp: IRectTransform): void {
+//         throw new Error("Method not implemented.");
+//     }
+// }

src/engine-components/ui/Layout.ts CHANGED Viewed

@@ -1,17 +1,316 @@
-import { Behaviour } from "../Component";
+import { ILayoutGroup, IRectTransform, IRectTransformChangedReceiver } from "./Interfaces";
+import { Behaviour, GameObject } from "../Component";
+import { serializable } from "../../engine/engine_serialization";
+import { Canvas } from "./Canvas";
+import { RectTransform } from "./RectTransform";
+import { getParam } from "../../engine/engine_utils";
-export class LayoutGroup extends Behaviour {
+const debug = getParam("debuguilayout");
+export class Padding {
+    @serializable()
+    left: number = 0;
+    @serializable()
+    right: number = 0;
+    @serializable()
+    top: number = 0;
+    @serializable()
+    bottom: number = 0;
+    get vertical() {
+        return this.top + this.bottom;
+    }
+    get horizontal() {
+        return this.left + this.right;
+    }
+}
+export enum TextAnchor {
+    UpperLeft = 0,
+    UpperCenter = 1,
+    UpperRight = 2,
+    MiddleLeft = 3,
+    MiddleCenter = 4,
+    MiddleRight = 5,
+    LowerLeft = 6,
+    LowerCenter = 7,
+    LowerRight = 8,
+    Custom = 9
+}
+enum Axis {
+    Horizontal = "x",
+    Vertical = "y"
+}
+export abstract class LayoutGroup extends Behaviour implements ILayoutGroup {
+    private _rectTransform: RectTransform | null = null;
+    private get rectTransform() {
+        return this._rectTransform;
+    }
+    onParentRectTransformChanged(_comp: IRectTransform): void {
+        this._needsUpdate = true;
+    }
+    private _needsUpdate: boolean = false;
+    get isDirty(): boolean {
+        return this._needsUpdate;
+    }
+    get isLayoutGroup(): boolean {
+        return true;
+    }
+    updateLayout() {
+        if (!this._rectTransform) return;
+        if (debug)
+            console.warn("Layout Update", this.context.time.frame, this.name);
+        this._needsUpdate = false;
+        this.onCalculateLayout(this._rectTransform);
+    }
+    // onBeforeRender(): void {
+    //     this.updateLayout();
+    // }
+    @serializable()
+    childAlignment: TextAnchor = TextAnchor.UpperLeft;
+    @serializable()
     reverseArrangement: boolean = false;
+    @serializable()
+    spacing: number = 0;
+    @serializable(Padding)
+    padding!: Padding;
+    @serializable()
+    minWidth: number = 0;
+    @serializable()
+    minHeight: number = 0;
+    @serializable()
+    flexibleHeight: number = 0;
+    @serializable()
+    flexibleWidth: number = 0;
+    @serializable()
+    preferredHeight: number = 0;
+    @serializable()
+    preferredWidth: number = 0;
+    start() {
+        this._needsUpdate = true;
+    }
+    onEnable(): void {
+        if(debug) console.log(this.name, this);
+        this._rectTransform = this.gameObject.getComponent(RectTransform);
+        const canvas = this.gameObject.getComponentInParent(Canvas);
+        if (canvas) {
+            canvas.registerLayoutGroup(this);
+        }
+        this._needsUpdate = true;
+    }
+    onDisable(): void {
+        const canvas = this.gameObject.getComponentInParent(Canvas);
+        if (canvas) {
+            canvas.unregisterLayoutGroup(this);
+        }
+    }
+    protected abstract onCalculateLayout(rt: RectTransform);
+    // for animation:
+    private set m_Spacing(val) {
+        if (val === this.spacing) return;
+        this._needsUpdate = true;
+        this.spacing = val;
+    }
+    get m_Spacing() {
+        return this.spacing;
+    }
 }
-export class VerticalLayoutGroup extends LayoutGroup {
+export abstract class HorizontalOrVerticalLayoutGroup extends LayoutGroup {
+    @serializable()
+    childControlHeight: boolean = true;
+    @serializable()
+    childControlWidth: boolean = true;
+    @serializable()
+    childForceExpandHeight: boolean = false;
+    @serializable()
+    childForceExpandWidth: boolean = false;
+    @serializable()
+    childScaleHeight: boolean = false;
+    @serializable()
+    childScaleWidth: boolean = false;
+    protected abstract get primaryAxis(): Axis;
+    protected onCalculateLayout(rect: RectTransform) {
+        const axis = this.primaryAxis;
+        const totalWidth = rect.width;
+        let actualWidth = totalWidth;
+        const totalHeight = rect.height;
+        let actualHeight = totalHeight;
+        actualWidth -= this.padding.horizontal;
+        actualHeight -= this.padding.vertical;
+        // console.log(rt.name, "width=" + totalWidth + ", height=" + totalHeight)
+        const paddingAxis = axis === Axis.Horizontal ? this.padding.horizontal : this.padding.vertical;
+        const isHorizontal = axis === Axis.Horizontal;
+        const isVertical = !isHorizontal;
+        const otherAxis = isHorizontal ? "y" : "x";
+        const controlSize = isHorizontal ? this.childControlWidth : this.childControlHeight;
+        const controlSizeOtherAxis = isHorizontal ? this.childControlHeight : this.childControlWidth;
+        const forceExpandSize = isHorizontal ? this.childForceExpandWidth : this.childForceExpandHeight;
+        const forceExpandSizeOtherAxis = isHorizontal ? this.childForceExpandHeight : this.childForceExpandWidth;
+        const actualExpandSize = isHorizontal ? actualHeight : actualWidth;
+        const totalSpace = isHorizontal ? totalWidth : totalHeight;
+        // 0 is left/top, 0.5 is middle, 1 is right/bottom
+        const alignmentOnAxis = 0.5 * (isHorizontal ? this.childAlignment % 3 : Math.floor(this.childAlignment / 3));
+        let start = 0;
+        if (isHorizontal) {
+            start += this.padding.left;
+        }
+        else
+            start += this.padding.top;
+        // Calculate total size of the elements
+        let totalChildSize = 0;
+        let actualRectTransformChildCount = 0;
+        for (let i = 0; i < this.gameObject.children.length; i++) {
+            const ch = this.gameObject.children[i];
+            const rt = GameObject.getComponent(ch, RectTransform);
+            if (rt?.activeAndEnabled) {
+                actualRectTransformChildCount += 1;
+                if (isHorizontal) {
+                    totalChildSize += rt.width;
+                }
+                else {
+                    totalChildSize += rt.height;
+                }
+            }
+        }
+        let sizePerChild = 0;
+        const totalSpacing = this.spacing * (actualRectTransformChildCount - 1)
+        if (forceExpandSize || controlSize) {
+            let size = 0;
+            if (isHorizontal) {
+                size = actualWidth -= totalSpacing;
+            }
+            else {
+                size = actualHeight -= totalSpacing;
+            }
+            if (actualRectTransformChildCount > 0)
+                sizePerChild = size / actualRectTransformChildCount;
+        }
+        let leftOffset = 0;
+        leftOffset += this.padding.left;
+        leftOffset -= this.padding.right;
+        if (alignmentOnAxis !== 0) {
+            start = totalSpace - totalChildSize;
+            start *= alignmentOnAxis;
+            start -= totalSpacing * alignmentOnAxis;
+            if (isHorizontal) {
+                start -= this.padding.right * alignmentOnAxis;
+                start += this.padding.left * (1 - alignmentOnAxis);
+                if (start < this.padding.left) {
+                    start = this.padding.left;
+                }
+            }
+            else {
+                start -= this.padding.bottom * alignmentOnAxis;
+                start += this.padding.top * (1 - alignmentOnAxis);
+                if (start < this.padding.top) {
+                    start = this.padding.top;
+                }
+            }
+        }
+        // Apply layout
+        let k = 0;
+        for (let i = 0; i < this.gameObject.children.length; i++) {
+            const ch = this.gameObject.children[i];
+            const rt = GameObject.getComponent(ch, RectTransform);
+            if (rt?.activeAndEnabled) {
+                rt.pivot?.set(.5, .5);
+                // Horizontal padding
+                const x = totalWidth * .5 + leftOffset * .5;
+                if (rt.anchoredPosition.x !== x)
+                    rt.anchoredPosition.x = x;
+                const y = totalHeight * -.5
+                if (rt.anchoredPosition.y !== y)
+                    rt.anchoredPosition.y = y;
+                // Set the size for the secondary axis (e.g. height for a horizontal layout group)
+                if (forceExpandSizeOtherAxis && controlSizeOtherAxis && rt.sizeDelta[otherAxis] !== actualExpandSize) {
+                    rt.sizeDelta[otherAxis] = actualExpandSize;
+                }
+                // Set the size for the primary axis (e.g. width for a horizontal layout group)
+                if (forceExpandSize && controlSize && rt.sizeDelta[axis] !== sizePerChild) {
+                    rt.sizeDelta[axis] = sizePerChild
+                }
+                const size = isHorizontal ? rt.width : rt.height;
+                let halfSize = size * .5;
+                start += halfSize;
+                if (forceExpandSize) {
+                    let preferredStart = sizePerChild * (k + 1) - sizePerChild * .5;
+                    if (preferredStart > start)
+                        start = preferredStart;
+                }
+                let value = start;
+                if (axis === Axis.Vertical)
+                    value = -value;
+                // Only set the position if it's not already the correct one to avoid triggering the rectTransform dirty event
+                if (rt.anchoredPosition[axis] !== value) {
+                    rt.anchoredPosition[axis] = value
+                }
+                start += halfSize;
+                start += this.spacing;
+                k += 1;
+            }
+        }
+    }
 }
-export class HorizontalLayoutGroup extends LayoutGroup {
+export class VerticalLayoutGroup extends HorizontalOrVerticalLayoutGroup {
+    protected get primaryAxis() {
+        return Axis.Vertical;
+    }
 }
+export class HorizontalLayoutGroup extends HorizontalOrVerticalLayoutGroup {
+    protected get primaryAxis() {
+        return Axis.Horizontal;
+    }
+}
 export class GridLayoutGroup extends LayoutGroup {
+    protected onCalculateLayout() {
+    }
 }

src/engine-components/utils/LookAt.ts CHANGED Viewed

@@ -1,21 +1,74 @@
 import { serializable } from "../../engine/engine_serialization";
 import { Behaviour } from "../Component";
-import { Object3D } from "three";
-import { getWorldPosition, lookAtInverse } from "../../engine/engine_three_utils";
+import { Matrix4, Object3D, Quaternion, Vector3 } from "three";
+import { getWorldPosition, getWorldQuaternion, setWorldQuaternion } from "../../engine/engine_three_utils";
-export class LookAt extends Behaviour {
+import { USDObject } from "../../engine-components/export/usdz/ThreeUSDZExporter";
+import { UsdzBehaviour } from "../../engine-components/export/usdz/extensions/behavior/Behaviour";
+import { ActionBuilder, BehaviorModel, TriggerBuilder, USDVec3 } from "../../engine-components/export/usdz/extensions/behavior/BehavioursBuilder";
+export class LookAt extends Behaviour implements UsdzBehaviour {
     @serializable(Object3D)
     target?: Object3D;
     @serializable()
     invertForward: boolean = false;
+    @serializable()
+    keepUpDirection: boolean = true;
+    @serializable()
+    copyTargetRotation: boolean = false;
+    private static flipYQuat: Quaternion = new Quaternion().setFromAxisAngle(new Vector3(0, 1, 0), Math.PI);
     onBeforeRender(): void {
-        if (!this.target) return;
-        if (!this.invertForward)
-            this.gameObject.lookAt(getWorldPosition(this.target!));
+        let target: Object3D | null | undefined = this.target;
+        if (!target) target = this.context.mainCamera;
+        if (!target) return;
+        const lookTarget = getWorldPosition(target);
+        const lookFrom = getWorldPosition(this.gameObject);
+        if (this.keepUpDirection)
+            lookTarget.y = lookFrom.y;
+        if (this.copyTargetRotation)
+            setWorldQuaternion(this.gameObject, getWorldQuaternion(target));
         else
-            lookAtInverse(this.gameObject, getWorldPosition(this.target!));
+            this.gameObject.lookAt(lookTarget);
+        if (this.invertForward)
+            this.gameObject.quaternion.multiply(LookAt.flipYQuat);
     }
+    createBehaviours(ext, model: USDObject, _context) {
+        if (model.uuid === this.gameObject.uuid) {
+            let alignmentTarget = model;
+            // not entirely sure why we need to do this - looks like LookAt with up vector doesn't work properly in
+            // QuickLook, so we need to introduce an empty parent and rotate the model by 90° around Y
+            if (this.keepUpDirection) {
+                const parent = USDObject.createEmptyParent(model);
+                alignmentTarget = parent;
+                // rotate by 90° - counter-rotation on the parent makes sure
+                // that without Preliminary Behaviours it still looks right
+                parent.matrix.multiply(new Matrix4().makeRotationZ(Math.PI / 2));
+                model.matrix.multiply(new Matrix4().makeRotationZ(-Math.PI / 2));
+            }
+            const lookAt = new BehaviorModel("lookat " + this.name,
+                TriggerBuilder.sceneStartTrigger(),
+                ActionBuilder.lookAtCameraAction(
+                    alignmentTarget,
+                    undefined,
+                    this.invertForward ? USDVec3.back : USDVec3.forward,
+                    this.keepUpDirection ? USDVec3.up : USDVec3.zero
+                ),
+            );
+            ext.addBehavior(lookAt);
+        }
+    }
 }

src/engine-components/postprocessing/PostProcessingHandler.ts CHANGED Viewed

@@ -1,4 +1,4 @@
-import { HalfFloatType, sRGBEncoding, WebGLRenderTarget } from "three";
+import { HalfFloatType } from "three";
 import { Context } from "../../engine/engine_setup";
 import { getParam, isMobileDevice } from "../../engine/engine_utils";
 import { BloomEffect, BrightnessContrastEffect, ChromaticAberrationEffect, DepthDownsamplingPass, DepthOfFieldEffect, Effect, EffectComposer, EffectPass, HueSaturationEffect, NormalPass, Pass, PixelationEffect, RenderPass, SelectiveBloomEffect, SSAOEffect, VignetteEffect } from "postprocessing";

src/engine-components/ui/RectTransform.ts CHANGED Viewed

@@ -7,10 +7,11 @@
 import { getParam } from "../../engine/engine_utils";
 import { onChange } from "./Utils";
 import { foreachComponentEnumerator } from "../../engine/engine_gameobject";
-import { ICanvas, IRectTransform, IRectTransformChangedReceiver } from "./Interfaces";
+import { ICanvas, IRectTransform, IRectTransformChangedReceiver, ILayoutGroup } from "./Interfaces";
 import { GameObject } from '../Component';
 const debug = getParam("debugui");
+const debugLayout = getParam("debuguilayout");
 export class Size {
     width!: number;
@@ -30,7 +31,7 @@
 export class RectTransform extends BaseUIComponent implements IRectTransform, IRectTransformChangedReceiver {
-    offset: number = 0.05;
+    offset: number = 0.1;
     // @serializable(Object3D)
     // root? : Object3D;
@@ -94,14 +95,13 @@
         return this.sizeDelta.y;
     }
-    private lastMatrixWorld!: Matrix4;
+    // private lastMatrixWorld!: Matrix4;
     private lastMatrix!: Matrix4;
     private rectBlock!: Object3D;
     private _transformNeedsUpdate: boolean = false;
     awake() {
         super.awake();
-        this.lastMatrixWorld = new Matrix4();
         this.lastMatrix = new Matrix4();
         this.rectBlock = new Object3D();;
         this.rectBlock.position.z = .1;
@@ -112,9 +112,10 @@
         // TODO: we need to replace this with the watch that e.g. Rigibody is using (or the one in utils?)
         // perhaps we can also just manually check the few properties in the update loops?
-        onChange(this, "_anchoredPosition", () => { this._transformNeedsUpdate = true; });
-        onChange(this, "sizeDelta", () => { this._transformNeedsUpdate = true; });
-        onChange(this, "pivot", () => { this._transformNeedsUpdate = true; });
+        // TODO: check if value actually changed, this is called on assignment
+        onChange(this, "_anchoredPosition", () => { this.markDirty(); });
+        onChange(this, "sizeDelta", () => { this.markDirty(); });
+        onChange(this, "pivot", () => { this.markDirty(); });
         // When exported with an anchored position offset we remove it here
         // because it would otherwise be applied twice when the anchoring is animated
@@ -135,29 +136,68 @@
         super.onEnable();
         this.addShadowComponent(this.rectBlock);
         this._transformNeedsUpdate = true;
+        this.Canvas?.registerTransform(this);
     }
     onDisable() {
         super.onDisable();
         this.removeShadowComponent();
+        this.Canvas?.unregisterTransform(this);
     }
-    onParentRectTransformChanged(_comp: IRectTransform) {
+    onParentRectTransformChanged(comp: IRectTransform) {
+        if (this._transformNeedsUpdate) return;
         // When the parent rect transform changes we have to to recalculate our transform
+        this.onApplyTransform(debugLayout ? `${comp.name} changed` : undefined);
+    }
+    get isDirty() {
+        if(!this._transformNeedsUpdate) this._transformNeedsUpdate = !this.lastMatrix.equals(this.gameObject.matrix);
+        return this._transformNeedsUpdate;
+    }
+    // private _copyMatrixAfterRender: boolean = false;
+    markDirty() {
+        if (this._transformNeedsUpdate) return;
+        if (debugLayout) console.warn("RectTransform markDirty()", this.name)
         this._transformNeedsUpdate = true;
-        this.applyTransform();
+        // If mark dirty is called explictly we want to allow updating the transform again when updateTransform is called
+        // if we dont reset it here we get delayed layout updates
+        this._lastUpdateFrame = -1;
     }
+    /** Will update the transforms if it changed or is dirty */
+    updateTransform() {
+        // TODO: instead of checking matrix again it would perhaps be better to test if position, rotation or scale have changed individually?
+        const transformChanged = this._transformNeedsUpdate || !this.lastMatrix.equals(this.gameObject.matrix);// || !this.lastMatrixWorld.equals(this.gameObject.matrixWorld);
+        if (transformChanged && this.canUpdate()) {
+            this.onApplyTransform(this._transformNeedsUpdate ? "Marked dirty" : "Matrix changed");
+        }
+    }
     private _parentRectTransform?: RectTransform;
+    private _lastUpdateFrame: number = -1;
-    private applyTransform() {
+    private canUpdate() {
+        return this._transformNeedsUpdate && this.activeAndEnabled && this._lastUpdateFrame !== this.context.time.frame;
+    }
+    private onApplyTransform(reason?: string) {
+        // TODO: need to improve the update logic, with this UI updates have some frame delay but dont happen exponentially per hierarchy
+        if (this.context.time.frameCount === this._lastUpdateFrame) return;
+        this._lastUpdateFrame = this.context.time.frameCount;
         const uiobject = this.shadowComponent;
         if (!uiobject) return;
-        this._transformNeedsUpdate = false;
         this._parentRectTransform = GameObject.getComponentInParent(this.gameObject.parent!, RectTransform) as RectTransform;
-        if (debug) console.log("RectTransform ApplyTransform", this.name, this.isRoot());
+        this._transformNeedsUpdate = false;
+        this.lastMatrix.copy(this.gameObject.matrix);
+        if (debugLayout) console.warn("RectTransform → ApplyTransform", this.name + " because " + reason);
         if (!this.isRoot()) {
             // Reset temp matrix
             uiobject.matrix.identity();
@@ -183,8 +223,7 @@
             tempMatrix.setPosition(tempVec.x, tempVec.y, tempVec.z);
             uiobject.matrix.premultiply(tempMatrix);
             // apply scale if necessary
-            if (this.gameObject.scale.x || this.gameObject.scale.y || this.gameObject.scale.z)
-                uiobject.matrix.scale(this.gameObject.scale);
+            uiobject.matrix.scale(this.gameObject.scale);
         }
         else {
             // We have to rotate the canvas when it's in worldspace
@@ -192,43 +231,29 @@
             if (!canvas.screenspace) uiobject.rotation.y = Math.PI;
         }
-        this._copyMatrixAfterRender = true;
-        this.lastMatrix.copy(this.gameObject.matrix);
         // iterate other components on this object that might need to know about the transform change
         // e.g. Graphic components should update their width and height
         const includeChildren = true;
-        for (const comp of foreachComponentEnumerator(this.gameObject, BaseUIComponent, includeChildren)) {
+        for (const comp of foreachComponentEnumerator(this.gameObject, BaseUIComponent, includeChildren, 1)) {
             if (comp === this) continue;
+            if (!comp.activeAndEnabled) continue;
             const callback = comp as any as IRectTransformChangedReceiver;
-            if (callback.onParentRectTransformChanged)
+            if (callback.onParentRectTransformChanged) {
+                // if (debugLayout) console.log(`RectTransform ${this.name} → call`, comp.name + "/" + comp.constructor.name)
                 callback.onParentRectTransformChanged(this);
+            }
         }
-    }
-    private _copyMatrixAfterRender: boolean = false;
-    markDirty() {
-        this._transformNeedsUpdate = true;
+        // const layout = GameObject.getComponentInParent(this.gameObject, ILayoutGroup);
     }
+    // onAfterRender() {
+    //     if (this._copyMatrixAfterRender) {
+    //         // can we only have this event when the transform changed in this frame? Otherwise all RectTransforms will be iterated. Not sure what is better
+    //         this.lastMatrixWorld.copy(this.gameObject.matrixWorld);
+    //     }
+    // }
-    onBeforeRender() {
-        // TODO: instead of checking matrix again it would perhaps be better to test if position, rotation or scale have changed individually?
-        const transformChanged = this.gameObject.matrixWorldNeedsUpdate || this._transformNeedsUpdate || !this.lastMatrixWorld.equals(this.gameObject.matrixWorld) || !this.lastMatrix.equals(this.gameObject.matrix);
-        if (transformChanged)
-        {
-            this.applyTransform();
-        }
-    }
-    onAfterRender() {
-        if (this._copyMatrixAfterRender) {
-            // can we only have this event when the transform changed in this frame? Otherwise all RectTransforms will be iterated. Not sure what is better
-            this.lastMatrixWorld.copy(this.gameObject.matrixWorld);
-        }
-    }
     /** applies the position offset to the passed in vector */
     private applyAnchoring(pos: Vector3) {
         pos.x += this.anchoredPosition.x;

src/engine-components/export/usdz/types.ts DELETED Viewed

@@ -1,39 +0,0 @@
-import { Object3D, Matrix4, Material, BufferGeometry } from "three";
-// keep in sync with USDZExporter.js
-/** implementation is in three */
-export declare class USDZDocument {
-    name: string;
-    get isDocumentRoot(): boolean;
-    add(obj: USDZObject);
-    remove(obj: USDZObject);
-    traverse(callback: (obj: USDZObject) => void);
-    findById(uuid: string): USDZObject | undefined;
-    get isDynamic(): boolean;
-}
-/** implementation is in three */
-export declare class USDZObject {
-    static createEmptyParent(parent: USDZObject);
-    uuid: string;
-    name: string;
-    matrix: Matrix4;
-    material: Material;
-    geometry: BufferGeometry;
-    parent: USDZObject | USDZDocument | null;
-    children: USDZObject[];
-    _eventListeners: { [event: string]: Function[] };
-    isDynamic: boolean;
-    is(obj: Object3D): boolean;
-    isEmpty(): boolean;
-    clone();
-    getPath();
-    add(child: USDZObject);
-    remove(child: USDZObject);
-    addEventListener(evt: string, listener: Function);
-    removeEventListener(evt: string, listener: Function);
-}

src/engine-components/export/usdz/USDZExporter.ts CHANGED Viewed

@@ -1,15 +1,15 @@
 import { delay, getParam, isiOS, isMobileDevice, isSafari } from "../../../engine/engine_utils";
-import { Object3D, Color, Mesh, Matrix4 } from "three";
-import { USDZExporter as ThreeUSDZExporter } from "three/examples/jsm/exporters/USDZExporter";
+import { Object3D, Mesh, Matrix4 } from "three";
+import { USDZExporter as ThreeUSDZExporter } from "./ThreeUSDZExporter";
 import { AnimationExtension } from "./extensions/Animation"
 import { ensureQuicklookLinkIsCreated } from "./utils/quicklook";
 import { getFormattedDate } from "./utils/timeutils";
 import { registerAnimatorsImplictly } from "./utils/animationutils";
-import { IUSDZExporterExtension } from "./Extension";
+import { IUSDExporterExtension } from "./Extension";
 import { Behaviour, GameObject } from "../../Component";
 import { WebXR } from "../../webxr/WebXR"
 import { serializable } from "../../../engine/engine_serialization";
-import { isDevEnvironment, showBalloonMessage, showBalloonWarning } from "../../../engine/debug/debug";
+import { showBalloonMessage, showBalloonWarning } from "../../../engine/debug/debug";
 import { Context } from "../../../engine/engine_setup";
 import { WebARSessionRoot } from "../../webxr/WebARSessionRoot";
 import { hasProLicense } from "../../../engine/engine_license";
@@ -32,7 +32,7 @@
 export class USDZExporter extends Behaviour {
     @serializable(Object3D)
-    objectToExport?: THREE.Object3D;
+    objectToExport?: Object3D;
     @serializable()
     autoExportAnimations: boolean = false;
@@ -43,13 +43,20 @@
     @serializable(QuickLookOverlay)
     overlay?: QuickLookOverlay;
-    extensions: IUSDZExporterExtension[] = [];
+    // Currently not exposed to integrations - not fully tested. Set from code (e.g. image tracking)
+    @serializable()
+    anchoringType: "plane" | "image" | "face" | "none" = "plane";
+    // Currently not exposed to integrations - not fully tested. Set from code (e.g. image tracking)
+    @serializable()
+    planeAnchoringAlignment: "horizontal" | "vertical" | "any" = "horizontal";
+    extensions: IUSDExporterExtension[] = [];
     private link!: HTMLAnchorElement;
     private webxr?: WebXR;
     private webARSessionRoot: WebARSessionRoot | undefined;
     start() {
         if (debug) {
             console.log(this);
@@ -72,13 +79,11 @@
             this.exportAsync();
         });
-        if (!this.objectToExport) this.objectToExport = this.gameObject;
-        if (isDevEnvironment() && (!this.objectToExport?.children?.length && !(this.objectToExport as Mesh)?.isMesh)) {
-            showBalloonWarning("USDZ Exporter has nothing to export");
-            console.warn("USDZExporter has no objects to export assigned:", this)
-        }
+        // fall back to this object or to the scene if it's empty and doesn't have a mesh
+        if (!this.objectToExport)
+            this.objectToExport = this.gameObject;
+        if (!this.objectToExport?.children?.length && !(this.objectToExport as Mesh)?.isMesh)
+            this.objectToExport = this.context.scene;
     }
@@ -135,13 +140,30 @@
         this.dispatchEvent(new CustomEvent("before-export", { detail: eventArgs }))
         let name = this.exportFileName ?? this.objectToExport?.name ?? this.name;
-        if (debug) name += "-" + getFormattedDate();
-        else if (!hasProLicense()) name = name + " - Made with Needle";
+        if (!hasProLicense()) name += "-MadeWithNeedle";
+        name += "-" + getFormattedDate(); // seems iOS caches the file in some cases, this ensures we always have a fresh file
         //@ts-ignore
         exporter.debug = debug;
+        // sanitize anchoring types
+        if (this.anchoringType !== "plane" && this.anchoringType !== "none" && this.anchoringType !== "image" && this.anchoringType !== "face")
+            this.anchoringType = "plane";
+        if (this.planeAnchoringAlignment !== "horizontal" && this.planeAnchoringAlignment !== "vertical" && this.planeAnchoringAlignment !== "any")
+            this.planeAnchoringAlignment = "horizontal";
         //@ts-ignore
-        const arraybuffer = await exporter.parse(this.objectToExport, extensions);
+        const arraybuffer = await exporter.parse(this.objectToExport, {
+            ar: {
+                anchoring: {
+                    type: this.anchoringType,
+                }
+            },
+            planeAnchoring: {
+                alignment: this.planeAnchoringAlignment,
+            },
+            extensions: extensions
+        });
         const blob = new Blob([arraybuffer], { type: 'application/octet-stream' });
         this.dispatchEvent(new CustomEvent("after-export", { detail: eventArgs }))

src/engine-components/webxr/WebXRImageTracking.ts CHANGED Viewed

@@ -1,10 +1,14 @@
-import { WebXR } from "./WebXR";
+import { WebXR, WebXREvent } from "./WebXR";
 import { serializable } from "../../engine/engine_serialization";
 import { Behaviour, GameObject } from "../Component";
 import { Object3D, Quaternion, Vector3 } from "three";
 import { CircularBuffer, getParam } from "../../engine/engine_utils";
 import { AssetReference } from "../../engine/engine_addressables";
+import { showBalloonWarning } from "../../engine/debug";
+import { USDZExporter } from "../../engine-components/export/usdz/USDZExporter";
+import { USDZExporterContext, USDWriter, imageToCanvas } from "../../engine-components/export/usdz/ThreeUSDZExporter";
 // https://github.com/immersive-web/marker-tracking/blob/main/explainer.md
 const debug = getParam("debugimagetracking");
@@ -18,16 +22,6 @@
     readonly measuredSize: number;
     readonly state: "tracked" | "emulated";
-    // private _matrix: Matrix4 | null = null;
-    // private get matrix(): Matrix4 {
-    //     if (!this._matrix) {
-    //         // this._matrix = WebXRTrackedImage._matrixBuffer.get();
-    //         // const matrix = this._pose.transform.matrix;
-    //         // this._matrix.fromArray(matrix);
-    //     }
-    //     return this._matrix!;
-    // }
     /** Copy the image position to a vector */
     getPosition(vec: Vector3) {
         this.ensureTransformData();
@@ -42,13 +36,23 @@
         return quat;
     }
-    applyToObject(object: Object3D) {
+    private static y180 = new Quaternion().setFromAxisAngle(new Vector3(0, 1, 0), Math.PI);
+    applyToObject(object: Object3D, t01: number | undefined = undefined) {
         this.ensureTransformData();
-        object.position.copy(this._position);
-        object.quaternion.copy(this._rotation);
+        // check if position/_position or rotation/_rotation changed more than just a little bit
+        const haveChanged = object.position.distanceToSquared(this._position) > 0.05 || object.quaternion.angleTo(this._rotation) > 0.05;
+        if (t01 === undefined || t01 >= 1 || haveChanged) {
+            object.position.copy(this._position);
+            object.quaternion.copy(this._rotation);
+        }
+        else {
+            t01 = Math.max(0, Math.min(1, t01));
+            object.position.lerp(this._position, t01);
+            object.quaternion.slerp(this._rotation, t01);
+        }
+        object.quaternion.multiply(WebXRTrackedImage.y180);
     }
-    // private static _matrixBuffer: CircularBuffer<Matrix4> = new CircularBuffer(() => new Matrix4(), 20);
     private static _positionBuffer: CircularBuffer<Vector3> = new CircularBuffer(() => new Vector3(), 20);
     private static _rotationBuffer: CircularBuffer<Quaternion> = new CircularBuffer(() => new Quaternion(), 20);
     private _position!: Vector3;
@@ -58,8 +62,14 @@
             this._position = WebXRTrackedImage._positionBuffer.get();
             this._rotation = WebXRTrackedImage._rotationBuffer.get();
             const t = this._pose.transform;
-            this._position.set(-t.position.x, t.position.y, -t.position.z);
-            this._rotation.set(-t.orientation.x, t.orientation.y, -t.orientation.z, t.orientation.w);
+            // when parented to the world, we need to flip data here
+            //this._position.set(-t.position.x, t.position.y, -t.position.z);
+            // this._rotation.set(-t.orientation.x, t.orientation.y, -t.orientation.z, t.orientation.w);
+            // for some reason when parented to the XRRig, we need the original data
+            this._position.set(t.position.x, t.position.y, t.position.z);
+            this._rotation.set(t.orientation.x, t.orientation.y, t.orientation.z, t.orientation.w);
         }
     }
@@ -99,6 +109,32 @@
     imageDoesNotMove: boolean = false;
 }
+class ImageTrackingExtension {
+    get extensionName() { return "image-tracking"; }
+    private filename: string;
+    private widthInMeters: number;
+    private imageData: Uint8Array;
+    constructor(filename: string, imageData: Uint8Array, widthInMeters: number) {
+        this.filename = filename;
+        this.imageData = imageData;
+        this.widthInMeters = widthInMeters;
+    }
+    onAfterHierarchy(_context: USDZExporterContext, writer: USDWriter) {
+        writer.beginBlock(`def Preliminary_ReferenceImage "AnchoringReferenceImage"`);
+        writer.appendLine(`uniform asset image = @tracker/` + this.filename + `@`);
+        writer.appendLine(`uniform double physicalWidth = ` + (this.widthInMeters * 100).toFixed(8));
+        writer.closeBlock();
+    }
+    onAfterSerialize(context: USDZExporterContext) {
+        context.files['tracker/' + this.filename] = this.imageData;
+    }
+}
 export class WebXRImageTracking extends Behaviour {
     @serializable(WebXRImageTrackingModel)
@@ -121,22 +157,43 @@
                     imageElement.addEventListener("load", async () => {
                         const img = await createImageBitmap(imageElement);
                         WebXRImageTracking._imageElements.set(url, img);
+                        // read back Uint8Array to use in USDZ -
+                        // TODO better would be to do that once we actually need it
+                        const canvas = await imageToCanvas( img );
+                        if (canvas) {
+                            const blob = await new Promise( resolve => canvas.toBlob( resolve, 'image/png', 1 ) ) as any;
+                            const arrayBuffer = await blob.arrayBuffer();
+                            const exporter = GameObject.findObjectOfType(USDZExporter);
+                            if (exporter) {
+                                exporter.extensions.push(
+                                    new ImageTrackingExtension("marker.png", new Uint8Array(arrayBuffer), this.trackedImages[0].widthInMeters)
+                                );
+                                exporter.anchoringType = "image";
+                            }
+                        }
                     });
                 }
             }
         }
     }
+    private xr: WebXR | null = null;
     onEnable(): void {
-        WebXR.addEventListener("modify-ar-options", this.onModifyAROptions);
-        WebXR.addEventListener("xrStarted", this.onXRStarted);
-        this.addEventListener("image-tracking", this.onImageTrackingUpdate)
+        this.xr = GameObject.findObjectOfType(WebXR);
+        WebXR.addEventListener(WebXREvent.ModifyAROptions, this.onModifyAROptions);
+        WebXR.addEventListener(WebXREvent.XRStarted, this.onXRStarted);
+        WebXR.addEventListener(WebXREvent.XRUpdate, this.onXRUpdate);
+        this.addEventListener("image-tracking", this.onImageTrackingUpdate);
     }
     onDisable(): void {
-        WebXR.removeEventListener("modify-ar-options", this.onModifyAROptions);
-        WebXR.removeEventListener("xrStarted", this.onXRStarted);
-        this.removeEventListener("image-tracking", this.onImageTrackingUpdate)
+        WebXR.removeEventListener(WebXREvent.ModifyAROptions, this.onModifyAROptions);
+        WebXR.removeEventListener(WebXREvent.XRStarted, this.onXRStarted);
+        WebXR.removeEventListener(WebXREvent.XRUpdate, this.onXRUpdate);
+        this.removeEventListener("image-tracking", this.onImageTrackingUpdate);
     }
     private onModifyAROptions = (event: any) => {
@@ -197,8 +254,12 @@
                     if (asset) {
                         trackedData!.object = asset;
-                        if (asset !== this.gameObject)
-                            this.gameObject.add(asset);
+                        // make sure to parent to the WebXR.rig
+                        if (this.xr) {
+                            this.xr.Rig.add(asset);
+                        }
                         image.applyToObject(asset);
                         if (!asset.activeSelf)
                             GameObject.setActive(asset, true);
@@ -215,6 +276,10 @@
                 if (!trackedData.object) continue;
+                if (this.xr) {
+                    this.xr.Rig.add(trackedData.object);
+                }
                 image.applyToObject(trackedData.object);
                 if (!trackedData.object.activeSelf)
                     GameObject.setActive(trackedData.object, true);
@@ -229,9 +294,17 @@
         }
     };
-    onBeforeRender(frame: XRFrame | null): void {
+    private onXRUpdate = (evt): void => {
+        const frame = evt.frame;
+        if (!frame) return;
+        if (frame.session && !("getImageTrackingResults" in frame)) {
+            showBalloonWarning("Image tracking is currently not supported on this device. On Chrome for Android, you can enable the <a href=\"chrome://flags/#webxr-incubations\">chrome://flags/#webxr-incubations</a> flag.");
+            return;
+        }
         //@ts-ignore
-        if (frame?.session && typeof frame.getImageTrackingResults === "function") {
+        if (frame.session && typeof frame.getImageTrackingResults === "function") {
             //@ts-ignore
             const results = frame.getImageTrackingResults();
             if (results.length) {

src/engine-components/export/usdz/extensions/behavior/Actions.ts ADDED Viewed

@@ -0,0 +1,99 @@
+import { Object3D, Matrix4, Material, BufferGeometry } from "three";
+import { ActionBuilder, ActionModel } from "./BehavioursBuilder";
+import { USDObject, USDDocument } from "../../ThreeUSDZExporter";
+export abstract class DocumentAction {
+    get id(): string { return this.object.uuid; }
+    protected object: Object3D;
+    protected model?: USDObject;
+    constructor(obj: Object3D) {
+        this.object = obj;
+    }
+    apply(document: USDDocument) {
+        if (!this.model) {
+            this.model = document.findById(this.object.uuid);
+            if (!this.model) {
+                console.error("could not find model with id " + this.object.uuid);
+                return;
+            }
+        }
+        this.onApply(document);
+    }
+    protected abstract onApply(document: USDDocument);
+}
+export class VariantAction extends DocumentAction {
+    constructor(obj: Object3D, matrix?: Matrix4, material?: Material, geometry?: BufferGeometry) {
+        super(obj);
+        this.matrix = matrix;
+        this.material = material;
+        this.geometry = geometry;
+    }
+    private matrix: Matrix4 | undefined;
+    private material: Material | undefined;
+    private geometry: BufferGeometry | undefined;
+    protected onApply(_: USDDocument) {
+        const model = this.model;
+        if (!model) return;
+        if (!model.parent?.isDynamic) {
+            USDObject.createEmptyParent(model);
+        }
+        const clone = model.clone();
+        if (this.matrix) clone.matrix = this.matrix;
+        if (this.material) clone.material = this.material;
+        if (this.geometry) clone.geometry = this.geometry;
+        model.parent?.add(clone);
+    }
+    private _enableAction?: ActionModel;
+    private _disableAction?: ActionModel;
+    enable(): ActionModel {
+        if (this._enableAction) return this._enableAction;
+        this._enableAction = ActionBuilder.fadeAction(this.object, 0, true);;
+        return this._enableAction;
+    }
+    disable(): ActionModel {
+        if (this._disableAction) return this._disableAction;
+        this._disableAction = ActionBuilder.fadeAction(this.object, 0, false);;
+        return this._disableAction;
+    }
+}
+export class ActionCollection {
+    private actions: DocumentAction[];
+    private sortedActions?: { [key: string]: DocumentAction[] };
+    constructor(actions: DocumentAction[]) {
+        this.actions = [...actions]
+    }
+    // organize is called once when getting an action for the first time
+    // the sorted actions are baked then and adding new actions will not be added anymore
+    private organize() {
+        this.sortedActions = {};
+        for (const action of this.actions) {
+            const id = action.id;
+            if (!this.sortedActions[id]) {
+                this.sortedActions[id] = [];
+            }
+            this.sortedActions[id].push(action);
+        }
+    }
+    /** returns all document actions affecting the object passed in */
+    getActions(obj: Object3D): DocumentAction[] | null {
+        if (!this.sortedActions) this.organize();
+        return this.sortedActions![obj.uuid];
+    }
+}

src/engine-components/export/usdz/extensions/behavior/Behaviour.ts ADDED Viewed

@@ -0,0 +1,181 @@
+import { Behaviour, GameObject } from "../../../../Component";
+import { USDZExporter } from "../../USDZExporter";
+import { IUSDExporterExtension } from "../../Extension";
+import { USDObject, USDWriter } from "../../ThreeUSDZExporter";
+import { BehaviorModel } from "./BehavioursBuilder";
+import { IContext } from "../../../../../engine/engine_types";
+export interface UsdzBehaviour {
+    createBehaviours?(ext: BehaviorExtension, model: USDObject, context: IContext): void;
+    beforeCreateDocument?(ext: BehaviorExtension, context: IContext): void;
+    afterCreateDocument?(ext: BehaviorExtension, context: IContext): void;
+}
+export class USDZBehaviours extends Behaviour {
+    start() {
+        const exporter = GameObject.findObjectOfType(USDZExporter);
+        if (exporter) {
+            exporter.extensions.push(new BehaviorExtension());
+        }
+    }
+}
+export class BehaviorExtension implements IUSDExporterExtension {
+    get extensionName(): string {
+        return "Behaviour";
+    }
+    private behaviours: BehaviorModel[] = [];
+    addBehavior(beh: BehaviorModel) {
+        this.behaviours.push(beh);
+    }
+    behaviourComponents: Array<UsdzBehaviour> = [];
+    onBeforeBuildDocument(context) {
+        context.root.traverse(e => {
+            GameObject.foreachComponent(e, (comp) => {
+                const c = comp as unknown as UsdzBehaviour;
+                if (
+                    typeof c.createBehaviours === "function" ||
+                    typeof c.beforeCreateDocument === "function" ||
+                    typeof c.afterCreateDocument === "function"
+                ) {
+                    this.behaviourComponents.push(c);
+                    c.beforeCreateDocument?.call(c, this, context);
+                }
+            }, false);
+        });
+    }
+    onExportObject(_object, model: USDObject, context) {
+        for (const beh of this.behaviourComponents) {
+            beh.createBehaviours?.call(beh, this, model, context);
+        }
+    }
+    onAfterBuildDocument(context) {
+        for (const beh of this.behaviourComponents) {
+            if (typeof beh.afterCreateDocument === "function")
+                beh.afterCreateDocument(this, context);
+        }
+        this.behaviourComponents.length = 0;
+    }
+    onAfterHierarchy(context, writer : USDWriter) {
+        if (this.behaviours?.length) {
+            // this.combineBehavioursWithSameTapActions();
+            writer.beginBlock('def Scope "Behaviors"');
+            for (const beh of this.behaviours)
+                beh.writeTo(this, context.document, writer);
+            writer.closeBlock();
+        }
+    }
+    // combine behaviours that have tap triggers on the same object
+    // private combineBehavioursWithSameTapActions() {
+    //     // TODO: if behaviours have different settings (e.g. one is exclusive and another one is not) this wont work - we need more logic for that
+    //     const combined: { [key: string]: { behaviorId: string, trigger: TriggerModel, actions: IBehaviorElement[] } } = {};
+    //     for (let i = this.behaviours.length - 1; i >= 0; i--) {
+    //         const beh = this.behaviours[i];
+    //         const trigger = beh.trigger as TriggerModel;
+    //         if (!Array.isArray(trigger) && TriggerBuilder.isTapTrigger(trigger)) {
+    //             const targetObject = trigger.targetId;
+    //             if (!targetObject) continue;
+    //             if (!combined[targetObject]) {
+    //                 combined[targetObject] = { behaviorId: beh.id, trigger: trigger, actions: [] };
+    //             }
+    //             const action = beh.action;
+    //             combined[targetObject].actions.push(action);
+    //             this.behaviours.splice(i, 1);
+    //         }
+    //     }
+    //     for (const key in combined) {
+    //         const val = combined[key];
+    //         console.log("Combine " + val.actions.length + " actions on " + val.trigger.id, val.actions);
+    //         const beh = new BehaviorModel(val.behaviorId, val.trigger, ActionBuilder.sequence(...val.actions));
+    //         this.behaviours.push(beh);
+    //     }
+    // }
+}
+        // const playAnimationOnTap = new BehaviorModel("b_" + model.name + "_playanim", TriggerBuilder.tapTrigger(model),
+        //     ActionBuilder.parallel(
+        //         ActionBuilder.lookAtCameraAction(model),
+        //         ActionBuilder.sequence(
+        //             //ActionBuilder.startAnimationAction(model, 0, 0, 1, false, true),
+        //             ActionBuilder.emphasize(model, 1, MotionType.Float),
+        //             ActionBuilder.waitAction(1),
+        //             ActionBuilder.emphasize(model, 1, MotionType.Blink),
+        //             ActionBuilder.waitAction(1),
+        //             ActionBuilder.emphasize(model, 1, MotionType.Jiggle),
+        //             ActionBuilder.waitAction(1),
+        //             ActionBuilder.emphasize(model, 1, MotionType.Pulse),
+        //             ActionBuilder.waitAction(1),
+        //             ActionBuilder.emphasize(model, 1, MotionType.Spin),
+        //             ActionBuilder.waitAction(1),
+        //             ActionBuilder.emphasize(model, 1, MotionType.Bounce),
+        //             ActionBuilder.waitAction(1),
+        //             ActionBuilder.emphasize(model, 1, MotionType.Flip),
+        //             ActionBuilder.waitAction(1),
+        //         ).makeLooping()
+        //     ).makeLooping()
+        // );
+        // this.behaviours.push(playAnimationOnTap);
+        // return;
+        // const identityMatrix = new Matrix4().identity();
+        // const emptyParent = new USDZObject(model.name + "_empty", model.matrix);
+        // const parent = model.parent;
+        // parent.add(emptyParent);
+        // model.matrix = identityMatrix;
+        // emptyParent.add(model);
+        // const geometry = new THREE.SphereGeometry(.6, 32, 16);
+        // const modelVariant = new USDZObject(model.name + "_variant", identityMatrix, geometry, new MeshStandardMaterial({ color: 0xff0000 }));
+        // emptyParent.add(modelVariant);
+        // const matrix2 = new Matrix4();
+        // matrix2.makeTranslation(.5, 0, 0);
+        // const modelVariant2 = new USDZObject(model.name + "_variant2", matrix2, geometry, new MeshStandardMaterial({ color: 0xffff00 }));
+        // emptyParent.add(modelVariant2);
+        // const hideVariantOnStart = new BehaviorModel("b_" + model.name + "_start", TriggerBuilder.sceneStartTrigger(), ActionBuilder.fadeAction(modelVariant, 0, false));
+        // this.behaviours.push(hideVariantOnStart);
+        // const showVariant = new BehaviorModel("b_" + model.name + "_show_variant", [TriggerBuilder.tapTrigger(model)], new GroupActionModel("group", [
+        //     ActionBuilder.fadeAction(model, 0, false),
+        //     ActionBuilder.fadeAction(modelVariant, 0, true),
+        // ]));
+        // this.behaviours.push(showVariant);
+        // const showOriginal = new BehaviorModel("b_" + model.name + "_show_original", [
+        //     TriggerBuilder.tapTrigger(modelVariant),
+        //     TriggerBuilder.tapTrigger(modelVariant2)
+        // ],
+        //     new GroupActionModel("group", [
+        //         ActionBuilder.fadeAction([modelVariant, modelVariant2], 0, false),
+        //         //ActionBuilder.waitAction(1),
+        //         ActionBuilder.fadeAction(model, 0, true),
+        //         //ActionBuilder.waitAction(.2),
+        //         ActionBuilder.startAnimationAction(model, 0, 1000, 1, false, true),
+        //         //ActionBuilder.lookAtCameraAction(model, 2, Vec3.forward, Vec3.up),
+        //         //ActionBuilder.waitAction(1),
+        //         //ActionBuilder.fadeAction(modelVariant2, 0, true),
+        //     ]).makeSequence());
+        // this.behaviours.push(showOriginal);

src/engine-components/export/usdz/extensions/behavior/BehaviourComponents.ts ADDED Viewed

@@ -0,0 +1,503 @@
+import { Behaviour, GameObject } from "../../../../Component";
+import { Animator } from "../../../../Animator";
+import { Renderer } from "../../../../Renderer";
+import { serializable } from "../../../../../engine/engine_serialization_decorator";
+import { IPointerClickHandler } from "../../../../ui/PointerEvents";
+import { RegisteredAnimationInfo, UsdzAnimation } from "../Animation";
+import { getWorldPosition, getWorldQuaternion, getWorldScale, setWorldPosition, setWorldQuaternion, setWorldScale } from "../../../../../engine/engine_three_utils";
+import { Object3D, Material, Vector3, Quaternion } from "three";
+import { USDObject } from "../../ThreeUSDZExporter";
+import { BehaviorExtension, UsdzBehaviour } from "./Behaviour";
+import { ActionBuilder, ActionModel, BehaviorModel, MotionType, Space, TriggerBuilder } from "./BehavioursBuilder";
+export class ChangeTransformOnClick extends Behaviour implements IPointerClickHandler, UsdzBehaviour {
+    @serializable(Object3D)
+    object?: Object3D;
+    @serializable(Object3D)
+    target?: Object3D;
+    @serializable()
+    duration: number = 1;
+    @serializable()
+    relativeMotion: boolean = false;
+    private coroutine: Generator | null = null;
+    private targetPos = new Vector3();
+    private targetRot = new Quaternion();
+    private targetScale = new Vector3();
+    private *moveToTarget() {
+        if (!this.target || !this.object) return;
+        const thisPos = getWorldPosition(this.object).clone();
+        const targetPos = getWorldPosition(this.target).clone();
+        const thisRot = getWorldQuaternion(this.object).clone();
+        const targetRot = getWorldQuaternion(this.target).clone();
+        const thisScale = getWorldScale(this.object).clone();
+        const targetScale = getWorldScale(this.target).clone();
+        const dist = thisPos.distanceTo(targetPos);
+        const rotDist = thisRot.angleTo(targetRot);
+        const scaleDist = thisScale.distanceTo(targetScale);
+        if (dist < 0.01 && rotDist < 0.01 && scaleDist < 0.01) {
+            setWorldPosition(this.object, targetPos);
+            setWorldQuaternion(this.object, targetRot);
+            setWorldScale(this.object, targetScale);
+            this.coroutine = null;
+            return;
+        }
+        let t01 = 0;
+        let eased = 0;
+        while (t01 < 1) {
+            t01 += this.context.time.deltaTime / this.duration;
+            if (t01 > 1) t01 = 1;
+            // apply ease-in-out
+            // https://easings.net/
+            eased =  t01 < 0.5 ? 4 * t01 * t01 * t01 : 1 - Math.pow(-2 * t01 + 2, 3) / 2;
+            this.targetPos.lerpVectors(thisPos, targetPos, eased);
+            this.targetRot.slerpQuaternions(thisRot, targetRot, eased);
+            this.targetScale.lerpVectors(thisScale, targetScale, eased);
+            setWorldPosition(this.object, this.targetPos);
+            setWorldQuaternion(this.object, this.targetRot);
+            setWorldScale(this.object, this.targetScale);
+            yield;
+        }
+        this.coroutine = null;
+    }
+    private *moveRelative() {
+        if (!this.target || !this.object) return;
+        const thisPos = this.object.position.clone();
+        const thisRot = this.object.quaternion.clone();
+        const thisScale = this.object.scale.clone();
+        const posOffset = this.target.position.clone();
+        const rotOffset = this.target.quaternion.clone();
+        const scaleOffset = this.target.scale.clone();
+        // convert into right space
+        posOffset.applyQuaternion(this.object.quaternion);
+        this.targetPos.copy(this.object.position).add(posOffset);
+        this.targetRot.copy(this.object.quaternion).multiply(rotOffset);
+        this.targetScale.copy(this.object.scale).multiply(scaleOffset);
+        let t01 = 0;
+        let eased = 0;
+        while (t01 < 1) {
+            t01 += this.context.time.deltaTime / this.duration;
+            if (t01 > 1) t01 = 1;
+            // apply ease-in-out
+            // https://easings.net/
+            eased =  t01 < 0.5 ? 4 * t01 * t01 * t01 : 1 - Math.pow(-2 * t01 + 2, 3) / 2;
+            this.object.position.lerpVectors(thisPos, this.targetPos, eased);
+            this.object.quaternion.slerpQuaternions(thisRot, this.targetRot, eased);
+            this.object.scale.lerpVectors(thisScale, this.targetScale, eased);
+            yield;
+        }
+        this.coroutine = null;
+    }
+    onPointerClick() {
+        if (this.coroutine) this.stopCoroutine(this.coroutine);
+        if (!this.relativeMotion)
+            this.coroutine = this.startCoroutine(this.moveToTarget());
+        else
+            this.coroutine = this.startCoroutine(this.moveRelative());
+    }
+    beforeCreateDocument(ext) {
+        if (this.target && this.object && this.gameObject) {
+            const moveForward = new BehaviorModel("Move to " + this.target?.name,
+                TriggerBuilder.tapTrigger(this.gameObject),
+                ActionBuilder.transformAction(this.object, this.target, this.duration, this.relativeMotion ? Space.Relative : Space.Absolute),
+            );
+            ext.addBehavior(moveForward);
+        }
+    }
+}
+export class ChangeMaterialOnClick extends Behaviour implements IPointerClickHandler, UsdzBehaviour {
+    @serializable(Material)
+    materialToSwitch?: Material;
+    @serializable(Material)
+    variantMaterial?: Material;
+    private _objectsWithThisMaterial: Renderer[] = [];
+    awake() {
+        if (this.variantMaterial && this.materialToSwitch) {
+            const renderer = GameObject.findObjectsOfType(Renderer);
+            for (const rend of renderer) {
+                if (rend.sharedMaterial === this.materialToSwitch) {
+                    this._objectsWithThisMaterial.push(rend);
+                }
+            }
+        }
+    }
+    onPointerClick() {
+        if (!this.variantMaterial) return;
+        for (const rend of this._objectsWithThisMaterial) {
+            rend.sharedMaterial = this.variantMaterial;
+        }
+    }
+    private selfModel!: USDObject;
+    private targetModels: USDObject[] = [];
+    private static _materialTriggersPerId: { [key: string]: ChangeMaterialOnClick[] } = {}
+    createBehaviours(_ext: BehaviorExtension, model: USDObject, _context) {
+        const shouldExport = this._objectsWithThisMaterial.find(o => o.gameObject.uuid === model.uuid);
+        if (shouldExport) {
+            this.targetModels.push(model);
+        }
+        if (this.gameObject.uuid === model.uuid) {
+            this.selfModel = model;
+            if (this.materialToSwitch) {
+                if (!ChangeMaterialOnClick._materialTriggersPerId[this.materialToSwitch.uuid])
+                    ChangeMaterialOnClick._materialTriggersPerId[this.materialToSwitch.uuid] = [];
+                ChangeMaterialOnClick._materialTriggersPerId[this.materialToSwitch.uuid].push(this);
+            }
+        }
+    }
+    afterCreateDocument(ext: BehaviorExtension, _context) {
+        if (!this.materialToSwitch) return;
+        const handlers = ChangeMaterialOnClick._materialTriggersPerId[this.materialToSwitch.uuid];
+        if (handlers) {
+            const variants: { [key: string]: USDObject[] } = {}
+            for (const handler of handlers) {
+                const createdVariants = handler.createVariants();
+                if (createdVariants && createdVariants.length > 0)
+                    variants[handler.selfModel.uuid] = createdVariants;
+            }
+            const otherVariants: any[] = [];
+            for (const handler of handlers) {
+                for (const key in variants) {
+                    if (key !== handler.selfModel.uuid) {
+                        otherVariants.push(variants[key]);
+                    }
+                }
+                handler.createAndAttachBehaviors(ext, variants[handler.selfModel.uuid], otherVariants);
+            }
+        }
+        delete ChangeMaterialOnClick._materialTriggersPerId[this.materialToSwitch.uuid];
+    }
+    private createAndAttachBehaviors(ext: BehaviorExtension, myVariants, otherVariants) {
+        const start: ActionModel[] = [];
+        const select: ActionModel[] = [];
+        // the order here matters
+        for (const target of this.targetModels) {
+            const hideOriginal = ActionBuilder.fadeAction(target, 0, false);
+            select.push(hideOriginal);
+        }
+        for (const v of otherVariants) {
+            select.push(ActionBuilder.fadeAction(v, 0, false));
+        }
+        for (const v of myVariants) {
+            start.push(ActionBuilder.fadeAction(v, 0, false));
+            select.push(ActionBuilder.fadeAction(v, 0, true));
+        }
+        ext.addBehavior(new BehaviorModel("Select " + this.selfModel.name,
+            TriggerBuilder.tapTrigger(this.selfModel),
+            ActionBuilder.parallel(...select))
+        );
+        ext.addBehavior(new BehaviorModel("Start hidden " + this.selfModel.name,
+            TriggerBuilder.sceneStartTrigger(),
+            ActionBuilder.parallel(...start))
+        );
+    }
+    private createVariants() {
+        if (!this.variantMaterial) return null;
+        const variantModels: USDObject[] = [];
+        for (const target of this.targetModels) {
+            const variant = target.clone();
+            variant.name += " variant_" + this.variantMaterial.name;
+            variant.name = variant.name.replace(/\s/g, "_");
+            variant.material = this.variantMaterial;
+            variant.geometry = target.geometry;
+            variant.matrix = target.matrix;
+            if (!target.parent || !target.parent.isEmpty()) {
+                USDObject.createEmptyParent(target);
+            }
+            if (target.parent) target.parent.add(variant);
+            variantModels.push(variant);
+        }
+        return variantModels;
+    }
+}
+export class SetActiveOnClick extends Behaviour implements IPointerClickHandler, UsdzBehaviour {
+    @serializable(Object3D)
+    target?: Object3D;
+    // TODO bring back, doesn't work yet
+    private toggleOnClick: boolean = false;
+    @serializable()
+    targetState: boolean = true;
+    @serializable()
+    hideSelf: boolean = true;
+    onPointerClick() {
+        if (!this.toggleOnClick && this.hideSelf)
+            this.gameObject.visible = false;
+        if (this.target)
+            this.target.visible = this.toggleOnClick ? !this.target.visible : this.targetState;
+    }
+    private selfModel!: USDObject;
+    private otherModel?: USDObject;
+    private toggleModel?: USDObject;
+    createBehaviours(_, model, _context) {
+        if (model.uuid === this.gameObject.uuid) {
+            this.selfModel = model;
+        }
+    }
+    private stateBeforeCreatingDocument: boolean = false;
+    beforeCreateDocument() {
+        this.stateBeforeCreatingDocument = this.gameObject.visible;
+        this.gameObject.visible = true;
+    }
+    afterCreateDocument(ext, context) {
+        if (!this.target) return;
+        this.otherModel = context.document.findById(this.target.uuid);
+        if (this.selfModel && this.otherModel) {
+            let hideClickedObject = this.hideSelf;
+            let targetState = this.targetState;
+            if (this.toggleOnClick) {
+                hideClickedObject = true;
+                targetState = !this.target.visible;
+                // TODO check where we have to create the clone; here it doesn't show up
+                this.toggleModel = this.selfModel.clone();
+                this.toggleModel.name += "_toggle";
+                if (this.selfModel.parent)
+                    this.selfModel.parent.add(this.toggleModel);
+            }
+            const sequence: ActionModel[] = [];
+            if (hideClickedObject)
+                sequence.push(ActionBuilder.fadeAction(this.selfModel, 0, false));
+            if (this.toggleModel)
+                sequence.push(ActionBuilder.fadeAction(this.toggleModel, 0, true));
+            sequence.push(ActionBuilder.fadeAction(this.otherModel, 0, this.targetState));
+            ext.addBehavior(new BehaviorModel("Toggle_" + this.selfModel.name + "_hideSelf",
+                TriggerBuilder.tapTrigger(this.selfModel),
+                ActionBuilder.sequence(...sequence)
+            ));
+            // TODO this is relatively similar to the material variant switching logic, can we reuse that?
+            if (this.toggleOnClick && this.toggleModel) {
+                const toggleSequence: ActionModel[] = [];
+                toggleSequence.push(ActionBuilder.fadeAction(this.toggleModel, 0, false));
+                toggleSequence.push(ActionBuilder.fadeAction(this.selfModel, 0, true));
+                toggleSequence.push(ActionBuilder.fadeAction(this.otherModel, 0, !this.targetState));
+                ext.addBehavior(new BehaviorModel("Toggle_" + this.selfModel.name + "_toggleSelf",
+                    TriggerBuilder.tapTrigger(this.toggleModel),
+                    ActionBuilder.sequence(...toggleSequence)
+                ));
+                ext.addBehavior(new BehaviorModel("HideOnStart_" + this.gameObject.name,
+                    TriggerBuilder.sceneStartTrigger(),
+                    ActionBuilder.fadeAction(this.toggleModel, 0, false)
+                ));
+            }
+        }
+        this.gameObject.visible = this.stateBeforeCreatingDocument;
+    }
+}
+export class HideOnStart extends Behaviour implements UsdzBehaviour {
+    start() {
+        this.gameObject.visible = false;
+    }
+    createBehaviours(ext, model, _context) {
+        if (model.uuid === this.gameObject.uuid)
+            ext.addBehavior(new BehaviorModel("HideOnStart_" + this.gameObject.name,
+                TriggerBuilder.sceneStartTrigger(),
+                ActionBuilder.fadeAction(model, 0, false)
+            ));
+    }
+    beforeCreateDocument() {
+        this.gameObject.visible = true;
+    }
+    afterCreateDocument() {
+        this.gameObject.visible = false;
+    }
+}
+export class EmphasizeOnClick extends Behaviour implements UsdzBehaviour {
+    @serializable()
+    target?: Object3D;
+    @serializable()
+    duration: number = 0.5;
+    @serializable()
+    motionType: MotionType = MotionType.bounce;
+    beforeCreateDocument() { }
+    createBehaviours(ext, model, _context) {
+        if (!this.target) return;
+        if (model.uuid === this.gameObject.uuid)
+        {
+            const emphasize = new BehaviorModel("emphasize " + this.name,
+                TriggerBuilder.tapTrigger(this.gameObject),
+                ActionBuilder.emphasize(this.target, this.duration, this.motionType, undefined, "basic"),
+            );
+            ext.addBehavior(emphasize);
+        }
+    }
+    afterCreateDocument(_ext, _context) { }
+}
+export class PlayAnimationOnClick extends Behaviour implements IPointerClickHandler, UsdzBehaviour, UsdzAnimation {
+    @serializable(Object3D)
+    target?: Object3D;
+    @serializable(Animator)
+    animator?: Animator;
+    @serializable()
+    stateName?: string;
+    onPointerClick() {
+        if (!this.target) return;
+        if (this.stateName)
+            this.animator?.play(this.stateName, 0, 0, .1);
+    }
+    private selfModel: any;
+    private registeredAnimationModel: any;
+    private registeredAnimation?: RegisteredAnimationInfo;
+    createBehaviours(_ext, model, _context) {
+        if (model.uuid === this.gameObject.uuid)
+            this.selfModel = model;
+    }
+    afterCreateDocument(ext, context) {
+        if (!this.registeredAnimation || !this.registeredAnimationModel) return;
+        const document = context.document;
+        document.traverse(model => {
+            if (model.uuid === this.target?.uuid && this.registeredAnimation) {
+                const playAnimationOnTap = new BehaviorModel("tap " + this.name + " for " + this.stateName + " on " + this.target?.name,
+                    TriggerBuilder.tapTrigger(this.selfModel),
+                    ActionBuilder.startAnimationAction(model, this.registeredAnimation.start, this.registeredAnimation.duration)
+                );
+                ext.addBehavior(playAnimationOnTap);
+            }
+        });
+    }
+    createAnimation(ext, model, _context) {
+        if (this.target && this.animator) {
+            const state = this.animator?.runtimeAnimatorController?.findState(this.stateName);
+            this.registeredAnimationModel = model;
+            this.registeredAnimation = ext.registerAnimation(this.target, state?.motion.clip);
+        }
+    }
+}
+export class PreliminaryAction extends Behaviour {
+    getType(): string | void { }
+    @serializable(Object3D)
+    target?: Object3D;
+    getDuration(): number | void { };
+}
+export class PreliminaryTrigger extends Behaviour {
+    @serializable(PreliminaryAction)
+    target?: PreliminaryAction;
+}
+export class VisibilityAction extends PreliminaryAction {
+    //@type int
+    @serializable()
+    type: VisibilityActionType = VisibilityActionType.Hide;
+    @serializable()
+    duration: number = 1;
+    getType() {
+        switch (this.type) {
+            case VisibilityActionType.Hide: return "hide";
+            case VisibilityActionType.Show: return "show";
+        }
+    }
+    getDuration() {
+        return this.duration;
+    }
+}
+export class TapGestureTrigger extends PreliminaryTrigger {
+}
+export enum VisibilityActionType {
+    Show = 0,
+    Hide = 1,
+}

src/engine-components/export/usdz/extensions/behavior/BehavioursBuilder.ts ADDED Viewed

@@ -0,0 +1,459 @@
+import { Object3D } from "three";
+import { USDDocument, USDObject, USDWriter, makeNameSafeForUSD } from "../../ThreeUSDZExporter";
+import { BehaviorExtension } from "./Behaviour";
+// TODO: rename to usdz element
+export interface IBehaviorElement {
+    id: string;
+    writeTo(document: USDDocument, writer: USDWriter);
+}
+export class BehaviorModel {
+    static global_id: number = 0;
+    id: string;
+    trigger: IBehaviorElement | IBehaviorElement[];
+    action: IBehaviorElement;
+    exclusive: boolean = false;
+    makeExclusive(exclusive: boolean): BehaviorModel {
+        this.exclusive = exclusive;
+        return this;
+    }
+    constructor(id: string, trigger: IBehaviorElement | IBehaviorElement[], action: IBehaviorElement) {
+        this.id = "Behavior_" + makeNameSafeForUSD(id) + "_" + BehaviorModel.global_id++;
+        this.trigger = trigger;
+        this.action = action;
+    }
+    writeTo(_ext: BehaviorExtension, document: USDDocument, writer: USDWriter) {
+        if (!this.trigger || !this.action) return;
+        writer.beginBlock(`def Preliminary_Behavior "${this.id}"`);
+        writer.appendLine(`rel actions = <${this.action.id}>`);
+        writer.appendLine(`uniform bool exclusive = ${this.exclusive}`);
+        let triggerString = "";
+        if (Array.isArray(this.trigger)) {
+            triggerString = "[";
+            for (let i = 0; i < this.trigger.length; i++) {
+                const tr = this.trigger[i];
+                triggerString += "<" + tr.id + ">";
+                if (i + 1 < this.trigger.length) triggerString += ", ";
+            }
+            triggerString += "]";
+        }
+        else
+            triggerString = `<${this.trigger.id}>`;
+        writer.appendLine(`rel triggers = ${triggerString} `);
+        writer.appendLine();
+        if (Array.isArray(this.trigger)) {
+            for (const trigger of this.trigger) {
+                trigger.writeTo(document, writer);
+                writer.appendLine();
+            }
+        }
+        else
+            this.trigger.writeTo(document, writer);
+        writer.appendLine();
+        this.action.writeTo(document, writer);
+        writer.closeBlock();
+    }
+}
+type Target = USDObject | USDObject[] | Object3D | Object3D[];
+/** called to resolve target objects to usdz paths */
+function resolve(targetObject: Target, document: USDDocument): string {
+    let result: string = "";
+    if (Array.isArray(targetObject)) {
+        let str = "[ ";
+        for (let i = 0; i < targetObject.length; i++) {
+            let obj = targetObject[i];
+            if (typeof obj === "string")
+                str += obj;
+            else if ( typeof obj === "object") {
+                //@ts-ignore
+                if (obj.isObject3D) {
+                    //@ts-ignore
+                    obj = document.findById(obj.uuid);
+                }
+                const res = (obj as any).getPath?.call(obj) as string;
+                str += res;
+            }
+            if (i + 1 < targetObject.length) str += ", ";
+        }
+        str += " ]";
+        result = str;
+    }
+    else if (typeof targetObject === "object") {
+        //@ts-ignore
+        if (targetObject.isObject3D) {
+            //@ts-ignore
+            targetObject = document.findById(targetObject.uuid);
+        }
+        result = (targetObject as any).getPath?.call(targetObject) as string;
+    }
+    // in three there's now a new "Scenes" parent and "Scene" xform that's injected;
+    // we need to add this to our path here so that we have full paths
+    result = result.replace(document.name, document.name + "/Scenes/Scene");
+    return result;
+}
+export class TriggerModel implements IBehaviorElement {
+    static global_id: number = 0;
+    id: string;
+    targetId?: string | Target;
+    tokenId?: string;
+    type?: string;
+    constructor(targetId?: string | Target, id?: string) {
+        if (targetId) this.targetId = targetId;
+        if (id) this.id = id;
+        else this.id = "Trigger_" + TriggerModel.global_id++;
+    }
+    writeTo(document: USDDocument, writer: USDWriter) {
+        writer.beginBlock(`def Preliminary_Trigger "${this.id}"`);
+        if (this.targetId) {
+            if (typeof this.targetId !== "string") this.targetId = resolve(this.targetId, document);
+            writer.appendLine(`rel affectedObjects = ` + this.targetId);
+        }
+        if (this.tokenId)
+            writer.appendLine(`token info:id = "${this.tokenId}"`);
+        if (this.type)
+            writer.appendLine(`token type = "${this.type}"`);
+        writer.closeBlock();
+    }
+}
+export class TriggerBuilder {
+    static sceneStartTrigger(): TriggerModel {
+        const trigger = new TriggerModel();
+        trigger.targetId = undefined;
+        trigger.tokenId = "SceneTransition";
+        trigger.type = "enter";
+        return trigger;
+    }
+    static tapTrigger(targetObject: Target): TriggerModel {
+        const trigger = new TriggerModel(targetObject);
+        trigger.tokenId = "TapGesture";
+        return trigger;
+    }
+    static isTapTrigger(trigger?: TriggerModel) {
+        return trigger?.tokenId === "TapGesture";
+    }
+}
+export class GroupActionModel implements IBehaviorElement {
+    static global_id: number = 0;
+    static getId(): number {
+        return this.global_id++;
+    }
+    id: string;
+    actions: IBehaviorElement[];
+    loops: number = 0;
+    performCount: number = 1;
+    type: string = "serial";
+    constructor(id: string, actions: IBehaviorElement[]) {
+        this.id = id;
+        this.actions = actions;
+    }
+    addAction(el: IBehaviorElement): GroupActionModel {
+        this.actions.push(el);
+        return this;
+    }
+    makeParallel(): GroupActionModel {
+        this.type = "parallel";
+        return this;
+    }
+    makeSequence(): GroupActionModel {
+        this.type = "serial";
+        return this;
+    }
+    makeLooping() {
+        this.loops = 1;
+        return this;
+    }
+    makeRepeat(count: number) {
+        this.performCount = count;
+        return this;
+    }
+    writeTo(document: USDDocument, writer: USDWriter) {
+        writer.beginBlock(`def Preliminary_Action "${this.id}"`);
+        writer.beginArray("rel actions");
+        for (const act of this.actions) {
+            if (!act) continue;
+            writer.appendLine("<" + act.id + ">,");
+        }
+        writer.closeArray();
+        writer.appendLine();
+        writer.appendLine(`token info:id = "Group"`);
+        writer.appendLine(`bool loops = ${this.loops} `);
+        writer.appendLine(`int performCount = ${this.performCount} `);
+        writer.appendLine(`token type = "${this.type}"`);
+        writer.appendLine();
+        for (const act of this.actions) {
+            if (!act) continue;
+            act.writeTo(document, writer);
+            writer.appendLine();
+        }
+        writer.closeBlock();
+    }
+}
+export enum MotionType {
+    pop = 0,
+    blink = 1,
+    bounce = 2,
+    flip = 3,
+    float = 4,
+    jiggle = 5,
+    pulse = 6,
+    spin = 7,
+};
+export enum Space {
+    Relative = "relative",
+    Absolute = "absolute"
+};
+export class ActionModel implements IBehaviorElement {
+    private static global_id: number = 0;
+    id: string;
+    tokenId?: string;
+    affectedObjects?: string | Target;
+    easeType?: string;;
+    motionType?: string;
+    duration?: number;
+    moveDistance?: number;
+    style?: string;
+    type?: string;
+    front?: Vec3;
+    up?: Vec3;
+    start?: number;
+    animationSpeed?: number;
+    reversed?: boolean;
+    pingPong?: boolean;
+    xFormTarget?: Target | string;
+    clone(): ActionModel {
+        const copy = new ActionModel();
+        const id = copy.id;
+        Object.assign(copy, this);
+        copy.id = id;
+        return copy;
+    }
+    constructor(affectedObjects?: string | Target, id?: string) {
+        if (affectedObjects) this.affectedObjects = affectedObjects;
+        if (id) this.id = id;
+        /*else if (affectedObjects) {
+            this.id = "Action_" + sanitizeId(affectedObjects);
+        }
+        else */
+        else
+            this.id = "Action";
+        this.id += "_" + ActionModel.global_id++;
+    }
+    writeTo(document: USDDocument, writer: USDWriter) {
+        writer.beginBlock(`def Preliminary_Action "${this.id}"`);
+        if (this.affectedObjects) {
+            if (typeof this.affectedObjects !== "string") this.affectedObjects = resolve(this.affectedObjects, document);
+            writer.appendLine('rel affectedObjects = ' + this.affectedObjects);
+        }
+        if (typeof this.duration === "number")
+            writer.appendLine(`double duration = ${this.duration} `);
+        if (this.easeType)
+            writer.appendLine(`token easeType = "${this.easeType}"`);
+        if (this.tokenId)
+            writer.appendLine(`token info:id = "${this.tokenId}"`);
+        if (this.motionType)
+            writer.appendLine(`token motionType = "${this.motionType}"`);
+        if (typeof this.moveDistance === "number")
+            writer.appendLine(`double moveDistance = ${this.moveDistance} `);
+        if (this.style)
+            writer.appendLine(`token style = "${this.style}"`);
+        if (this.type)
+            writer.appendLine(`token type = "${this.type}"`);
+        if (this.front)
+            writer.appendLine(`vector3d front = (${this.front.x}, ${this.front.y}, ${this.front.z})`);
+        if (this.up)
+            writer.appendLine(`vector3d upVector = (${this.up.x}, ${this.up.y}, ${this.up.z})`);
+        if (typeof this.start === "number") {
+            writer.appendLine(`double start = ${this.start} `);
+        }
+        if (typeof this.animationSpeed === "number") {
+            writer.appendLine(`double animationSpeed = ${this.animationSpeed} `);
+        }
+        if (typeof this.reversed === "boolean") {
+            writer.appendLine(`bool reversed = ${this.reversed}`)
+        }
+        if (typeof this.pingPong === "boolean") {
+            writer.appendLine(`bool reverses = ${this.pingPong}`)
+        }
+        if (this.xFormTarget) {
+            if (typeof this.xFormTarget !== "string")
+                this.xFormTarget = resolve(this.xFormTarget, document);
+            writer.appendLine(`rel xformTarget = ${this.xFormTarget}`)
+        }
+        writer.closeBlock();
+    }
+}
+class Vec3 {
+    x: number = 0;
+    y: number = 0;
+    z: number = 0;
+    constructor(x: number, y: number, z: number) {
+        this.x = x;
+        this.y = y;
+        this.z = z;
+    }
+    static get up(): Vec3 {
+        return new Vec3(0, 1, 0);
+    }
+    static get right(): Vec3 {
+        return new Vec3(1, 0, 0);
+    }
+    static get forward(): Vec3 {
+        return new Vec3(0, 0, 1);
+    }
+    static get back(): Vec3 {
+        return new Vec3(0, 0, -1);
+    }
+    static get zero(): Vec3 {
+        return new Vec3(0, 0, 0);
+    }
+}
+export class ActionBuilder {
+    static sequence(...params: IBehaviorElement[]) {
+        const group = new GroupActionModel("group_" + GroupActionModel.getId(), params);
+        return group.makeSequence();
+    }
+    static parallel(...params: IBehaviorElement[]) {
+        const group = new GroupActionModel("group_" + GroupActionModel.getId(), params);
+        return group.makeParallel();
+    }
+    static fadeAction(targetObject: Target, duration: number, show: boolean): ActionModel {
+        const act = new ActionModel(targetObject);
+        act.tokenId = "Visibility";
+        act.type = show ? "show" : "hide";
+        act.duration = duration;
+        act.style = "basic";
+        act.motionType = "none";
+        act.moveDistance = 0;
+        act.easeType = "none";
+        return act;
+    }
+    /**
+     * creates an action that plays an animation
+     * @param start offset in seconds!
+     * @param duration in seconds! 0 means play to end
+     */
+    static startAnimationAction(targetObject: Target, start: number, duration: number = 0, animationSpeed: number = 1, reversed: boolean = false, pingPong: boolean = false): IBehaviorElement {
+        const act = new ActionModel(targetObject);
+        act.tokenId = "StartAnimation";
+        act.start = start;
+        // start is time in seconds, the documentation is not right here
+        act.duration = duration;
+        // duration of 0 is play to end
+        act.animationSpeed = animationSpeed;
+        act.reversed = reversed;
+        act.pingPong = pingPong;
+        if (reversed) {
+            act.start -= duration;
+            //console.warn("Reversed animation does currently not work. The resulting file will most likely not playback.", act.id, targetObject);
+        }
+        if (pingPong) {
+            act.pingPong = false;
+            const back = act.clone();
+            back.reversed = !reversed;
+            back.start = act.start;
+            if (back.reversed) {
+                back.start -= duration;
+            }
+            const group = ActionBuilder.sequence(act, back);
+            return group;
+        }
+        return act;
+    }
+    static waitAction(duration: number): ActionModel {
+        const act = new ActionModel();
+        act.tokenId = "Wait";
+        act.duration = duration;
+        return act;
+    }
+    static lookAtCameraAction(targets: Target, duration: number = 9999999999999, front?: Vec3, up?: Vec3): ActionModel {
+        const act = new ActionModel(targets);
+        act.tokenId = "LookAtCamera";
+        act.duration = duration;
+        act.front = front ?? Vec3.forward;
+        // 0,0,0 is a special case for "free look"
+        // 0,1,0 is for "y-locked look-at"
+        act.up = up ?? Vec3.up;
+        return act;
+    }
+    static emphasize(targets: Target, duration: number, motionType: MotionType = MotionType.bounce, moveDistance: number = 1, style: string = "basic") {
+        const act = new ActionModel(targets);
+        act.tokenId = "Emphasize";
+        act.duration = duration;
+        act.style = style ?? "basic";
+        act.motionType = MotionType[motionType];
+        act.moveDistance = moveDistance;
+        return act;
+    }
+    static transformAction(targets: Target, transformTarget: Target, duration: number, transformType: Space, easeType: string = "inout") {
+        const act = new ActionModel(targets);
+        act.tokenId = "Transform";
+        act.duration = duration;
+        act.type = transformType;
+        act.easeType = easeType;
+        if (Array.isArray(transformTarget)) {
+            console.error("Transform target must not be an array", transformTarget);
+        }
+        act.xFormTarget = transformTarget;
+        return act;
+    }
+}
+export { Vec3 as USDVec3 }

src/engine-components/export/usdz/extensions/DocumentExtension.ts ADDED Viewed

@@ -0,0 +1,10 @@
+import { IUSDExporterExtension } from "../Extension";
+export class DocumentExtension implements IUSDExporterExtension {
+    get extensionName(): string {
+        return "DocumentExtension";
+    }
+    onAfterBuildDocument(_context: any) { }
+}

src/engine-components/export/usdz/ThreeUSDZExporter.ts ADDED Viewed

@@ -0,0 +1,1280 @@
+import {
+	PlaneGeometry,
+	Texture,
+	Uniform,
+	PerspectiveCamera,
+	Scene,
+	Mesh,
+	ShaderMaterial,
+	WebGLRenderer,
+	MathUtils,
+	Matrix4,
+	RepeatWrapping,
+	MirroredRepeatWrapping,
+	DoubleSide,
+	BufferGeometry,
+	Material,
+	Camera,
+	Color,
+	MeshStandardMaterial,
+} from 'three';
+import * as fflate from 'three/examples/jsm/libs/fflate.module.js';
+function makeNameSafe( str ) {
+	str = str.replace( /[^a-zA-Z0-9_]/g, '' );
+	// if str does not start with a-zA-Z_ add _ to the beginning
+	if ( !str.match( /^[a-zA-Z_]/ ) )
+		str = '_' + str;
+	return str;
+}
+class USDObject {
+	static USDObject_export_id = 0;
+	uuid: string;
+	name: string;
+	matrix: Matrix4;
+	private _isDynamic: boolean;
+	get isDynamic() { return this._isDynamic; }
+	private set isDynamic( value ) { this._isDynamic = value; }
+	geometry: BufferGeometry | null;
+	material: Material | null;
+	camera: Camera | null;
+	parent: USDObject | null;
+	children: Array<USDObject | null> = [];
+	_eventListeners: {};
+	mesh: any;
+	static createEmptyParent( object ) {
+		const emptyParent = new USDObject( MathUtils.generateUUID(), object.name + '_empty_' + ( USDObject.USDObject_export_id ++ ), object.matrix );
+		const parent = object.parent;
+		parent.add( emptyParent );
+		emptyParent.add( object );
+		emptyParent.isDynamic = true;
+		object.matrix = new Matrix4().identity();
+		return emptyParent;
+	}
+	constructor( id, name, matrix, mesh: BufferGeometry | null = null, material: Material | null = null, camera: Camera | null = null ) {
+		this.uuid = id;
+		this.name = makeNameSafe( name );
+		this.matrix = matrix;
+		this.geometry = mesh;
+		this.material = material;
+		this.camera = camera;
+		this.parent = null;
+		this.children = [];
+		this._eventListeners = {};
+		this._isDynamic = false;
+	}
+	is( obj ) {
+		if ( ! obj ) return false;
+		return this.uuid === obj.uuid;
+	}
+	isEmpty() {
+		return ! this.geometry;
+	}
+	clone() {
+		const clone = new USDObject( MathUtils.generateUUID(), this.name, this.matrix, this.mesh, this.material );
+		clone.isDynamic = this.isDynamic;
+		return clone;
+	}
+	getPath() {
+		let current = this.parent;
+		let path = this.name;
+		while ( current ) {
+			path = current.name + '/' + path;
+			current = current.parent;
+		}
+		return '</' + path + '>';
+	}
+	add( child ) {
+		if ( child.parent ) {
+			child.parent.remove( child );
+		}
+		child.parent = this;
+		this.children.push( child );
+	}
+	remove( child ) {
+		const index = this.children.indexOf( child );
+		if ( index >= 0 ) {
+			if ( child.parent === this ) child.parent = null;
+			this.children.splice( index, 1 );
+		}
+	}
+	addEventListener( evt, listener ) {
+		if ( ! this._eventListeners[ evt ] ) this._eventListeners[ evt ] = [];
+		this._eventListeners[ evt ].push( listener );
+	}
+	removeEventListener( evt, listener ) {
+		if ( ! this._eventListeners[ evt ] ) return;
+		const index = this._eventListeners[ evt ].indexOf( listener );
+		if ( index >= 0 ) {
+			this._eventListeners[ evt ].splice( index, 1 );
+		}
+	}
+	onSerialize( writer, context ) {
+		const listeners = this._eventListeners[ 'serialize' ];
+		if ( listeners ) listeners.forEach( listener => listener( writer, context ) );
+	}
+}
+class USDDocument extends USDObject {
+	stageLength: number;
+	get isDocumentRoot() {
+		return true;
+	}
+	get isDynamic() {
+		return false;
+	}
+	constructor() {
+		super(undefined, 'StageRoot', new Matrix4(), null, null, null);
+		this.children = [];
+		this.stageLength = 200;
+	}
+	add( child: USDObject ) {
+		child.parent = this;
+		this.children.push( child );
+	}
+	remove( child: USDObject ) {
+		const index = this.children.indexOf( child );
+		if ( index >= 0 ) {
+			if ( child.parent === this ) child.parent = null;
+			this.children.splice( index, 1 );
+		}
+	}
+	traverse( callback, current: USDObject | null = null ) {
+		if ( current !== null ) callback( current );
+		else current = this;
+		if ( current.children ) {
+			for ( const child of current.children ) {
+				this.traverse( callback, child );
+			}
+		}
+	}
+	findById( uuid ) {
+		let found = false;
+		function search( current ) {
+			if ( found ) return;
+			if ( current.uuid === uuid ) {
+				found = true;
+				return current;
+			}
+			if ( current.children ) {
+				for ( const child of current.children ) {
+					const res = search( child );
+					if ( res ) return res;
+				}
+			}
+		}
+		return search( this );
+	}
+	buildHeader() {
+		return `#usda 1.0
+(
+	customLayerData = {
+		string creator = "Three.js USDZExporter"
+	}
+	defaultPrim = "${makeNameSafe( this.name )}"
+	metersPerUnit = 1
+	upAxis = "Y"
+	startTimeCode = 0
+	endTimeCode = ${this.stageLength}
+	timeCodesPerSecond = 60
+	framesPerSecond = 60
+)
+`;
+	}
+}
+const newLine = '\n';
+class USDWriter {
+	str: string;
+	indent: number;
+	constructor() {
+		this.str = '';
+		this.indent = 0;
+	}
+	clear() {
+		this.str = '';
+		this.indent = 0;
+	}
+	beginBlock( str ) {
+		str = this.applyIndent( str );
+		this.str += str;
+		this.str += newLine;
+		this.str += this.applyIndent( '{' );
+		this.str += newLine;
+		this.indent += 1;
+	}
+	closeBlock() {
+		this.indent -= 1;
+		this.str += this.applyIndent( '}' ) + newLine;
+	}
+	beginArray( str ) {
+		str = this.applyIndent( str + ' = [' );
+		this.str += str;
+		this.str += newLine;
+		this.indent += 1;
+	}
+	closeArray() {
+		this.indent -= 1;
+		this.str += this.applyIndent( ']' ) + newLine;
+	}
+	appendLine( str = '' ) {
+		str = this.applyIndent( str );
+		this.str += str;
+		this.str += newLine;
+	}
+	toString() {
+		return this.str;
+	}
+	applyIndent( str ) {
+		let indents = '';
+		for ( let i = 0; i < this.indent; i ++ ) indents += '\t';
+		return indents + str;
+	}
+}
+class USDZExporterContext {
+	root: any;
+	exporter: any;
+	extensions: any;
+	materials: {};
+	textures: {};
+	files: {};
+	document: USDDocument;
+	output: string;
+	constructor( root, exporter, extensions ) {
+		this.root = root;
+		this.exporter = exporter;
+		if ( extensions )
+			this.extensions = extensions;
+		this.materials = {};
+		this.textures = {};
+		this.files = {};
+		this.document = new USDDocument();
+		this.output = '';
+	}
+}
+class USDZExporterOptions {
+	ar: { anchoring: { type: string } } = { anchoring: { type: 'plane' } };
+	planeAnchoring: { alignment: string } = { alignment: 'horizontal' };
+	extensions: any[] = [];
+}
+class USDZExporter {
+	debug: boolean;
+	sceneAnchoringOptions: {} = {};
+	extensions: any;
+	constructor() {
+		this.debug = false;
+	}
+	async parse( scene, options: USDZExporterOptions = new USDZExporterOptions() ) {
+		options = Object.assign( {
+			ar: {
+				anchoring: { type: 'plane' },
+				planeAnchoring: { alignment: 'horizontal' }
+			},
+			extensions: []
+		}, options );
+		this.sceneAnchoringOptions = options;
+		// @ts-ignore
+		const context = new USDZExporterContext( scene, this, options.extensions );
+		this.extensions = context.extensions;
+		const files = context.files;
+		const modelFileName = 'model.usda';
+		// model file should be first in USDZ archive so we init it here
+		files[ modelFileName ] = null;
+		const materials = context.materials;
+		const textures = context.textures;
+		invokeAll( context, 'onBeforeBuildDocument' );
+		traverseVisible( scene, context.document, context );
+		invokeAll( context, 'onAfterBuildDocument' );
+		parseDocument( context );
+		invokeAll( context, 'onAfterSerialize' );
+		context.output += buildMaterials( materials, textures );
+		const header = context.document.buildHeader();
+		const final = header + '\n' + context.output;
+		// full output file
+		if ( this.debug )
+			console.log( final );
+		files[ modelFileName ] = fflate.strToU8( final );
+		context.output = '';
+		for ( const id in textures ) {
+			let texture = textures[ id ];
+			const isRGBA = texture.format === 1023;
+			if ( texture.isCompressedTexture ) {
+				texture = copyTexture( texture );
+			}
+			// TODO add readback options for textures that don't have texture.image
+			const canvas = await imageToCanvas( texture.image );
+			if ( canvas ) {
+				const blob = await new Promise( resolve => canvas.toBlob( resolve, isRGBA ? 'image/png' : 'image/jpeg', 1 ) ) as any;
+				files[ `textures/Texture_${id}.${isRGBA ? 'png' : 'jpg'}` ] = new Uint8Array( await blob.arrayBuffer() );
+			} else {
+				console.warn( 'Can`t export texture: ', texture );
+			}
+		}
+		// 64 byte alignment
+		// https://github.com/101arrowz/fflate/issues/39#issuecomment-777263109
+		let offset = 0;
+		for ( const filename in files ) {
+			const file = files[ filename ];
+			const headerSize = 34 + filename.length;
+			offset += headerSize;
+			const offsetMod64 = offset & 63;
+			if ( offsetMod64 !== 4 ) {
+				const padLength = 64 - offsetMod64;
+				const padding = new Uint8Array( padLength );
+				files[ filename ] = [ file, { extra: { 12345: padding } } ];
+			}
+			offset = file.length;
+		}
+		return fflate.zipSync( files, { level: 0 } );
+	}
+}
+function traverseVisible( object, parentModel, context ) {
+	if ( ! object.visible ) return;
+	let model: USDObject | undefined = undefined;
+	const geometry = object.geometry;
+	const material = object.material;
+	if ( object.isMesh && material && (material.isMeshStandardMaterial || material.isMeshBasicMaterial) && ! object.isSkinnedMesh ) {
+		const name = getObjectId( object );
+		model = new USDObject( object.uuid, name, object.matrix, geometry, material );
+	} else if ( object.isCamera ) {
+		const name = getObjectId( object );
+		model = new USDObject( object.uuid, name, object.matrix, undefined, undefined, object );
+	} else {
+		const name = getObjectId( object );
+		model = new USDObject( object.uuid, name, object.matrix );
+	}
+	if ( model ) {
+		if ( parentModel ) {
+			parentModel.add( model );
+		}
+		parentModel = model;
+		if ( context.extensions ) {
+			for ( const ext of context.extensions ) {
+				if ( ext.onExportObject ) ext.onExportObject.call( ext, object, model, context );
+			}
+		}
+	} else {
+		const name = getObjectId( object );
+		const empty = new USDObject( object.uuid, name, object.matrix );
+		if ( parentModel ) {
+			parentModel.add( empty );
+		}
+		parentModel = empty;
+	}
+	for ( const ch of object.children ) {
+		traverseVisible( ch, parentModel, context );
+	}
+}
+function parseDocument( context: USDZExporterContext ) {
+	for ( const child of context.document.children ) {
+		addResources( child, context );
+	}
+	const writer = new USDWriter();
+	writer.beginBlock( `def Xform "${context.document.name}"` );
+	writer.beginBlock( `def Scope "Scenes" (
+			kind = "sceneLibrary"
+		)` );
+	writer.beginBlock( `def Xform "Scene" (
+			apiSchemas = ["Preliminary_AnchoringAPI"]
+			customData = {
+				bool preliminary_collidesWithEnvironment = 0
+				string sceneName = "Scene"
+			}
+			sceneName = "Scene"
+		)` );
+	writer.appendLine( `token preliminary:anchoring:type = "${context.exporter.sceneAnchoringOptions.ar.anchoring.type}"` );
+	if (context.exporter.sceneAnchoringOptions.ar.anchoring.type === 'plane')
+		writer.appendLine( `token preliminary:planeAnchoring:alignment = "${context.exporter.sceneAnchoringOptions.ar.planeAnchoring.alignment}"` );
+	// bit hacky as we don't have a callback here yet. Relies on the fact that the image is named identical in the ImageTracking extension.
+	if (context.exporter.sceneAnchoringOptions.ar.anchoring.type === 'image')
+		writer.appendLine( `rel preliminary:imageAnchoring:referenceImage = </${context.document.name}/Scenes/Scene/AnchoringReferenceImage>` );
+	writer.appendLine();
+	for ( const child of context.document.children ) {
+		buildXform( child, writer, context );
+	}
+	invokeAll( context, 'onAfterHierarchy', writer );
+	writer.closeBlock();
+	writer.closeBlock();
+	writer.closeBlock();
+	context.output += writer.toString();
+}
+function addResources( object, context: USDZExporterContext ) {
+	const geometry = object.geometry;
+	let material = object.material;
+	if ( geometry ) {
+		if ( material.isMeshStandardMaterial ) { // || material.isMeshBasicMaterial // TODO convert unlit to lit+emissive
+			const geometryFileName = 'geometries/Geometry_' + geometry.id + '.usd';
+			if ( ! ( geometryFileName in context.files ) ) {
+				const meshObject = buildMeshObject( geometry );
+				context.files[ geometryFileName ] = buildUSDFileAsString( meshObject, context );
+			}
+			if ( ! ( material.uuid in context.materials ) ) {
+				context.materials[ material.uuid ] = material;
+			}
+		} else {
+			console.warn( 'THREE.USDZExporter: Unsupported material type (USDZ only supports MeshStandardMaterial)', name );
+		}
+	}
+	for ( const ch of object.children ) {
+		addResources( ch, context );
+	}
+}
+function invokeAll( context: USDZExporterContext, name: string, writer: USDWriter | null = null ) {
+	if ( context.extensions ) {
+		for ( const ext of context.extensions ) {
+			if ( typeof ext[ name ] === 'function' )
+				ext[ name ]( context, writer );
+		}
+	}
+}
+function copyTexture( texture ) {
+	const geometry = new PlaneGeometry( 2, 2, 1, 1 );
+	const material = new ShaderMaterial( {
+		uniforms: { blitTexture: new Uniform( texture ) },
+		vertexShader: `
+			varying vec2 vUv;
+			void main(){
+				vUv = uv;
+				gl_Position = vec4(position.xy * 1.0,0.,.999999);
+			}`,
+		fragmentShader: `
+			uniform sampler2D blitTexture;
+			varying vec2 vUv;
+			void main(){
+				gl_FragColor = vec4(vUv.xy, 0, 1);
+				gl_FragColor = texture2D( blitTexture, vUv);
+			}`
+	} );
+	const mesh = new Mesh( geometry, material );
+	mesh.frustumCulled = false;
+	const cam = new PerspectiveCamera();
+	const scene = new Scene();
+	scene.add( mesh );
+	const renderer = new WebGLRenderer( { antialias: false } );
+	renderer.setSize( texture.image.width, texture.image.height );
+	renderer.clear();
+	renderer.render( scene, cam );
+	return new Texture( renderer.domElement );
+}
+function isImageBitmap( image ) {
+	return ( typeof HTMLImageElement !== 'undefined' && image instanceof HTMLImageElement ) ||
+		( typeof HTMLCanvasElement !== 'undefined' && image instanceof HTMLCanvasElement ) ||
+		( typeof OffscreenCanvas !== 'undefined' && image instanceof OffscreenCanvas ) ||
+		( typeof ImageBitmap !== 'undefined' && image instanceof ImageBitmap );
+}
+async function imageToCanvas( image, color: string | undefined = undefined, flipY = false ) {
+	if ( isImageBitmap( image ) ) {
+		const scale = 1024 / Math.max( image.width, image.height );
+		const canvas = document.createElement( 'canvas' );
+		canvas.width = image.width * Math.min( 1, scale );
+		canvas.height = image.height * Math.min( 1, scale );
+		const context = canvas.getContext( '2d' );
+		if (!context) throw new Error('Could not get canvas 2D context');
+		if ( flipY === true ) {
+			context.translate( 0, canvas.height );
+			context.scale( 1, - 1 );
+		}
+		context.drawImage( image, 0, 0, canvas.width, canvas.height );
+		// TODO remove, not used anymore
+		if ( color !== undefined ) {
+			const hex = parseInt( color, 16 );
+			const r = ( hex >> 16 & 255 ) / 255;
+			const g = ( hex >> 8 & 255 ) / 255;
+			const b = ( hex & 255 ) / 255;
+			const imagedata = context.getImageData( 0, 0, canvas.width, canvas.height );
+			const data = imagedata.data;
+			for ( let i = 0; i < data.length; i += 4 ) {
+				data[ i + 0 ] = data[ i + 0 ] * r;
+				data[ i + 1 ] = data[ i + 1 ] * g;
+				data[ i + 2 ] = data[ i + 2 ] * b;
+			}
+			context.putImageData( imagedata, 0, 0 );
+		}
+		return canvas;
+	} else {
+		throw new Error( 'THREE.USDZExporter: No valid image data found. Unable to process texture.' );
+	}
+}
+//
+const PRECISION = 7;
+function buildHeader() {
+	return `#usda 1.0
+(
+    customLayerData = {
+        string creator = "Three.js USDZExporter"
+    }
+    metersPerUnit = 1
+    upAxis = "Y"
+)
+`;
+}
+function buildUSDFileAsString( dataToInsert, _context: USDZExporterContext ) {
+	let output = buildHeader();
+	output += dataToInsert;
+	return fflate.strToU8( output );
+}
+function getObjectId( object ) {
+	return object.name.replace( /[-<>\(\)\[\]§$%&\/\\\=\?\,\;]/g, '' ) + '_' + object.id;
+}
+// Xform
+export function buildXform( model, writer, context ) {
+	const matrix = model.matrix;
+	const geometry = model.geometry;
+	const material = model.material;
+	const camera = model.camera;
+	const name = model.name;
+	const transform = buildMatrix( matrix );
+	if ( matrix.determinant() < 0 ) {
+		console.warn( 'THREE.USDZExporter: USDZ does not support negative scales', name );
+	}
+	if ( geometry )
+		writer.beginBlock( `def Xform "${name}" (prepend references = @./geometries/Geometry_${geometry.id}.usd@</Geometry>)` );
+	else if ( camera )
+		writer.beginBlock( `def Camera "${name}"` );
+	else
+		writer.beginBlock( `def Xform "${name}"` );
+	if ( material )
+		writer.appendLine( `rel material:binding = </Materials/Material_${material.id}>` );
+	writer.appendLine( `matrix4d xformOp:transform = ${transform}` );
+	writer.appendLine( 'uniform token[] xformOpOrder = ["xformOp:transform"]' );
+	if ( camera ) {
+		if ( camera.isOrthographicCamera ) {
+			writer.appendLine( `float2 clippingRange = (${camera.near}, ${camera.far})` );
+			writer.appendLine( `float horizontalAperture = ${( ( Math.abs( camera.left ) + Math.abs( camera.right ) ) * 10 ).toPrecision( PRECISION )}` );
+			writer.appendLine( `float verticalAperture = ${( ( Math.abs( camera.top ) + Math.abs( camera.bottom ) ) * 10 ).toPrecision( PRECISION )}` );
+			writer.appendLine( 'token projection = "orthographic"' );
+		} else {
+			writer.appendLine( `float2 clippingRange = (${camera.near.toPrecision( PRECISION )}, ${camera.far.toPrecision( PRECISION )})` );
+			writer.appendLine( `float focalLength = ${camera.getFocalLength().toPrecision( PRECISION )}` );
+			writer.appendLine( `float focusDistance = ${camera.focus.toPrecision( PRECISION )}` );
+			writer.appendLine( `float horizontalAperture = ${camera.getFilmWidth().toPrecision( PRECISION )}` );
+			writer.appendLine( 'token projection = "perspective"' );
+			writer.appendLine( `float verticalAperture = ${camera.getFilmHeight().toPrecision( PRECISION )}` );
+		}
+	}
+	if ( model.onSerialize ) {
+		model.onSerialize( writer, context );
+	}
+	if ( model.children ) {
+		writer.appendLine();
+		for ( const ch of model.children ) {
+			buildXform( ch, writer, context );
+		}
+	}
+	writer.closeBlock();
+}
+function fn( num ) {
+	return num.toFixed( 10 );
+}
+function buildMatrix( matrix ) {
+	const array = matrix.elements;
+	return `( ${buildMatrixRow( array, 0 )}, ${buildMatrixRow( array, 4 )}, ${buildMatrixRow( array, 8 )}, ${buildMatrixRow( array, 12 )} )`;
+}
+function buildMatrixRow( array, offset ) {
+	return `(${fn( array[ offset + 0 ] )}, ${fn( array[ offset + 1 ] )}, ${fn( array[ offset + 2 ] )}, ${fn( array[ offset + 3 ] )})`;
+}
+// Mesh
+function buildMeshObject( geometry ) {
+	const mesh = buildMesh( geometry );
+	return `
+def "Geometry"
+{
+  ${mesh}
+}
+`;
+}
+function buildMesh( geometry ) {
+	const name = 'Geometry';
+	const attributes = geometry.attributes;
+	const count = attributes.position.count;
+	return `
+    def Mesh "${name}"
+    {
+        int[] faceVertexCounts = [${buildMeshVertexCount( geometry )}]
+        int[] faceVertexIndices = [${buildMeshVertexIndices( geometry )}]
+        normal3f[] normals = [${buildVector3Array( attributes.normal, count )}] (
+            interpolation = "vertex"
+        )
+        point3f[] points = [${buildVector3Array( attributes.position, count )}]
+        ${attributes.uv ?
+		`float2[] primvars:st = [${buildVector2Array( attributes.uv, count )}] (
+            interpolation = "vertex"
+        )` : '' }
+		${attributes.uv2 ?
+		`float2[] primvars:st2 = [${buildVector2Array( attributes.uv2, count )}] (
+            interpolation = "vertex"
+        )` : '' }
+        uniform token subdivisionScheme = "none"
+    }
+`;
+}
+function buildMeshVertexCount( geometry ) {
+	const count = geometry.index !== null ? geometry.index.count : geometry.attributes.position.count;
+	return Array( count / 3 ).fill( 3 ).join( ', ' );
+}
+function buildMeshVertexIndices( geometry: BufferGeometry ) {
+	const index = geometry.index;
+	const array: Array<number> = [];
+	if ( index !== null ) {
+		for ( let i = 0; i < index.count; i ++ ) {
+			array.push( index.getX( i ) );
+		}
+	} else {
+		const length = geometry.attributes.position.count;
+		for ( let i = 0; i < length; i ++ ) {
+			array.push( i );
+		}
+	}
+	return array.join( ', ' );
+}
+function buildVector3Array( attribute, count ) {
+	if ( attribute === undefined ) {
+		console.warn( 'USDZExporter: Normals missing.' );
+		return Array( count ).fill( '(0, 0, 0)' ).join( ', ' );
+	}
+	const array: Array<string> = [];
+	for ( let i = 0; i < attribute.count; i ++ ) {
+		const x = attribute.getX( i );
+		const y = attribute.getY( i );
+		const z = attribute.getZ( i );
+		array.push( `(${x.toPrecision( PRECISION )}, ${y.toPrecision( PRECISION )}, ${z.toPrecision( PRECISION )})` );
+	}
+	return array.join( ', ' );
+}
+function buildVector2Array( attribute, count ) {
+	if ( attribute === undefined ) {
+		console.warn( 'USDZExporter: UVs missing.' );
+		return Array( count ).fill( '(0, 0)' ).join( ', ' );
+	}
+	const array: Array<string> = [];
+	for ( let i = 0; i < attribute.count; i ++ ) {
+		const x = attribute.getX( i );
+		const y = attribute.getY( i );
+		array.push( `(${x.toPrecision( PRECISION )}, ${1 - y.toPrecision( PRECISION )})` );
+	}
+	return array.join( ', ' );
+}
+// Materials
+function buildMaterials( materials, textures ) {
+	const array: Array<string> = [];
+	for ( const uuid in materials ) {
+		const material = materials[ uuid ];
+		array.push( buildMaterial( material, textures ) );
+	}
+	return `def "Materials"
+{
+${array.join( '' )}
+}
+`;
+}
+function buildMaterial( material, textures ) {
+	// https://graphics.pixar.com/usd/docs/UsdPreviewSurface-Proposal.html
+	const pad = '            ';
+	const inputs: Array<string> = [];
+	const samplers: Array<string> = [];
+	const exportForQuickLook = true;
+	function buildTexture( texture, mapType, color: Color | undefined = undefined, opacity: number | undefined = undefined ) {
+		const id = texture.id + ( color ? '_' + color.getHexString() : '' ) + ( opacity ? '_' + opacity : '' );
+		const isRGBA = texture.format === 1023;
+		const wrapS = ( texture.wrapS == RepeatWrapping ) ? 'repeat' : ( texture.wrapS == MirroredRepeatWrapping ? 'mirror' : 'clamp' );
+		const wrapT = ( texture.wrapT == RepeatWrapping ) ? 'repeat' : ( texture.wrapT == MirroredRepeatWrapping ? 'mirror' : 'clamp' );
+		const repeat = texture.repeat.clone();
+		const offset = texture.offset.clone();
+		// texture coordinates start in the opposite corner, need to correct
+		offset.y = 1 - offset.y - repeat.y;
+		// turns out QuickLook is buggy and interprets texture repeat inverted.
+		// Apple Feedback: 	FB10036297 and FB11442287
+		if ( exportForQuickLook ) {
+			offset.x = offset.x / repeat.x;
+			offset.y = offset.y / repeat.y;
+		}
+		textures[ id ] = texture;
+		const uvReader = mapType == 'occlusion' ? 'uvReader_st2' : 'uvReader_st';
+		const needsTextureTransform = ( repeat.x != 1 || repeat.y != 1 || offset.x != 0 || offset.y != 0 );
+		const textureTransformInput = `</Materials/Material_${material.id}/${uvReader}.outputs:result>`;
+		const textureTransformOutput = `</Materials/Material_${material.id}/Transform2d_${mapType}.outputs:result>`;
+		return `
+        ${needsTextureTransform ? `def Shader "Transform2d_${mapType}" (
+            sdrMetadata = {
+                string role = "math"
+            }
+        )
+        {
+            uniform token info:id = "UsdTransform2d"
+            float2 inputs:in.connect = ${textureTransformInput}
+            float2 inputs:scale = ${buildVector2( repeat )}
+            float2 inputs:translation = ${buildVector2( offset )}
+            float2 outputs:result
+        }
+		` : '' }
+		def Shader "Texture_${texture.id}_${mapType}"
+        {
+            uniform token info:id = "UsdUVTexture"
+            asset inputs:file = @textures/Texture_${id}.${isRGBA ? 'png' : 'jpg'}@
+            float2 inputs:st.connect = ${needsTextureTransform ? textureTransformOutput : textureTransformInput}
+			float4 inputs:scale = (${color ? color.r + ', ' + color.g + ', ' + color.b : '1, 1, 1'}, ${opacity ? opacity : '1'})
+            token inputs:wrapS = "${wrapS}"
+            token inputs:wrapT = "${wrapT}"
+            float outputs:r
+            float outputs:g
+            float outputs:b
+            float3 outputs:rgb
+            ${material.transparent || material.alphaTest > 0.0 ? 'float outputs:a' : ''}
+        }`;
+	}
+	const effectiveOpacity = ( material.transparent || material.alphaTest ) ? material.opacity : 1;
+	if ( material.side === DoubleSide ) {
+		console.warn( 'THREE.USDZExporter: USDZ does not support double sided materials', material );
+	}
+	if ( material.map !== null ) {
+		inputs.push( `${pad}color3f inputs:diffuseColor.connect = </Materials/Material_${material.id}/Texture_${material.map.id}_diffuse.outputs:rgb>` );
+		if ( material.transparent ) {
+			inputs.push( `${pad}float inputs:opacity.connect = </Materials/Material_${material.id}/Texture_${material.map.id}_diffuse.outputs:a>` );
+		} else if ( material.alphaTest > 0.0 ) {
+			inputs.push( `${pad}float inputs:opacity.connect = </Materials/Material_${material.id}/Texture_${material.map.id}_diffuse.outputs:a>` );
+			inputs.push( `${pad}float inputs:opacityThreshold = ${material.alphaTest}` );
+		}
+		samplers.push( buildTexture( material.map, 'diffuse', material.color, effectiveOpacity ) );
+	} else {
+		inputs.push( `${pad}color3f inputs:diffuseColor = ${buildColor( material.color )}` );
+	}
+	if ( material.emissiveMap ) {
+		inputs.push( `${pad}color3f inputs:emissiveColor.connect = </Materials/Material_${material.id}/Texture_${material.emissiveMap.id}_emissive.outputs:rgb>` );
+		samplers.push( buildTexture( material.emissiveMap, 'emissive' ) );
+	} else if ( material.emissive?.getHex() > 0 ) {
+		inputs.push( `${pad}color3f inputs:emissiveColor = ${buildColor( material.emissive )}` );
+	} else {
+		inputs.push( `${pad}color3f inputs:emissiveColor = (0, 0, 0)` );
+	}
+	if ( material.normalMap ) {
+		inputs.push( `${pad}normal3f inputs:normal.connect = </Materials/Material_${material.id}/Texture_${material.normalMap.id}_normal.outputs:rgb>` );
+		samplers.push( buildTexture( material.normalMap, 'normal' ) );
+	}
+	if ( material.aoMap ) {
+		inputs.push( `${pad}float inputs:occlusion.connect = </Materials/Material_${material.id}/Texture_${material.aoMap.id}_occlusion.outputs:r>` );
+		samplers.push( buildTexture( material.aoMap, 'occlusion' ) );
+	}
+	if ( material.roughnessMap && material.roughness === 1 ) {
+		inputs.push( `${pad}float inputs:roughness.connect = </Materials/Material_${material.id}/Texture_${material.roughnessMap.id}_roughness.outputs:g>` );
+		samplers.push( buildTexture( material.roughnessMap, 'roughness' ) );
+	} else {
+		inputs.push( `${pad}float inputs:roughness = ${material.roughness}` );
+	}
+	if ( material.metalnessMap && material.metalness === 1 ) {
+		inputs.push( `${pad}float inputs:metallic.connect = </Materials/Material_${material.id}/Texture_${material.metalnessMap.id}_metallic.outputs:b>` );
+		samplers.push( buildTexture( material.metalnessMap, 'metallic' ) );
+	} else {
+		inputs.push( `${pad}float inputs:metallic = ${material.metalness}` );
+	}
+	if ( material.alphaMap ) {
+		inputs.push( `${pad}float inputs:opacity.connect = </Materials/Material_${material.id}/Texture_${material.alphaMap.id}_opacity.outputs:r>` );
+		inputs.push( `${pad}float inputs:opacityThreshold = 0.0001` );
+		samplers.push( buildTexture( material.alphaMap, 'opacity' ) );
+	} else {
+		inputs.push( `${pad}float inputs:opacity = ${effectiveOpacity}` );
+	}
+	if ( material.isMeshPhysicalMaterial ) {
+		inputs.push( `${pad}float inputs:clearcoat = ${material.clearcoat}` );
+		inputs.push( `${pad}float inputs:clearcoatRoughness = ${material.clearcoatRoughness}` );
+		inputs.push( `${pad}float inputs:ior = ${material.ior}` );
+	}
+	return `
+    def Material "Material_${material.id}"
+    {
+        def Shader "PreviewSurface"
+        {
+            uniform token info:id = "UsdPreviewSurface"
+${inputs.join( '\n' )}
+            int inputs:useSpecularWorkflow = 0
+            token outputs:surface
+        }
+        token outputs:surface.connect = </Materials/Material_${material.id}/PreviewSurface.outputs:surface>
+        def Shader "uvReader_st"
+        {
+            uniform token info:id = "UsdPrimvarReader_float2"
+            token inputs:varname = "st"
+            float2 inputs:fallback = (0.0, 0.0)
+            float2 outputs:result
+        }
+		def Shader "uvReader_st2"
+        {
+            uniform token info:id = "UsdPrimvarReader_float2"
+            token inputs:varname = "st2"
+            float2 inputs:fallback = (0.0, 0.0)
+            float2 outputs:result
+        }
+${samplers.join( '\n' )}
+    }
+`;
+}
+function buildColor( color ) {
+	return `(${color.r}, ${color.g}, ${color.b})`;
+}
+function buildVector2( vector ) {
+	return `(${ vector.x }, ${ vector.y })`;
+}
+export { USDZExporter, USDZExporterContext, USDWriter, USDObject, buildMatrix, USDDocument, makeNameSafe as makeNameSafeForUSD, imageToCanvas };

src/engine-components/export/usdz/extensions/USDZText.ts ADDED Viewed

@@ -0,0 +1,142 @@
+import { IBehaviorElement } from "../extensions/behavior/BehavioursBuilder";
+import { USDDocument, USDWriter } from "../ThreeUSDZExporter";
+export enum TextWrapMode {
+    singleLine = "singleLine",
+    hardBreaks = "hardBreaks",
+    flowing = "flowing",
+}
+export enum HorizontalAlignment {
+    left = "left",
+    center = "center",
+    right = "right",
+    justified = "justified"
+}
+export enum VerticalAlignment {
+    top = "top",
+    middle = "middle",
+    lowerMiddle = "lowerMiddle",
+    baseline = "baseline",
+    bottom = "bottom"
+}
+export class USDZText implements IBehaviorElement {
+    static global_id: number = 0;
+    static getId(): number {
+        return this.global_id++;
+    }
+    id: string;
+    content: string = "";
+    font?: string[] = [];
+    pointSize: number = 144;
+    width?: number;
+    height?: number;
+    depth?: number;
+    wrapMode?: TextWrapMode;
+    horizontalAlignment?: HorizontalAlignment;
+    verticalAlignment?: VerticalAlignment;
+    setDepth(depth: number): USDZText {
+        this.depth = depth;
+        return this;
+    }
+    setPointSize(pointSize: number): USDZText {
+        this.pointSize = pointSize;
+        return this;
+    }
+    setHorizontalAlignment(align: HorizontalAlignment) {
+        this.horizontalAlignment = align;
+        return this;
+    }
+    setVerticalAlignment(align: VerticalAlignment) {
+        this.verticalAlignment = align;
+        return this;
+    }
+    constructor(id: string) {
+        this.id = id;
+    }
+    writeTo(_document: USDDocument | undefined, writer: USDWriter) {
+        writer.beginBlock(`def Preliminary_Text "${this.id}"`);
+        if (this.content)
+            writer.appendLine(`string content = "${this.content}"`);
+        if (!this.font || this.font.length <= 0) {
+            this.font ||= [];
+            this.font?.push("sans-serif");
+        }
+        const str = this.font.map(s => `"${s}"`).join(", ");
+        writer.appendLine(`string[] font = [ ${str} ]`);
+        writer.appendLine(`double pointSize = ${this.pointSize}`);
+        if (typeof this.width === "number")
+            writer.appendLine(`double width = ${this.width}`);
+        if (typeof this.height === "number")
+            writer.appendLine(`double height = ${this.height}`);
+        if (typeof this.depth === "number")
+            writer.appendLine(`double depth = ${this.depth}`);
+        if (this.wrapMode)
+            writer.appendLine(`token wrapMode = "${this.wrapMode}"`);
+        if (this.horizontalAlignment)
+            writer.appendLine(`token horizontalAlignment = "${this.horizontalAlignment}"`);
+        if (this.verticalAlignment)
+            writer.appendLine(`token verticalAlignment = "${this.verticalAlignment}"`);
+        writer.closeBlock();
+    }
+}
+export class TextBuilder {
+    static singleLine(str: string, pointSize?: number, depth?: number): USDZText {
+        const text = new USDZText("text_" + USDZText.getId());
+        text.content = str;
+        if (pointSize)
+            text.pointSize = pointSize;
+        if (depth)
+            text.depth = depth;
+        return text;
+    }
+    static multiLine(str: string, width: number, height: number, horizontal: HorizontalAlignment, vertical: VerticalAlignment, wrapMode?: TextWrapMode) {
+        const text = new USDZText("text_" + USDZText.getId());
+        text.content = str;
+        text.width = width;
+        text.height = height;
+        text.horizontalAlignment = horizontal;
+        text.verticalAlignment = vertical;
+        if (wrapMode)
+            text.wrapMode = wrapMode;
+        return text;
+    }
+}
+export class TextExtension {
+    onExportObject(_object, model, _context) {
+        model.addEventListener("serialize", (writer, _context) => {
+            const text = TextBuilder.multiLine("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.",
+                1, 1, HorizontalAlignment.justified, VerticalAlignment.top);
+            text.pointSize = 300;
+            text.depth = .01;
+            text.writeTo(undefined, writer);
+        });
+    }
+}