Needle Engine 3.21.0-alpha

Files changed (9) hide show

src/engine-components/Collider.ts +27 -4
src/engine/engine_context.ts +39 -17
src/engine/engine_physics_rapier.ts +66 -21
src/engine/engine_types.ts +11 -3
src/engine/engine_util_decorator.ts +21 -1
src/engine/engine_utils.ts +41 -0
src/engine/extensions/NEEDLE_lightmaps.ts +3 -1
src/engine-components/RendererLightmap.ts +5 -8
src/engine-components/webxr/WebARSessionRoot.ts +262 -28

src/engine-components/Collider.ts CHANGED Viewed

@@ -3,9 +3,11 @@
 import { serializable } from "../engine/engine_serialization_decorator.js";
 import { Event, Group, Mesh, Object3D, Vector3 } from "three"
 // import { IColliderProvider, registerColliderProvider } from "../engine/engine_physics.js";
-import { ICollider } from "../engine/engine_types.js";
+import { IBoxCollider, ICollider, ISphereCollider } from "../engine/engine_types.js";
 import { getWorldScale } from "../engine/engine_three_utils.js";
 import { PhysicsMaterial } from "../engine/engine_physics.types.js";
+import { validate } from "../engine/engine_util_decorator.js";
+import { unwatchWrite, watchWrite } from "../engine/engine_utils.js";
 export class Collider extends Behaviour implements ICollider {
@@ -48,26 +50,43 @@
         return this.context.physics.engine?.getBody(this);
     }
+    updateProperties = () => {
+        this.context.physics.engine?.updateProperties(this);
+    }
 }
-export class SphereCollider extends Collider {
+export class SphereCollider extends Collider implements ISphereCollider {
+    @validate()
     @serializable()
     radius: number = .5;
     @serializable(Vector3)
     center: Vector3 = new Vector3(0, 0, 0);
     onEnable() {
         super.onEnable();
-        this.context.physics.engine?.addSphereCollider(this, this.center, this.radius);
+        this.context.physics.engine?.addSphereCollider(this, this.center);
+        watchWrite(this.gameObject.scale, this.updateProperties);
     }
+    onDisable(): void {
+        super.onDisable();
+        unwatchWrite(this.gameObject.scale, this.updateProperties);
+    }
+    onValidate(): void {
+        this.updateProperties();
+    }
 }
-export class BoxCollider extends Collider {
+export class BoxCollider extends Collider implements IBoxCollider {
+    @validate()
     @serializable(Vector3)
     size: Vector3 = new Vector3(1, 1, 1);
     @serializable(Vector3)
     center: Vector3 = new Vector3(0, 0, 0);
@@ -75,6 +94,10 @@
         super.onEnable();
         this.context.physics.engine?.addBoxCollider(this, this.center, this.size);
     }
+    onValidate(): void {
+        this.updateProperties();
+    }
 }

src/engine/engine_context.ts CHANGED Viewed

@@ -156,7 +156,10 @@
 	name: string;
 	alias: string | undefined | null;
-	/** When the renderer or camera are managed by an external process (e.g. when running in r3f context) */
+	/** When the renderer or camera are managed by an external process (e.g. when running in r3f context).
+	 * When this is false you are responsible to call update(timestamp, xframe.
+	 * It is also currently assumed that rendering is handled performed by an external process
+	 * */
 	isManagedExternally: boolean = false;
 	/** set to true to pause the update loop. You can receive an event for it in your components.
 	 * Note that script updates will not be called when paused */
@@ -903,15 +906,19 @@
 			console.warn("Can not start render loop while creating context");
 			return false;
 		}
-		this.renderer.setAnimationLoop((timestamp, frame: XRFrame | null) => this.update(timestamp, frame));
+		this.renderer.setAnimationLoop((timestamp, frame: XRFrame | null) => {
+			if (this.isManagedExternally) return;
+			this.update(timestamp, frame)
+		});
 		return true;
 	}
+	/** Performs a full update step including script callbacks, rendering (unless isManagedExternally is set to false) and post render callbacks */
 	public update(timestamp: DOMHighResTimeStamp, frame?: XRFrame | null) {
 		if (frame === undefined) frame = null;
 		if (isDevEnvironment() || debug || looputils.hasNewScripts()) {
 			try {
-				this.internalRender(timestamp, frame);
+				this.internalStep(timestamp, frame);
 			}
 			catch (err) {
 				if ((isDevEnvironment() || debug) && err instanceof Error)
@@ -923,7 +930,7 @@
 			}
 		}
 		else {
-			this.internalRender(timestamp, frame);
+			this.internalStep(timestamp, frame);
 		}
 	}
@@ -931,7 +938,15 @@
 	private _accumulatedTime = 0;
 	private _dispatchReadyAfterFrame = false;
-	private internalRender(timestamp: DOMHighResTimeStamp, frame: XRFrame | null) {
+	// TODO: we need to skip after render callbacks if the render loop is managed externally. When changing this we also need to to update the r3f sample
+	private internalStep(timestamp: DOMHighResTimeStamp, frame: XRFrame | null) {
+		if (this.internalOnBeforeRender(timestamp, frame) === false) return;
+		this.internalOnRender();
+		this.internalOnAfterRender();
+	}
+	private internalOnBeforeRender(timestamp: DOMHighResTimeStamp, frame: XRFrame | null) {
 		this._xrFrame = frame;
 		this._currentFrameEvent = FrameEvent.Undefined;
@@ -944,7 +959,7 @@
 			if (typeof targetFrameRate === "object") targetFrameRate = targetFrameRate.value!;
 			// if(debug) console.log(this._accumulatedTime, (1 / (targetFrameRate)))
 			if (this._accumulatedTime < (1 / (targetFrameRate + 1))) {
-				return;
+				return false;
 			}
 			this._accumulatedTime = 0;
 		}
@@ -952,7 +967,7 @@
 		this._stats?.begin();
 		Context.Current = this;
-		if (this.onHandlePaused()) return;
+		if (this.onHandlePaused()) return false;
 		Context.Current = this;
 		this.time.update();
@@ -987,7 +1002,7 @@
 			}
 		}
 		this.executeCoroutines(FrameEvent.EarlyUpdate);
-		if (this.onHandlePaused()) return;
+		if (this.onHandlePaused()) return false;
 		this._currentFrameEvent = FrameEvent.Update;
@@ -1000,7 +1015,7 @@
 			}
 		}
 		this.executeCoroutines(FrameEvent.Update);
-		if (this.onHandlePaused()) return;
+		if (this.onHandlePaused()) return false;
 		this._currentFrameEvent = FrameEvent.LateUpdate;
@@ -1015,7 +1030,7 @@
 		// this.mainLight = null;
 		this.executeCoroutines(FrameEvent.LateUpdate);
-		if (this.onHandlePaused()) return;
+		if (this.onHandlePaused()) return false;
 		if (this.physics.engine) {
 			const physicsSteps = 1;
@@ -1030,7 +1045,7 @@
 			this.physics.engine.postStep();
 		}
-		if (this.onHandlePaused()) return;
+		if (this.onHandlePaused()) return false;
 		if (this.isVisibleToUser || this.runInBackground) {
@@ -1058,15 +1073,22 @@
 				}
 			}
+		}
-			if (!this.isManagedExternally) {
-				looputils.runPrewarm(this);
-				this._currentFrameEvent = FrameEvent.Undefined;
-				this.renderNow();
-				this._currentFrameEvent = FrameEvent.OnAfterRender;
-			}
+		return true;
+	}
+	private internalOnRender() {
+		if (!this.isManagedExternally) {
+			looputils.runPrewarm(this);
+			this._currentFrameEvent = FrameEvent.Undefined;
+			this.renderNow();
+			this._currentFrameEvent = FrameEvent.OnAfterRender;
+		}
+	}
+	private internalOnAfterRender() {
+		if (this.isVisibleToUser || this.runInBackground) {
 			for (let i = 0; i < this.scripts_onAfterRender.length; i++) {
 				const script = this.scripts_onAfterRender[i];
 				if (!script.activeAndEnabled) continue;

src/engine/engine_physics_rapier.ts CHANGED Viewed

@@ -13,10 +13,12 @@
     IGameObject,
     Vec2,
     IContext,
+    ISphereCollider,
+    IBoxCollider,
 } from './engine_types.js';
 import { foreachComponent } from './engine_gameobject.js';
-import { ActiveCollisionTypes, ActiveEvents, CoefficientCombineRule, Ball, Collider, ColliderDesc, EventQueue, JointData, QueryFilterFlags, RigidBody, RigidBodyType, ShapeColliderTOI, World, Ray } from '@dimforge/rapier3d-compat';
+import { ActiveCollisionTypes, ActiveEvents, CoefficientCombineRule, Ball, Collider, ColliderDesc, EventQueue, JointData, QueryFilterFlags, RigidBody, RigidBodyType, ShapeColliderTOI, World, Ray, ShapeType, Cuboid } from '@dimforge/rapier3d-compat';
 import { CollisionDetectionMode, PhysicsMaterialCombine } from '../engine/engine_physics.types.js';
 import { Gizmos } from './engine_gizmos.js';
 import { Mathf } from './engine_math.js';
@@ -40,8 +42,8 @@
 let RAPIER: undefined | any = undefined;
 declare const NEEDLE_USE_RAPIER: boolean;
 globalThis["NEEDLE_USE_RAPIER"] = globalThis["NEEDLE_USE_RAPIER"] !== undefined ? globalThis["NEEDLE_USE_RAPIER"] : true;
-if(debugPhysics)
-    console.log("Use Rapier", NEEDLE_USE_RAPIER, globalThis["NEEDLE_USE_RAPIER"] )
+if (debugPhysics)
+    console.log("Use Rapier", NEEDLE_USE_RAPIER, globalThis["NEEDLE_USE_RAPIER"])
 if (NEEDLE_USE_RAPIER) {
     ContextRegistry.registerCallback(ContextEvent.ContextCreationStart, evt => {
@@ -79,6 +81,7 @@
                 this.bodies.splice(index, 1);
                 this.objects.splice(index, 1);
                 // Remove the collider from the physics world
                 if (rapierBody instanceof Collider) {
                     const rapierCollider = rapierBody as Collider;
@@ -88,7 +91,13 @@
                     const rapierRigidbody: RigidBody | null = rapierCollider.parent();
                     if (rapierRigidbody && rapierRigidbody.numColliders() <= 0) {
                         const rigidbody = rapierRigidbody[$componentKey] as IRigidbody;
-                        this.removeBody(rigidbody);
+                        if (rigidbody) {
+                            // If the collider was attached to a rigidbody and this rigidbody now has no colliders anymore we should ignore it - because the Rigidbody component will delete itself
+                        }
+                        else {
+                            // But if there is no explicit rigidbody needle component then the colliders did create it implictly and thus we need to remove it here:
+                            this.world?.removeRigidBody(rapierRigidbody);
+                        }
                     }
                 }
                 // Remove the rigidbody from the physics world
@@ -133,12 +142,23 @@
         }
     }
-    updateProperties(rigidbody: IRigidbody) {
+    updateProperties(obj: IRigidbody | ICollider) {
         this.validate();
-        const physicsBody = this.internal_getRigidbody(rigidbody);
-        if (physicsBody) {
-            this.internalUpdateProperties(rigidbody, physicsBody);
+        if ((obj as ICollider).isCollider) {
+            const col = obj as ICollider;
+            const body = col[$bodyKey];
+            if (body) {
+                this.internalUpdateColliderProperties(col, body);
+            }
         }
+        else {
+            const rb = obj as IRigidbody;
+            const physicsBody = this.internal_getRigidbody(rb);
+            if (physicsBody) {
+                this.internalUpdateRigidbodyProperties(rb, physicsBody);
+            }
+        }
     }
     addForce(rigidbody: IRigidbody, force: Vec3, wakeup: boolean) {
         this.validate();
@@ -212,7 +232,7 @@
     }
     private async internalInitialization() {
-        if(debugPhysics) console.log("Initialize rapier physics engine");
+        if (debugPhysics) console.log("Initialize rapier physics engine");
         // NEEDLE_PHYSICS_INIT_START
         // use .env file with VITE_NEEDLE_USE_RAPIER=false to treeshake rapier
         // @ts-ignore
@@ -229,7 +249,7 @@
         this._hasCreatedWorld = true;
         if (RAPIER === undefined) {
             if (debugPhysics) console.log("Import Rapier");
-            const _rapier  = await import("@dimforge/rapier3d-compat");
+            const _rapier = await import("@dimforge/rapier3d-compat");
             if (debugPhysics) console.log("Init Rapier");
             await _rapier.init();
             // only assign after all loads are done to avoid a race condition
@@ -512,7 +532,7 @@
         this.createCollider(collider, desc, center);
     }
-    async addSphereCollider(collider: ICollider, center: Vector3, radius: number) {
+    async addSphereCollider(collider: ICollider, center: Vector3) {
         if (!this._isInitialized)
             await this.initialize(collider.context);
         if (!collider.activeAndEnabled) return;
@@ -520,12 +540,9 @@
             if (debugPhysics) console.warn("Physics are disabled");
             return;
         }
-        const obj = collider.gameObject;
-        const scale = getWorldScale(obj, this._tempPosition).multiplyScalar(radius);
-        // Prevent negative scales
-        scale.x = Math.abs(scale.x);
-        const desc = ColliderDesc.ball(scale.x);
+        const desc = ColliderDesc.ball(.5);
         this.createCollider(collider, desc, center);
+        this.updateProperties(collider);
     }
     async addCapsuleCollider(collider: ICollider, center: Vector3, height: number, radius: number) {
@@ -572,7 +589,7 @@
         if (Math.abs(scale.x - 1) > 0.0001 || Math.abs(scale.y - 1) > 0.0001 || Math.abs(scale.z - 1) > 0.0001) {
             const key = geo.uuid + "_" + scale.x + "_" + scale.y + "_" + scale.z + "_" + convex;
             if (this._meshCache.has(key)) {
-                if(debugPhysics) console.warn("Use cached mesh collider")
+                if (debugPhysics) console.warn("Use cached mesh collider")
                 positions = this._meshCache.get(key)!;
             }
             else {
@@ -606,8 +623,8 @@
     }
     /** Get the Needle Engine component for a rapier object */
-    getComponent(rapierObject:object) : IComponent | null {
-        if(!rapierObject) return null;
+    getComponent(rapierObject: object): IComponent | null {
+        if (!rapierObject) return null;
         const component = rapierObject[$componentKey];
         return component;
     }
@@ -714,7 +731,7 @@
             }
             rigidBody[$componentKey] = rb;
             rb[$bodyKey] = rigidBody;
-            this.internalUpdateProperties(rb, rigidBody);
+            this.internalUpdateRigidbodyProperties(rb, rigidBody);
             this.getRigidbodyRelativeMatrix(collider.gameObject, rb.gameObject, _matrix);
         }
@@ -739,7 +756,35 @@
         return rb[$bodyKey] as RigidBody;
     }
-    private internalUpdateProperties(rb: IRigidbody, rigidbody: RigidBody) {
+    private internalUpdateColliderProperties(col: ICollider, collider: Collider) {
+        const shape = collider.shape;
+        switch (shape.type) {
+            // Sphere Collider
+            case ShapeType.Ball:
+                {
+                    const ball = shape as Ball;
+                    const sc = col as ISphereCollider;
+                    const changed = ball.radius !== sc.radius;
+                    const obj = col.gameObject;
+                    const scale = getWorldScale(obj, this._tempPosition).multiplyScalar(sc.radius);
+                    // Prevent negative scales
+                    ball.radius = Math.abs(scale.x);
+                    if (changed)
+                        collider.setShape(ball);
+                    break;
+                }
+            case ShapeType.Cuboid:
+                const cuboid = shape as Cuboid;
+                const sc = col as IBoxCollider;
+                cuboid.halfExtents.x = sc.size.x * 0.5;
+                cuboid.halfExtents.y = sc.size.y * 0.5;
+                cuboid.halfExtents.z = sc.size.z * 0.5;
+                collider.setShape(cuboid);
+                break;
+        }
+    }
+    private internalUpdateRigidbodyProperties(rb: IRigidbody, rigidbody: RigidBody) {
         // continuous collision detection
         // https://rapier.rs/docs/user_guides/javascript/rigid_bodies#continuous-collision-detection
         rigidbody.enableCcd(rb.collisionDetectionMode !== CollisionDetectionMode.Discrete);

src/engine/engine_types.ts CHANGED Viewed

@@ -244,6 +244,14 @@
     sharedMaterial?: PhysicsMaterial;
 }
+export declare interface ISphereCollider extends ICollider {
+    radius: number;
+}
+export declare interface IBoxCollider extends ICollider {
+    size: Vec3;
+}
 export declare interface IRigidbody extends IComponent {
     constraints: RigidbodyConstraints;
     isKinematic: boolean;
@@ -255,7 +263,7 @@
     useGravity: boolean;
     gravityScale: number;
     dominanceGroup: number;
     collisionDetectionMode: CollisionDetectionMode;
     lockPositionX: boolean;
@@ -422,14 +430,14 @@
     sphereOverlap(point: Vector3, radius: number): Array<SphereOverlapResult>;
     // Collider methods
-    addSphereCollider(collider: ICollider, center: Vector3, radius: number);
+    addSphereCollider(collider: ICollider, center: Vector3);
     addBoxCollider(collider: ICollider, center: Vector3, size: Vector3);
     addCapsuleCollider(collider: ICollider, center: Vector3, radius: number, height: number);
     addMeshCollider(collider: ICollider, mesh: Mesh, convex: boolean, scale: Vector3);
     // Rigidbody methods
     wakeup(rb: IRigidbody);
-    updateProperties(rb: IRigidbody);
+    updateProperties(rb: IRigidbody | ICollider);
     resetForces(rb: IRigidbody, wakeup: boolean);
     resetTorques(rb: IRigidbody, wakeup: boolean);
     addForce(rb: IRigidbody, vec: Vec3, wakeup: boolean);

src/engine/engine_util_decorator.ts CHANGED Viewed

@@ -1,6 +1,8 @@
 import { $isAssigningProperties } from "./engine_serialization_core.js";
-import { LogType, showBalloonMessage } from "./debug/index.js";
+import { LogType, isDevEnvironment, showBalloonMessage } from "./debug/index.js";
 import { Constructor, IComponent } from "./engine_types.js";
+import { Quaternion, Vector2, Vector3, Vector4 } from "three";
+import { Watch, unwatchWrite, watchWrite } from "./engine_utils.js";
 declare type setter = (v: any) => void;
@@ -41,12 +43,30 @@
         const awake = target.__internalAwake;
         target.__internalAwake = function () {
+            if (!this.onValidate) {
+                if(isDevEnvironment()) console.warn("Usage of @validate decorate detected but there is no onValidate method in your class: \"" + target.constructor?.name + "\"")
+                return;
+            }
             // only build wrapper once per type
             if (this[$prop] === undefined) {
                 // make sure the field is initialized in a hidden property
                 this[$prop] = this[propertyKey];
+                // For complex types we need to watch the write operation (the underlying values)
+                // Since the object itself doesnt change (normally)
+                // This is relevant if we want to use @validate() on e.g. a Vector3 which is animated from an animationclip
+                const _val = this[propertyKey];
+                if (_val instanceof Vector2 ||
+                    _val instanceof Vector3 ||
+                    _val instanceof Vector4 ||
+                    _val instanceof Quaternion) {
+                    const vec = this[propertyKey];
+                    const cb = () => { this.onValidate(propertyKey); }
+                    watchWrite(vec, cb)
+                }
                 Object.defineProperty(this, propertyKey, {
                     set: function (v) {
                         if (this[$isAssigningProperties] === true) {

src/engine/engine_utils.ts CHANGED Viewed

@@ -1,4 +1,5 @@
 // use for typesafe interface method calls
+import { Quaternion, Vector, Vector2, Vector3, Vector4 } from "three";
 import { SourceIdentifier } from "./engine_types.js";
 // https://schneidenbach.gitbooks.io/typescript-cookbook/content/nameof-operator.html
@@ -276,6 +277,7 @@
 export interface IWatch {
     subscribeWrite(callback: WriteCallback);
+    unsubscribeWrite(callback: WriteCallback);
     apply();
     revoke();
     dispose();
@@ -287,6 +289,11 @@
     subscribeWrite(callback: WriteCallback) {
         this.writeCallbacks.push(callback);
     }
+    unsubscribeWrite(callback: WriteCallback) {
+        const i = this.writeCallbacks.indexOf(callback);
+        if (i === -1) return;
+        this.writeCallbacks.splice(i, 1);
+    }
     private writeCallbacks: (WriteCallback)[] = [];
     constructor(object: object, prop: string) {
@@ -372,6 +379,11 @@
             w.subscribeWrite(callback);
         }
     }
+    unsubscribeWrite(callback: WriteCallback) {
+        for (const w of this._watches) {
+            w.unsubscribeWrite(callback);
+        }
+    }
     apply() {
         for (const w of this._watches) {
@@ -393,7 +405,36 @@
     }
 }
+const watchesKey = Symbol("needle:watches");
+/** Subscribe to an object being written to
+ * Currently supporting Vector3
+ */
+export function watchWrite(vec: Vector, cb: Function) {
+    if (!vec[watchesKey]) {
+        if (vec instanceof Vector2) {
+            vec[watchesKey] = new Watch(vec, ["x", "y"]);
+        }
+        else if (vec instanceof Vector3) {
+            vec[watchesKey] = new Watch(vec, ["x", "y", "z"]);
+        }
+        else if (vec instanceof Vector4 || vec instanceof Quaternion) {
+            vec[watchesKey] = new Watch(vec, ["x", "y", "z", "w"]);
+        }
+        else {
+            return false;
+        }
+    }
+    vec[watchesKey].subscribeWrite(cb);
+    return true;
+}
+export function unwatchWrite(vec: Vector, cb: Function) {
+    if (!vec) return;
+    const watch = vec[watchesKey];
+    if (!watch) return;
+    watch.unsubscribeWrite(cb);
+};
 export function isMobileDevice() {
     return (typeof window.orientation !== "undefined") || (navigator.userAgent.indexOf('IEMobile') !== -1);
 }

src/engine/extensions/NEEDLE_lightmaps.ts CHANGED Viewed

@@ -13,7 +13,7 @@
 // should we split it into multiple extensions?
 export const EXTENSION_NAME = "NEEDLE_lightmaps";
-const debug = getParam("debuglightmapsextension");
+const debug = getParam("debuglightmapsextension") || getParam("debuglightmaps")
 export enum LightmapType {
     Lightmap = 0,
@@ -65,6 +65,8 @@
                     const dependencies: Array<Promise<any>> = [];
                     for (const entry of arr) {
                         if (entry.pointer) {
+                            if (debug)
+                                console.log(entry);
                             let res: Promise<any> | null = null;
                             // Check if the pointer is a json pointer:
                             if (entry.pointer.startsWith("/textures/")) {

src/engine-components/RendererLightmap.ts CHANGED Viewed

@@ -42,7 +42,10 @@
         this.lightmapScaleOffset = lightmapScaleOffset;
         this.lightmapTexture = lightmapTexture;
-        if (debug) this.setLightmapDebugMaterial();
+        if (debug) {
+            console.log("Lightmap:", this.gameObject.name, lightmapIndex, "\nScaleOffset:", lightmapScaleOffset, "\nTexture:", lightmapTexture)
+            this.setLightmapDebugMaterial();
+        }
         this.applyLightmap();
     }
@@ -134,14 +137,8 @@
                     vec2 lUv = vUv1.xy * lightmapScaleOffset.xy + vec2(lightmapScaleOffset.z, (1. - (lightmapScaleOffset.y + lightmapScaleOffset.w)));
                     vec4 lightMapTexel = texture2D( lightMap, lUv);
-                    // The range of RGBM lightmaps goes from 0 to 34.49 (5^2.2) in linear space, and from 0 to 5 in gamma space.
-                    // lightMapTexel.rgb *= lightMapTexel.a * 8.; // no idea where that "8" comes from... heuristically derived
-                    // lightMapTexel.a = 1.;
-                    // lightMapTexel = conv_sRGBToLinear(lightMapTexel);
-                    // lightMapTexel.rgb = vec3(1.);
-                    // gl_FragColor = vec4(vUv1.xy, 0, 1);
                     gl_FragColor = lightMapTexel;
+                    gl_FragColor.a = 1.;
                 }
                 `,
             defines: { USE_LIGHTMAP: '' }

src/engine-components/webxr/WebARSessionRoot.ts CHANGED Viewed

@@ -1,11 +1,14 @@
 import { Behaviour, GameObject } from "../Component.js";
-import { Matrix4, Object3D } from "three";
+import { Matrix4, Object3D, Plane, Quaternion, Ray, Raycaster, Vector2, Vector3 } from "three";
 import { WebAR, WebXR } from "./WebXR.js";
 import { InstancingUtil } from "../../engine/engine_instancing.js";
 import { serializable } from "../../engine/engine_serialization_decorator.js";
+import { Context } from "../../engine/engine_context.js";
 // https://github.com/takahirox/takahirox.github.io/blob/master/js.mmdeditor/examples/js/controls/DeviceOrientationControls.js
+const tempMatrix = new Matrix4();
 export class WebARSessionRoot extends Behaviour {
     webAR: WebAR | null = null;
@@ -17,7 +20,11 @@
     @serializable()
     invertForward: boolean = false;
+    /** Preview feature: enable touch transform */
     @serializable()
+    arTouchTransform: boolean = false;
+    @serializable()
     get arScale(): number {
         return this._arScale;
     }
@@ -30,6 +37,7 @@
     private readonly _initalMatrix = new Matrix4();
     private readonly _selectStartFn = this.onSelectStart.bind(this);
     private readonly _selectEndFn = this.onSelectEnd.bind(this);
+    private userInput?: WebXRSessionRootUserInput;
     start() {
         const xr = GameObject.findObjectOfType(WebXR);
@@ -49,6 +57,7 @@
     private _anchor: XRAnchor | null = null;
     onBegin(session: XRSession) {
         this._placementPose = null;
         this.gameObject.visible = false;
         this.gameObject.matrixAutoUpdate = false;
@@ -77,9 +86,7 @@
     onUpdate(rig: Object3D | null, _session: XRSession, hit: XRHitTestResult | null, pose: XRPose | null | undefined): boolean {
         if (pose && !this._placementPose) {
             if (!this._gotFirstHitTestResult) {
                 this._gotFirstHitTestResult = true;
                 this.dispatchEvent(new CustomEvent('canPlaceSession', { detail: { pose: pose } }));
@@ -94,19 +101,11 @@
                 this.placeAt(rig, poseMatrix);
                 if (hit && pose)
                     this.onCreatePlacementAnchor(hit, pose);
                 return true;
             }
         }
         return false;
-        // if (this._placementPose) {
-        //     this.gameObject.matrixAutoUpdate = false;
-        //     const matrix = pose?.transform.matrix;
-        //     if (matrix) {
-        //         this.gameObject.matrix.fromArray(matrix);
-        //     }
-        //     this.gameObject.visible = true;
-        // }
     }
     private onCreatePlacementAnchor(hit: XRHitTestResult, pose: XRPose) {
@@ -118,45 +117,55 @@
     }
     private _anchorMatrix: Matrix4 = new Matrix4();
     onBeforeRender(frame: XRFrame | null): void {
-        if (frame && this._anchor && this._rig) {
-            const referenceSpace = this.context.renderer.xr.getReferenceSpace();
-            if (referenceSpace) {
-                const pose = frame.getPose(this._anchor.anchorSpace, referenceSpace);
-                if (pose) {
-                    const poseMatrix = this._anchorMatrix.fromArray(pose.transform.matrix).invert();
-                    this.placeAt(this._rig, poseMatrix);
+        if (frame && this._rig) {
+            if (this._anchor) {
+                const referenceSpace = this.context.renderer.xr.getReferenceSpace();
+                if (referenceSpace) {
+                    const pose = frame.getPose(this._anchor.anchorSpace, referenceSpace);
+                    if (pose) {
+                        const poseMatrix = this._anchorMatrix.fromArray(pose.transform.matrix).invert();
+                        this.placeAt(this._rig, poseMatrix);
+                        return;
+                    }
                 }
             }
+            else if (this._placementPose) {
+                this.placeAt(this._rig, this._placementPose!);
+            }
         }
     }
     private _invertedSessionRootMatrix: Matrix4 = new Matrix4();
+    private _invertForwardMatrix: Matrix4 = new Matrix4().makeRotationY(Math.PI);
     placeAt(rig: Object3D | null, mat: Matrix4) {
         if (!this._placementPose) this._placementPose = new Matrix4();
         this._placementPose.copy(mat);
         // apply session root offset
         const invertedSessionRoot = this._invertedSessionRootMatrix.copy(this.gameObject.matrixWorld).invert();
         this._placementPose.premultiply(invertedSessionRoot);
         if (rig) {
             if (this.invertForward) {
-                const rot = new Matrix4().makeRotationY(Math.PI);
-                this._placementPose.premultiply(rot);
+                this._placementPose.premultiply(this._invertForwardMatrix);
             }
+            if (!this.userInput) this.userInput = new WebXRSessionRootUserInput(this.context);
+            this.userInput.enable();
             this._rig = rig;
             this.setScale(this.arScale);
         }
         else this._rig = null;
-        // this.gameObject.matrix.copy(this._placementPose);
-        // if (rig) {
-        //     this.gameObject.matrix.premultiply(rig.matrixWorld)
-        // }
         this.gameObject.visible = true;
     }
     onEnd(rig: Object3D | null, _session: XRSession) {
+        this.userInput?.disable();
+        this.userInput?.reset();
         this._placementPose = null;
         this.gameObject.visible = false;
         this.gameObject.matrixAutoUpdate = false;
@@ -200,10 +209,235 @@
         }
         // we apply the transform to the rig because we want to move the user's position for easy networking
         rig.matrixAutoUpdate = false;
+        if (this.arTouchTransform && this.userInput) {
+            this.userInput.applyMatrixTo(this._placementPose);
+            // rig.matrix.premultiply(this.userInput.offset);
+        }
         rig.matrix.multiplyMatrices(tempMatrix.makeScale(scale, scale, scale), this._placementPose);
         rig.matrix.decompose(rig.position, rig.quaternion, rig.scale);
         rig.updateMatrixWorld();
     }
 }
-const tempMatrix = new Matrix4();
+class WebXRSessionRootUserInput {
+    private static up = new Vector3(0, 1, 0);
+    private static zero = new Vector3(0, 0, 0);
+    private static one = new Vector3(1, 1, 1);
+    oneFingerDrag: boolean = true;
+    twoFingerRotate: boolean = true;
+    twoFingerScale: boolean = true;
+    readonly context: Context;
+    readonly offset: Matrix4;
+    readonly plane: Plane;
+    private _scale: number = 1;
+    // readonly translate: Vector3 = new Vector3();
+    // readonly rotation: Quaternion = new Quaternion();
+    // readonly scale: Vector3 = new Vector3(1, 1, 1);
+    constructor(context: Context) {
+        this.context = context;
+        this.offset = new Matrix4()
+        this.plane = new Plane();
+        this.plane.setFromNormalAndCoplanarPoint(WebXRSessionRootUserInput.up, WebXRSessionRootUserInput.zero);
+    }
+    private _enabled: boolean = false;
+    enable() {
+        if (this._enabled) return;
+        this._enabled = true;
+        window.addEventListener('touchstart', this.touchStart, { passive: false });
+        window.addEventListener('touchmove', this.touchMove, { passive: false });
+        window.addEventListener('touchend', this.touchEnd, { passive: false });
+    }
+    disable() {
+        if (!this._enabled) return;
+        this._enabled = false;
+        window.removeEventListener('touchstart', this.touchStart);
+        window.removeEventListener('touchmove', this.touchMove);
+        window.removeEventListener('touchend', this.touchEnd);
+    }
+    reset() {
+        this._scale = 1;
+        this.offset.identity();
+    }
+    applyMatrixTo(matrix: Matrix4) {
+        matrix.premultiply(this.offset);
+        // if (this._needsUpdate)
+        //     this.updateMatrix();
+        // matrix.premultiply(this._rotationMatrix);
+        // matrix.premultiply(this.offset).premultiply(this._rotationMatrix)
+    }
+    // private _needsUpdate: boolean = true;
+    // private _rotationMatrix: Matrix4 = new Matrix4();
+    // private updateMatrix() {
+    //     this._needsUpdate = false;
+    //     this._rotationMatrix.makeRotationFromQuaternion(this.rotation);
+    //     this.offset.compose(this.translate, new Quaternion(), this.scale);
+    //     // const rot = this._tempMatrix.makeRotationY(this.angle);
+    //     // this.translate.applyMatrix4(rot);
+    //     // this.offset.elements[12] = this.translate.x;
+    //     // this.offset.elements[13] = this.translate.y;
+    //     // this.offset.elements[14] = this.translate.z;
+    //     // this.offset.premultiply(rot);
+    //     // const s = this.scale;
+    //     // this.offset.premultiply(this._tempMatrix.makeScale(s, s, s));
+    // }
+    private prev: Map<number, { x: number, z: number, screenx: number, screeny: number }> = new Map();
+    private _didMultitouch: boolean = false;
+    private touchStart = (evt: TouchEvent) => {
+        for (let i = 0; i < evt.changedTouches.length; i++) {
+            const touch = evt.changedTouches[i];
+            if (!this.prev.has(touch.identifier))
+                this.prev.set(touch.identifier, {
+                    x: 0,
+                    z: 0,
+                    screenx: 0,
+                    screeny: 0,
+                });
+            const prev = this.prev.get(touch.identifier);
+            if (prev) {
+                const pos = this.getPositionOnPlane(touch.clientX, touch.clientY);
+                prev.x = pos.x;
+                prev.z = pos.z;
+                prev.screenx = touch.clientX;
+                prev.screeny = touch.clientY;
+            }
+        }
+    }
+    private touchEnd = (evt: TouchEvent) => {
+        if (evt.touches.length <= 0) {
+            this._didMultitouch = false;
+        }
+    }
+    private touchMove = (evt: TouchEvent) => {
+        if (evt.defaultPrevented) return;
+        if (evt.touches.length === 1) {
+            // if we had multiple touches before due to e.g. pinching / rotating
+            // and stopping one of the touches, we don't want to move the scene suddenly
+            // this will be resettet when all touches stop
+            if (this._didMultitouch) {
+                return;
+            }
+            const touch = evt.touches[0];
+            const prev = this.prev.get(touch.identifier);
+            if (!prev) return;
+            const pos = this.getPositionOnPlane(touch.clientX, touch.clientY);
+            const dx = pos.x - prev.x;
+            const dy = pos.z - prev.z;
+            if (dx === 0 && dy === 0) return;
+            if (this.oneFingerDrag)
+                this.addMovement(dx, dy);
+            prev.x = pos.x;
+            prev.z = pos.z;
+            prev.screenx = touch.clientX;
+            prev.screeny = touch.clientY;
+            return;
+        }
+        else if (evt.touches.length === 2) {
+            this._didMultitouch = true;
+            const touch1 = evt.touches[0];
+            const touch2 = evt.touches[1];
+            const prev1 = this.prev.get(touch1.identifier);
+            const prev2 = this.prev.get(touch2.identifier);
+            if (!prev1 || !prev2) return;
+            if (this.twoFingerRotate) {
+                const angle1 = Math.atan2(touch1.clientY - touch2.clientY, touch1.clientX - touch2.clientX);
+                const lastAngle = Math.atan2(prev1.screeny - prev2.screeny, prev1.screenx - prev2.screenx);
+                const angleDiff = angle1 - lastAngle;
+                if (Math.abs(angleDiff) > 0.001) {
+                    this.addRotation(angleDiff);
+                }
+            }
+            if (this.twoFingerScale) {
+                const distx = touch1.clientX - touch2.clientX;
+                const disty = touch1.clientY - touch2.clientY;
+                const dist = Math.sqrt(distx * distx + disty * disty);
+                const lastDistx = prev1.screenx - prev2.screenx;
+                const lastDisty = prev1.screeny - prev2.screeny;
+                const lastDist = Math.sqrt(lastDistx * lastDistx + lastDisty * lastDisty);
+                const distDiff = dist - lastDist;
+                if (Math.abs(distDiff) > 2) {
+                    this.addScale(distDiff)
+                }
+            }
+            prev1.screenx = touch1.clientX;
+            prev1.screeny = touch1.clientY;
+            prev2.screenx = touch2.clientX;
+            prev2.screeny = touch2.clientY;
+        }
+    }
+    private readonly _raycaster: Raycaster = new Raycaster();
+    private readonly _intersection: Vector3 = new Vector3();
+    private readonly _screenPos: Vector3 = new Vector3();
+    private getPositionOnPlane(tx: number, ty: number): Vector3 {
+        const camera = this.context.mainCamera!;
+        this._screenPos.x = (tx / window.innerWidth) * 2 - 1;
+        this._screenPos.y = -(ty / window.innerHeight) * 2 + 1;
+        this._screenPos.z = 1;
+        this._screenPos.unproject(camera);
+        this._raycaster.set(camera.position, this._screenPos.sub(camera.position));
+        this._raycaster.ray.intersectPlane(this.plane, this._intersection);
+        return this._intersection;
+    }
+    private addMovement(dx: number, dz: number) {
+        // this.translate.x -= dx;
+        // this.translate.z -= dz;
+        // this._needsUpdate = true;
+        // return
+        // some arbitrary factor
+        dx *= .75;
+        dz *= .75;
+        // increase diff if the scene is scaled small
+        dx /= this._scale;
+        dz /= this._scale;
+        // apply it
+        this.offset.elements[12] -= dx;
+        this.offset.elements[14] -= dz;
+    };
+    private readonly _tempMatrix: Matrix4 = new Matrix4();
+    private addScale(diff: number) {
+        diff /= window.innerWidth
+        // this.scale.x *= 1 + diff;
+        // this.scale.y *= 1 + diff;
+        // this.scale.z *= 1 + diff;
+        // this._needsUpdate = true;
+        // return;
+        // we use this factor to modify the translation factor (in apply movement)
+        this._scale *= 1 + diff;
+        // apply the scale
+        this._tempMatrix.makeScale(1 - diff, 1 - diff, 1 - diff);
+        this.offset.premultiply(this._tempMatrix);
+    }
+    private addRotation(rot: number) {
+        // this.rotation.multiply(new Quaternion().setFromAxisAngle(WebXRSessionRootUserInput.up, rot));
+        // this._needsUpdate = true;
+        // return;
+        this._tempMatrix.makeRotationY(rot);
+        this.offset.premultiply(this._tempMatrix);
+    }
+}