Needle Engine 3.20.0

Files changed (6) hide show

src/engine/engine_context.ts +29 -21
src/engine/engine_input.ts +7 -2
src/engine/engine_physics_rapier.ts +12 -2
src/engine/engine_types.ts +6 -0
src/engine-components/export/gltf/GltfExport.ts +8 -2
src/engine-components/webxr/WebXRPlaneTracking.ts +156 -61

src/engine/engine_context.ts CHANGED Viewed

@@ -131,6 +131,14 @@
 		Context._defaultTargetFramerate.value = val;
 	}
+	private static _defaultWebglRendererParameters: WebGLRendererParameters = {
+		antialias: true,
+		alpha: false,
+	};
+	static get DefaultWebGLRendererParameters(): WebGLRendererParameters {
+		return Context._defaultWebglRendererParameters;
+	}
 	/** the needle engine version */
 	get version() {
 		return VERSION;
@@ -324,10 +332,6 @@
 			this.renderer = args.renderer;
 			this.isManagedExternally = true;
 		}
-		else {
-			this.createRenderer();
-		}
 		if (args?.runInBackground !== undefined) this.runInBackground = args.runInBackground;
 		if (args?.scene) this.scene = args.scene;
 		else this.scene = new Scene();
@@ -361,15 +365,16 @@
 		ContextRegistry.register(this);
 	}
-	private createRenderer() {
+	private createNewRenderer() {
 		this.renderer?.dispose();
-		const params = this.getWebGLRendererParameters();
+		const params = Context.DefaultWebGLRendererParameters;
+		if (!params.canvas) {
+			// get canvas already configured in the Needle Engine Web Component
+			const canvas = this.domElement?.shadowRoot?.querySelector("canvas");
+			if (canvas) params.canvas = canvas;
+		}
-		// get canvas already configured in the Needle Engine Web Component
-		const canvas = this.domElement?.shadowRoot?.querySelector("canvas");
-		if (canvas) params.canvas = canvas;
 		this.renderer = new WebGLRenderer(params);
 		this.renderer.debug.checkShaderErrors = isDevEnvironment() || getParam("checkshadererrors") === true;
@@ -389,14 +394,7 @@
 		this.renderer.useLegacyLights = false;
 	}
-	private getWebGLRendererParameters(): WebGLRendererParameters {
-		return {
-			antialias: true,
-			alpha: false,
-		};
-	}
 	private _intersectionObserver: IntersectionObserver | null = null;
 	private internalOnUpdateVisible() {
 		this._intersectionObserver?.disconnect();
@@ -470,9 +468,11 @@
 		this.physics?.engine?.clearCaches();
 		if (!this.isManagedExternally) {
-			this.renderer.renderLists.dispose();
-			this.renderer.state.reset();
-			this.renderer.resetState();
+			if (this.renderer) {
+				this.renderer.renderLists.dispose();
+				this.renderer.state.reset();
+				this.renderer.resetState();
+			}
 		}
 		// We do not want to clear the renderer here because when switching src we want to keep the last rendered frame in case the loading screen is not visible
 		// if a user wants to see the background they can still call setClearAlpha(0) and clear manually
@@ -661,8 +661,10 @@
 		this.clear();
 		// stop the animation loop if its running during creation
 		// since we do not want to start enabling scripts etc before they are deserialized
-		if (this.isManagedExternally === false)
+		if (this.isManagedExternally === false) {
+			this.createNewRenderer();
 			this.renderer?.setAnimationLoop(null);
+		}
 		await delay(1);
@@ -764,6 +766,12 @@
 		Context.Current = this;
 		looputils.processNewScripts(this);
+		// We have to step once so that colliders that have been created in onEnable can be raycasted in start
+		if (this.physics.engine) {
+			this.physics.engine?.step(0);
+			this.physics.engine?.postStep();
+		}
 		// const mainCam = this.mainCameraComponent as Camera;
 		// if (mainCam) {
 		// 	mainCam.applyClearFlagsIfIsActiveCamera();

src/engine/engine_input.ts CHANGED Viewed

@@ -633,8 +633,13 @@
         lf.copy(this._pointerPositions[evt.button]);
         // accumulate delta (it's reset in end of frame), if we just write it here it's not correct when the browser console is open
         const delta = this._pointerPositionsDelta[evt.button];
-        const dx = evt.clientX - lf.x;
-        const dy = evt.clientY - lf.y;
+        let dx = evt.clientX - lf.x;
+        let dy = evt.clientY - lf.y;
+        // if pointer is locked, clientX and Y are not changed, but Movement is.
+        if(dx === 0 && evt.movementX !== 0)
+            dx = evt.movementX || 0;
+        if(dy === 0 && evt.movementY !== 0)
+            dy = evt.movementY || 0;
         delta.x += dx;
         delta.y += dy;

src/engine/engine_physics_rapier.ts CHANGED Viewed

@@ -234,7 +234,7 @@
             await RAPIER.init()
         }
         if (debugPhysics) console.log("Physics engine initialized, creating world...");
-        this.world = new World(this._gravity);
+        this._world = new World(this._gravity);
         this.enabled = true;
         this._isInitialized = true;
         if (debugPhysics) console.log("Physics world created");
@@ -427,6 +427,8 @@
     // physics simulation
     enabled: boolean = false;
+    /** Get access to the rapier world */
+    public get world(): World | undefined { return this._world };
     private _tempPosition: Vector3 = new Vector3();
     private _tempQuaternion: Quaternion = new Quaternion();
@@ -439,7 +441,7 @@
     get isUpdating(): boolean { return this._isUpdatingPhysicsWorld; }
-    private world?: World;
+    private _world?: World;
     private _hasCreatedWorld: boolean = false;
     private eventQueue?: EventQueue;
     private collisionHandler?: PhysicsCollisionHandler;
@@ -593,12 +595,20 @@
         }
     }
+    /** Get the rapier body for a Needle component */
     getBody(obj: ICollider | IRigidbody): null | any {
         if (!obj) return null;
         const body = obj[$bodyKey];
         return body;
     }
+    /** Get the Needle Engine component for a rapier object */
+    getComponent(rapierObject:object) : IComponent | null {
+        if(!rapierObject) return null;
+        const component = rapierObject[$componentKey];
+        return component;
+    }
     private createCollider(collider: ICollider, desc: ColliderDesc, center?: Vector3) {
         if (!this.world) throw new Error("Physics world not initialized");
         const matrix = this._tempMatrix;

src/engine/engine_types.ts CHANGED Viewed

@@ -395,9 +395,15 @@
     clearCaches();
     enabled: boolean;
+    get world(): any;
     set gravity(vec3: Vec3);
     get gravity(): Vec3;
+    /** Get the rapier body for a Needle component */
+    getBody(obj: ICollider | IRigidbody): null | any;
+    /** Get the Needle Engine component for a rapier object */
+    getComponent(rapierObject:object) : IComponent | null;
     // raycasting
     /** Fast raycast against physics colliders

src/engine-components/export/gltf/GltfExport.ts CHANGED Viewed

@@ -1,9 +1,11 @@
+import { Object3D, Vector3 } from "three";
+import { GLTFExporter } from 'three/examples/jsm/exporters/GLTFExporter.js';
 import { Behaviour, GameObject } from "../../Component.js";
-import { GLTFExporter } from 'three/examples/jsm/exporters/GLTFExporter.js';
 import GLTFMeshGPUInstancingExtension from '../../../include/three/EXT_mesh_gpu_instancing_exporter.js';
 import { Renderer } from "../../Renderer.js";
-import { Object3D, Vector3 } from "three";
 import { SerializationContext } from "../../../engine/engine_serialization_core.js";
+import { serializable } from "../../../engine/engine_serialization_decorator.js";
 import { NEEDLE_components } from "../../../engine/extensions/NEEDLE_components.js";
 import { getWorldPosition } from "../../../engine/engine_three_utils.js";
 import { BoxHelperComponent } from "../../BoxHelperComponent.js";
@@ -36,7 +38,11 @@
 }
 export class GltfExport extends Behaviour {
+    @serializable()
     binary: boolean = true;
+    @serializable(Object3D)
     objects: Object3D[] = [];
     private exporter?: GLTFExporter;

src/engine-components/webxr/WebXRPlaneTracking.ts CHANGED Viewed

@@ -1,4 +1,4 @@
-import { BufferAttribute, BufferGeometry, Group, Mesh, Object3D, Vector3 } from "three";
+import { BufferAttribute, BufferGeometry, Group, Material, Mesh, Object3D, TypedArray, Vector3 } from "three";
 import { MeshCollider } from "../Collider.js";
 import { Behaviour, GameObject } from "../Component.js";
 import { WebXR, WebXREvent } from "./WebXR.js";
@@ -9,13 +9,22 @@
 const debug = getParam("debugplanetracking");
+declare type XRMesh = {
+    meshSpace: XRSpace;
+    lastChangedTime: number;
+    vertices: Float32Array;
+    indices: Uint32Array;
+    semanticLabel?: string;
+}
 declare type XRFramePlanes = XRFrame & {
     detectedPlanes?: Set<XRPlane>;
+    detectedMeshes?: Set<XRMesh>;
 }
 export declare type XRPlaneContext = {
     id: number;
-    xrPlane: XRPlane;
+    xrData: XRPlane | XRMesh;
     timestamp: number;
     mesh?: Mesh | Group;
     collider?: MeshCollider;
@@ -28,13 +37,21 @@
 export class WebXRPlaneTracking extends Behaviour {
-    /** Optional: if assigned it will be instantiated per tracked plane */
+    /** Optional: if assigned it will be instantiated per tracked plane/tracked mesh */
     @serializable(Object3D)
-    planeTemplate?: Object3D;
+    dataTemplate?: Object3D;
     @serializable()
-    initiateRoomCaptureIfNoPlanes = true;
+    initiateRoomCaptureIfNoData = true;
+    @serializable()
+    usePlaneData: boolean = true;
+    @serializable()
+    useMeshData: boolean = true;
     get trackedPlanes() { return this._allPlanes.values(); }
+    get trackedMeshes() { return this._allMeshes.values(); }
     onEnable(): void {
         WebXR.addEventListener(WebXREvent.XRUpdate, this.onXRUpdate);
@@ -49,60 +66,40 @@
     private onModifyAROptions = (event: any) => {
         const options = event.detail;
         const features = options.optionalFeatures || [];
-        if (!features.includes("plane-detection"))
+        if (this.usePlaneData && !features.includes("plane-detection"))
             features.push("plane-detection");
+        if (this.useMeshData && !features.includes("mesh-detection"))
+            features.push("mesh-detection");
         options.optionalFeatures = features;
     }
     private onXRUpdate = (evt) => {
-        this.processPlanes(evt.rig, evt.frame);
-    }
-    private _planeId = 1;
-    private readonly _allPlanes = new Map<XRPlane, XRPlaneContext>();
-    private firstTimeNoPlanesDetected = -100;
-    private processPlanes(rig: Object3D, frame: XRFramePlanes) {
-        const renderer = this.context.renderer;
         // parenting tracked planes to the XR rig ensures that they synced with the real-world user data;
         // otherwise they would "swim away" when the user rotates / moves / teleports and so on.
         // There may be cases where we want that! E.g. a user walks around on their own table in castle builder
-        if (!rig) return;
+        if (!evt.rig) return;
-        // If we dont have the detectedPlanes field this means the user didnt add the option
-        if (frame.detectedPlanes === undefined) return;
+        const frame = evt.frame as XRFramePlanes;
+        const renderer = this.context.renderer;
         const referenceSpace = renderer.xr.getReferenceSpace();
         if (!referenceSpace) return;
+        const planes = frame.detectedPlanes;
+        const meshes = frame.detectedMeshes;
+        const hasAnyPlanes = planes !== undefined && planes.size > 0;
+        const hasAnyMeshes = meshes !== undefined && meshes.size > 0;
-        for (const plane of this._allPlanes.keys()) {
-            if (!frame.detectedPlanes.has(plane)) {
-                const planeContext = this._allPlanes.get(plane)!;
-                // plane was removed
-                this._allPlanes.delete(plane);
-                if (debug) console.log("Plane no longer tracked, id=" + planeContext.id);
-                if (planeContext.mesh)
-                    rig?.remove(planeContext.mesh);
-                    const evt = new CustomEvent<WebXRPlaneTrackingEvent>("plane-tracking", {
-                        detail: {
-                            type: "plane-removed",
-                            context: planeContext
-                        }
-                    })
-                    this.dispatchEvent(evt);
-            }
-        }
         // When no planes are found and we haven't already run the coroutine,
         // we start it and then wait for 2s before opening the settings.
         // This only works on Quest through a magic method on the frame,
         // see https://developer.oculus.com/documentation/web/webxr-mixed-reality/#:~:text=Because%20few%20people,once%20per%20session.
-        if (this.initiateRoomCaptureIfNoPlanes) {
-            if (frame.detectedPlanes.size == 0 && this.firstTimeNoPlanesDetected < -10)
+        if (this.initiateRoomCaptureIfNoData) {
+            if (!hasAnyPlanes && !hasAnyMeshes && this.firstTimeNoPlanesDetected < -10)
                 this.firstTimeNoPlanesDetected = Date.now();
-            if (frame.detectedPlanes.size > 0)
+            if (hasAnyPlanes || hasAnyMeshes)
                 this.firstTimeNoPlanesDetected = -1; // we're done
             if (this.firstTimeNoPlanesDetected > 0 && Date.now() - this.firstTimeNoPlanesDetected > 2500) {
                 if ("initiateRoomCapture" in frame.session) {
@@ -113,31 +110,91 @@
             }
         }
-        for (const plane of frame.detectedPlanes) {
-            const planePose = frame.getPose(plane.planeSpace, referenceSpace);
+        if (planes !== undefined)
+            this.processFrameData(evt.rig, evt.frame, planes, this._allPlanes);
+        if (meshes !== undefined)
+            this.processFrameData(evt.rig, evt.frame, meshes, this._allMeshes);
+    }
+    private _dataId = 1;
+    private readonly _allPlanes = new Map<XRPlane, XRPlaneContext>();
+    private readonly _allMeshes = new Map<XRMesh, XRPlaneContext>();
+    private firstTimeNoPlanesDetected = -100;
+    private processFrameData(rig: Object3D, frame: XRFramePlanes, detected: Set<XRPlane | XRMesh>, _all: Map<XRPlane | XRMesh, XRPlaneContext>) {
+        const renderer = this.context.renderer;
+        const referenceSpace = renderer.xr.getReferenceSpace();
+        if (!referenceSpace) return;
+        for (const data of _all.keys()) {
+            if (!detected.has(data)) {
+                const dataContext = _all.get(data)!;
+                // plane was removed
+                _all.delete(data);
+                if (debug) console.log("Plane no longer tracked, id=" + dataContext.id);
+                if (dataContext.mesh)
+                    rig?.remove(dataContext.mesh);
+                const evt = new CustomEvent<WebXRPlaneTrackingEvent>("plane-tracking", {
+                    detail: {
+                        type: "plane-removed",
+                        context: dataContext
+                    }
+                })
+                this.dispatchEvent(evt);
+            }
+        }
+        for (const data of detected) {
+            const space = "planeSpace" in data ? data.planeSpace
+                : ("meshSpace" in data ? data.meshSpace
+                : undefined);
+            if (!space) continue;
+            const planePose = frame.getPose(space, referenceSpace);
             let planeMesh: Object3D | undefined;
+            const makeBlockerMaterials = (m: Material | Array<Material>) => {
+                if (!m) return;
+                if (m instanceof Array) {
+                    for (const m0 of m)
+                        makeBlockerMaterials(m0);
+                    return;
+                }
+                if (!m.name.includes("Occlu")) return;
+                m.colorWrite = false;
+                m.depthWrite = true;
+                m.transparent = false;
+                m.polygonOffset = true;
+                m.polygonOffsetFactor = 1;
+                m.polygonOffsetUnits = 0.1;
+                m["_renderOrder"] = -1000;
+            }
             // If the plane already existed just update it
-            if (this._allPlanes.has(plane)) {
-                const planeContext = this._allPlanes.get(plane)!;
+            if (_all.has(data)) {
+                const planeContext = _all.get(data)!;
                 planeMesh = planeContext.mesh;
-                if (planeContext.timestamp < plane.lastChangedTime) {
-                    planeContext.timestamp = plane.lastChangedTime;
+                if (planeContext.timestamp < data.lastChangedTime) {
+                    planeContext.timestamp = data.lastChangedTime;
                     // Update the mesh geometry
                     if (planeContext.mesh) {
-                        const geometry = this.createGeometry(plane.polygon);
+                        const geometry = this.createGeometry(data);
                         if (planeContext.mesh instanceof Mesh) {
                             planeContext.mesh.geometry.dispose();
                             planeContext.mesh.geometry = geometry;
+                            makeBlockerMaterials(planeContext.mesh.material);
                         }
                         else if (planeContext.mesh instanceof Group) {
                             for (const ch of planeContext.mesh.children) {
                                 if (ch instanceof Mesh) {
                                     ch.geometry.dispose();
                                     ch.geometry = geometry;
+                                    makeBlockerMaterials(ch.material);
                                 }
                             }
                         }
@@ -164,19 +221,30 @@
             else {
                 // if we don't have any template assigned we just use a simple mesh object
-                if (!this.planeTemplate) {
-                    this.planeTemplate = new Mesh();
+                if (!this.dataTemplate) {
+                    this.dataTemplate = new Mesh();
                 }
-                if (this.planeTemplate) {
+                if (this.dataTemplate) {
                     // Create instance
-                    const newPlane = GameObject.instantiate(this.planeTemplate) as GameObject;
+                    const newPlane = GameObject.instantiate(this.dataTemplate) as GameObject;
                     planeMesh = newPlane;
                     if (newPlane instanceof Mesh) {
                         disposeObjectResources(newPlane.geometry);
-                        newPlane.geometry = this.createGeometry(plane.polygon);
+                        newPlane.geometry = this.createGeometry(data);
+                        makeBlockerMaterials(newPlane.material);
                     }
+                    else if (newPlane instanceof Group) {
+                        for (const ch of newPlane.children) {
+                            if (ch instanceof Mesh) {
+                                disposeObjectResources(ch.geometry);
+                                ch.geometry = this.createGeometry(data);
+                                makeBlockerMaterials(ch.material);
+                            }
+                        }
+                    }
                     const mc = newPlane.getComponent(MeshCollider) as MeshCollider;
                     if (mc) {
@@ -194,13 +262,13 @@
                     rig.add(newPlane);
                     const planeContext: XRPlaneContext = {
-                        id: this._planeId++,
-                        xrPlane: plane,
-                        timestamp: plane.lastChangedTime,
+                        id: this._dataId++,
+                        xrData: data,
+                        timestamp: data.lastChangedTime,
                         mesh: newPlane as unknown as Mesh,
                         collider: mc
                     };
-                    this._allPlanes.set(plane, planeContext);
+                    _all.set(data, planeContext);
                     if (debug) console.log("New plane detected, id=" + planeContext.id);
@@ -230,9 +298,36 @@
         };
     }
-    createGeometry(polygon: Vec3[]) {
+    createGeometry(data: XRPlane | XRMesh) {
+        if ("polygon" in data) {
+            return this.createPlaneGeometry(data.polygon);
+        }
+        else if ("vertices" in data && "indices" in data) {
+            return this.createMeshGeometry(data.vertices, data.indices);
+        }
+        return new BufferGeometry();
+    }
+    createMeshGeometry(vertices: Float32Array, indices: Uint32Array) {
         const geometry = new BufferGeometry();
+        geometry.setIndex(new BufferAttribute(indices, 1));
+        geometry.setAttribute('position', new BufferAttribute(vertices, 3));
+        // set UVs in worldspace
+        const uvs = Array<number>();
+        for (let i = 0; i < vertices.length; i+=3) {
+            uvs.push(vertices[i], vertices[i + 2]);
+        }
+        geometry.setAttribute('uv', new BufferAttribute(new Float32Array(uvs), 2));
+        geometry.computeVertexNormals();
+        geometry.computeTangents();
+        return geometry;
+      }
+    createPlaneGeometry(polygon: Vec3[]) {
+        const geometry = new BufferGeometry();
         const vertices: number[] = [];
         const uvs: number[] = [];
         polygon.forEach(point => {
@@ -268,4 +363,4 @@
         return geometry;
     }
-}
+}