Needle Engine 3.1.0-alpha

Files changed (30) hide show

src/include/three/ARButton.js +2 -2
plugins/vite/config.js +8 -0
plugins/vite/copyfiles.js +12 -3
plugins/vite/drop.js +1 -0
plugins/vite/index.js +2 -0
src/engine/codegen/register_types.js +8 -0
src/engine-components/AnimatorController.ts +18 -3
src/engine/api.ts +1 -1
src/engine-components/AudioSource.ts +2 -2
src/engine-components/Component.ts +1 -1
src/engine-components/codegen/components.ts +4 -0
src/engine-components/postprocessing/Effects/DepthOfField.ts +3 -0
src/engine/engine_addressables.ts +102 -8
src/engine/engine_serialization_builtin_serializer.ts +23 -3
src/engine/engine_serialization_core.ts +27 -20
src/engine/engine_serialization.ts +5 -4
src/engine/engine_utils.ts +35 -14
src/engine/extensions/NEEDLE_animator_controller_model.ts +1 -0
src/engine/extensions/NEEDLE_progressive.ts +2 -2
src/engine-components/ParticleSystem.ts +34 -4
src/engine-components/ParticleSystemModules.ts +1 -1
src/engine-components/Skybox.ts +2 -2
src/engine-components/ui/Text.ts +7 -7
src/engine-components/timeline/TimelineTracks.ts +2 -2
src/engine-components/VideoPlayer.ts +6 -6
src/engine-components/WebXR.ts +11 -2
src/engine-components/WebXRController.ts +2 -2
plugins/vite/transform-codegen.js +45 -0
src/engine-components/postprocessing/Effects/TiltShiftEffect.ts +56 -0
src/engine-components/WebXRImageTracking.ts +192 -0

src/include/three/ARButton.js CHANGED Viewed

@@ -1,6 +1,6 @@
 class ARButton {
-	static createButton( renderer, options = {} ) {
+	static createButton( renderer, options = {}, beforeRequestSession ) {
 		const button = document.createElement( 'button' );
 		let ARButtonControlsDomOverlay = false;
@@ -126,7 +126,7 @@
 			button.onclick = function () {
 				if ( currentSession === null ) {
+					beforeRequestSession?.call(this, options);
 					navigator.xr.requestSession( 'immersive-ar', options ).then( onSessionStarted );
 				} else {

plugins/vite/config.js CHANGED Viewed

@@ -1,5 +1,6 @@
 import { existsSync, readFileSync } from 'fs';
+/** the codegen meta file */
 export async function loadConfig(path) {
     try {
         // First try to get the path from the config
@@ -28,6 +29,7 @@
     }
 }
+/** get the needle.config.json */
 export function tryLoadProjectConfig() {
     try {
         const root = process.cwd();
@@ -45,4 +47,10 @@
     }
     return null;
+}
+/** "assets" -> the directory name inside the output directory to put e.g. glb files into */
+export function builtAssetsDirectory(){
+    return "assets";
 }

plugins/vite/copyfiles.js CHANGED Viewed

@@ -1,6 +1,7 @@
 import { resolve, join } from 'path'
 import { existsSync, statSync, mkdirSync, readdirSync, copyFileSync, mkdir } from 'fs';
+import { builtAssetsDirectory, tryLoadProjectConfig } from './config.js';
 /** copy files on build from assets to dist */
@@ -19,7 +20,14 @@
             const baseDir = process.cwd();
             const pluginName = "needle-copy-files";
-            const outdirName = "dist";
+            let assetsDirName = "assets";
+            let outdirName = "dist";
+            const needleConfig = tryLoadProjectConfig();
+            if(needleConfig){
+                assetsDirName = needleConfig.assetsDirectory;
+            }
             const outDir = resolve(baseDir, outdirName);
             if (!existsSync(outDir)) {
                 mkdirSync(outDir);
@@ -36,9 +44,9 @@
             }
             // copy assets dir
-            const assetsDir = resolve(baseDir, 'assets');
+            const assetsDir = resolve(baseDir, assetsDirName);
             if (existsSync(assetsDir)) {
-                console.log(`[${pluginName}] - Copy assets to ${outdirName}/assets`)
+                console.log(`[${pluginName}] - Copy assets to ${outdirName}/${builtAssetsDirectory()}`)
                 const targetDir = resolve(outDir, 'assets');
                 copyRecursiveSync(assetsDir, targetDir);
             }
@@ -53,6 +61,7 @@
     }
 }
 function copyRecursiveSync(src, dest) {
     var exists = existsSync(src);
     var stats = exists && statSync(src);

plugins/vite/drop.js CHANGED Viewed

@@ -6,6 +6,7 @@
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
+/** experimental, allow dropping files from Unity into the running scene */
 export const needleDrop = (command, config, userSettings) => {
     if (command === "build") return;

plugins/vite/index.js CHANGED Viewed

@@ -6,6 +6,7 @@
 import { editorConnection } from "./editor-connection.js";
 import { needleCopyFiles } from "./copyfiles.js";
 import { needleViteAlias } from "./alias.js";
+import { needleTransformCodegen } from "./transform-codegen.js";
 import { needleLicense } from "./license.js";
 export * from "./gzip.js";
@@ -27,6 +28,7 @@
         needleReload(command, config, userSettings),
         needleBuild(command, config, userSettings),
         needleCopyFiles(command, config, userSettings),
+        needleTransformCodegen(command, config, userSettings),
         needleDrop(command, config, userSettings),
     ];
     array.push(await editorConnection(command, config, userSettings, array));

src/engine/codegen/register_types.js CHANGED Viewed

@@ -150,6 +150,7 @@
 import { TestSimulateUserData } from "../../engine-components/TestRunner";
 import { Text } from "../../engine-components/ui/Text";
 import { TextureSheetAnimationModule } from "../../engine-components/ParticleSystemModules";
+import { TiltShiftEffect } from "../../engine-components/postprocessing/Effects/TiltShiftEffect";
 import { ToneMapping } from "../../engine-components/postprocessing/Effects/Tonemapping";
 import { TrailModule } from "../../engine-components/ParticleSystemModules";
 import { TransformData } from "../../engine-components/export/usdz/extensions/Animation";
@@ -173,7 +174,10 @@
 import { WebXR } from "../../engine-components/WebXR";
 import { WebXRAvatar } from "../../engine-components/WebXRAvatar";
 import { WebXRController } from "../../engine-components/WebXRController";
+import { WebXRImageTracking } from "../../engine-components/WebXRImageTracking";
+import { WebXRImageTrackingModel } from "../../engine-components/WebXRImageTracking";
 import { WebXRSync } from "../../engine-components/WebXRSync";
+import { WebXRTrackedImage } from "../../engine-components/WebXRImageTracking";
 import { XRFlag } from "../../engine-components/XRFlag";
 import { XRGrabModel } from "../../engine-components/WebXRGrabRendering";
 import { XRGrabRendering } from "../../engine-components/WebXRGrabRendering";
@@ -330,6 +334,7 @@
 TypeStore.add("TestSimulateUserData", TestSimulateUserData);
 TypeStore.add("Text", Text);
 TypeStore.add("TextureSheetAnimationModule", TextureSheetAnimationModule);
+TypeStore.add("TiltShiftEffect", TiltShiftEffect);
 TypeStore.add("ToneMapping", ToneMapping);
 TypeStore.add("TrailModule", TrailModule);
 TypeStore.add("TransformData", TransformData);
@@ -353,7 +358,10 @@
 TypeStore.add("WebXR", WebXR);
 TypeStore.add("WebXRAvatar", WebXRAvatar);
 TypeStore.add("WebXRController", WebXRController);
+TypeStore.add("WebXRImageTracking", WebXRImageTracking);
+TypeStore.add("WebXRImageTrackingModel", WebXRImageTrackingModel);
 TypeStore.add("WebXRSync", WebXRSync);
+TypeStore.add("WebXRTrackedImage", WebXRTrackedImage);
 TypeStore.add("XRFlag", XRFlag);
 TypeStore.add("XRGrabModel", XRGrabModel);
 TypeStore.add("XRGrabRendering", XRGrabRendering);

src/engine-components/AnimatorController.ts CHANGED Viewed

@@ -249,9 +249,16 @@
             if (action) {
                 const dur = state.motion.clip!.duration;
-                const normalizedTime = dur <= 0 ? 1 : action.time / dur;
-                const makeTransition = transition.hasExitTime ? normalizedTime >= transition.exitTime : true;
-                // console.log(state.name, makeTransition, transition.hasExitTime, normalizedTime, transition.exitTime)
+                const normalizedTime = dur <= 0 ? 1 : Math.abs(action.time / dur);
+                let makeTransition = false;
+                if (transition.hasExitTime) {
+                    if (action.timeScale > 0) makeTransition = normalizedTime >= transition.exitTime;
+                    // When the animation is playing backwards we need to check exit time inverted
+                    else if(action.timeScale < 0) makeTransition = 1 - normalizedTime >= transition.exitTime;
+                }
+                else {
+                    makeTransition = true;
+                }
                 if (makeTransition) {
                     // if (transition.hasExitTime && transition.exitTime >= .9999)
                     action.clampWhenFinished = true;
@@ -283,6 +290,12 @@
                 action.time = 0;
                 action.play();
             }
+            else if (action.time <= 0 && action.timeScale < 0) {
+                didTriggerLooping = true;
+                action.reset();
+                action.time = action.getClip().duration;
+                action.play();
+            }
         }
         // call update state behaviours:
@@ -367,9 +380,11 @@
                 action.stop();
             action.reset();
             action.timeScale = this._speed;
+            if (state.speed !== undefined) action.timeScale *= state.speed;
             action.enabled = true;
             const duration = state.motion.clip!.duration;
             action.time = offsetNormalized * duration;
+            if(action.timeScale < 0) action.time = duration - action.time;
             action.clampWhenFinished = true;
             action.setLoop(LoopOnce, 0);
             if (durationInSec > 0)

src/engine/api.ts CHANGED Viewed

@@ -6,7 +6,7 @@
 export * from "./engine_gameobject";
 export * from "./engine_components";
 export * from "./engine_components_internal";
-export { AssetReference } from "./engine_addressables";
+export { AssetReference, ImageReference } from "./engine_addressables";
 export { Context, FrameEvent } from "./engine_setup";
 export * from "./debug/debug";
 export { validate } from "./engine_util_decorator"

src/engine-components/AudioSource.ts CHANGED Viewed

@@ -82,7 +82,7 @@
         document.addEventListener('touchstart', fn);
     }
-    @serializable()
+    @serializable(URL)
     clip: string = "";
     @serializable()
@@ -202,7 +202,7 @@
                 }
                 break;
             case "visible":
-                console.log("visible", this.enabled, this.playOnAwake, !this.isPlaying, AudioSource._userInteractionRegistered, this.wasPlaying);
+                if (debug) console.log("visible", this.enabled, this.playOnAwake, !this.isPlaying, AudioSource._userInteractionRegistered, this.wasPlaying);
                 if (this.enabled && this.playOnAwake && !this.isPlaying && AudioSource._userInteractionRegistered && this.wasPlaying) {
                     this.play();
                 }

src/engine-components/Component.ts CHANGED Viewed

@@ -9,6 +9,7 @@
 import { ConstructorConcrete, SourceIdentifier, IComponent, IGameObject, Constructor, GuidsMap, UIDProvider, Collision, ICollider } from "../engine/engine_types";
 import { addNewComponent, destroyComponentInstance, findObjectOfType, findObjectsOfType, getComponent, getComponentInChildren, getComponentInParent, getComponents, getComponentsInChildren, getComponentsInParent, getOrAddComponent, moveComponentInstance, removeComponent } from "../engine/engine_components";
 import { findByGuid, destroy, InstantiateOptions, instantiate, HideFlags, foreachComponent, markAsInstancedRendered, isActiveInHierarchy, isActiveSelf, isUsingInstancing, setActive, isDestroyed } from "../engine/engine_gameobject";
+import { resolveUrl } from "../engine/engine_utils";
 // export interface ISerializationCallbackReceiver {
@@ -641,7 +642,6 @@
         return false;
     }
 }
 class Behaviour extends Component {

src/engine-components/codegen/components.ts CHANGED Viewed

@@ -145,6 +145,7 @@
 export { TestSimulateUserData } from "../TestRunner";
 export { Text } from "../ui/Text";
 export { TextureSheetAnimationModule } from "../ParticleSystemModules";
+export { TiltShiftEffect } from "../postprocessing/Effects/TiltShiftEffect";
 export { ToneMapping } from "../postprocessing/Effects/Tonemapping";
 export { TrailModule } from "../ParticleSystemModules";
 export { TransformData } from "../export/usdz/extensions/Animation";
@@ -168,7 +169,10 @@
 export { WebXR } from "../WebXR";
 export { WebXRAvatar } from "../WebXRAvatar";
 export { WebXRController } from "../WebXRController";
+export { WebXRImageTracking } from "../WebXRImageTracking";
+export { WebXRImageTrackingModel } from "../WebXRImageTracking";
 export { WebXRSync } from "../WebXRSync";
+export { WebXRTrackedImage } from "../WebXRImageTracking";
 export { XRFlag } from "../XRFlag";
 export { XRGrabModel } from "../WebXRGrabRendering";
 export { XRGrabRendering } from "../WebXRGrabRendering";

src/engine-components/postprocessing/Effects/DepthOfField.ts CHANGED Viewed

@@ -36,6 +36,9 @@
     @serializable(VolumeParameter)
     resolutionScale?: VolumeParameter;
+    @serializable(VolumeParameter)
+    bokehScale?: VolumeParameter;
     init() {
         this.focalLength.valueProcessor = v => {
             const t = v / 300;

src/engine/engine_addressables.ts CHANGED Viewed

@@ -1,8 +1,8 @@
-import { getParam, getPath } from "../engine/engine_utils";
+import { getParam, resolveUrl } from "../engine/engine_utils";
 // import { loadSync, parseSync } from "./engine_scenetools";
 import { SerializationContext, TypeSerializer } from "./engine_serialization_core";
 import { Context } from "./engine_setup";
-import { Group, Object3D, Scene } from "three";
+import { Group, Object3D, Scene, Texture } from "three";
 import { processNewScripts } from "./engine_mainloop_utils";
 import { registerPrefabProvider, syncInstantiate } from "./engine_networking_instantiate";
 import { download, hash } from "./engine_web_api";
@@ -50,7 +50,7 @@
 export class AssetReference {
     static getOrCreate(sourceId: SourceIdentifier, uri: string, context: Context): AssetReference {
-        const fullPath = getPath(sourceId, uri);
+        const fullPath = resolveUrl(sourceId, uri);
         if (debug) console.log("GetOrCreate Addressable from", sourceId, uri, "FinalPath=", fullPath);
         const addressables = context.addressables;
         const existing = addressables.findAssetReference(fullPath);
@@ -72,15 +72,16 @@
     private _loading?: PromiseLike<any>;
+    // TODO: rename to url
     get uri(): string {
-        return this._uri;
+        return this._url;
     }
     get rawAsset(): any { return this._asset; }
     private _asset: any;
     private _glbRoot?: Object3D | null;
-    private _uri: string;
+    private _url: string;
     private _progressListeners: ProgressCallback[] = [];
     private _hash?: string;
@@ -90,14 +91,14 @@
     private _rawBinary?: ArrayBuffer | null;
     constructor(uri: string, hash?: string) {
-        this._uri = uri;
+        this._url = uri;
         this._hash = hash;
         if (uri.includes("?v="))
             this._hashedUri = uri;
         else
             this._hashedUri = hash ? uri + "?v=" + hash : uri;
-        registerPrefabProvider(this._uri, this.onResolvePrefab.bind(this));
+        registerPrefabProvider(this._url, this.onResolvePrefab.bind(this));
     }
     private async onResolvePrefab(uri: string): Promise<IGameObject | null> {
@@ -330,4 +331,97 @@
     }
 }
-new AddressableSerializer();
+new AddressableSerializer();
+export class ImageReference {
+    private static imageReferences = new Map<string, ImageReference>();
+    static getOrCreate(url: string) {
+        let ref = ImageReference.imageReferences.get(url);
+        if (!ref) {
+            ref = new ImageReference(url);
+            ImageReference.imageReferences.set(url, ref);
+        }
+        return ref;
+    }
+    constructor(url: string) {
+        this.url = url;
+    }
+    readonly url!: string;
+    private _bitmap?: Promise<ImageBitmap | null>;
+    private _bitmapObject?: ImageBitmap;
+    dispose() {
+        if (this._bitmapObject) {
+            this._bitmapObject.close();
+        }
+        this._bitmap = undefined;
+    }
+    createHTMLImage(): HTMLImageElement {
+        const img = new Image();
+        img.src = this.url;
+        return img;
+    }
+    createTexture(): Promise<Texture | null> {
+        return this.getBitmap().then((bitmap) => {
+            if (bitmap) {
+                const texture = new Texture(bitmap);
+                texture.needsUpdate = true;
+                return texture;
+            }
+            return null;
+        });
+    }
+    /** Loads the bitmap data of the image */
+    getBitmap(): Promise<ImageBitmap | null> {
+        if (this._bitmap) return this._bitmap;
+        this._bitmap = new Promise((res, _) => {
+            const imageElement = document.createElement("img") as HTMLImageElement;
+            imageElement.addEventListener("load", () => {
+                this._bitmap = createImageBitmap(imageElement).then((bitmap) => {
+                    this._bitmapObject = bitmap;
+                    res(bitmap);
+                    return bitmap;
+                })
+            });
+            imageElement.addEventListener("error", err => {
+                console.error("Failed to load image:" + this.url, err);
+                res(null);
+            });
+            imageElement.src = this.url;
+        });
+        return this._bitmap;
+    }
+}
+export class ImageReferenceSerializer extends TypeSerializer {
+    constructor() {
+        super([ImageReference]);
+    }
+    onSerialize(_data: string, _context: SerializationContext) {
+        return null;
+    }
+    onDeserialize(data: string, _context: SerializationContext) {
+        if (typeof data === "string") {
+            const url = resolveUrl(_context.gltfId, data)
+            return ImageReference.getOrCreate(url);
+        }
+        return undefined;
+    }
+}
+new ImageReferenceSerializer();

src/engine/engine_serialization_builtin_serializer.ts CHANGED Viewed

@@ -7,6 +7,7 @@
 import { Color, Object3D, Texture, WebGLRenderTarget } from "three";
 import { RenderTexture } from "./engine_texture";
 import { isDevEnvironment } from "../engine/debug/debug";
+import { resolveUrl } from "./engine_utils";
 // export class SourcePath {
 //     src?:string
@@ -300,8 +301,8 @@
         return undefined;
     }
-    private createEventMethod(target : object, methodName: string, args?: any) : Function | undefined {
+    private createEventMethod(target: object, methodName: string, args?: any): Function | undefined {
         return (...forwardedArgs) => {
             const method = target[methodName];
             if (typeof method === "function") {
@@ -339,4 +340,23 @@
         return undefined;
     }
 }
-new RenderTextureSerializer();
+new RenderTextureSerializer();
+export class UriSerializer extends TypeSerializer {
+    constructor() {
+        super([URL]);
+    }
+    onSerialize(_data: string, _context: SerializationContext) {
+        return null;
+    }
+    onDeserialize(data: string, _context: SerializationContext) {
+        if (typeof data === "string") {
+            return resolveUrl(_context.gltfId, data);
+        }
+        return undefined;
+    }
+}
+new UriSerializer();

src/engine/engine_serialization_core.ts CHANGED Viewed

@@ -173,6 +173,7 @@
     root: THREE.Object3D;
     gltf?: GLTF;
+    /** the url of the glb that is currently being loaded */
     gltfId?: SourceIdentifier;
     object!: THREE.Object3D;
     target?: object;
@@ -290,16 +291,21 @@
                 context.type = undefined;
                 context.path = key;
+                if (obj.onBeforeDeserializeMember !== undefined) {
+                    // callback to the instance, if it returns true assume it's done all the things itself
+                    if (obj.onBeforeDeserializeMember(key, data, context) === true) continue;
+                }
                 if (serializedEntryInfo === null) {
                     obj[key] = data;
+                    // if(typeof data === "string"){
+                    //     const serializer = helper.getSerializerForConstructor(String);
+                    //     const res = serializer?.onDeserialize(data, context);
+                    //     if(res !== undefined) obj[key] = res;
+                    // }
                 }
                 else {
-                    if (obj.onBeforeDeserializeMember !== undefined) {
-                        // callback to the instance, if it returns true assume it's done all the things itself
-                        if (obj.onBeforeDeserializeMember(key, data, context) === true) continue;
-                    }
                     if (Array.isArray(serializedEntryInfo)) {
                         for (let i = 0; i < serializedEntryInfo.length; i++) {
                             const typeInfoOrConstructor = serializedEntryInfo[i];
@@ -314,25 +320,26 @@
                         obj[key] = tryResolve(serializedEntryInfo);
                     }
-                    function tryResolve(typeInfoOrConstructor) {
-                        const typeInformationOrConstructor = typeInfoOrConstructor as ITypeInformation;
-                        // if the entry does specify an object of type ITypeInformation and has the type field set
-                        const type = typeInformationOrConstructor.type;
-                        if (type) {
-                            return deserializeObjectWithType(data, type, context, undefined, obj[key]);
-                        }
-                        // it can also just contain a constructor
-                        else {
-                            const constructor = typeInfoOrConstructor as Constructor<any>;
-                            return deserializeObjectWithType(data, constructor, context, undefined, obj[key]);
-                        }
-                    }
                     buffer.length = 0;
+                }
-                    if (obj.onAfterDeserializeMember !== undefined) {
-                        obj.onAfterDeserializeMember(key, data, context);
+                if (obj.onAfterDeserializeMember !== undefined) {
+                    obj.onAfterDeserializeMember(key, data, context);
+                }
+                function tryResolve(typeInfoOrConstructor) {
+                    const typeInformationOrConstructor = typeInfoOrConstructor as ITypeInformation;
+                    // if the entry does specify an object of type ITypeInformation and has the type field set
+                    const type = typeInformationOrConstructor.type;
+                    if (type) {
+                        return deserializeObjectWithType(data, type, context, undefined, obj[key]);
                     }
+                    // it can also just contain a constructor
+                    else {
+                        const constructor = typeInfoOrConstructor as Constructor<any>;
+                        return deserializeObjectWithType(data, constructor, context, undefined, obj[key]);
+                    }
                 }
             }
         }

src/engine/engine_serialization.ts CHANGED Viewed

@@ -1,6 +1,7 @@
 import { serializeObject, deserializeObject } from "./engine_serialization_core";
-import * as builtin from "./engine_serialization_builtin_serializer";
-// export builtin so it will be called and registered
-export { serializeObject, deserializeObject, builtin };
-export { serializable, serializeable } from "./engine_serialization_decorator"
+export { serializeObject, deserializeObject };
+export { serializable, serializeable } from "./engine_serialization_decorator"
+export * from "./engine_serialization_builtin_serializer";

src/engine/engine_utils.ts CHANGED Viewed

@@ -203,30 +203,51 @@
     });
 }
-// if a timeline is exported via menu item the audio clip path is relative to the glb (same folder)
+// 1) if a timeline is exported via menu item the audio clip path is relative to the glb (same folder)
 // we need to detect that here and build the new audio source path relative to the new glb location
 // the same is/might be true for any file that is/will be exported via menu item
-const debugGetPath = getParam("debugsourcepath");
-export function getPath(source: SourceIdentifier | undefined, uri: string): string {
-    if (source === undefined) {
-        if (debugGetPath) console.warn("getPath: source is undefined, returning uri", uri);
+// 2) if the needle.config assetDirectory is modified (from e.g. /assets to /needle/assets) when building a distributable our vite transform and copy plugin will move the files to dist/assets hence we cannot use project-relative paths (because the path changes). What we do instead if make all paths serialized in a glb relative to the glb. The rel: prefix is used to detect urls that need to be resolved.
+const debugGetPath = getParam("debugresolveurl");
+export const relativePathPrefix = "rel:";
+/** @deprecated use resolveUrl instead */
+export function getPath(source:SourceIdentifier|undefined, uri:string) : string {
+    return resolveUrl(source, uri);
+}
+/**
+ * Use to resolve a url serialized in a glTF file
+ * @param source The uri of the loading file
+ * @param uri The uri of the file to resolve, can be absolute or relative
+ * @returns The resolved uri
+ */
+export function resolveUrl(source: SourceIdentifier | undefined, uri: string): string {
+    if (uri === undefined) {
+        if (debugGetPath) console.warn("getPath: uri is undefined, returning uri", uri);
         return uri;
     }
+    if(uri.startsWith("./")) {
+        return uri;
+    }
     if (uri.startsWith("http")) {
         if (debugGetPath) console.warn("getPath: uri is absolute, returning uri", uri);
         return uri;
     }
+    if (source === undefined) {
+        if (debugGetPath) console.warn("getPath: source is undefined, returning uri", uri);
+        return uri;
+    }
+    if(uri.startsWith(relativePathPrefix)){
+        uri = uri.substring(4);
+    }
     const pathIndex = source.lastIndexOf("/");
     if (pathIndex >= 0) {
-        let newUri = source.substring(0, pathIndex + 1);
-        const uriDirectoryIndex = uri.lastIndexOf("/");
-        if (uriDirectoryIndex >= 0) {
-            newUri += uri.substring(uriDirectoryIndex + 1);
-        } else {
-            newUri += uri;
-        }
-        if (debugGetPath) console.log("getPath:", source, " - changed uri from\n", uri, "\n→ ", newUri);
+        // Take the source uri as the base path
+        const basePath = source.substring(0, pathIndex + 1);
+        // Append the relative uri
+        let newUri = basePath + uri;
+        // newUri = new URL(newUri, globalThis.location.href).href;
+        if (debugGetPath) console.log("source:", source, "- changed uri \nfrom", uri, "\n→ ", newUri, "\n" + basePath);
         return newUri;
     }
     return uri;

src/engine/extensions/NEEDLE_animator_controller_model.ts CHANGED Viewed

@@ -32,6 +32,7 @@
 export declare type State = {
     name: string,
     hash: number;
+    speed?: number;
     motion: Motion,
     transitions: Transition[],
     behaviours: StateMachineBehaviourModel[],

src/engine/extensions/NEEDLE_progressive.ts CHANGED Viewed

@@ -3,7 +3,7 @@
 import { SourceIdentifier } from "../engine_types";
 import { Context } from "../engine_setup";
 import { addDracoAndKTX2Loaders } from "../engine_loaders";
-import { delay, getParam, getPath } from "../engine_utils";
+import { delay, getParam, resolveUrl } from "../engine_utils";
 export const EXTENSION_NAME = "NEEDLE_progressive";
@@ -130,7 +130,7 @@
         if (progressiveInfo) {
             if (debug)
                 console.log(key, progressiveInfo.uri, progressiveInfo.guid);
-            const uri = getPath(source, progressiveInfo.uri);
+            const uri = resolveUrl(source, progressiveInfo.uri);
             if (uri.endsWith(".glb") || uri.endsWith(".gltf")) {
                 if (!progressiveInfo.guid) {
                     console.warn("missing pointer for glb/gltf texture", progressiveInfo);

src/engine-components/ParticleSystem.ts CHANGED Viewed

@@ -193,6 +193,7 @@
     }
     genValue(): number {
+        if (!this.system.isPlaying) return 0;
         if (!this.system.emission.enabled) return 0;
         if (this.system.currentParticles >= this.system.maxParticles) return 0;
         // emission over time
@@ -220,6 +221,7 @@
 class ParticleSystemEmissionOverDistance extends BaseValueGenerator {
     genValue(): number {
+        if (!this.system.isPlaying) return 0;
         // this seems not be called yet
         return 0;
         // if (this.system.currentParticles >= this.system.maxParticles) return 0;
@@ -547,7 +549,7 @@
         }
         switch (this.system.renderer.renderMode) {
             case ParticleSystemRenderMode.Billboard: return RenderMode.BillBoard;
-            // case ParticleSystemRenderMode.Stretch: return RenderMode.Stretch;
+            case ParticleSystemRenderMode.Stretch: return RenderMode.StretchedBillBoard;
             case ParticleSystemRenderMode.HorizontalBillboard: return RenderMode.LocalSpace;
             case ParticleSystemRenderMode.VerticalBillboard: return RenderMode.LocalSpace;
             case ParticleSystemRenderMode.Mesh: return RenderMode.LocalSpace;
@@ -597,8 +599,8 @@
                 }
             }, true)
         }
         this._isPlaying = true;
-        this._time = 0;
         // https://github.com/Alchemist0823/three.quarks/pull/35
         if (this._particleSystem) {
@@ -608,14 +610,42 @@
         this.emission?.reset();
     }
-    pause() {
+    pause(includeChildren = true) {
+        if (includeChildren) {
+            GameObject.foreachComponent(this.gameObject, comp => {
+                if (comp instanceof ParticleSystem && comp !== this) {
+                    comp.pause(false);
+                }
+            }, true)
+        }
         this._isPlaying = false;
     }
-    stop() {
+    /** clear=true removes all emitted particles */
+    stop(includeChildren = true, clear: boolean = false) {
+        if (includeChildren) {
+            GameObject.foreachComponent(this.gameObject, comp => {
+                if (comp instanceof ParticleSystem && comp !== this) {
+                    comp.stop(false, clear);
+                }
+            }, true)
+        }
         this._isPlaying = false;
         this._time = 0;
+        if (clear) this.reset();
     }
+    /** remove emitted particles and reset time */
+    reset() {
+        this._time = 0;
+        if (this._particleSystem) {
+            this._particleSystem.particleNum = 0;
+            this._particleSystem["emissionState"].time = 0;
+            this._particleSystem["emitEnded"] = false;
+            this.emission?.reset();
+        }
+    }
     private _state?: ParticlesEmissionState;
     emit(count: number) {
         if (this._particleSystem) {

src/engine-components/ParticleSystemModules.ts CHANGED Viewed

@@ -35,7 +35,7 @@
 export enum ParticleSystemRenderMode {
     Billboard = 0,
-    //   Stretch = 1,
+    Stretch = 1,
     HorizontalBillboard = 2,
     VerticalBillboard = 3,
     Mesh = 4,

src/engine-components/Skybox.ts CHANGED Viewed

@@ -5,7 +5,7 @@
 import { EquirectangularRefractionMapping, sRGBEncoding, Texture, TextureLoader } from "three"
 import { syncField } from "../engine/engine_networking_auto";
 import { Camera } from "./Camera";
-import { getParam, getPath } from "../engine/engine_utils";
+import { getParam, resolveUrl } from "../engine/engine_utils";
 const debug = getParam("debugskybox");
@@ -56,7 +56,7 @@
         if(debug) console.log("Remote skybox url?: " + url);
         if (!url.startsWith("http") && !url.startsWith("www.") && !url.startsWith("data:")) {
-            url = getPath(this.sourceId, url);
+            url = resolveUrl(this.sourceId, url);
             if(debug) console.log("Remote skybox resolved to " + url);
         }

src/engine-components/ui/Text.ts CHANGED Viewed

@@ -7,7 +7,7 @@
 import { updateRenderSettings } from './Utils';
 import { Canvas } from './Canvas';
 import { serializable } from '../../engine/engine_serialization_decorator';
-import { getParam, getPath } from '../../engine/engine_utils';
+import { getParam, resolveUrl } from '../../engine/engine_utils';
 const debug = getParam("debugtext");
@@ -481,24 +481,24 @@
         // if a font path has a known suffix we remove it
         if (fontName.endsWith("-regular")) {
-            if (style === FontStyle.Normal) return getPath(this.sourceId, fontName);
+            if (style === FontStyle.Normal) return resolveUrl(this.sourceId, fontName);
                 fontName = fontName.substring(0, fontName.length - "-regular".length);
         }
         else if (fontName.endsWith("-bold")) {
-            if (style === FontStyle.Bold)return getPath(this.sourceId, fontName);
+            if (style === FontStyle.Bold)return resolveUrl(this.sourceId, fontName);
                 fontName = fontName.substring(0, fontName.length - "-bold".length);
         }
         else if (fontName.endsWith("-italic")) {
-            if (style === FontStyle.Italic)return getPath(this.sourceId, fontName);
+            if (style === FontStyle.Italic)return resolveUrl(this.sourceId, fontName);
                 fontName = fontName.substring(0, fontName.length - "-italic".length);
         }
         else if (fontName.endsWith("-bolditalic")) {
-            if (style === FontStyle.BoldAndItalic)return getPath(this.sourceId, fontName);
+            if (style === FontStyle.BoldAndItalic)return resolveUrl(this.sourceId, fontName);
                 fontName = fontName.substring(0, fontName.length - "-bolditalic".length);
         }
         else
             // If a font does not have a specific style suffic we dont support getting the correct font style
-            return getPath(this.sourceId, fontName);
+            return resolveUrl(this.sourceId, fontName);
         switch (style) {
             case FontStyle.Normal:
@@ -515,7 +515,7 @@
                 break;
         }
-        return getPath(this.sourceId, fontName);
+        return resolveUrl(this.sourceId, fontName);
     }
 }

src/engine-components/timeline/TimelineTracks.ts CHANGED Viewed

@@ -5,7 +5,7 @@
 import { Context } from "../../engine/engine_setup";
 import { SignalReceiver } from "./SignalAsset";
 import { AnimationClip, Quaternion, Vector3 } from "three";
-import { getParam, getPath } from "../../engine/engine_utils";
+import { getParam, resolveUrl } from "../../engine/engine_utils";
 import { AudioSource } from "../AudioSource";
 import { Animator } from "../Animator"
@@ -517,7 +517,7 @@
     private getAudioFilePath(path: string) {
         // TODO: this should be the timeline asset location probably which MIGHT be different
         const glbLocation = this.director.sourceId;
-        return getPath(glbLocation, path);
+        return resolveUrl(glbLocation, path);
     }
     onAllowAudioChanged(allow: boolean) {

src/engine-components/VideoPlayer.ts CHANGED Viewed

@@ -3,7 +3,7 @@
 import { serializable } from "../engine/engine_serialization_decorator";
 import { LinearFilter, Material, Mesh, Object3D, RawShaderMaterial, ShaderMaterial, Texture, TextureLoader, Vector2, Vector4, VideoTexture } from "three";
 import { awaitInput } from "../engine/engine_input_utils";
-import { getParam } from "../engine/engine_utils";
+import { getParam, resolveUrl } from "../engine/engine_utils";
 import { Renderer } from "./Renderer";
 import { getWorldScale } from "../engine/engine_three_utils";
 import { ObjectUtils, PrimitiveType } from "../engine/engine_create_objects";
@@ -49,11 +49,12 @@
     renderer: THREE.Object3D | null = null;
     @serializable()
     playOnAwake: boolean = true;
-    @serializable()
-    playOnEnable?: boolean;
     @serializable()
     aspectMode: AspectMode = AspectMode.None;
+    @serializable(URL)
+    private clip?: string | MediaStream | null = null;
     @serializable()
     private renderMode?: VideoRenderMode;
@@ -153,7 +154,6 @@
     private audioOutputMode: VideoAudioOutputMode = VideoAudioOutputMode.AudioSource;
     private source!: VideoSource;
-    private clip?: string | MediaStream | null = null;
     private url?: string | null = null;
     private _videoElement: HTMLVideoElement | null = null;
@@ -209,7 +209,7 @@
     }
     onEnable(): void {
-        if (this.playOnEnable === true) {
+        if (this.playOnAwake === true) {
             this.handleBeginPlaying(true);
         }
         if (this.screenspace) {
@@ -422,7 +422,7 @@
         let muted = !this._receivedInput && this.audioOutputMode !== VideoAudioOutputMode.None;
         if(!muted && this._muted) muted = true;
         this._videoElement.muted = muted;
-        if (this.playOnAwake || this.playOnEnable)
+        if (this.playOnAwake)
             this._videoElement.autoplay = true;
     }

src/engine-components/WebXR.ts CHANGED Viewed

@@ -47,6 +47,7 @@
     XRStopped = "xrStopped",
     XRUpdate = "xrUpdate",
     RequestVRSession = "requestVRSession",
+    ModifyAROptions = "modify-ar-options",
 }
 export declare type CreateButtonOptions = {
@@ -91,6 +92,9 @@
         this.events.removeEventListener(type, listener);
         return listener;
     }
+    private static dispatchEvent(type: string, event: any): void {
+        this.events.dispatchEvent({ type, detail: event });
+    }
     public static createVRButton(webXR: WebXR, opts?: CreateButtonOptions): HTMLButtonElement | HTMLAnchorElement {
         if (!WebXR.XRSupported) {
@@ -122,7 +126,7 @@
             console.warn("No dom overlay root found, HTML overlays on top of screen-based AR will not work.");
         }
-        const arButton = ARButton.createButton(webXR.context.renderer, options);
+        const arButton = ARButton.createButton(webXR.context.renderer, options, this.onModifyAROptions.bind(this));
         arButton.classList.add('webxr-ar-button');
         arButton.classList.add('webxr-button');
         WebXR.resetButtonStyles(arButton);
@@ -131,6 +135,10 @@
         return arButton;
     }
+    private static onModifyAROptions(options){
+        WebXR.dispatchEvent(WebXREvent.ModifyAROptions, options);
+    }
     public static resetButtonStyles(button) {
         if (!button) return;
         button.style.position = "";
@@ -233,7 +241,8 @@
         this.context.domElement.append(buttonsContainer);
         // AR support
-        if (this.enableAR && this.createARButton && arSupported) {
+        // if (this.enableAR && this.createARButton && arSupported)
+        {
             arButton = WebXR.createARButton(this);
             this._arButton = arButton;
             buttonsContainer.appendChild(arButton);

src/engine-components/WebXRController.ts CHANGED Viewed

@@ -8,7 +8,7 @@
 import { Mathf } from "../engine/engine_math";
 import { RaycastOptions } from "../engine/engine_physics";
 import { getWorldPosition, getWorldQuaternion, setWorldPosition, setWorldQuaternion } from "../engine/engine_three_utils";
-import { getParam, getPath } from "../engine/engine_utils";
+import { getParam, resolveUrl } from "../engine/engine_utils";
 import { addDracoAndKTX2Loaders } from "../engine/engine_loaders";
 import { Avatar_POI } from "./avatar/Avatar_Brain_LookAt";
@@ -85,7 +85,7 @@
         const loader = new GLTFLoader();
         addDracoAndKTX2Loaders(loader, context);
         if (ctrl.webXR.handModelPath && ctrl.webXR.handModelPath !== "")
-            loader.setPath(getPath(owner.sourceId, ctrl.webXR.handModelPath));
+            loader.setPath(resolveUrl(owner.sourceId, ctrl.webXR.handModelPath));
         else
             // from XRHandMeshModel.js
             loader.setPath('https://cdn.jsdelivr.net/npm/@webxr-input-profiles/[email protected]/dist/profiles/generic-hand/');

plugins/vite/transform-codegen.js ADDED Viewed

@@ -0,0 +1,45 @@
+import { builtAssetsDirectory, tryLoadProjectConfig } from './config.js';
+/**
+ * modify the glb load path in codegen files
+ * this is necessary if the assets directory is not the default (changed by the user in needle.config.json)
+ */
+export const needleTransformCodegen = (command, config, userSettings) => {
+    if (config?.noCodegenTransform === true || userSettings?.noCodegenTransform === true) {
+        return;
+    }
+    let codegenDirectory = "src/generated";
+    const needleConfig = tryLoadProjectConfig();
+    if (needleConfig?.codegenDirectory?.length)
+        codegenDirectory = needleConfig.codegenDirectory;
+    let configuredAssetsDirectory = "assets";
+    if (needleConfig?.assetsDirectory?.length)
+        configuredAssetsDirectory = needleConfig.assetsDirectory;
+    // https://regex101.com/r/Y05z9P/1
+    // const matchCodegenFilePaths = /\"(.+)\/.+?\.(glb|gltf)/g;
+    return [
+        {
+            name: 'needle-transform-files',
+            apply: 'build',
+            transform(src, id) {
+                if (id.endsWith(codegenDirectory + "/gen.js")) {
+                    const assetsDir = builtAssetsDirectory();
+                    if (assetsDir !== configuredAssetsDirectory) {
+                        console.log(`[needle-transform-files] - Transform codegen paths \"${configuredAssetsDirectory}\" → \"${assetsDir}\"`)
+                        // replace codegen paths
+                        src = src.replaceAll(configuredAssetsDirectory, assetsDir);
+                        return {
+                            code: src,
+                            map: null
+                        }
+                    }
+                }
+            }
+        }
+    ];
+}

src/engine-components/postprocessing/Effects/TiltShiftEffect.ts ADDED Viewed

@@ -0,0 +1,56 @@
+import { registerCustomEffectType } from "../VolumeProfile";
+import { EffectProviderResult, PostProcessingEffect } from "../PostProcessingEffect";
+import { KernelSize, TiltShiftEffect as TiltShift } from "postprocessing";
+import { VolumeParameter } from "../VolumeParameter";
+import { serializable } from "../../../engine/engine_serialization";
+export class TiltShiftEffect extends PostProcessingEffect {
+    get typeName(): string {
+        return "TiltShiftEffect";
+    }
+    @serializable(VolumeParameter)
+    offset!: VolumeParameter;
+    @serializable(VolumeParameter)
+    rotation!: VolumeParameter;
+    @serializable(VolumeParameter)
+    focusArea!: VolumeParameter;
+    @serializable(VolumeParameter)
+    feather!: VolumeParameter
+    @serializable(VolumeParameter)
+    kernelSize!: VolumeParameter;
+    @serializable(VolumeParameter)
+    resolutionScale!: VolumeParameter;
+    init(): void {
+        this.offset.defaultValue = 0;
+        this.rotation.defaultValue = 0;
+        this.focusArea.defaultValue = 0.4;
+        this.feather.defaultValue = 0.3;
+        this.kernelSize.defaultValue = KernelSize.MEDIUM;
+        this.resolutionScale.defaultValue = 1 / window.devicePixelRatio;
+    }
+    onCreateEffect(): EffectProviderResult | undefined {
+        console.log(this);
+        const effect = new TiltShift({
+            kernelSize: KernelSize.VERY_LARGE,
+        });
+        this.offset.onValueChanged = v => effect.offset = v;
+        this.rotation.onValueChanged = v => effect.rotation = v;
+        this.focusArea.onValueChanged = v => effect.focusArea = v;
+        this.feather.onValueChanged = v => effect.feather = v;
+        this.kernelSize.onValueChanged = v => effect.blurPass.kernelSize = v;
+        this.resolutionScale.onValueChanged = v => effect.resolution.scale = v / window.devicePixelRatio;
+        return effect;
+    }
+}
+registerCustomEffectType("TiltShiftEffect", TiltShiftEffect);

src/engine-components/WebXRImageTracking.ts ADDED Viewed

@@ -0,0 +1,192 @@
+import { WebXR } from "./WebXR";
+import { serializable } from "../engine/engine_serialization";
+import { Behaviour } from "./Component";
+import { Matrix4, Object3D, Quaternion, Vector, Vector3 } from "three";
+import { CircularBuffer, getParam } from "../engine/engine_utils";
+// https://github.com/immersive-web/marker-tracking/blob/main/explainer.md
+const debug = getParam("debugimagetracking");
+const _scaleTemp = new Vector3();
+export class WebXRTrackedImage {
+    get url(): string { return this._trackedImage.image ?? ""; }
+    get widthInMeters() { return this._trackedImage.widthInMeters ?? undefined; }
+    get bitmap(): ImageBitmap { return this._bitmap; }
+    readonly measuredSize: number;
+    readonly state: "tracked" | "emulated";
+    // private _matrix: Matrix4 | null = null;
+    // private get matrix(): Matrix4 {
+    //     if (!this._matrix) {
+    //         // this._matrix = WebXRTrackedImage._matrixBuffer.get();
+    //         // const matrix = this._pose.transform.matrix;
+    //         // this._matrix.fromArray(matrix);
+    //     }
+    //     return this._matrix!;
+    // }
+    /** Copy the image position to a vector */
+    getPosition(vec: Vector3) {
+        this.ensureTransformData();
+        vec.copy(this._position);
+        return vec;
+    }
+    /** Copy the image rotation to a quaternion */
+    getQuaternion(quat: Quaternion) {
+        this.ensureTransformData();
+        quat.copy(this._rotation);
+        return quat;
+    }
+    applyToObject(object: Object3D) {
+        this.ensureTransformData();
+        object.position.copy(this._position);
+        object.quaternion.copy(this._rotation);
+    }
+    // private static _matrixBuffer: CircularBuffer<Matrix4> = new CircularBuffer(() => new Matrix4(), 20);
+    private static _positionBuffer: CircularBuffer<Vector3> = new CircularBuffer(() => new Vector3(), 20);
+    private static _rotationBuffer: CircularBuffer<Quaternion> = new CircularBuffer(() => new Quaternion(), 20);
+    private _position!: Vector3;
+    private _rotation!: Quaternion;
+    private ensureTransformData() {
+        if (!this._position) {
+            this._position = WebXRTrackedImage._positionBuffer.get();
+            this._rotation = WebXRTrackedImage._rotationBuffer.get();
+            const t = this._pose.transform;
+            this._position.set(-t.position.x, t.position.y, -t.position.z);
+            this._rotation.set(-t.orientation.x, t.orientation.y, -t.orientation.z, t.orientation.w);
+        }
+    }
+    private readonly _trackingComponent: WebXRImageTracking;;
+    private readonly _trackedImage: WebXRImageTrackingModel;
+    private readonly _bitmap: ImageBitmap;
+    private readonly _pose: any;
+    constructor(context: WebXRImageTracking, trackedImage: WebXRImageTrackingModel, bitmap: ImageBitmap, measuredSize: number, state: "tracked" | "emulated", pose: any) {
+        this._trackingComponent = context;;
+        this._trackedImage = trackedImage;
+        this._bitmap = bitmap;
+        this.measuredSize = measuredSize;
+        this.state = state;
+        this._pose = pose;
+    }
+}
+declare type WebXRImageTrackingEvent = (images: WebXRImageTrackingEvent[]) => void;
+export class WebXRImageTrackingModel {
+    @serializable()
+    image?: string;
+    @serializable()
+    widthInMeters!: number;
+}
+export class WebXRImageTracking extends Behaviour {
+    @serializable(WebXRImageTrackingModel)
+    trackedImages!: WebXRImageTrackingModel[];
+    private readonly trackedImageIndexMap: Map<number, WebXRImageTrackingModel> = new Map();
+    private static _imageElements: Map<string, ImageBitmap | null> = new Map();
+    awake(): void {
+        for (const trackedImage of this.trackedImages) {
+            if (trackedImage.image) {
+                if (WebXRImageTracking._imageElements.has(trackedImage.image)) {
+                }
+                else {
+                    const url = trackedImage.image;
+                    WebXRImageTracking._imageElements.set(url, null);
+                    const imageElement = document.createElement("img") as HTMLImageElement;
+                    imageElement.src = url;
+                    imageElement.addEventListener("load", async () => {
+                        const img = await createImageBitmap(imageElement);
+                        WebXRImageTracking._imageElements.set(url, img);
+                    });
+                }
+            }
+        }
+    }
+    onEnable(): void {
+        WebXR.addEventListener("modify-ar-options", this.onModifyAROptions);
+    }
+    onDisable(): void {
+        WebXR.removeEventListener("modify-ar-options", this.onModifyAROptions);
+    }
+    private onModifyAROptions = (event: any) => {
+        const options = event.detail;
+        const features = options.optionalFeatures || [];
+        features.push("image-tracking");
+        options.optionalFeatures = features;
+        options.trackedImages = [];
+        for (const trackedImage of this.trackedImages) {
+            if (trackedImage.image?.length && trackedImage.widthInMeters > 0) {
+                const bitmap = WebXRImageTracking._imageElements.get(trackedImage.image);
+                if (bitmap) {
+                    this.trackedImageIndexMap.set(options.trackedImages.length, trackedImage);
+                    options.trackedImages.push({
+                        image: bitmap,
+                        widthInMeters: trackedImage.widthInMeters
+                    });
+                }
+            }
+        }
+    }
+    onBeforeRender(frame: XRFrame | null): void {
+        //@ts-ignore
+        if (frame?.session && typeof frame.getImageTrackingResults === "function") {
+            //@ts-ignore
+            const results = frame.getImageTrackingResults();
+            if (results.length) {
+                const space = this.context.renderer.xr.getReferenceSpace();
+                if (space) {
+                    const images: WebXRTrackedImage[] = [];
+                    for (const result of results) {
+                        const imageIndex = result.index;
+                        const trackedImage = this.trackedImageIndexMap.get(imageIndex);
+                        if (trackedImage) {
+                            const pose = frame.getPose(result.imageSpace, space);
+                            const state = result.trackingState;
+                            const imageData = new WebXRTrackedImage(this, trackedImage, result.image, result.measuredSize, state, pose);
+                            images.push(imageData);
+                        }
+                        else {
+                            if (debug) {
+                                console.warn("No tracked image for index", imageIndex);
+                            }
+                        }
+                    }
+                    if (images.length > 0) {
+                        try {
+                            this.dispatchEvent(new CustomEvent("image-tracking", { detail: images }));
+                        }
+                        catch (e) {
+                            console.error(e);
+                        }
+                    }
+                }
+            }
+        }
+    }
+}