From 5ceb7e55b81b4c846532b4ab5c454f72e1c876e6 Mon Sep 17 00:00:00 2001 From: benStre Date: Wed, 11 Sep 2024 17:54:00 +0200 Subject: [PATCH 01/27] set Runtime.OPTIONS.PROTECT_POINTERS default to true --- runtime/runtime.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/runtime.ts b/runtime/runtime.ts index 8f8fd9e..f8898cf 100644 --- a/runtime/runtime.ts +++ b/runtime/runtime.ts @@ -222,7 +222,7 @@ export class Runtime { // can be changed public static OPTIONS = { - PROTECT_POINTERS: false, // explicit permissions are required for remote endpoints to read/write pointers (current default: false) + PROTECT_POINTERS: true, // explicit permissions are required for remote endpoints to read/write pointers (default: true) INDIRECT_REFERENCES: false, // enable support for indirect references to pointers from other pointers DEFAULT_REQUEST_TIMEOUT: 5000, // default timeout for DATEX requests in ms GARBAGE_COLLECTION_TIMEOUT: 1000, // time after which a value can get garbage collected From c09708e5b3af091e1fa3ccb19d4f1ed9e0d6a51b Mon Sep 17 00:00:00 2001 From: benStre Date: Thu, 19 Sep 2024 00:03:51 +0200 Subject: [PATCH 02/27] refactor types --- compiler/compiler.ts | 8 +- datex_short.ts | 49 ++++---- functions.ts | 26 +++-- runtime/endpoint_config.ts | 4 +- runtime/io_handler.ts | 2 +- runtime/pointers.ts | 214 ++++++++++++++++++----------------- runtime/runtime.ts | 104 ++++++++--------- storage/storage.ts | 4 +- types/function.ts | 4 +- types/logic.ts | 16 +-- types/type.ts | 6 +- utils/global_types.ts | 4 +- utils/history.ts | 6 +- utils/interface-generator.ts | 2 +- utils/iterable-handler.ts | 24 ++-- 15 files changed, 244 insertions(+), 229 deletions(-) diff --git a/compiler/compiler.ts b/compiler/compiler.ts index 76c1429..ce575d7 100644 --- a/compiler/compiler.ts +++ b/compiler/compiler.ts @@ -15,7 +15,7 @@ const logger = new Logger("datex compiler"); import { ReadableStream, Runtime } from "../runtime/runtime.ts"; import { Endpoint, IdEndpoint, Target, WildcardTarget, Institution, Person, BROADCAST, target_clause, endpoints, LOCAL_ENDPOINT } from "../types/addressing.ts"; -import { Pointer, PointerProperty, Ref } from "../runtime/pointers.ts"; +import { Pointer, PointerProperty, ReactiveValue } from "../runtime/pointers.ts"; import { CompilerError, RuntimeError, Error as DatexError, ValueError } from "../types/errors.ts"; import { Function as DatexFunction } from "../types/function.ts"; @@ -2764,7 +2764,7 @@ export class Compiler { // make sure normal pointers are collapsed (ignore error if uninitialized pointer is passed in) try { - value = Ref.collapseValue(value); + value = ReactiveValue.collapseValue(value); } catch {} @@ -2864,7 +2864,7 @@ export class Compiler { const skip_first_collapse = !SCOPE.options._first_insert_done&&SCOPE.options.collapse_first_inserted; const option_collapse = SCOPE.options.collapse_pointers && !(SCOPE.options.keep_external_pointers && value instanceof Pointer && !value.is_origin); - const no_proxify = value instanceof Ref && (((value instanceof Pointer && value.is_anonymous) || option_collapse) || skip_first_collapse); + const no_proxify = value instanceof ReactiveValue && (((value instanceof Pointer && value.is_anonymous) || option_collapse) || skip_first_collapse); // proxify pointer exceptions: if (no_proxify) { @@ -2903,7 +2903,7 @@ export class Compiler { // indirect reference pointer if (indirectReferencePtr) { SCOPE.options._first_insert_done = true; - Compiler.builder.insert(Ref.collapseValue(value, true), SCOPE, is_root, parents, unassigned_children); + Compiler.builder.insert(ReactiveValue.collapseValue(value, true), SCOPE, is_root, parents, unassigned_children); return; } diff --git a/datex_short.ts b/datex_short.ts index b18cf5c..3802d8c 100644 --- a/datex_short.ts +++ b/datex_short.ts @@ -1,7 +1,7 @@ // shortcut functions // import { Datex } from "./datex.ts"; -import { baseURL, Runtime, PrecompiledDXB, Type, Pointer, Ref, PointerProperty, primitive, Target, IdEndpoint, Markdown, MinimalJSRef, RefOrValue, PartialRefOrValueObject, datex_meta, ObjectWithDatexValues, Compiler, endpoint_by_endpoint_name, endpoint_name, Storage, compiler_scope, datex_scope, DatexResponse, target_clause, ValueError, logger, Class, getUnknownMeta, Endpoint, INSERT_MARK, CollapsedValueAdvanced, CollapsedValue, SmartTransformFunction, compiler_options, activePlugins, METADATA, handleDecoratorArgs, RefOrValueObject, PointerPropertyParent, InferredPointerProperty, RefLike, dc } from "./datex_all.ts"; +import { baseURL, Runtime, PrecompiledDXB, Type, Pointer, ReactiveValue, PointerProperty, primitive, Target, IdEndpoint, Markdown, MinimalJSRef, RefOrValue, PartialRefOrValueObject, datex_meta, ObjectWithDatexValues, Compiler, endpoint_by_endpoint_name, endpoint_name, Storage, compiler_scope, datex_scope, DatexResponse, target_clause, ValueError, logger, Class, getUnknownMeta, Endpoint, INSERT_MARK, CollapsedValueAdvanced, CollapsedValue, SmartTransformFunction, compiler_options, activePlugins, METADATA, handleDecoratorArgs, RefOrValueObject, PointerPropertyParent, InferredPointerProperty, RefLike, dc } from "./datex_all.ts"; /** make decorators global */ import { assert as _assert, timeout as _timeout, entrypoint as _entrypoint, ref as _ref, entrypointProperty as _entrypointProperty, property as _property, struct as _struct, endpoint as _endpoint, sync as _sync, allow as _allow} from "./datex_all.ts"; @@ -41,14 +41,14 @@ declare global { const selectProperty: typeof _selectProperty; const not: typeof _not; const effect: typeof _effect; - const observe: typeof Ref.observe - const observeAndInit: typeof Ref.observeAndInit - const unobserve: typeof Ref.unobserve + const observe: typeof ReactiveValue.observe + const observeAndInit: typeof ReactiveValue.observeAndInit + const unobserve: typeof ReactiveValue.unobserve /** * Prevents any values accessed within the callback function from * being captured by a transform function (e.g. always) */ - const isolate: typeof Ref.disableCapturing + const isolate: typeof ReactiveValue.disableCapturing /** * The local endpoint of the current runtime (alias for Datex.Runtime.endpoint) @@ -323,7 +323,7 @@ export function pointer(value:RefOrValue, property?:unknown): unknown { export function prop>(parent:RefOrValue, propertyKey: T extends Map ? K : unknown): PointerProperty ? V : unknown>|(T extends Map ? V : unknown) export function prop>(parent:RefOrValue, propertyKey: keyof T): PointerProperty|T[keyof T] export function prop(parent:Map|Record, propertyKey: unknown): any { - if (Ref.isRef(parent)) return PointerProperty.get(parent, propertyKey); + if (ReactiveValue.isRef(parent)) return PointerProperty.get(parent, propertyKey); else if (parent instanceof Map) return parent.get(propertyKey); else return parent[propertyKey as keyof typeof parent]; } @@ -361,11 +361,10 @@ export function revokeAccess(value: any, endpoint: string|Endpoint) { export const $$ = pointer; -interface $fn { - (value:T): MinimalJSRef -} -type $type = (Record|Promise>>) & $fn; +type $type = (Record|Promise>>) & { + (value:T): MinimalJSRef +}; /** * Compiled reactivity syntax ($()) - throws an error when called directly and not compiled to $$() or always() @@ -385,10 +384,8 @@ export const $ = new Proxy(function(){} as unknown as $type, { } }, - apply() { - // TODO: change errors, not UIX specific - if (client_type == "deno") throw new Error("Experimental $() syntax is currently not supported on the backend"); - else throw new Error("Experimental $() syntax is not enabled per default. Add \"experimentalFeatures: ['embedded-reactivity']\" to your app.dx to enable this feature."); + apply(_target, _thisArg, args) { + return $$(...args as [any, any]); }, }) @@ -399,23 +396,23 @@ export const $ = new Proxy(function(){} as unknown as $type, { * @returns */ export function val(val: RefOrValue):T { // TODO: return inferred type instead of T (ts resolution error, too deep) - return Ref.collapseValue(val, true, true) + return ReactiveValue.collapseValue(val, true, true) } // generate primitive pointers export function decimal(value:RefOrValue = 0): Pointer { - if (value instanceof Ref) value = value.val; // collapse + if (value instanceof ReactiveValue) value = value.val; // collapse return Pointer.create(undefined, Number(value)) // adds pointer or returns existing pointer } export function integer(value:RefOrValue = 0n): Pointer { - if (value instanceof Ref) value = value.val; // collapse + if (value instanceof ReactiveValue) value = value.val; // collapse return Pointer.create(undefined, BigInt(Math.floor(Number(value)))) // adds pointer or returns existing pointer } export function text(string:TemplateStringsArray, ...vars:any[]):Promise> export function text(value?:RefOrValue): Pointer export function text(value:RefOrValue|TemplateStringsArray = "", ...vars:any[]): Pointer|Promise> { - if (value instanceof Ref) value = value.val; // collapse + if (value instanceof ReactiveValue) value = value.val; // collapse // template transform if (value instanceof Array) { return >>_datex(`always '${value.raw.map(s=>s.replace(/\(/g, '\\(').replace(/\'/g, "\\'")).join(INSERT_MARK)}'`, vars) @@ -423,7 +420,7 @@ export function text(value:RefOrValue|TemplateStringsArray = "", ...vars else return Pointer.create(undefined, String(value)) // adds pointer or returns existing pointer } export function boolean(value:RefOrValue = false): Pointer { - if (value instanceof Ref) value = value.val; // collapse + if (value instanceof ReactiveValue) value = value.val; // collapse return Pointer.create(undefined, Boolean(value)) // adds pointer or returns existing pointer } @@ -433,7 +430,7 @@ export function md(string:TemplateStringsArray, ...vars:any[]):Promise export function md(value?:RefOrValue): Markdown export function md(value:RefOrValue|TemplateStringsArray = "", ...vars:any[]): Markdown|Promise { // transform string reference - if (value instanceof Ref) return > _datex `always ${value}` + if (value instanceof ReactiveValue) return > _datex `always ${value}` // template transform else if (value instanceof Array) return >_datex(`always '${value.raw.map(s=>s.replace(/\(/g, '\\(').replace(/\'/g, "\\'")).join(INSERT_MARK)}'`, vars) // pointer from string @@ -485,7 +482,7 @@ export function static_pointer(value:RefOrValue, endpoint:IdEndpoint, uniq const static_id = Pointer.getStaticPointerId(endpoint, unique_id); const pointer = Pointer.create(static_id, value) if (label) pointer.addLabel(typeof label == "string" ? label.replace(/^\$/, '') : label); - return Ref.collapseValue(pointer); + return ReactiveValue.collapseValue(pointer); } // similar to pointer(), but also adds a label @@ -619,7 +616,7 @@ export function props(parent:RefOrValue, strong_pa // other DatexValues can also be returned -> check if property already a DatexValue if (!strong_parent_bounding) { const property = pointer.getProperty(key); - if (property instanceof Ref) return property; + if (property instanceof ReactiveValue) return property; } // create a DatexPointerProperty return PointerProperty.get(pointer, >key); @@ -688,10 +685,10 @@ Object.defineProperty(globalThis, 'equals', {value:_equals, configurable:false}) Object.defineProperty(globalThis, 'selectProperty', {value:_selectProperty, configurable:false}) Object.defineProperty(globalThis, 'not', {value:_not, configurable:false}) Object.defineProperty(globalThis, 'effect', {value:_effect, configurable:false}) -Object.defineProperty(globalThis, 'observe', {value:Ref.observe.bind(Ref), configurable:false}) -Object.defineProperty(globalThis, 'observeAndInit', {value:Ref.observeAndInit.bind(Ref), configurable:false}) -Object.defineProperty(globalThis, 'unobserve', {value:Ref.unobserve.bind(Ref), configurable:false}) -Object.defineProperty(globalThis, 'isolate', {value:Ref.disableCapturing.bind(Ref), configurable:false}) +Object.defineProperty(globalThis, 'observe', {value:ReactiveValue.observe.bind(ReactiveValue), configurable:false}) +Object.defineProperty(globalThis, 'observeAndInit', {value:ReactiveValue.observeAndInit.bind(ReactiveValue), configurable:false}) +Object.defineProperty(globalThis, 'unobserve', {value:ReactiveValue.unobserve.bind(ReactiveValue), configurable:false}) +Object.defineProperty(globalThis, 'isolate', {value:ReactiveValue.disableCapturing.bind(ReactiveValue), configurable:false}) Object.defineProperty(globalThis, 'grantAccess', {value:grantAccess, configurable:false}) Object.defineProperty(globalThis, 'grantPublicAccess', {value:grantPublicAccess, configurable:false}) diff --git a/functions.ts b/functions.ts index 1be199e..0e7329a 100644 --- a/functions.ts +++ b/functions.ts @@ -4,13 +4,12 @@ */ -import { AsyncTransformFunction, INSERT_MARK, MaybeObjectRef, MinimalJSRef, Pointer, Ref, RefLike, RefOrValue, SmartTransformFunction, SmartTransformOptions, TransformFunction, TransformFunctionInputs, logger } from "./datex_all.ts"; +import { AsyncTransformFunction, INSERT_MARK, MaybeObjectRef, MinimalJSRef, Pointer, ReactiveValue, RefLike, RefOrValue, SmartTransformFunction, SmartTransformOptions, TransformFunction, TransformFunctionInputs, logger } from "./datex_all.ts"; import { Datex } from "./mod.ts"; import { PointerError } from "./types/errors.ts"; import { IterableHandler } from "./utils/iterable-handler.ts"; import { AsyncSmartTransformFunction, MinimalJSRefWithIndirectRef, RestrictSameType } from "./runtime/pointers.ts"; - /** * A generic transform function, creates a new pointer containing the result of the callback function. * At any point in time, the pointer is the result of the callback function. @@ -24,6 +23,11 @@ import { AsyncSmartTransformFunction, MinimalJSRefWithIndirectRef, RestrictSameT * ``` */ export function always(transform:SmartTransformFunction, options?: SmartTransformOptions): MinimalJSRefWithIndirectRef // return signature from Value.collapseValue(Pointer.smartTransform()) + + +// only works with JUSIX compilation +export function always(value:T, options?: SmartTransformOptions): MinimalJSRef + /** * Shortcut for datex `always (...)` * @param script @@ -42,11 +46,11 @@ export function always(scriptOrJSTransform:TemplateStringsArray|SmartTransformFu throw new PointerError(`Promises cannot be returned from always transforms - use 'asyncAlways' instead`); } else { - return Ref.collapseValue(ptr); + return ReactiveValue.collapseValue(ptr); } } // datex script - else return (async ()=>Ref.collapseValue(await datex(`always (${scriptOrJSTransform.raw.join(INSERT_MARK)})`, vars)))() + else return (async ()=>ReactiveValue.collapseValue(await datex(`always (${scriptOrJSTransform.raw.join(INSERT_MARK)})`, vars)))() } @@ -72,7 +76,7 @@ export async function asyncAlways(transform:AsyncSmartTransformFunction, o else { logger.warn("asyncAlways: transform function did not return a Promise, you should use 'always' instead") } - return Ref.collapseValue(ptr) as MinimalJSRef + return ReactiveValue.collapseValue(ptr) as MinimalJSRef } /** @@ -95,12 +99,12 @@ export async function asyncAlways(transform:AsyncSmartTransformFunction, o */ export function reactiveFn(fn: (...args: Args) => Awaited>>) { return (...args: MapToRefOrVal) => always(() => { - const collapsedArgs = args.map(arg => Ref.collapseValue(arg, true, true)) as Args; + const collapsedArgs = args.map(arg => ReactiveValue.collapseValue(arg, true, true)) as Args; return fn(...collapsedArgs) }); } -type MapToRefOrVal = {[K in keyof T]: T[K] extends Ref ? T[K] : RefOrValue} +type MapToRefOrVal = {[K in keyof T]: T[K] extends ReactiveValue ? T[K] : RefOrValue} const getGreetingMessage = (country: RefOrValue) => { @@ -183,7 +187,7 @@ export function effect|undefined>(handler:W ex * @returns */ export function transform(dependencies:V, transform:TransformFunction, persistent_datex_transform?:string) { - return Ref.collapseValue(Pointer.createTransform(dependencies, transform, persistent_datex_transform)); + return ReactiveValue.collapseValue(Pointer.createTransform(dependencies, transform, persistent_datex_transform)); } /** * A generic transform function, creates a new pointer containing the result of the callback function. @@ -196,7 +200,7 @@ export function transform(dependencies:V, t * @returns */ export async function transformAsync(dependencies:V, transform:AsyncTransformFunction, persistent_datex_transform?:string) { - return Ref.collapseValue(await Pointer.createTransformAsync(dependencies, transform, persistent_datex_transform)); + return ReactiveValue.collapseValue(await Pointer.createTransformAsync(dependencies, transform, persistent_datex_transform)); } @@ -204,7 +208,7 @@ export function map(iterable: Iterable< let mapped:U[]|Map // live map - if (Datex.Ref.isRef(iterable)) { + if (Datex.ReactiveValue.isRef(iterable)) { // return map if (options?.outType == "map") { @@ -339,7 +343,7 @@ export const select = toggle; * @param b input value */ export function equals(a:RefLike|T, b: RefLike|V): Datex.Pointer { - return transform([a, b], (a,b) => Datex.Ref.collapseValue(a, true, true) === Datex.Ref.collapseValue(b, true, true), + return transform([a, b], (a,b) => Datex.ReactiveValue.collapseValue(a, true, true) === Datex.ReactiveValue.collapseValue(b, true, true), // dx transforms not working correctly (with uix) /*`always (${Runtime.valueToDatexString(a)} === ${Runtime.valueToDatexString(b)})`*/) as any; } diff --git a/runtime/endpoint_config.ts b/runtime/endpoint_config.ts index 6f1587a..fdb96c9 100644 --- a/runtime/endpoint_config.ts +++ b/runtime/endpoint_config.ts @@ -8,7 +8,7 @@ import { Runtime } from "./runtime.ts"; import { Tuple } from "../types/tuple.ts"; import { cache_path } from "./cache_path.ts"; import { DatexObject } from "../types/object.ts"; -import { Ref } from "./pointers.ts"; +import { ReactiveValue } from "./pointers.ts"; import { normalizePath } from "../utils/normalize-path.ts"; type channel_type = 'websocket'|'http' @@ -48,7 +48,7 @@ class EndpointConfig implements EndpointConfigData { public publicNodes?: Map public get endpoint() { - return Ref.collapseValue(this.#endpoint, true, true)!; + return ReactiveValue.collapseValue(this.#endpoint, true, true)!; } public set endpoint(endpoint: Endpoint|string|undefined) { this.#endpoint = typeof endpoint == "string" ? Endpoint.get(endpoint) as Endpoint : endpoint; diff --git a/runtime/io_handler.ts b/runtime/io_handler.ts index ba817bd..f0b9de4 100644 --- a/runtime/io_handler.ts +++ b/runtime/io_handler.ts @@ -89,7 +89,7 @@ export class IOHandler { else if (this.std_outf) await this.std_outf(params, meta); } public static stdOut(params:any[], meta: datex_meta){ - for (let i=0;i = (value:V extends RefLike ? T : V, key?:K, type?:Ref.UPDATE_TYPE, transform?:boolean, is_child_update?:boolean, previous?: any, atomic_id?:symbol)=>void|boolean -export type observe_options = {types?:Ref.UPDATE_TYPE[], ignore_transforms?:boolean, recursive?:boolean} +export type observe_handler = (value:V extends RefLike ? T : V, key?:K, type?:ReactiveValue.UPDATE_TYPE, transform?:boolean, is_child_update?:boolean, previous?: any, atomic_id?:symbol)=>void|boolean +export type observe_options = {types?:ReactiveValue.UPDATE_TYPE[], ignore_transforms?:boolean, recursive?:boolean} const arrayProtoNames = Object.getOwnPropertyNames(Array.prototype); const objectProtoNames = Object.getOwnPropertyNames(Object.prototype) @@ -49,7 +49,7 @@ export type RefLike = Pointer|PointerProperty export type RefLikeOut = PointerWithPrimitive|PointerProperty // root class for pointers and pointer properties, value changes can be observed -export abstract class Ref extends EventTarget { +export abstract class ReactiveValue extends EventTarget { #observerCount = 0; @@ -63,7 +63,7 @@ export abstract class Ref extends EventTarget { constructor(value?:RefOrValue) { super(); - value = Ref.collapseValue(value); + value = ReactiveValue.collapseValue(value); if (value!=undefined) this.val = value; } @@ -74,7 +74,7 @@ export abstract class Ref extends EventTarget { } public set val(value: T|undefined) { const previous = this.#val; - this.#val = Ref.collapseValue(value, true, true); + this.#val = ReactiveValue.collapseValue(value, true, true); if (previous !== this.#val) this.triggerValueInitEvent(false, previous) } @@ -90,7 +90,7 @@ export abstract class Ref extends EventTarget { // same as val setter, but can be awaited public setVal(value:T, trigger_observers = true, is_transform?:boolean) { const previous = this.#val; - this.#val = Ref.collapseValue(value, true, true); + this.#val = ReactiveValue.collapseValue(value, true, true); if (trigger_observers && previous !== this.#val) return this.triggerValueInitEvent(is_transform, previous) } @@ -98,11 +98,11 @@ export abstract class Ref extends EventTarget { const value = this.current_val; const promises = []; for (const [o, options] of this.#observers??[]) { - if ((!options?.types || options.types.includes(Ref.UPDATE_TYPE.INIT)) && !(is_transform && options?.ignore_transforms)) promises.push(o(value, VOID, Ref.UPDATE_TYPE.INIT, is_transform, undefined, previous)); + if ((!options?.types || options.types.includes(ReactiveValue.UPDATE_TYPE.INIT)) && !(is_transform && options?.ignore_transforms)) promises.push(o(value, VOID, ReactiveValue.UPDATE_TYPE.INIT, is_transform, undefined, previous)); } for (const [object, observers] of this.#observers_bound_objects??[]) { for (const [o, options] of observers??[]) { - if ((!options?.types || options.types.includes(Ref.UPDATE_TYPE.INIT)) && !(is_transform && options?.ignore_transforms)) promises.push(o.call(object, value, VOID, Ref.UPDATE_TYPE.INIT, is_transform, undefined, previous)); + if ((!options?.types || options.types.includes(ReactiveValue.UPDATE_TYPE.INIT)) && !(is_transform && options?.ignore_transforms)) promises.push(o.call(object, value, VOID, ReactiveValue.UPDATE_TYPE.INIT, is_transform, undefined, previous)); } } return Promise.allSettled(promises); @@ -175,7 +175,7 @@ export abstract class Ref extends EventTarget { if (!(pointer.val instanceof Array) && ![...pointer.getKeys()].includes(key)) { throw new ValueError("Property "+key.toString()+" does not exist in value"); } - if (pointer instanceof Pointer && Pointer.pointerifyValue(pointer.shadow_object?.[key]) instanceof Ref) return Pointer.pointerifyValue(pointer.shadow_object?.[key]); + if (pointer instanceof Pointer && Pointer.pointerifyValue(pointer.shadow_object?.[key]) instanceof ReactiveValue) return Pointer.pointerifyValue(pointer.shadow_object?.[key]); else return PointerProperty.get(pointer, key, true); } } @@ -184,11 +184,11 @@ export abstract class Ref extends EventTarget { // handle function as transform if (useFunction) { handler.apply = (_target, thisRef, args) => { - const thisVal = Ref.collapseValue(thisRef, true, true); + const thisVal = ReactiveValue.collapseValue(thisRef, true, true); if (typeof thisVal != "function") throw new Error("Cannot create a reference transform, not a function"); if (thisRef instanceof PointerProperty) { - return Ref.collapseValue(Pointer.createTransform([thisRef.pointer], ()=>{ + return ReactiveValue.collapseValue(Pointer.createTransform([thisRef.pointer], ()=>{ return thisVal(...args); })); } @@ -223,7 +223,7 @@ export abstract class Ref extends EventTarget { public static observe(value: V, handler:observe_handler, bound_object?:object, key?:K, options?:observe_options):void { const pointer = Pointer.pointerifyValue(value); if (pointer instanceof Pointer) pointer.observe(handler, bound_object, key, options); - else if (pointer instanceof Ref) pointer.observe(handler, bound_object, options); + else if (pointer instanceof ReactiveValue) pointer.observe(handler, bound_object, options); else throw new ValueError("Cannot observe this value because it has no pointer") } @@ -235,15 +235,15 @@ export abstract class Ref extends EventTarget { this.observe(value, handler, bound_object, key, options); } catch {} // throws if value does not have a DATEX reference, can be ignored - in this case no observer is set, only the initial handler call is triggered const val = this.collapseValue(value, true, true); - if (handler.call) handler.call(bound_object, val, undefined, Ref.UPDATE_TYPE.INIT); - else handler(val, undefined, Ref.UPDATE_TYPE.INIT); + if (handler.call) handler.call(bound_object, val, undefined, ReactiveValue.UPDATE_TYPE.INIT); + else handler(val, undefined, ReactiveValue.UPDATE_TYPE.INIT); } // call handler when value changes public static unobserve(value: V, handler:observe_handler, bound_object?:object, key?:K):void { const pointer = Pointer.pointerifyValue(value); if (pointer instanceof Pointer) pointer.unobserve(handler, bound_object, key); - else if (pointer instanceof Ref) pointer.unobserve(handler, bound_object); + else if (pointer instanceof ReactiveValue) pointer.unobserve(handler, bound_object); else throw new ValueError("Cannot unobserve this value because it has no pointer") } @@ -325,7 +325,7 @@ export abstract class Ref extends EventTarget { if (collapse_indirect_references == undefined) collapse_indirect_references = false; if ( - value instanceof Ref && + value instanceof ReactiveValue && ( collapse_primitive_pointers || !(value instanceof Pointer && value.is_js_primitive) @@ -367,7 +367,7 @@ export abstract class Ref extends EventTarget { * Returns true if the value has a bound pointer or is a Datex.Ref */ public static isRef(value: unknown) { - return (value instanceof Ref || Pointer.pointer_value_map.has(value)); + return (value instanceof ReactiveValue || Pointer.pointer_value_map.has(value)); } // copy the value of a primitive datex value to another primitive value @@ -377,7 +377,7 @@ export abstract class Ref extends EventTarget { - protected static capturedGetters? = new Set() + protected static capturedGetters? = new Set() protected static capturedGettersWithKeys? = new Map>().setAutoDefault(Set) /** @@ -426,26 +426,26 @@ export abstract class Ref extends EventTarget { * * toString() for any value */ handleBeforeNonReferencableGet(key:any = NOT_EXISTING) { - if (Ref.freezeCapturing) return; + if (ReactiveValue.freezeCapturing) return; // remember previous capture state - const previousGetters = Ref.capturedGetters; - const previousGettersWithKeys = Ref.capturedGettersWithKeys; - const previousCapturing = Ref.isCapturing; + const previousGetters = ReactiveValue.capturedGetters; + const previousGettersWithKeys = ReactiveValue.capturedGettersWithKeys; + const previousCapturing = ReactiveValue.isCapturing; // trigger transform update if not live if (this.#transformSource && !this.#liveTransform && !this.#forceLiveTransform) { - Ref.capturedGetters = new Set(); - Ref.capturedGettersWithKeys = new Map().setAutoDefault(Set); + ReactiveValue.capturedGetters = new Set(); + ReactiveValue.capturedGettersWithKeys = new Map().setAutoDefault(Set); this.#transformSource.update(); } if (previousCapturing) { - Ref.isCapturing = true; - Ref.capturedGetters = previousGetters ?? new Set(); - Ref.capturedGettersWithKeys = previousGettersWithKeys ?? new Map().setAutoDefault(Set); + ReactiveValue.isCapturing = true; + ReactiveValue.capturedGetters = previousGetters ?? new Set(); + ReactiveValue.capturedGettersWithKeys = previousGettersWithKeys ?? new Map().setAutoDefault(Set); - if (key === NOT_EXISTING) Ref.capturedGetters.add(this); - else if (this instanceof Pointer) Ref.capturedGettersWithKeys.getAuto(this).add(key) + if (key === NOT_EXISTING) ReactiveValue.capturedGetters.add(this); + else if (this instanceof Pointer) ReactiveValue.capturedGettersWithKeys.getAuto(this).add(key) else { logger.warn("invalid capture, must be a pointer or property") } @@ -536,7 +536,7 @@ export abstract class Ref extends EventTarget { /** * @deprecated use Datex.Ref instead */ -export const Value = Ref; +export const Value = ReactiveValue; /** * @deprecated use Datex.RefLike instead */ @@ -560,7 +560,7 @@ export type TransformSource = { update: ()=>void // dependency values - deps: IterableWeakSet + deps: IterableWeakSet keyedDeps: IterableWeakMap> } @@ -568,7 +568,7 @@ export type PointerPropertyParent = Map | Record = PointerProperty ? MV : Parent[Key&keyof Parent]> // interface to access (read/write) pointer value properties -export class PointerProperty extends Ref { +export class PointerProperty extends ReactiveValue { // override hasInstance from Ref static [Symbol.hasInstance]: (val: unknown) => val is PointerProperty @@ -668,7 +668,7 @@ export class PointerProperty extends Ref { console.warn("Cannot set value of lazy pointer property"); return; } - this.pointer!.handleSet(this.key, Ref.collapseValue(value, true, true)); + this.pointer!.handleSet(this.key, ReactiveValue.collapseValue(value, true, true)); } public override get current_val():T { @@ -683,7 +683,7 @@ export class PointerProperty extends Ref { console.warn("Cannot set value of lazy pointer property"); return; } - return this.pointer!.handleSet(this.key, Ref.collapseValue(value, true, true)); + return this.pointer!.handleSet(this.key, ReactiveValue.collapseValue(value, true, true)); } #observer_internal_handlers = new WeakMap() @@ -696,14 +696,14 @@ export class PointerProperty extends Ref { return; } const value_pointer = Pointer.pointerifyValue(this.current_val); - if (value_pointer instanceof Ref) value_pointer.observe(handler, bound_object, options); // also observe internal value changes + if (value_pointer instanceof ReactiveValue) value_pointer.observe(handler, bound_object, options); // also observe internal value changes const internal_handler = (v:unknown)=>{ const value_pointer = Pointer.pointerifyValue(v); - if (value_pointer instanceof Ref) value_pointer.observe(handler, bound_object, options); // also update observe for internal value changes - if (handler.call) handler.call(bound_object, v,undefined,Ref.UPDATE_TYPE.INIT) + if (value_pointer instanceof ReactiveValue) value_pointer.observe(handler, bound_object, options); // also update observe for internal value changes + if (handler.call) handler.call(bound_object, v,undefined,ReactiveValue.UPDATE_TYPE.INIT) // if arrow function - else handler(v,undefined,Ref.UPDATE_TYPE.INIT) + else handler(v,undefined,ReactiveValue.UPDATE_TYPE.INIT) }; this.pointer!.observe(internal_handler, bound_object, this.key, options) @@ -720,7 +720,7 @@ export class PointerProperty extends Ref { return; } const value_pointer = Pointer.pointerifyValue(this.current_val); - if (value_pointer instanceof Ref) value_pointer.unobserve(handler, bound_object); // also unobserve internal value changes + if (value_pointer instanceof ReactiveValue) value_pointer.unobserve(handler, bound_object); // also unobserve internal value changes let internal_handler:observe_handler|undefined @@ -750,7 +750,9 @@ export type ReadonlyRef = Readonly>; * @deprecated Use Datex.RefOrValue instead */ export type CompatValue = RefLike|T; -export type RefOrValue = RefLike|T; // TODO: remove RefLike, only workaround +export type RefOrValue = RefLike|T; + + /** * object with refs or values as properties */ @@ -767,7 +769,7 @@ export type CollapsedValue> = // generic value classes T extends PointerProperty ? TT : T extends Pointer ? TT : - T extends Ref ? TT : + T extends ReactiveValue ? TT : T; // collapsed value that still has a reference in JS @@ -776,7 +778,7 @@ export type CollapsedValueJSCompatible> = // generic value classes T extends PointerProperty ? (TT extends primitive ? T : TT) : T extends Pointer ? (TT extends primitive ? T : TT) : - T extends Ref ? (TT extends primitive ? T : TT) : + T extends ReactiveValue ? (TT extends primitive ? T : TT) : T; // number -> Number, ..., get prototype methods @@ -876,13 +878,26 @@ export type MinimalJSRefWithIndirectRef> = ObjectRef<_C> // collapsed object ) -export type MinimalJSRef> = +export type Ref> = _C&{} extends symbol ? symbol : ( _C&{} extends WrappedPointerValue ? PointerWithPrimitive<_C>: // keep pointer reference ObjectRef<_C> // collapsed object ) +/** + * @deprecated use Ref + */ +export type MinimalJSRef> = Ref + +// same as MinimalJSRef, but objects don't have $ and $$ properties +export type MinimalJSRefNoObjRef> = +_C&{} extends symbol ? symbol : ( + _C&{} extends WrappedPointerValue ? + PointerWithPrimitive<_C>: // keep pointer reference + _C // collapsed object +) + // return Pointer&T for primitives (excluding boolean) and Pointer otherwise export type PointerWithPrimitive = T&{} extends WrappedPointerValue ? T&{} extends primitive ? @@ -891,9 +906,6 @@ export type PointerWithPrimitive = T&{} extends WrappedPointerValue ? Pointer // e.g. Pointer> -type a = MinimalJSRef - - export type CollapsedValueAdvanced, COLLAPSE_POINTER_PROPERTY extends boolean|undefined = true, COLLAPSE_PRIMITIVE_POINTER extends boolean|undefined = true, _C = CollapsedValue> = // if _C extends primitive ? @@ -911,7 +923,7 @@ export type ProxifiedValue> = // already a proxified value T extends PointerProperty ? T : T extends Pointer? T : - T extends Ref ? T : + T extends ReactiveValue ? T : // proxify RefLike @@ -1108,7 +1120,7 @@ export type SmartTransformOptions = { type TransformState = { isLive: boolean; isFirst: boolean; - deps: IterableWeakSet>; + deps: IterableWeakSet>; keyedDeps: AutoMap>; returnCache: Map; getDepsHash: () => string; @@ -1147,7 +1159,7 @@ const observableArrayMethods = new Set([ /** Wrapper class for all pointer values ($xxxxxxxx) */ -export class Pointer extends Ref { +export class Pointer extends ReactiveValue { // override hasInstance from Ref static [Symbol.hasInstance]: (val: unknown) => val is Pointer @@ -1405,7 +1417,7 @@ export class Pointer extends Ref { * @deprecated use Ref.isRef */ public static isReference(value:unknown) { - return Ref.isRef(value); + return ReactiveValue.isRef(value); } // returns the existing pointer for a value, or the value, if no pointer exists @@ -1672,9 +1684,9 @@ export class Pointer extends Ref { // create/get DatexPointer for value if possible (not primitive) and return value static proxifyValue(value:unknown, sealed = false, allowed_access?:target_clause, anonymous = false, persistant= false, check_proxify_as_child = false) { if ((value instanceof Pointer && value.is_js_primitive) || value instanceof PointerProperty) return value; // return by reference - else if (value instanceof Ref) return value.val; // return by value + else if (value instanceof ReactiveValue) return value.val; // return by value const type = Type.ofValue(value) - const collapsed_value = Ref.collapseValue(value,true,true) + const collapsed_value = ReactiveValue.collapseValue(value,true,true) // if proxify_as_child=false: don't create pointer for this value, return original value // e.g.: primitive values if ((check_proxify_as_child && !type.proxify_as_child) || type.is_primitive) { @@ -1690,7 +1702,7 @@ export class Pointer extends Ref { if (value instanceof LazyPointer) throw new PointerError("Lazy Pointer not supported in this context"); if (value instanceof Pointer) return >value; // return pointer by reference //if (value instanceof PointerProperty) return value; // return pointerproperty TODO: handle pointer properties? - value = Ref.collapseValue(value, true, true); + value = ReactiveValue.collapseValue(value, true, true); const ptr = Pointer.getByValue(value); // try proxify @@ -1740,7 +1752,7 @@ export class Pointer extends Ref { let p:Pointer; // DatexValue: DatexPointer or DatexPointerProperty not valid as object, get the actual value instead - value = Ref.collapseValue(value,true,true) + value = ReactiveValue.collapseValue(value,true,true) // is js primitive value if (Object(value) !== value && typeof value !== "symbol") { @@ -1963,7 +1975,7 @@ export class Pointer extends Ref { // remove property observers for (const [value, handler] of this.#active_property_observers.values()) { - Ref.unobserve(value, handler, this.#unique); + ReactiveValue.unobserve(value, handler, this.#unique); } // delete labels @@ -2111,7 +2123,7 @@ export class Pointer extends Ref { endpoint = endpoint?.main; if ( Runtime.OPTIONS.PROTECT_POINTERS - && !(endpoint == Runtime.endpoint) + && !(endpoint == Runtime.endpoint.main) && this.is_origin && ( !endpoint || @@ -2252,14 +2264,14 @@ export class Pointer extends Ref { * create a new transformed pointer from an existing pointer */ public transform(transform:TransformFunction<[this],R>) { - return Ref.collapseValue(Pointer.createTransform([this], transform)); + return ReactiveValue.collapseValue(Pointer.createTransform([this], transform)); } /** * create a new transformed pointer from an existing pointer (Async transform function) */ public transformAsync(transform:AsyncTransformFunction<[this],R>) { - return Ref.collapseValue(Pointer.createTransformAsync([this], transform)); + return ReactiveValue.collapseValue(Pointer.createTransformAsync([this], transform)); } @@ -2539,14 +2551,14 @@ export class Pointer extends Ref { // TODO: await promises? for (const [key, entry] of this.change_observers) { for (const [o, options] of entry) { - if ((!options?.types || options.types.includes(Ref.UPDATE_TYPE.INIT))) o(value, key, Ref.UPDATE_TYPE.INIT); + if ((!options?.types || options.types.includes(ReactiveValue.UPDATE_TYPE.INIT))) o(value, key, ReactiveValue.UPDATE_TYPE.INIT); } } for (const [object, entries] of this.bound_change_observers) { for (const [key, handlers] of entries) { for (const [handler, options] of handlers) { - if ((!options?.types || options.types.includes(Ref.UPDATE_TYPE.INIT))) { - const res = handler.call(object, value, key, Ref.UPDATE_TYPE.INIT); + if ((!options?.types || options.types.includes(ReactiveValue.UPDATE_TYPE.INIT))) { + const res = handler.call(object, value, key, ReactiveValue.UPDATE_TYPE.INIT); if (res === false) this.unobserve(handler, object, key); } } @@ -2570,7 +2582,7 @@ export class Pointer extends Ref { * @param v initial value */ protected initializeValue(v:RefOrValue, is_transform?:boolean) { - let val = Ref.collapseValue(v,true,true); + let val = ReactiveValue.collapseValue(v,true,true); if (typeof val == "symbol" && Symbol.keyFor(val) !== undefined) { throw new Error("Global and well-known symbols (e.g. Symbol.for('name') or Symbol.iterator) are no yet supported as pointer values") @@ -2720,7 +2732,7 @@ export class Pointer extends Ref { * @returns promise which resolves when all update side effects are resolved */ protected updateValue(v:RefOrValue, trigger_observers = true, is_transform?:boolean) { - const val = Ref.collapseValue(v,true,true); + const val = ReactiveValue.collapseValue(v,true,true); const newType = Type.ofValue(val); // not changed (relevant for primitive values) @@ -2768,7 +2780,7 @@ export class Pointer extends Ref { if (!didCustomUpdate) { // is indirect reference, set new value - if (this.supportsIndirectRefs && (this.#indirectReference || Ref.isRef(val))) { + if (this.supportsIndirectRefs && (this.#indirectReference || ReactiveValue.isRef(val))) { this.#indirectReference = Pointer.getByValue(val); super.setVal(val, trigger_observers, is_transform); } @@ -2847,7 +2859,7 @@ export class Pointer extends Ref { const transformMethod = transform instanceof Function ? transform : ()=>transform.execute(Runtime.endpoint); - const initialValue = await (observe_values.length==1 ? transformMethod(...>[Ref.collapseValue(observe_values[0], true, true)]) : transformMethod(...>observe_values.map(v=>Ref.collapseValue(v, true, true)))); // transform current value + const initialValue = await (observe_values.length==1 ? transformMethod(...>[ReactiveValue.collapseValue(observe_values[0], true, true)]) : transformMethod(...>observe_values.map(v=>ReactiveValue.collapseValue(v, true, true)))); // transform current value if (initialValue === VOID) throw new ValueError("initial tranform value cannot be void"); this.setVal(initialValue, true, true); @@ -2859,8 +2871,8 @@ export class Pointer extends Ref { // transform updates for (const value of observe_values) { - if (value instanceof Ref) value.observe(async ()=>{ - const newValue = await (observe_values.length==1 ? transformMethod(...>[Ref.collapseValue(observe_values[0], true, true)]) : transformMethod(...>observe_values.map(v=>Ref.collapseValue(v, true, true)))); // update value + if (value instanceof ReactiveValue) value.observe(async ()=>{ + const newValue = await (observe_values.length==1 ? transformMethod(...>[ReactiveValue.collapseValue(observe_values[0], true, true)]) : transformMethod(...>observe_values.map(v=>ReactiveValue.collapseValue(v, true, true)))); // update value if (newValue !== VOID) this.setVal(newValue, true, true) }); } @@ -2868,7 +2880,7 @@ export class Pointer extends Ref { } protected handleTransform(observe_values:V, transform:TransformFunction, persistent_datex_transform?:string): Pointer { - const initialValue = observe_values.length==1 ? transform(...>[Ref.collapseValue(observe_values[0], true, true)]) : transform(...>observe_values.map(v=>Ref.collapseValue(v, true, true))); // transform current value + const initialValue = observe_values.length==1 ? transform(...>[ReactiveValue.collapseValue(observe_values[0], true, true)]) : transform(...>observe_values.map(v=>ReactiveValue.collapseValue(v, true, true))); // transform current value if (initialValue === VOID) throw new ValueError("initial tranform value cannot be void"); this.setVal(initialValue, true, true); @@ -2878,8 +2890,8 @@ export class Pointer extends Ref { // transform updates for (const value of observe_values) { - if (value instanceof Ref) value.observe(()=>{ - const newValue = observe_values.length==1 ? transform(...>[Ref.collapseValue(observe_values[0], true, true)]) : transform(...>observe_values.map(v=>Ref.collapseValue(v, true, true))); // update value + if (value instanceof ReactiveValue) value.observe(()=>{ + const newValue = observe_values.length==1 ? transform(...>[ReactiveValue.collapseValue(observe_values[0], true, true)]) : transform(...>observe_values.map(v=>ReactiveValue.collapseValue(v, true, true))); // update value // await promise (should avoid in non-async transform) if (newValue instanceof Promise) newValue.then((val)=>this.setVal(val, true, true)); else if (newValue !== VOID) this.setVal(newValue, true, true); @@ -2919,7 +2931,7 @@ export class Pointer extends Ref { const state: TransformState = { isLive: false, isFirst: true, - deps: new IterableWeakSet(), + deps: new IterableWeakSet(), keyedDeps: new IterableWeakMap>().setAutoDefault(Set), returnCache: new Map(), @@ -2945,7 +2957,7 @@ export class Pointer extends Ref { state.isFirst = false; let val!: T - let capturedGetters: Set> | undefined; + let capturedGetters: Set> | undefined; let capturedGettersWithKeys: AutoMap, Set> | undefined; if (options?.cache) { @@ -2958,12 +2970,12 @@ export class Pointer extends Ref { // no cached value found, run transform function if (val === undefined) { - Ref.captureGetters(); + ReactiveValue.captureGetters(); try { val = transform() as T; // also trigger getter if pointer is returned - Ref.collapseValue(val, true, true); + ReactiveValue.collapseValue(val, true, true); } catch (e) { if (e !== Pointer.WEAK_EFFECT_DISPOSED) console.error(e); @@ -2972,7 +2984,7 @@ export class Pointer extends Ref { } // always cleanup capturing finally { - ({capturedGetters, capturedGettersWithKeys} = Ref.getCapturedGetters()); + ({capturedGetters, capturedGettersWithKeys} = ReactiveValue.getCapturedGetters()); } } @@ -3116,7 +3128,7 @@ export class Pointer extends Ref { private handleTransformValue( val: T, - capturedGetters: Set>|undefined, + capturedGetters: Set>|undefined, capturedGettersWithKeys: AutoMap, Set>|undefined, state: TransformState, ignoreReturnValue: boolean, @@ -3124,7 +3136,7 @@ export class Pointer extends Ref { ) { if (!ignoreReturnValue && !this.value_initialized) { if (val == undefined) this.#is_js_primitive = true; - else this.#updateIsJSPrimitive(Ref.collapseValue(val,true,true)); + else this.#updateIsJSPrimitive(ReactiveValue.collapseValue(val,true,true)); } // set isLive to true, if not primitive @@ -3642,7 +3654,7 @@ export class Pointer extends Ref { } // already proxified child - set observers - else if (value[name] instanceof Ref) { + else if (value[name] instanceof ReactiveValue) { this.initShadowObjectPropertyObserver(name, value[name]); } high_priority_keys.add(name); @@ -3654,8 +3666,8 @@ export class Pointer extends Ref { while ((prototype = Object.getPrototypeOf(prototype)) != Object.prototype) { for (const name of this.visible_children ?? Object.keys(prototype)) { try { - if (prototype[name] instanceof Ref && !high_priority_keys.has(name)) { // only observer Values, and ignore if already observed higher up in prototype chain - this.initShadowObjectPropertyObserver(name, prototype[name]); + if (prototype[name] instanceof ReactiveValue && !high_priority_keys.has(name)) { // only observer Values, and ignore if already observed higher up in prototype chain + this.initShadowObjectPropertyObserver(name, prototype[name]); } } catch (e) { logger.warn("could not check prototype property:",name) @@ -3768,7 +3780,7 @@ export class Pointer extends Ref { get: () => { this.handleBeforeNonReferencableGet(name); // important: reference shadow_object, not this.shadow_object here, otherwise it might get garbage collected - return Ref.collapseValue(shadow_object[name], true, true); + return ReactiveValue.collapseValue(shadow_object[name], true, true); } }); @@ -3816,7 +3828,7 @@ export class Pointer extends Ref { get: (_target, key) => { if (key == DX_PTR) return this; if (this.#custom_prop_getter && (!this.shadow_object || !(key in this.shadow_object)) && !(typeof key == "symbol")) return this.#custom_prop_getter(key); - const val:any = Ref.collapseValue(this.shadow_object?.[key], true, true); + const val:any = ReactiveValue.collapseValue(this.shadow_object?.[key], true, true); if (key != "$" && key != "$$") { if (is_array) this.handleBeforeNonReferenceableGetArray(key); @@ -3959,7 +3971,7 @@ export class Pointer extends Ref { return (typeof property_value == "function") ? (...args:unknown[])=>(property_value).apply(this.current_val, args) : property_value; } // restricted to DATEX properties - else if (this.shadow_object && key in this.shadow_object) property_value = Ref.collapseValue(this.shadow_object[key], true, true) + else if (this.shadow_object && key in this.shadow_object) property_value = ReactiveValue.collapseValue(this.shadow_object[key], true, true) } return property_value; } @@ -4070,7 +4082,7 @@ export class Pointer extends Ref { } // inform observers - return this.callObservers(value, key, Ref.UPDATE_TYPE.SET, false, false, previous) + return this.callObservers(value, key, ReactiveValue.UPDATE_TYPE.SET, false, false, previous) } @@ -4114,7 +4126,7 @@ export class Pointer extends Ref { // inform observers - return this.callObservers(value, VOID, Ref.UPDATE_TYPE.ADD) + return this.callObservers(value, VOID, ReactiveValue.UPDATE_TYPE.ADD) } @@ -4198,7 +4210,7 @@ export class Pointer extends Ref { } // inform observers - this.callObservers(VOID, VOID, Ref.UPDATE_TYPE.CLEAR) + this.callObservers(VOID, VOID, ReactiveValue.UPDATE_TYPE.CLEAR) } /** all values are removed */ @@ -4228,7 +4240,7 @@ export class Pointer extends Ref { // array splice // trigger BEFORE_DELETE for (let i = obj.length - netDeleteCount; i < obj.length; i++) { - this.callObservers(obj[i], i, Ref.UPDATE_TYPE.BEFORE_DELETE) + this.callObservers(obj[i], i, ReactiveValue.UPDATE_TYPE.BEFORE_DELETE) } // previous entries @@ -4275,11 +4287,11 @@ export class Pointer extends Ref { for (let i = originalLength-1; i>=start_index; i--) { // element moved here? if (i < obj.length) { - this.callObservers(obj[i], i, Ref.UPDATE_TYPE.SET, undefined, undefined, previous[i], atomicId) + this.callObservers(obj[i], i, ReactiveValue.UPDATE_TYPE.SET, undefined, undefined, previous[i], atomicId) } // end of array, trigger delete else { - this.callObservers(VOID, i, Ref.UPDATE_TYPE.DELETE, undefined, undefined, previous[i], atomicId) + this.callObservers(VOID, i, ReactiveValue.UPDATE_TYPE.DELETE, undefined, undefined, previous[i], atomicId) } } @@ -4300,7 +4312,7 @@ export class Pointer extends Ref { const previous = this.getProperty(key); // inform observers before delete - this.callObservers(previous, key, Ref.UPDATE_TYPE.BEFORE_DELETE) + this.callObservers(previous, key, ReactiveValue.UPDATE_TYPE.BEFORE_DELETE) const res = JSInterface.handleDeletePropertySilently(obj, key, this, this.type); if (res == INVALID || res == NOT_EXISTING) { @@ -4331,7 +4343,7 @@ export class Pointer extends Ref { } // inform observers - return this.callObservers(VOID, key, Ref.UPDATE_TYPE.DELETE, undefined, undefined, previous) + return this.callObservers(VOID, key, ReactiveValue.UPDATE_TYPE.DELETE, undefined, undefined, previous) } @@ -4342,7 +4354,7 @@ export class Pointer extends Ref { let obj = this.current_val; // inform observers before remove - this.callObservers(value, undefined, Ref.UPDATE_TYPE.BEFORE_REMOVE) + this.callObservers(value, undefined, ReactiveValue.UPDATE_TYPE.BEFORE_REMOVE) // try set on custom pseudo class try { @@ -4370,7 +4382,7 @@ export class Pointer extends Ref { } // inform observers - return this.callObservers(value, VOID, Ref.UPDATE_TYPE.REMOVE) + return this.callObservers(value, VOID, ReactiveValue.UPDATE_TYPE.REMOVE) } @@ -4409,7 +4421,7 @@ export class Pointer extends Ref { this.shadow_object[key] = value; // add observer for internal changes - if (value instanceof Ref) { + if (value instanceof ReactiveValue) { this.initShadowObjectPropertyObserver(key, value); } @@ -4425,21 +4437,21 @@ export class Pointer extends Ref { // remove previous observer for property if (this.#active_property_observers.has(key)) { const [value, handler] = this.#active_property_observers.get(key)!; - Ref.unobserve(value, handler, this.#unique); // xxxxxx + ReactiveValue.unobserve(value, handler, this.#unique); // xxxxxx } // new observer // TODO: is weak pointer reference correct here? const ref = new WeakRef(this); - const handler = (_value: unknown, _key?: unknown, _type?: Ref.UPDATE_TYPE, _is_transform?: boolean, _is_child_update?: boolean, previous?: any) => { + const handler = (_value: unknown, _key?: unknown, _type?: ReactiveValue.UPDATE_TYPE, _is_transform?: boolean, _is_child_update?: boolean, previous?: any) => { const self = ref.deref(); if (!self) return; // console.warn(_value,_key,_type,_is_transform) // inform observers (TODO: more update event info?, currently just acting as if it was a SET) - self.callObservers(_value, key, Ref.UPDATE_TYPE.SET, _is_transform, true, previous) + self.callObservers(_value, key, ReactiveValue.UPDATE_TYPE.SET, _is_transform, true, previous) }; - Ref.observeAndInit(value, handler, this.#unique); + ReactiveValue.observeAndInit(value, handler, this.#unique); this.#active_property_observers.set(key, [value, handler]); } @@ -4496,7 +4508,7 @@ export class Pointer extends Ref { - public override unobserve(handler:(value:any, key?:K, type?:Ref.UPDATE_TYPE)=>void, bound_object?:object, key?:K):void { + public override unobserve(handler:(value:any, key?:K, type?:ReactiveValue.UPDATE_TYPE)=>void, bound_object?:object, key?:K):void { // unobserve all changes if (key == undefined) { super.unobserve(handler, bound_object); // default observer @@ -4520,7 +4532,7 @@ export class Pointer extends Ref { } - private callObservers(value:any, key:any, type:Ref.UPDATE_TYPE, is_transform = false, is_child_update = false, previous?: any, atomic_id?: symbol) { + private callObservers(value:any, key:any, type:ReactiveValue.UPDATE_TYPE, is_transform = false, is_child_update = false, previous?: any, atomic_id?: symbol) { const promises = []; // key specific observers if (key!=undefined) { @@ -4570,7 +4582,7 @@ export class Pointer extends Ref { } -export namespace Ref { +export namespace ReactiveValue { export enum UPDATE_TYPE { INIT, // set (initial) reference UPDATE, // update value diff --git a/runtime/runtime.ts b/runtime/runtime.ts index f8898cf..1e5a8bf 100644 --- a/runtime/runtime.ts +++ b/runtime/runtime.ts @@ -27,7 +27,7 @@ Symbol.prototype.toJSON = function(){return globalThis.String(this)} /***** imports */ import { Compiler, compiler_options, PrecompiledDXB, ProtocolDataTypesMap, DatexResponse} from "../compiler/compiler.ts"; // Compiler functions -import { Pointer, PointerProperty, RefOrValue, Ref, ObjectWithDatexValues, JSValueWith$, MinimalJSRef, ObjectRef, RefLike, UpdateScheduler} from "./pointers.ts"; +import { Pointer, PointerProperty, RefOrValue, ReactiveValue, ObjectWithDatexValues, JSValueWith$, MinimalJSRef, ObjectRef, RefLike, UpdateScheduler} from "./pointers.ts"; import { BROADCAST, Endpoint, endpoints, IdEndpoint, LOCAL_ENDPOINT, Target, target_clause, WildcardTarget } from "../types/addressing.ts"; import { RuntimePerformance } from "./performance_measure.ts"; import { NetworkError, PermissionError, PointerError, RuntimeError, SecurityError, ValueError, Error as DatexError, CompilerError, TypeError, SyntaxError, AssertionError } from "../types/errors.ts"; @@ -89,7 +89,7 @@ function static_pointer(value:RefOrValue, endpoint:IdEndpoint, unique_id:n const static_id = Pointer.getStaticPointerId(endpoint, unique_id); const pointer = Pointer.create(static_id, value) if (label) pointer.addLabel(typeof label == "string" ? label.replace(/^\$/, '') : label); - return Ref.collapseValue(pointer); + return ReactiveValue.collapseValue(pointer); } // -------------------------------------------------------------- @@ -140,6 +140,8 @@ export class StaticScope { return this[name]; } setVariable(name: string, value: any) { + const ptr = Pointer.pointerifyValue(value); + if (ptr) ptr.grantPublicAccess(true); return this[name] = value; } hasVariable(name: string) { @@ -309,13 +311,13 @@ export class Runtime { // create new transform pointer else { - const keyVal = Ref.collapseValue(key, true, true); + const keyVal = ReactiveValue.collapseValue(key, true, true); const value_available = !local_map[Runtime.ENV.LANG] || (keyVal in local_map[Runtime.ENV.LANG]); // in map or auto translate const transformValues:RefLike[] = [PointerProperty.get(Runtime.ENV, 'LANG')]; - if (key instanceof Ref) transformValues.push(key) + if (key instanceof ReactiveValue) transformValues.push(key) const string_transform = Pointer.createTransform(transformValues, - (lang:string) => this.transformLangValue(lang, local_map, Ref.collapseValue(key, true, true)), // sync js mapping + (lang:string) => this.transformLangValue(lang, local_map, ReactiveValue.collapseValue(key, true, true)), // sync js mapping value_available ? ` var lang = #env->LANG; var local_map = ${Runtime.valueToDatexString(local_map)}; @@ -356,7 +358,7 @@ export class Runtime { // force override transform values (currently unknown) for (const key of this.#not_loaded_local_strings.get(local_map)??[]) { - const keyVal = Ref.collapseValue(key, true, true); + const keyVal = ReactiveValue.collapseValue(key, true, true); if (lang in local_map && keyVal in local_map[lang]) { local_strings_map.get(key)!.val = await this.transformLangValue(lang, local_map, keyVal); local_strings_map.get(key)!.setDatexTransform(` @@ -1806,7 +1808,7 @@ export class Runtime { public static async simpleScopeExecution(scope:datex_scope) { // run scope, result is saved in 'scope' object await this.run(scope); - return Ref.collapseValue(scope.result); + return ReactiveValue.collapseValue(scope.result); } static #cryptoProxies = new Map() @@ -2906,8 +2908,8 @@ export class Runtime { */ public static async equalValues(a:any, b:any) { // collapse (primitive) pointers - a = Ref.collapseValue(a,true,true); - b = Ref.collapseValue(b,true,true); + a = ReactiveValue.collapseValue(a,true,true); + b = ReactiveValue.collapseValue(b,true,true); // empty Tuple equals void if (a === VOID && b instanceof Tuple && Object.keys(b).length == 0) return true; @@ -2985,7 +2987,7 @@ export class Runtime { // proxyify pointers if (!collapse_pointers && !deep_collapse) value = Pointer.pointerifyValue(value); - if (collapse_pointers && value instanceof Ref) value = value.val; + if (collapse_pointers && value instanceof ReactiveValue) value = value.val; // don't show anonymous pointers as pointers if (value instanceof Pointer && value.is_anonymous) value = value.original_value; @@ -3657,7 +3659,7 @@ export class Runtime { if (o_parent instanceof Pointer) o_parent.assertEndpointCanRead(SCOPE?.sender) - key = Ref.collapseValue(key,true,true); + key = ReactiveValue.collapseValue(key,true,true); // check read permission (throws an error) Runtime.runtime_actions.checkValueReadPermission(SCOPE, parent, key) @@ -3713,7 +3715,7 @@ export class Runtime { return Iterator.map(parent, (child)=>Runtime.runtime_actions.getProperty(SCOPE, child, key)) } - parent = Ref.collapseValue(parent,true,true); + parent = ReactiveValue.collapseValue(parent,true,true); // custom types get let new_obj = JSInterface.handleGetProperty(parent, key) @@ -3766,7 +3768,7 @@ export class Runtime { if (!(o_parent instanceof Pointer)) o_parent = null; else o_parent.assertEndpointCanRead(SCOPE?.sender) - key = Ref.collapseValue(key,true,true); + key = ReactiveValue.collapseValue(key,true,true); // check read/write permission (throws an error) Runtime.runtime_actions.checkValueUpdatePermission(SCOPE, parent, key) @@ -3778,7 +3780,7 @@ export class Runtime { // key is * - set for all matching keys (recursive) if (key === WILDCARD) { - parent = Ref.collapseValue(parent,true,true); + parent = ReactiveValue.collapseValue(parent,true,true); // handle custom pseudo class if (JSInterface.hasPseudoClass(parent)) { // void => clear @@ -3839,8 +3841,8 @@ export class Runtime { } - parent = Ref.collapseValue(parent,true,true); - value = Ref.collapseValue(value,true); + parent = ReactiveValue.collapseValue(parent,true,true); + value = ReactiveValue.collapseValue(value,true); // TODO permission handling // if (parent[DX_PERMISSIONS]?.[key] && !(parent[DX_PERMISSIONS][key]).test(SCOPE.sender)) { @@ -3933,14 +3935,14 @@ export class Runtime { if (!(o_parent instanceof Pointer)) o_parent = null; else o_parent.assertEndpointCanRead(SCOPE?.sender) - key = Ref.collapseValue(key,true,true); + key = ReactiveValue.collapseValue(key,true,true); // check read/write permission (throws an error) if (parent) Runtime.runtime_actions.checkValueUpdatePermission(SCOPE, parent, key) // key is * - add for all matching keys (recursive) if (key === WILDCARD) { - parent = Ref.collapseValue(parent); + parent = ReactiveValue.collapseValue(parent); let keys:Iterable; // handle custom pseudo class if (JSInterface.hasPseudoClass(parent)) { @@ -4000,9 +4002,9 @@ export class Runtime { - current_val = Ref.collapseValue(current_val); // first make sure that current_val is actual value - parent = Ref.collapseValue(parent); - value = Ref.collapseValue(value); + current_val = ReactiveValue.collapseValue(current_val); // first make sure that current_val is actual value + parent = ReactiveValue.collapseValue(parent); + value = ReactiveValue.collapseValue(value); if (SCOPE.header.type==ProtocolDataType.UPDATE) o_parent?.excludeEndpointFromUpdates(SCOPE.sender); @@ -4018,19 +4020,19 @@ export class Runtime { else if (assigned == NOT_EXISTING) { // DatexPrimitivePointers also collapsed - const current_val_prim = Ref.collapseValue(current_val,true,true); - const value_prim = Ref.collapseValue(value,true,true); // DatexPrimitivePointers also collapsed + const current_val_prim = ReactiveValue.collapseValue(current_val,true,true); + const value_prim = ReactiveValue.collapseValue(value,true,true); // DatexPrimitivePointers also collapsed try { - const currentValIsIntegerRef = current_val instanceof Ref && typeof current_val.val == "bigint"; - const currentValIsDecimalRef = current_val instanceof Ref && typeof current_val.val == "number"; + const currentValIsIntegerRef = current_val instanceof ReactiveValue && typeof current_val.val == "bigint"; + const currentValIsDecimalRef = current_val instanceof ReactiveValue && typeof current_val.val == "number"; // x.current_val ?= value switch (action_type) { case BinaryCode.ADD: if (current_val instanceof Array && !(current_val instanceof Tuple)) current_val.push(value); // Array push (TODO array extend?) - else if (current_val instanceof Ref && typeof current_val.val == "string" && typeof value_prim == "string") await current_val.setVal(current_val.val + value_prim); // primitive pointer operations + else if (current_val instanceof ReactiveValue && typeof current_val.val == "string" && typeof value_prim == "string") await current_val.setVal(current_val.val + value_prim); // primitive pointer operations else if (currentValIsDecimalRef && typeof value_prim == "number") await current_val.setVal(current_val.val + value_prim); else if (currentValIsIntegerRef && typeof value_prim == "bigint") await current_val.setVal(current_val.val + value_prim); else if (currentValIsDecimalRef && typeof value_prim == "bigint") await current_val.setVal(current_val.val + Number(value_prim)); @@ -4371,7 +4373,7 @@ export class Runtime { // make sure promise wrappers are collapsed el = Runtime.collapseValueCast(el) // first make sure pointers are collapsed - el = Ref.collapseValue(el) + el = ReactiveValue.collapseValue(el) // collapse Maybes (TODO) //if (el instanceof Maybe) el = await el.value; @@ -4465,7 +4467,7 @@ export class Runtime { el = new Negation(el) } - else if (typeof el == "boolean" || typeof (el = Ref.collapseValue(el, true, true)) == "boolean" ) { + else if (typeof el == "boolean" || typeof (el = ReactiveValue.collapseValue(el, true, true)) == "boolean" ) { el = !el; } else throw(new ValueError("Cannot negate this value ("+Runtime.valueToDatexString(el)+")", SCOPE)) @@ -4747,7 +4749,7 @@ export class Runtime { const pointer = Pointer.pointerifyValue(el); - if (!(pointer instanceof Ref) || (pointer instanceof Pointer && pointer.is_anonymous)) { + if (!(pointer instanceof ReactiveValue) || (pointer instanceof Pointer && pointer.is_anonymous)) { throw new ValueError("sync expects a reference value", SCOPE); } @@ -4784,9 +4786,9 @@ export class Runtime { // convert value to stream (writes to stream everytime the pointer value changes) else if (to instanceof Stream) { - to.write(Ref.collapseValue(pointer, true, true)); // current value as initial value + to.write(ReactiveValue.collapseValue(pointer, true, true)); // current value as initial value pointer.observe((v,k,t)=>{ - if (t == Ref.UPDATE_TYPE.INIT) to.write(v); + if (t == ReactiveValue.UPDATE_TYPE.INIT) to.write(v); }); } @@ -4802,7 +4804,7 @@ export class Runtime { let pointer = Pointer.pointerifyValue(el); - if (!(pointer instanceof Ref) || (pointer instanceof Pointer && pointer.is_anonymous)) throw new ValueError("stop sync expects a reference value", SCOPE); + if (!(pointer instanceof ReactiveValue) || (pointer instanceof Pointer && pointer.is_anonymous)) throw new ValueError("stop sync expects a reference value", SCOPE); // is a sync consumer const to = INNER_SCOPE.active_value; @@ -5007,7 +5009,7 @@ export class Runtime { // stream else if (SCOPE.inner_scope.stream_consumer) { - el = Ref.collapseValue(el, true, true); // collapse primitive values + el = ReactiveValue.collapseValue(el, true, true); // collapse primitive values // pipe stream if (el instanceof Stream) { @@ -5027,8 +5029,8 @@ export class Runtime { let is_true = false; - let a = Ref.collapseValue(INNER_SCOPE.active_value,true); // only collapse pointer properties, keep primitive pointers - let b = Ref.collapseValue(el,true); + let a = ReactiveValue.collapseValue(INNER_SCOPE.active_value,true); // only collapse pointer properties, keep primitive pointers + let b = ReactiveValue.collapseValue(el,true); let compared; @@ -5091,8 +5093,8 @@ export class Runtime { // + else if (INNER_SCOPE.operator === BinaryCode.ADD || INNER_SCOPE.operator === BinaryCode.SUBTRACT) { - el = Ref.collapseValue(el, true, true); // collapse primitive values - let val = Ref.collapseValue(INNER_SCOPE.active_value, true, true); + el = ReactiveValue.collapseValue(el, true, true); // collapse primitive values + let val = ReactiveValue.collapseValue(INNER_SCOPE.active_value, true, true); // negate for subtract if (INNER_SCOPE.operator === BinaryCode.SUBTRACT && (typeof el == "number" || typeof el == "bigint")) el = -el; @@ -5130,8 +5132,8 @@ export class Runtime { else if (INNER_SCOPE.operator === BinaryCode.AND) { // collapse primitive values - const val = Ref.collapseValue(INNER_SCOPE.active_value, true, true); - el = Ref.collapseValue(el, true, true); + const val = ReactiveValue.collapseValue(INNER_SCOPE.active_value, true, true); + el = ReactiveValue.collapseValue(el, true, true); // logical if (val instanceof Logical) { @@ -5181,8 +5183,8 @@ export class Runtime { else if (INNER_SCOPE.operator === BinaryCode.OR) { // collapse primitive values - let val = Ref.collapseValue(INNER_SCOPE.active_value, true, true); - el = Ref.collapseValue(el, true, true); + let val = ReactiveValue.collapseValue(INNER_SCOPE.active_value, true, true); + el = ReactiveValue.collapseValue(el, true, true); // logical @@ -5216,8 +5218,8 @@ export class Runtime { else if (INNER_SCOPE.operator === BinaryCode.MULTIPLY) { // collapse primitive values - let val = Ref.collapseValue(INNER_SCOPE.active_value, true, true); - el = Ref.collapseValue(el, true, true); + let val = ReactiveValue.collapseValue(INNER_SCOPE.active_value, true, true); + el = ReactiveValue.collapseValue(el, true, true); if (typeof val == "bigint" && typeof el == "bigint") { INNER_SCOPE.active_value *= el; @@ -5269,8 +5271,8 @@ export class Runtime { else if (INNER_SCOPE.operator === BinaryCode.DIVIDE) { // collapse primitive values - let val = Ref.collapseValue(INNER_SCOPE.active_value, true, true); - el = Ref.collapseValue(el, true, true); + let val = ReactiveValue.collapseValue(INNER_SCOPE.active_value, true, true); + el = ReactiveValue.collapseValue(el, true, true); if (typeof val == "bigint" && typeof el == "bigint") { if (el === 0n) throw new ValueError("Division by zero", SCOPE); @@ -5298,8 +5300,8 @@ export class Runtime { else if (INNER_SCOPE.operator === BinaryCode.POWER) { // collapse primitive values - let val = Ref.collapseValue(INNER_SCOPE.active_value, true, true); - el = Ref.collapseValue(el, true, true); + let val = ReactiveValue.collapseValue(INNER_SCOPE.active_value, true, true); + el = ReactiveValue.collapseValue(el, true, true); if (typeof val == "bigint" && typeof el == "bigint") { if (el < 0) throw new ValueError("Cannot use a negative exponent with an integer") @@ -5327,8 +5329,8 @@ export class Runtime { else if (INNER_SCOPE.operator === BinaryCode.MODULO) { // collapse primitive values - let val = Ref.collapseValue(INNER_SCOPE.active_value, true, true); - el = Ref.collapseValue(el, true, true); + let val = ReactiveValue.collapseValue(INNER_SCOPE.active_value, true, true); + el = ReactiveValue.collapseValue(el, true, true); if (typeof val == "bigint" && typeof el == "bigint") { INNER_SCOPE.active_value %= el; @@ -5412,7 +5414,7 @@ export class Runtime { // handle all ValueConsumers (, TODO?, ...) if (val instanceof DatexFunction || val instanceof Target /*|| val instanceof Filter*/ || val instanceof Assertion) { // insert el or [el], or [] if el==VOID (call without parameters) - if (val.handleApply) INNER_SCOPE.active_value = await val.handleApply(Ref.collapseValue(el), SCOPE); + if (val.handleApply) INNER_SCOPE.active_value = await val.handleApply(ReactiveValue.collapseValue(el), SCOPE); else throw new ValueError("Cannot apply values to this value", SCOPE); } @@ -6130,7 +6132,7 @@ export class Runtime { else { await this.runtime_actions.insertToScope( SCOPE, - Ref.collapseValue(await Pointer.createTransformAsync(INNER_SCOPE.scope_block_vars, code_block)) + ReactiveValue.collapseValue(await Pointer.createTransformAsync(INNER_SCOPE.scope_block_vars, code_block)) ) } diff --git a/storage/storage.ts b/storage/storage.ts index ade6ce4..7c73d9c 100644 --- a/storage/storage.ts +++ b/storage/storage.ts @@ -5,7 +5,7 @@ import type { ExecConditions, PointerSource } from "../utils/global_types.ts"; import { logger } from "../utils/global_values.ts"; import { client_type } from "../utils/constants.ts"; import { NOT_EXISTING } from "../runtime/constants.ts"; -import { Pointer, type MinimalJSRef, Ref } from "../runtime/pointers.ts"; +import { Pointer, type MinimalJSRef, ReactiveValue } from "../runtime/pointers.ts"; import { localStorage } from "./storage-locations/local-storage-compat.ts"; import { MessageLogger } from "../utils/message_logger.ts"; import { displayFatalError } from "../runtime/display.ts" @@ -863,7 +863,7 @@ export class Storage { let saving = false; - const handler = (v:unknown,key:unknown,t?:Ref.UPDATE_TYPE)=>{ + const handler = (v:unknown,key:unknown,t?:ReactiveValue.UPDATE_TYPE)=>{ if (saving) return; // don't block saving if only partial update diff --git a/types/function.ts b/types/function.ts index 8085738..c151548 100644 --- a/types/function.ts +++ b/types/function.ts @@ -1,4 +1,4 @@ -import { Pointer, Ref } from "../runtime/pointers.ts"; +import { Pointer, ReactiveValue } from "../runtime/pointers.ts"; import { Runtime } from "../runtime/runtime.ts"; import { logger } from "../utils/global_values.ts"; import type { StreamConsumer, ValueConsumer } from "./abstract_types.ts"; @@ -366,7 +366,7 @@ export class Function any = (...args: any) => any> e throw new RuntimeError("Cannot apply values to a with no executable DATEX or valid native target"); } - const context = (this.context instanceof Ref ? this.context.val : this.context); + const context = (this.context instanceof ReactiveValue ? this.context.val : this.context); let params:any[]; // record diff --git a/types/logic.ts b/types/logic.ts index b6879b6..f5a9211 100644 --- a/types/logic.ts +++ b/types/logic.ts @@ -4,7 +4,7 @@ import type { Class } from "../utils/global_types.ts"; import { Runtime } from "../runtime/runtime.ts"; -import { Ref } from "../runtime/pointers.ts"; +import { ReactiveValue } from "../runtime/pointers.ts"; import { RuntimeError } from "./errors.ts"; import { Assertion } from "./assertion.ts"; @@ -66,8 +66,8 @@ export class Logical extends Set { // TODO: empty - does not match? if (against === undefined) return false; - value = > Datex.Ref.collapseValue(value, true, true); - against = > Datex.Ref.collapseValue(against, true, true); + value = > Datex.ReactiveValue.collapseValue(value, true, true); + against = > Datex.ReactiveValue.collapseValue(against, true, true); // auto infer atomic class if (atomic_class === undefined) { @@ -111,14 +111,14 @@ export class Logical extends Set { } // default - return this.matchesSingle(Ref.collapseValue(value, true, true), against, atomic_class, assertionValue, throwInvalidAssertion); + return this.matchesSingle(ReactiveValue.collapseValue(value, true, true), against, atomic_class, assertionValue, throwInvalidAssertion); } private static matchesSingle(atomic_value:T, against: clause, atomic_class:Class&LogicalComparator, assertionValue = atomic_value, throwInvalidAssertion = false): boolean { - atomic_value = Datex.Ref.collapseValue(atomic_value, true, true); - against = > Datex.Ref.collapseValue(against, true, true); + atomic_value = Datex.ReactiveValue.collapseValue(atomic_value, true, true); + against = > Datex.ReactiveValue.collapseValue(against, true, true); // wrong atomic type for atomic_value at runtime if (atomic_class && !(atomic_value instanceof atomic_class)) throw new RuntimeError(`Invalid match check: atomic value has wrong type (expected ${Type.getClassDatexType(atomic_class)}, found ${Type.ofValue(atomic_value)})`); @@ -161,7 +161,7 @@ export class Logical extends Set { } // match - return atomic_class.logicalMatch(Ref.collapseValue(atomic_value, true, true), against); + return atomic_class.logicalMatch(ReactiveValue.collapseValue(atomic_value, true, true), against); } @@ -206,7 +206,7 @@ export class Logical extends Set { // default - value = Ref.collapseValue(value, true, true) + value = ReactiveValue.collapseValue(value, true, true) if (!(value instanceof atomic_class)) throw new RuntimeError(`logical collapse: atomic value has wrong type (expected ${Type.getClassDatexType(atomic_class)}, found ${Type.ofValue(value)})`); list.add(value); diff --git a/types/type.ts b/types/type.ts index 456e571..e45954e 100644 --- a/types/type.ts +++ b/types/type.ts @@ -12,7 +12,7 @@ import { Quantity } from "./quantity.ts"; import { Function as DatexFunction } from "./function.ts"; import { logger, TypedArray } from "../utils/global_values.ts"; import { BinaryCode } from "../compiler/binary_codes.ts" -import { RefOrValue, Pointer, Ref, PointerProperty } from "../runtime/pointers.ts"; +import { RefOrValue, Pointer, ReactiveValue, PointerProperty } from "../runtime/pointers.ts"; import { clause, Conjunction, Disjunction, Logical, Negation } from "./logic.ts"; import { Debugger } from "../runtime/debugger.ts"; import { Time } from "./time.ts"; @@ -755,7 +755,7 @@ export class Type extends ExtensibleFunction { // check if root type of value matches exactly public static matches(value:RefOrValue, type:type_clause, throwInvalidAssertion = false): value is (T extends Type ? TT : any) { - value = Ref.collapseValue(value, true, true); + value = ReactiveValue.collapseValue(value, true, true); // value has a matching DX_TEMPLATE if (type instanceof Type && type.template && value?.[DX_TEMPLATE] && this.matchesTemplate(value[DX_TEMPLATE], type.template)) return true; // compare types @@ -833,7 +833,7 @@ export class Type extends ExtensibleFunction { return value.type as Type; } - value = Ref.collapseValue(value,true,true) + value = ReactiveValue.collapseValue(value,true,true) // // should not happen // else if (value instanceof Pointer) { diff --git a/utils/global_types.ts b/utils/global_types.ts index 87a4250..7388658 100644 --- a/utils/global_types.ts +++ b/utils/global_types.ts @@ -1,5 +1,5 @@ import type { Type } from "../types/type.ts" -import type { Pointer, Ref } from "../runtime/pointers.ts" +import type { Pointer, ReactiveValue } from "../runtime/pointers.ts" import type { BinaryCode } from "../compiler/binary_codes.ts" import type { StreamConsumer } from "../types/abstract_types.ts" import { ProtocolDataType } from "../compiler/protocol_types.ts" @@ -64,7 +64,7 @@ export type datex_sub_scope = { keys?: boolean, // get keys for next value get?: boolean, // get url (next value) template?: boolean|Type, // set type template - observe?: boolean|Ref, // observe value + observe?: boolean|ReactiveValue, // observe value scope_block_for?: BinaryCode, // type of scope block scope_block_vars?: any[], // #0, #1, ... for scope block wait_await?: boolean, // await diff --git a/utils/history.ts b/utils/history.ts index 8ea09e1..13214f2 100644 --- a/utils/history.ts +++ b/utils/history.ts @@ -1,10 +1,10 @@ import { NOT_EXISTING } from "../runtime/constants.ts"; -import { Pointer, Ref, RefOrValue } from "../runtime/pointers.ts"; +import { Pointer, ReactiveValue, RefOrValue } from "../runtime/pointers.ts"; import { logger } from "./global_values.ts"; type HistoryStateChange = { pointer: Pointer, - type: Ref.UPDATE_TYPE, + type: ReactiveValue.UPDATE_TYPE, value: unknown, previous: unknown, key?: unknown, @@ -49,7 +49,7 @@ export class History { const pointer = Pointer.pointerifyValue(val); if (!(pointer instanceof Pointer)) throw new Error("Cannot include non-pointer value in history"); - Ref.observe(pointer, (value, key, type, _transform, _isChildUpdate, previous, atomicId) => { + ReactiveValue.observe(pointer, (value, key, type, _transform, _isChildUpdate, previous, atomicId) => { if (type == Pointer.UPDATE_TYPE.BEFORE_DELETE) return; // ignore if (type == Pointer.UPDATE_TYPE.BEFORE_REMOVE) return; // ignore diff --git a/utils/interface-generator.ts b/utils/interface-generator.ts index cba2b3f..28db8d2 100644 --- a/utils/interface-generator.ts +++ b/utils/interface-generator.ts @@ -138,7 +138,7 @@ async function getValueTSCode(module_name:string, name:string, value: any, no_po const is_datex_module = module_name.endsWith(".dx") || module_name.endsWith(".dxb") const type = Datex.Type.ofValue(value) - const is_pointer = (value instanceof Datex.Ref) || !!(Datex.Pointer.getByValue(value)); + const is_pointer = (value instanceof Datex.ReactiveValue) || !!(Datex.Pointer.getByValue(value)); const original_value = value; // if (no_pointer) { diff --git a/utils/iterable-handler.ts b/utils/iterable-handler.ts index fb31b33..8b179b1 100644 --- a/utils/iterable-handler.ts +++ b/utils/iterable-handler.ts @@ -64,14 +64,14 @@ export class IterableHandler { } deref.onValueChanged(v, k, t) } - Datex.Ref.observeAndInit(iterable, callback); + Datex.ReactiveValue.observeAndInit(iterable, callback); return callback; }, (callback) => { use (iterableRef, Datex); const deref = iterableRef.deref() - if (deref) Datex.Ref.unobserve(deref, callback); + if (deref) Datex.ReactiveValue.unobserve(deref, callback); } ); } @@ -133,13 +133,13 @@ export class IterableHandler { if (iterable instanceof Object) return Object.values(iterable); } - protected onValueChanged(value: Iterable|T, key: number|undefined, type:Datex.Ref.UPDATE_TYPE) { - if (type == Datex.Ref.UPDATE_TYPE.DELETE) return; // ignore DELETE event, only use BEFORE_DELETE event - if (type == Datex.Ref.UPDATE_TYPE.REMOVE) return; // ignore REMOVE event, only use BEFORE_REMOVE event + protected onValueChanged(value: Iterable|T, key: number|undefined, type:Datex.ReactiveValue.UPDATE_TYPE) { + if (type == Datex.ReactiveValue.UPDATE_TYPE.DELETE) return; // ignore DELETE event, only use BEFORE_DELETE event + if (type == Datex.ReactiveValue.UPDATE_TYPE.REMOVE) return; // ignore REMOVE event, only use BEFORE_REMOVE event // compatibility with key-value iterables // Map or Object - if (type != Datex.Ref.UPDATE_TYPE.INIT && type != Datex.Ref.UPDATE_TYPE.CLEAR && (this.iterable instanceof Map || !(this.iterable instanceof Set || this.iterable instanceof Array))) { + if (type != Datex.ReactiveValue.UPDATE_TYPE.INIT && type != Datex.ReactiveValue.UPDATE_TYPE.CLEAR && (this.iterable instanceof Map || !(this.iterable instanceof Set || this.iterable instanceof Array))) { const original_value = value; // TODO: required? if (this.iterable instanceof Map) value = >[key, value] @@ -151,10 +151,10 @@ export class IterableHandler { } // single property update - if (type == Datex.Ref.UPDATE_TYPE.SET) this.handleNewEntry(value, key) - else if (type == Datex.Ref.UPDATE_TYPE.ADD) this.handleNewEntry(value, this.getPseudoIndex(key, value)); + if (type == Datex.ReactiveValue.UPDATE_TYPE.SET) this.handleNewEntry(value, key) + else if (type == Datex.ReactiveValue.UPDATE_TYPE.ADD) this.handleNewEntry(value, this.getPseudoIndex(key, value)); // clear all - else if (type == Datex.Ref.UPDATE_TYPE.CLEAR) { + else if (type == Datex.ReactiveValue.UPDATE_TYPE.CLEAR) { // handle onEmpty if (this.onEmpty) { this.onEmpty.call ? this.onEmpty.call(this) : this.onEmpty(); @@ -166,10 +166,10 @@ export class IterableHandler { } } } - else if (type == Datex.Ref.UPDATE_TYPE.BEFORE_DELETE) this.handleRemoveEntry(key); - else if (type == Datex.Ref.UPDATE_TYPE.BEFORE_REMOVE) this.handleRemoveEntry(this.getPseudoIndex(key, value)); + else if (type == Datex.ReactiveValue.UPDATE_TYPE.BEFORE_DELETE) this.handleRemoveEntry(key); + else if (type == Datex.ReactiveValue.UPDATE_TYPE.BEFORE_REMOVE) this.handleRemoveEntry(this.getPseudoIndex(key, value)); // completely new value - else if (type == Datex.Ref.UPDATE_TYPE.INIT) { + else if (type == Datex.ReactiveValue.UPDATE_TYPE.INIT) { for (const e of this.entries.keys()) this.handleRemoveEntry(e); // clear all entries let key = 0; for (const child of >this.iterator(value??[])) this.handleNewEntry(child, key++); From a1d97aa8a562f200912fbabe7bd4ad77aac909b2 Mon Sep 17 00:00:00 2001 From: benStre Date: Wed, 25 Sep 2024 00:00:46 +0200 Subject: [PATCH 03/27] rename window to globalThis, improve always handling --- datex_short.ts | 10 +++- docs/manual/16 Communication Interfaces.md | 6 +-- functions.ts | 34 +++++++++++-- iframes/iframe-init.ts | 2 +- lib/localforage/utils/isIndexedDBValid.dev.js | 2 +- lib/localforage/utils/isIndexedDBValid.js | 2 +- .../window-interface.ts | 22 ++++---- network/datex-http-channel.ts | 2 +- runtime/pointers.ts | 51 +++++++++++++++++-- runtime/runtime.ts | 2 +- storage/storage.ts | 2 +- types/function-utils.ts | 2 +- utils/error-handling.ts | 2 +- utils/error-reporting.ts | 2 +- utils/path.ts | 2 +- 15 files changed, 110 insertions(+), 33 deletions(-) diff --git a/datex_short.ts b/datex_short.ts index 3802d8c..3c31a51 100644 --- a/datex_short.ts +++ b/datex_short.ts @@ -672,12 +672,20 @@ export function translocate|Set|Array)=>_always(cb, {allowStatic: true}), configurable:false}) +Object.defineProperty(globalThis, '_$', {value: (cb:SmartTransformFunction) => _always(cb, {allowStatic: true, _allowAsync: true, _collapseStatic: true}), configurable:false}) +Object.defineProperty(globalThis, '_$method', {value: _$method, configurable:false}) Object.defineProperty(globalThis, 'reactiveFn', {value:_reactiveFn, configurable:false}) Object.defineProperty(globalThis, 'toggle', {value:_toggle, configurable:false}) Object.defineProperty(globalThis, 'map', {value:_map, configurable:false}) diff --git a/docs/manual/16 Communication Interfaces.md b/docs/manual/16 Communication Interfaces.md index c4596dc..3b0fe2e 100644 --- a/docs/manual/16 Communication Interfaces.md +++ b/docs/manual/16 Communication Interfaces.md @@ -37,9 +37,9 @@ The interface constructor signature can be different for other communication int - ### Window -DATEX provides a simple communication interface that makes use of the [window.postMessage()](https://developer.mozilla.org/en-US/docs/Web/API/Window/postMessage) API to enable cross-origin communication between Window objects; e.g., between a page and a pop-up that it spawned, or between a page and an iframe embedded within it. +DATEX provides a simple communication interface that makes use of the [globalThis.postMessage()](https://developer.mozilla.org/en-US/docs/Web/API/Window/postMessage) API to enable cross-origin communication between Window objects; e.g., between a page and a pop-up that it spawned, or between a page and an iframe embedded within it. -The `WindowInterface.createWindow` method can open a window (page, popup or tab) and behaves similar to the [window.open](https://developer.mozilla.org/en-US/docs/Web/API/Window/open) API with the difference that it is asynchronously and returns an object including the remote endpoint (the endpoint of the popup) and the actual [Window](https://developer.mozilla.org/en-US/docs/Web/API/Window) containing the DOM document. +The `WindowInterface.createWindow` method can open a window (page, popup or tab) and behaves similar to the [globalThis.open](https://developer.mozilla.org/en-US/docs/Web/API/Window/open) API with the difference that it is asynchronously and returns an object including the remote endpoint (the endpoint of the popup) and the actual [Window](https://developer.mozilla.org/en-US/docs/Web/API/Window) containing the DOM document. ```ts import { WindowInterface } from "datex-core-legacy/network/communication-interfaces/window-interface.ts"; @@ -72,7 +72,7 @@ The `createWindow` utility is a wrapper method that opens the new window by pass ```ts const url = new URL("https://popup.com"); -const myWindow = window.open(url); +const myWindow = globalThis.open(url); const windowInterface = WindowInterface.createChildWindowInterface(myWindow, url); const connected = await communicationHub.addInterface(windowInterface); diff --git a/functions.ts b/functions.ts index 0e7329a..e816ae6 100644 --- a/functions.ts +++ b/functions.ts @@ -9,6 +9,8 @@ import { Datex } from "./mod.ts"; import { PointerError } from "./types/errors.ts"; import { IterableHandler } from "./utils/iterable-handler.ts"; import { AsyncSmartTransformFunction, MinimalJSRefWithIndirectRef, RestrictSameType } from "./runtime/pointers.ts"; +import { handleError } from "./utils/error-handling.ts"; +import { KnownError } from "./utils/error-handling.ts"; /** * A generic transform function, creates a new pointer containing the result of the callback function. @@ -41,19 +43,45 @@ export function always(scriptOrJSTransform:TemplateStringsArray|SmartTransformFu if (scriptOrJSTransform.constructor.name == "AsyncFunction") { throw new Error("Async functions are not allowed as always transforms") } - const ptr = Pointer.createSmartTransform(scriptOrJSTransform, undefined, undefined, undefined, vars[0]); + const options: SmartTransformOptions|undefined = typeof vars[0] == "object" ? vars[0] : undefined; + const ptr = Pointer.createSmartTransform(scriptOrJSTransform, undefined, undefined, undefined, options); + if (options?._allowAsync && !ptr.value_initialized && ptr.waiting_for_always_promise) { + return ptr.waiting_for_always_promise.then(()=>collapseTransformPointer(ptr, options?._collapseStatic)); + } if (!ptr.value_initialized && ptr.waiting_for_always_promise) { throw new PointerError(`Promises cannot be returned from always transforms - use 'asyncAlways' instead`); } else { - return ReactiveValue.collapseValue(ptr); + return collapseTransformPointer(ptr, options?._collapseStatic); } } // datex script - else return (async ()=>ReactiveValue.collapseValue(await datex(`always (${scriptOrJSTransform.raw.join(INSERT_MARK)})`, vars)))() + else if (scriptOrJSTransform.raw instanceof Array) { + return (async ()=>collapseTransformPointer(await datex(`always (${scriptOrJSTransform.raw.join(INSERT_MARK)})`, vars)))() + } + else { + handleError(new KnownError("You called 'always' with invalid arguments. It seems like you are not using deno for UIX.", [ + "Install deno for UIX, see https://docs.unyt.org/manual/uix/getting-started#install-deno", + "Call 'always' with a function: always(() => ...)" + ])) + } } +function collapseTransformPointer(ptr: Pointer, collapseStatic = false) { + let collapse = false; + if (collapseStatic) { + // check if transform function is static and value is a js primitive + if (ptr.isStaticTransform && ptr.is_js_primitive) { + collapse = true; + } + } + + const val = ReactiveValue.collapseValue(ptr, false, collapse); + if (collapse) ptr.delete(); + return val; +} + /** * A generic transform function, creates a new pointer containing the result of the callback function. * At any point in time, the pointer is the result of the callback function. diff --git a/iframes/iframe-init.ts b/iframes/iframe-init.ts index 5aefa7e..7d0ba2a 100644 --- a/iframes/iframe-init.ts +++ b/iframes/iframe-init.ts @@ -8,7 +8,7 @@ import { communicationHub } from "../network/communication-hub.ts"; import { WindowInterface } from "../network/communication-interfaces/window-interface.ts"; await Datex.Supranet.init(); -const windowInterface = WindowInterface.createParentInterface(window.parent) +const windowInterface = WindowInterface.createParentInterface(globalThis.parent) // use as default interface only if no other default interface active const useAsDefaultInterface = !communicationHub.defaultSocket await communicationHub.addInterface(windowInterface, useAsDefaultInterface) \ No newline at end of file diff --git a/lib/localforage/utils/isIndexedDBValid.dev.js b/lib/localforage/utils/isIndexedDBValid.dev.js index d0ca7ab..364e1f6 100644 --- a/lib/localforage/utils/isIndexedDBValid.dev.js +++ b/lib/localforage/utils/isIndexedDBValid.dev.js @@ -25,7 +25,7 @@ function isIndexedDBValid() { var hasFetch = typeof fetch === 'function' && fetch.toString().indexOf('[native code') !== -1; // Safari <10.1 does not meet our requirements for IDB support // (see: https://github.com/pouchdb/pouchdb/issues/5572). // Safari 10.1 shipped with fetch, we can use that to detect it. - // Note: this creates issues with `window.fetch` polyfills and + // Note: this creates issues with `globalThis.fetch` polyfills and // overrides; see: // https://github.com/localForage/localForage/issues/856 diff --git a/lib/localforage/utils/isIndexedDBValid.js b/lib/localforage/utils/isIndexedDBValid.js index 9c5089b..f54ecac 100644 --- a/lib/localforage/utils/isIndexedDBValid.js +++ b/lib/localforage/utils/isIndexedDBValid.js @@ -24,7 +24,7 @@ function isIndexedDBValid() { // Safari <10.1 does not meet our requirements for IDB support // (see: https://github.com/pouchdb/pouchdb/issues/5572). // Safari 10.1 shipped with fetch, we can use that to detect it. - // Note: this creates issues with `window.fetch` polyfills and + // Note: this creates issues with `globalThis.fetch` polyfills and // overrides; see: // https://github.com/localForage/localForage/issues/856 return ( diff --git a/network/communication-interfaces/window-interface.ts b/network/communication-interfaces/window-interface.ts index 96f96c3..6a5ddc1 100644 --- a/network/communication-interfaces/window-interface.ts +++ b/network/communication-interfaces/window-interface.ts @@ -26,8 +26,8 @@ export class WindowInterfaceSocket extends CommunicationInterfaceSocket { send(dxb: ArrayBuffer) { try { - if (this.transmissionMode == "json") this.window.postMessage(arrayBufferToBase64(dxb), this.windowOrigin) - else this.window.postMessage(dxb, this.windowOrigin) + if (this.transmissionMode == "json") this.globalThis.postMessage(arrayBufferToBase64(dxb), this.windowOrigin) + else this.globalThis.postMessage(dxb, this.windowOrigin) return true; } catch { @@ -72,13 +72,13 @@ export class WindowInterface extends CommunicationInterface { this.#isChild = false; // Modifying the sandbox attr does not make sense since here // since the src is already set and iframe is sandboxed on load - // window.setAttribute("sandbox", "allow-popups-to-escape-sandbox allow-modals allow-forms allow-popups allow-scripts allow-same-origin allow-top-navigation") - this.#windowOrigin = new URL(window.src).origin; - windowOriginURL = new URL(window.src); + // globalThis.setAttribute("sandbox", "allow-popups-to-escape-sandbox allow-modals allow-forms allow-popups allow-scripts allow-same-origin allow-top-navigation") + this.#windowOrigin = new URL(globalThis.src).origin; + windowOriginURL = new URL(globalThis.src); this.logger.debug("initializing as parent window, child iframe origin: " + this.#windowOrigin) } // is opened child window or inside iframe - else if (type !== "parent" && (type === "child" || window === self.window.opener || globalThis.self !== globalThis.top)) { + else if (type !== "parent" && (type === "child" || window === self.globalThis.opener || globalThis.self !== globalThis.top)) { this.#isChild = true; // explicitly set window origin @@ -86,9 +86,9 @@ export class WindowInterface extends CommunicationInterface { this.#windowOrigin = windowOriginURL.origin; } else { - // first try window.location.origin + // first try globalThis.location.origin try { - this.#windowOrigin = window.location.origin; + this.#windowOrigin = globalThis.location.origin; } // try document.referrer catch { @@ -135,7 +135,7 @@ export class WindowInterface extends CommunicationInterface { private sendInit() { - this.window.postMessage({ + this.globalThis.postMessage({ type: "INIT", endpoint: Runtime.endpoint.toString() }, this.#windowOrigin); @@ -144,7 +144,7 @@ export class WindowInterface extends CommunicationInterface { onClose?: ()=>void private handleClose() { - // check window.closed every second + // check globalThis.closed every second const interval = setInterval(() => { if (this.window?.closed) { clearInterval(interval); @@ -195,7 +195,7 @@ export class WindowInterface extends CommunicationInterface { * The WindowInterface is automatically removed when the window is closed. */ static createWindow(url: string | URL, target?: string, features?: string, connectionTimeout?: number) { - const newWindow = window.open(url, target, features); + const newWindow = globalThis.open(url, target, features); if (!newWindow) return Promise.resolve({window: null, endpoint: null}); const windowInterface = this.createChildWindowInterface(newWindow, url) diff --git a/network/datex-http-channel.ts b/network/datex-http-channel.ts index 84a7cba..b4b3959 100644 --- a/network/datex-http-channel.ts +++ b/network/datex-http-channel.ts @@ -6,7 +6,7 @@ * This is not the same as datex-over-http implemented in UIX * that send unsigned/unencrypted DATEX scripts to an HTTP server */ -export function sendDatexViaHTTPChannel(dxb: ArrayBuffer, origin = window.location.origin) { +export function sendDatexViaHTTPChannel(dxb: ArrayBuffer, origin = globalThis.location.origin) { // fallback to sendBeacon for firefox until fetch keepalive implemented if (navigator.userAgent.includes("Firefox/")) { diff --git a/runtime/pointers.ts b/runtime/pointers.ts index 4b7e73e..a2de7b3 100644 --- a/runtime/pointers.ts +++ b/runtime/pointers.ts @@ -455,6 +455,19 @@ export abstract class ReactiveValue extends EventTarget { #liveTransform = false; #forceLiveTransform = false; #transformSource?: TransformSource + #isStaticTransform = false; + + /** + * if true, there are no dependencies and the value is never updated + * (only used when a transform source exists, e.g. for effects and always) + */ + get isStaticTransform() { + return this.#isStaticTransform; + } + protected set _isStaticTransform(val: boolean) { + this.#isStaticTransform = val; + } + get transformSource() { return this.#transformSource @@ -564,6 +577,7 @@ export type TransformSource = { keyedDeps: IterableWeakMap> } + export type PointerPropertyParent = Map | Record; export type InferredPointerProperty = PointerProperty ? MV : Parent[Key&keyof Parent]> @@ -1115,11 +1129,16 @@ export type SmartTransformOptions = { initial?: T, cache?: boolean, allowStatic?: boolean, + // allow async when using always instead of asyncAlways + _allowAsync?: boolean, + // collapse primitive pointer if value has no reactive dependencies and garbage-collect pointer + _collapseStatic: boolean, } type TransformState = { isLive: boolean; isFirst: boolean; + executingEffect: boolean; deps: IterableWeakSet>; keyedDeps: AutoMap>; returnCache: Map; @@ -2735,14 +2754,16 @@ export class Pointer extends ReactiveValue { const val = ReactiveValue.collapseValue(v,true,true); const newType = Type.ofValue(val); + const current_val = this.current_val; + // not changed (relevant for primitive values) - if (this.current_val === val) { + if (Object.is(current_val, val)) { return; } // also check if array is equal - if (this.current_val instanceof Array && val instanceof Array) { - if (this.current_val.length == val.length && this.current_val.every((v,i)=>v===val[i])) { + if (current_val instanceof Array && val instanceof Array) { + if (current_val.length == val.length && current_val.every((v,i)=>v===val[i])) { return; } } @@ -2931,6 +2952,7 @@ export class Pointer extends ReactiveValue { const state: TransformState = { isLive: false, isFirst: true, + executingEffect: false, deps: new IterableWeakSet(), keyedDeps: new IterableWeakMap>().setAutoDefault(Set), returnCache: new Map(), @@ -2946,11 +2968,16 @@ export class Pointer extends ReactiveValue { }, update: () => { + // currently executing effect, skip update + if (state.executingEffect) return; + // no live transforms needed, just get current value // capture getters in first update() call to check if there // is a static transform and show a warning if (!state.isLive && !state.isFirst) { + state.executingEffect = true; this.setVal(transform() as T, true, true); + state.executingEffect = false; } // get transform value and update dependency observers else { @@ -2973,7 +3000,9 @@ export class Pointer extends ReactiveValue { ReactiveValue.captureGetters(); try { + state.executingEffect = true; val = transform() as T; + state.executingEffect = false; // also trigger getter if pointer is returned ReactiveValue.collapseValue(val, true, true); } @@ -3157,8 +3186,9 @@ export class Pointer extends ReactiveValue { const gettersCount = (capturedGetters?.size??0) + (capturedGettersWithKeys?.size??0); // no dependencies, will never change, this is not the intention of the transform - if (!ignoreReturnValue && hasGetters && !gettersCount && !options?.allowStatic) { - logger.warn("The transform value for " + this.idString() + " is a static value:", val); + if (!ignoreReturnValue && hasGetters && !gettersCount) { + this._isStaticTransform = true + if (!options?.allowStatic) logger.warn("The transform value for " + this.idString() + " is a static value:", val); // TODO: cleanup stuff not needed if no reactive transform } @@ -3640,6 +3670,7 @@ export class Pointer extends ReactiveValue { const high_priority_keys = new Set();// remember higher priority keys in prototype chain, don't override observer with older properties in the chain const value = this.shadow_object!; + for (const name of this.visible_children ?? Object.keys(value)) { const type = Type.ofValue(value[name]) @@ -4533,6 +4564,16 @@ export class Pointer extends ReactiveValue { private callObservers(value:any, key:any, type:ReactiveValue.UPDATE_TYPE, is_transform = false, is_child_update = false, previous?: any, atomic_id?: symbol) { + // disable unintentional capturing of dependencies for smart transforms that are triggered by getters inside observer callbacks + + // @ReactiveValue.disableCapturing + ReactiveValue.freezeCapturing = true; + const res = this._callObservers(value, key, type, is_transform, is_child_update, previous, atomic_id); + ReactiveValue.freezeCapturing = false; + return res; + } + + private _callObservers(value:any, key:any, type:ReactiveValue.UPDATE_TYPE, is_transform = false, is_child_update = false, previous?: any, atomic_id?: symbol) { const promises = []; // key specific observers if (key!=undefined) { diff --git a/runtime/runtime.ts b/runtime/runtime.ts index 1e5a8bf..d3bb6e7 100644 --- a/runtime/runtime.ts +++ b/runtime/runtime.ts @@ -603,7 +603,7 @@ export class Runtime { public static async getURLContent(url:URL, raw?:RAW, cached?:boolean, potentialDatexAsJsModule?:boolean):Promise public static async getURLContent(url_string:string|URL, raw:RAW=false, cached = false, potentialDatexAsJsModule = true):Promise { - if (url_string.toString().startsWith("route:") && window.location?.origin) url_string = new URL(url_string.toString().replace("route:", ""), window.location.origin) + if (url_string.toString().startsWith("route:") && globalThis.location?.origin) url_string = new URL(url_string.toString().replace("route:", ""), globalThis.location.origin) // catch fatal route errors here if (url_string.toString().startsWith("fatal:")) { diff --git a/storage/storage.ts b/storage/storage.ts index 7c73d9c..eaef4d6 100644 --- a/storage/storage.ts +++ b/storage/storage.ts @@ -1513,7 +1513,7 @@ export class Storage { Storage.allowExitWithoutSave(); if (client_type === "deno") Deno.exit(1); else if (globalThis.window?.location) { - window.location.reload(); + globalThis.location.reload(); } else logger.error("Could not reload in non-browser or Deno context") } diff --git a/types/function-utils.ts b/types/function-utils.ts index 3d08dbf..b131af9 100644 --- a/types/function-utils.ts +++ b/types/function-utils.ts @@ -221,7 +221,7 @@ export function createFunctionWithDependencyInjections(source: string, dependenc const createStaticFn = ` const freezedObjects = new WeakSet(); function createStaticObject(val) { - if (val && typeof val == "object" && !globalThis.Datex?.Ref.isRef(val)) { + if (val && typeof val == "object" && !globalThis.Datex?.ReactiveValue.isRef(val)) { if (freezedObjects.has(val)) return val; freezedObjects.add(val); for (const key of Object.keys(val)) val[key] = createStaticObject(val[key]); diff --git a/utils/error-handling.ts b/utils/error-handling.ts index 19f33bd..7d4ef60 100644 --- a/utils/error-handling.ts +++ b/utils/error-handling.ts @@ -28,7 +28,7 @@ export async function handleError(error: Error|string, logger = defaultLogger, e logger.error(error.message); if (error.solutions.length > 0) { console.log(); - logger.info(`Suggested Problem Solutions\n${error.solutions.map(s => `- ${s}`).join("\n")}`); + logger.info(`Suggested Problem Solutions:\n${error.solutions.map(s => `- ${s}`).join("\n")}\n`); } if (error.quickFixes) { for (const fix of error.quickFixes) { diff --git a/utils/error-reporting.ts b/utils/error-reporting.ts index 9676f82..43f48ab 100644 --- a/utils/error-reporting.ts +++ b/utils/error-reporting.ts @@ -18,7 +18,7 @@ export async function sendReport(identifier: string, reportData:Record Date: Sun, 29 Sep 2024 17:05:43 +0200 Subject: [PATCH 04/27] fix INIT_PROPS for inheritance --- types/type.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/types/type.ts b/types/type.ts index e45954e..681da67 100644 --- a/types/type.ts +++ b/types/type.ts @@ -293,7 +293,7 @@ export class Type extends ExtensibleFunction { public cast(value: any, context?:any, origin:Endpoint = Runtime.endpoint, make_pointer = false, ignore_js_config = false, assigningPtrId?: string, strict = false):T { // unknown type (no template or config) //if (!this.interface_config && !this.template) return UNKNOWN_TYPE; - + // has a JS configuration if (!ignore_js_config && this.interface_config){ // generate default value @@ -336,12 +336,13 @@ export class Type extends ExtensibleFunction { /** returns an object with a [INIT_PROPS] function that can be passed to newJSInstance() or called manually */ public getPropertyInitializer(value:any, useTemplate = true, strict = false) { - const initialized = {i:false}; + // TODO: is it okay to call INIT_PROPS multiple times? required for inherited classes + //const initialized = {i:false}; // property initializer - sets existing property for pointer object (is passed as first constructor argument when reconstructing) return Object.freeze({ [INIT_PROPS]: (instance:any)=>{ - if (initialized.i) return; - initialized.i=true; + // if (initialized.i) return; + // initialized.i=true; this.initProperties(instance, value, useTemplate, strict) } }) From 1f268bc13b1ce2143c6e0f1de6c5a5cc0de33ab3 Mon Sep 17 00:00:00 2001 From: benStre Date: Sun, 29 Sep 2024 18:12:06 +0200 Subject: [PATCH 05/27] add optimizations for _$ --- datex_short.ts | 2 +- functions.ts | 14 +++++++++++--- runtime/pointers.ts | 11 +++++++++++ 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/datex_short.ts b/datex_short.ts index 3c31a51..0ba784b 100644 --- a/datex_short.ts +++ b/datex_short.ts @@ -684,7 +684,7 @@ Object.defineProperty(globalThis, 'once', {value:once, configurable:false}) Object.defineProperty(globalThis, 'always', {value:_always, configurable:false}) Object.defineProperty(globalThis, 'asyncAlways', {value:_asyncAlways, configurable:false}) // used internally for reactive $ syntax -Object.defineProperty(globalThis, '_$', {value: (cb:SmartTransformFunction) => _always(cb, {allowStatic: true, _allowAsync: true, _collapseStatic: true}), configurable:false}) +Object.defineProperty(globalThis, '_$', {value: (cb:SmartTransformFunction) => _always(cb, {allowStatic: true, _allowAsync: true, _collapseStatic: true, _returnWrapper: true, _allowAnyType: true}), configurable:false}) Object.defineProperty(globalThis, '_$method', {value: _$method, configurable:false}) Object.defineProperty(globalThis, 'reactiveFn', {value:_reactiveFn, configurable:false}) Object.defineProperty(globalThis, 'toggle', {value:_toggle, configurable:false}) diff --git a/functions.ts b/functions.ts index e816ae6..c434399 100644 --- a/functions.ts +++ b/functions.ts @@ -46,13 +46,13 @@ export function always(scriptOrJSTransform:TemplateStringsArray|SmartTransformFu const options: SmartTransformOptions|undefined = typeof vars[0] == "object" ? vars[0] : undefined; const ptr = Pointer.createSmartTransform(scriptOrJSTransform, undefined, undefined, undefined, options); if (options?._allowAsync && !ptr.value_initialized && ptr.waiting_for_always_promise) { - return ptr.waiting_for_always_promise.then(()=>collapseTransformPointer(ptr, options?._collapseStatic)); + return ptr.waiting_for_always_promise.then(()=>collapseTransformPointer(ptr, options?._collapseStatic, options?._returnWrapper, options?._allowAnyType)); } if (!ptr.value_initialized && ptr.waiting_for_always_promise) { throw new PointerError(`Promises cannot be returned from always transforms - use 'asyncAlways' instead`); } else { - return collapseTransformPointer(ptr, options?._collapseStatic); + return collapseTransformPointer(ptr, options?._collapseStatic, options?._returnWrapper, options?._allowAnyType); } } // datex script @@ -68,7 +68,7 @@ export function always(scriptOrJSTransform:TemplateStringsArray|SmartTransformFu } -function collapseTransformPointer(ptr: Pointer, collapseStatic = false) { +function collapseTransformPointer(ptr: Pointer, collapseStatic = false, alwaysReturnWrapper = false, _allowAnyType = false) { let collapse = false; if (collapseStatic) { // check if transform function is static and value is a js primitive @@ -76,6 +76,14 @@ function collapseTransformPointer(ptr: Pointer, collapseStatic = false) { collapse = true; } } + + if (_allowAnyType) { + ptr.allowAnyType(true); + } + + if (alwaysReturnWrapper && !collapse) { + return ptr; + } const val = ReactiveValue.collapseValue(ptr, false, collapse); if (collapse) ptr.delete(); diff --git a/runtime/pointers.ts b/runtime/pointers.ts index a2de7b3..206b4c0 100644 --- a/runtime/pointers.ts +++ b/runtime/pointers.ts @@ -1133,6 +1133,10 @@ export type SmartTransformOptions = { _allowAsync?: boolean, // collapse primitive pointer if value has no reactive dependencies and garbage-collect pointer _collapseStatic: boolean, + // always return the wrapper instead of the collapsed value, even for non-primitive pointers + _returnWrapper?: boolean, + // set the pointer type to allow any value + _allowAnyType?: boolean, } type TransformState = { @@ -3335,10 +3339,17 @@ export class Pointer extends ReactiveValue { return (>this.#shadow_object)?.deref() } + #any_type = false; + get type():Type { + if (this.#any_type) return Type.std.Any; return this.#unwrapped_transform_type ?? this.#type; } + allowAnyType(any_type = true) { + this.#any_type = any_type; + } + public extended_pointers = new Set() From f44e7973be1fdf03812e14fc4d3a79b3447ff16d Mon Sep 17 00:00:00 2001 From: benStre Date: Sun, 29 Sep 2024 18:57:26 +0200 Subject: [PATCH 06/27] update value reference if this.#any_type is set to true for pointer --- runtime/pointers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/pointers.ts b/runtime/pointers.ts index 206b4c0..b0a3fcc 100644 --- a/runtime/pointers.ts +++ b/runtime/pointers.ts @@ -2795,7 +2795,7 @@ export class Pointer extends ReactiveValue { let updatePromise: Promise|undefined; // set primitive value, reference not required - if (this.is_js_primitive) { + if (this.is_js_primitive || this.#any_type) { const didCustomUpdate = this.customTransformUpdate(val) if (!didCustomUpdate) updatePromise = super.setVal(val, trigger_observers, is_transform); } From 255846e59986282ed815a5055563e166b2e5160f Mon Sep 17 00:00:00 2001 From: benStre Date: Sun, 29 Sep 2024 23:12:51 +0200 Subject: [PATCH 07/27] work on sqlite compatibility --- datex_short.ts | 11 +- deno.lock | 96 ++++++++- init.ts | 13 +- storage/storage-locations/mysql-db.ts | 65 ++++++ storage/storage-locations/sql-db.ts | 215 ++++++++++++------- storage/storage-locations/sql-definitions.ts | 8 - storage/storage-locations/sqlite-db.ts | 42 ++++ 7 files changed, 353 insertions(+), 97 deletions(-) create mode 100644 storage/storage-locations/mysql-db.ts create mode 100644 storage/storage-locations/sqlite-db.ts diff --git a/datex_short.ts b/datex_short.ts index 0ba784b..5620fd9 100644 --- a/datex_short.ts +++ b/datex_short.ts @@ -280,7 +280,7 @@ export function pointer>( */ export function pointer(value:RefLike): MinimalJSRef // defined 2x with Ref and T for correct type inference export function pointer(value:T): MinimalJSRef -export function pointer(value:RefOrValue, property?:unknown): unknown { +export function pointer(value:RefOrValue, property?:unknown, callStackIndex = 0): unknown { // pointer property if (property !== undefined) { @@ -292,7 +292,7 @@ export function pointer(value:RefOrValue, property?:unknown): unknown { const pointer = Pointer.createOrGet(value).js_value; // store as eternal? if (waitingEternals.size) { - const info = getCallerInfo()?.[0]; + const info = getCallerInfo()?.[callStackIndex]; if (!info) throw new Error("eternal values are not supported in this runtime environment"); const unique = `${info.file}:${info.row}`; if (waitingEternals.has(unique)) { @@ -301,7 +301,7 @@ export function pointer(value:RefOrValue, property?:unknown): unknown { } } if (waitingLazyEternals.size) { - const info = getCallerInfo()?.[0]; + const info = getCallerInfo()?.[callStackIndex]; if (!info) throw new Error("eternal values are not supported in this runtime environment"); const unique = `${info.file}:${info.row}`; if (waitingLazyEternals.has(unique)) { @@ -385,7 +385,7 @@ export const $ = new Proxy(function(){} as unknown as $type, { }, apply(_target, _thisArg, args) { - return $$(...args as [any, any]); + return pointer(args[0], args[1], 1); }, }) @@ -528,6 +528,9 @@ type revokeAccess = typeof revokeAccess declare global { const eternal: undefined const lazyEternal: undefined + /** + * @deprecated use $() + */ const $$: typeof pointer const $: $type diff --git a/deno.lock b/deno.lock index f1062c6..fd04a5b 100644 --- a/deno.lock +++ b/deno.lock @@ -1,7 +1,8 @@ { "version": "3", "redirects": { - "https://deno.land/std/async/mod.ts": "https://deno.land/std@0.208.0/async/mod.ts" + "https://deno.land/std/async/mod.ts": "https://deno.land/std@0.208.0/async/mod.ts", + "https://deno.land/x/sqlite/mod.ts": "https://deno.land/x/sqlite@v3.9.0/mod.ts" }, "remote": { "https://cdn.unyt.org/datex-core-js-legacy@0.0.x/VERSION.ts": "6322468cc91022075407fbf74e4799f751c2f9659064e7a5ff0442934223cd41", @@ -108,8 +109,47 @@ "https://cdn.unyt.org/datex-core-js-legacy@0.0.x/utils/sha256.ts": "52bfee6d3276da5fad865886a323198827c2c86e6491711f4875c5fa50be2248", "https://cdn.unyt.org/datex-core-js-legacy@0.0.x/utils/utils.ts": "86fe78e869c00d17f88a38dbe1f22a7d0197ac981c77dd635d3950dfc5a41337", "https://cdn.unyt.org/datex-core-js-legacy@0.0.x/wasm/adapter/pkg/datex_wasm.js": "e9416983fb275de1cd28849c3cb85ba3d3be05032e9246c73fe217e7a2d422c5", + "https://deno.land/std@0.104.0/_util/assert.ts": "2f868145a042a11d5ad0a3c748dcf580add8a0dbc0e876eaa0026303a5488f58", + "https://deno.land/std@0.104.0/async/deadline.ts": "1d6ac7aeaee22f75eb86e4e105d6161118aad7b41ae2dd14f4cfd3bf97472b93", + "https://deno.land/std@0.104.0/async/debounce.ts": "b2f693e4baa16b62793fd618de6c003b63228db50ecfe3bd51fc5f6dc0bc264b", + "https://deno.land/std@0.104.0/async/deferred.ts": "ce81070ad3ba3294f3f34c032af884ccde1a20922b648f6eaee54bd8fd951a1e", + "https://deno.land/std@0.104.0/async/delay.ts": "9de1d8d07d1927767ab7f82434b883f3d8294fb19cad819691a2ad81a728cf3d", + "https://deno.land/std@0.104.0/async/mod.ts": "78425176fabea7bd1046ce3819fd69ce40da85c83e0f174d17e8e224a91f7d10", + "https://deno.land/std@0.104.0/async/mux_async_iterator.ts": "62abff3af9ff619e8f2adc96fc70d4ca020fa48a50c23c13f12d02ed2b760dbe", + "https://deno.land/std@0.104.0/async/pool.ts": "353ce4f91865da203a097aa6f33de8966340c91b6f4a055611c8c5d534afd12f", + "https://deno.land/std@0.104.0/async/tee.ts": "6b8f1322b6dd2396202cfbe9cde9cab158d1e962cfd9197b0a97c6657bee79ce", + "https://deno.land/std@0.104.0/bytes/bytes_list.ts": "a13287edb03f19d27ba4927dec6d6de3e5bd46254cd4aee6f7e5815810122673", + "https://deno.land/std@0.104.0/bytes/mod.ts": "1ae1ccfe98c4b979f12b015982c7444f81fcb921bea7aa215bf37d84f46e1e13", + "https://deno.land/std@0.104.0/encoding/base64.ts": "eecae390f1f1d1cae6f6c6d732ede5276bf4b9cd29b1d281678c054dc5cc009e", + "https://deno.land/std@0.104.0/encoding/hex.ts": "5bc7df19af498c315cdaba69e2fce1b2aef5fc57344e8c21c08991aa8505a260", + "https://deno.land/std@0.104.0/fmt/colors.ts": "d2f8355f00a74404668fc5a1e4a92983ce1a9b0a6ac1d40efbd681cb8f519586", + "https://deno.land/std@0.104.0/fs/exists.ts": "b0d2e31654819cc2a8d37df45d6b14686c0cc1d802e9ff09e902a63e98b85a00", + "https://deno.land/std@0.104.0/hash/_wasm/hash.ts": "313a4820227f1c45fa7204d9c28731b4f8ce97cdcc5f1e7e4efcdf2d70540d32", + "https://deno.land/std@0.104.0/hash/_wasm/wasm.js": "792f612fbb9998e267f9ae3f82ed72444305cb9c77b5bbf7ff6517fd3b606ed1", + "https://deno.land/std@0.104.0/hash/hasher.ts": "57a9ec05dd48a9eceed319ac53463d9873490feea3832d58679df6eec51c176b", + "https://deno.land/std@0.104.0/hash/mod.ts": "dd339a26b094032f38d71311b85745e8d19f2085364794c1877057e057902dd9", + "https://deno.land/std@0.104.0/io/buffer.ts": "3ead6bb11276ebcf093c403f74f67fd2205a515dbbb9061862c468ca56f37cd8", + "https://deno.land/std@0.104.0/io/bufio.ts": "6024117aa37f8d21a116654bd5ca5191d803f6492bbc744e3cee5054d0e900d1", + "https://deno.land/std@0.104.0/io/util.ts": "85c33d61b20fd706acc094fe80d4c8ae618b04abcf3a96ca2b47071842c1c8ac", + "https://deno.land/std@0.104.0/log/handlers.ts": "8c7221a2408b4097e186b018f3f1a18865d20b98761aa1dccaf1ee3d57298355", + "https://deno.land/std@0.104.0/log/levels.ts": "088a883039ece5fa0da5f74bc7688654045ea7cb01bf200b438191a28d728eae", + "https://deno.land/std@0.104.0/log/logger.ts": "6b2dd8cbe6f407100b9becfe61595d7681f8ce3692412fad843de84d617a038e", + "https://deno.land/std@0.104.0/log/mod.ts": "91711789b28803082b1bdfb123d2c9685a7e01767f2e79c0a82706063ad964d8", + "https://deno.land/std@0.104.0/testing/_diff.ts": "5d3693155f561d1a5443ac751ac70aab9f5d67b4819a621d4b96b8a1a1c89620", + "https://deno.land/std@0.104.0/testing/asserts.ts": "e4311d45d956459d4423bc267208fe154b5294989da2ed93257b6a85cae0427e", "https://deno.land/std@0.168.0/_util/asserts.ts": "d0844e9b62510f89ce1f9878b046f6a57bf88f208a10304aab50efcb48365272", "https://deno.land/std@0.168.0/flags/mod.ts": "4f50ec6383c02684db35de38b3ffb2cd5b9fcfcc0b1147055d1980c49e82521c", + "https://deno.land/std@0.172.0/_util/asserts.ts": "178dfc49a464aee693a7e285567b3d0b555dc805ff490505a8aae34f9cfb1462", + "https://deno.land/std@0.172.0/_util/os.ts": "d932f56d41e4f6a6093d56044e29ce637f8dcc43c5a90af43504a889cf1775e3", + "https://deno.land/std@0.172.0/path/_constants.ts": "e49961f6f4f48039c0dfed3c3f93e963ca3d92791c9d478ac5b43183413136e0", + "https://deno.land/std@0.172.0/path/_interface.ts": "6471159dfbbc357e03882c2266d21ef9afdb1e4aa771b0545e90db58a0ba314b", + "https://deno.land/std@0.172.0/path/_util.ts": "86c2375a996c1931b2f2ac71fefd5ddf0cf0e579fa4ab12d3e4c552d4223b8d8", + "https://deno.land/std@0.172.0/path/common.ts": "ee7505ab01fd22de3963b64e46cff31f40de34f9f8de1fff6a1bd2fe79380000", + "https://deno.land/std@0.172.0/path/glob.ts": "d479e0a695621c94d3fd7fe7abd4f9499caf32a8de13f25073451c6ef420a4e1", + "https://deno.land/std@0.172.0/path/mod.ts": "4b83694ac500d7d31b0cdafc927080a53dc0c3027eb2895790fb155082b0d232", + "https://deno.land/std@0.172.0/path/posix.ts": "0874b341c2c6968ca38d323338b8b295ea1dae10fa872a768d812e2e7d634789", + "https://deno.land/std@0.172.0/path/separator.ts": "0fb679739d0d1d7bf45b68dacfb4ec7563597a902edbaf3c59b50d5bcadd93b1", + "https://deno.land/std@0.172.0/path/win32.ts": "672942919dd66ce1b8c224e77d3447e2ad8254eaff13fe6946420a9f78fa141e", "https://deno.land/std@0.208.0/assert/assert.ts": "9a97dad6d98c238938e7540736b826440ad8c1c1e54430ca4c4e623e585607ee", "https://deno.land/std@0.208.0/assert/assertion_error.ts": "4d0bde9b374dfbcbe8ac23f54f567b77024fb67dbb1906a852d67fe050d42f56", "https://deno.land/std@0.208.0/async/_util.ts": "7fef43ce1949b36ebd092a735c4659f3e63411eb4a3ed0c381bdc6e8f261d835", @@ -123,13 +163,65 @@ "https://deno.land/std@0.208.0/async/pool.ts": "47c1841cfa9c036144943d11747ddd44064f5baf8cb7ece25473ba873c6aceb0", "https://deno.land/std@0.208.0/async/retry.ts": "b80e37cf1701fe1edf1bec25440e0a870a320193c5dd1990ae0cadd0fc3ab886", "https://deno.land/std@0.208.0/async/tee.ts": "47e42d35f622650b02234d43803d0383a89eb4387e1b83b5a40106d18ae36757", + "https://deno.land/std@0.51.0/fmt/colors.ts": "127ce39ca2ad9714d4ada8d61367f540d76b5b0462263aa839166876b522d3de", + "https://deno.land/std@0.51.0/testing/asserts.ts": "213fedbb90a60ae232932c45bd62668f0c5cd17fc0f2a273e96506cba416d181", + "https://deno.land/std@0.51.0/testing/diff.ts": "8f591074fad5d35c0cafa63b1c5334dc3a17d5b934f3b9e07172eed9d5b55553", + "https://deno.land/std@0.77.0/fmt/colors.ts": "c5665c66f1a67228f21c5989bbb04b36d369b98dd7ceac06f5e26856c81c2531", + "https://deno.land/x/bytes_formater@v1.4.0/deps.ts": "4f98f74e21145423b873a5ca6ead66dc3e674fa81e230a0a395f9b86aafeceea", + "https://deno.land/x/bytes_formater@v1.4.0/format.ts": "657c41b9f180c3ed0f934dcf75f77b09b6a610be98bb07525bffe2acfd5af4d5", + "https://deno.land/x/bytes_formater@v1.4.0/mod.ts": "c6bf35303f53d74e9134eb13f666fb388fb4c62c6b12b17542bbadade250a864", + "https://deno.land/x/caller_metadata@v0.0.3/src/main.ts": "2258e7eb41a54ac9a99058f40a487c6313232feafcb5f02f22d51fe6b526dd73", "https://deno.land/x/input@2.0.3/history.ts": "3cad3fee1e2f86d4202ca6ab49f1a5609aff2323ff762cc637e1da3edbef9608", "https://deno.land/x/input@2.0.3/index.ts": "f9539661733886f9497873a640a79be4f996ea2d40408b68e74691353f5a5539", "https://deno.land/x/input@2.0.3/printer.ts": "60339de0f3d98ca9ad2cb4e3b047d0b680e27ed7ceb987d82d96fc00f7ab04d1", "https://deno.land/x/input@2.0.3/types.ts": "915079a7187073c9c9c99863ab4e4a4c7341e25bf5d4a2e02c33f8b896dd0c69", "https://deno.land/x/mimetypes@v1.0.0/mod.ts": "abd1ea614b32cfa7fd50b46f6e070617696664698b8a34935dc807f3035ad408", "https://deno.land/x/mimetypes@v1.0.0/src/mime.ts": "7cd7590eae6127dc1abfe710900fc749e94d5c715ffe8a148e59fe20ee21a9a5", + "https://deno.land/x/mysql@v2.12.1/deps.ts": "68635959a41bb08bc87db007679fb8449febc55d48202dff20b93cc23ef5820d", + "https://deno.land/x/mysql@v2.12.1/mod.ts": "3246c9c259434563be69cc95d5b792f8aac7ef5d10b8a6c6589aa54ebf1bd266", + "https://deno.land/x/mysql@v2.12.1/src/auth.ts": "129ea08b180d3e90e567c3f71e60432bb266304c224e17ea39d604bbcc1160d8", + "https://deno.land/x/mysql@v2.12.1/src/auth_plugin/caching_sha2_password.ts": "aab89e272382e6f408406f860ae6e79628275f4511e27a565049033543c4bdec", + "https://deno.land/x/mysql@v2.12.1/src/auth_plugin/crypt.ts": "8798819cce1171d95cfee8edda15fe6a652068cad4dc91f81b6e91cf90a13617", + "https://deno.land/x/mysql@v2.12.1/src/auth_plugin/index.ts": "8617e520ad854e38470aeefd07becdb3397c4cde16c2397dd48d5c10fdd5ab09", + "https://deno.land/x/mysql@v2.12.1/src/buffer.ts": "59f7e08e196f1b7e58cf5c3cf8ae8f4d0d47d1ae31430076fc468d974d3b59e7", + "https://deno.land/x/mysql@v2.12.1/src/client.ts": "30912964986667a2ce108c14f7153dd38e8089e55f8068e8d07697f75f2ac22f", + "https://deno.land/x/mysql@v2.12.1/src/connection.ts": "1d104c05441f8c94ee73123497fbbae28499f3badb0d9fef8cc82540688ada6e", + "https://deno.land/x/mysql@v2.12.1/src/constant/capabilities.ts": "2324c0e46ac43f59b7b03bdd878d7a14ecc5202b9e133c7e8769345a8290f2a1", + "https://deno.land/x/mysql@v2.12.1/src/constant/charset.ts": "253d7233679c774df623d1f974ebb358f3678c18fd6a623e25983311d97d959b", + "https://deno.land/x/mysql@v2.12.1/src/constant/errors.ts": "923bab27d524e43199fa21fdfcbe025580ca76d8b32254ad9505765c502f238a", + "https://deno.land/x/mysql@v2.12.1/src/constant/mysql_types.ts": "79c50de8eb5919b897e81e2ff2366ee1ffdbb4297f711e15003bdb787bbc8e6c", + "https://deno.land/x/mysql@v2.12.1/src/constant/packet.ts": "a1e7e00ce30c551c5f95c05d233b8d83f8e1fc865de97be3b317058e173630a9", + "https://deno.land/x/mysql@v2.12.1/src/deferred.ts": "35d087619d919961e849e382c33b2bfea15b4119f55eca2d9c9047f30512a2cb", + "https://deno.land/x/mysql@v2.12.1/src/logger.ts": "eb5feb3efdb9fd4887f6eccd5c06b5702591ac032af9857a12bbae86ceefe21b", + "https://deno.land/x/mysql@v2.12.1/src/packets/builders/auth.ts": "0b53dd5fa0269427aa54c3f6909bd830ffb426009061df89df262c504d6c9b70", + "https://deno.land/x/mysql@v2.12.1/src/packets/builders/client_capabilities.ts": "1000f2c1a20e0e119b9a416eb4ea4553cc1c5655d289a66e9077bf7a5993d52d", + "https://deno.land/x/mysql@v2.12.1/src/packets/builders/query.ts": "caf426a72ebe545ff5bab14c8b7b5e412dd8827c091322959cdf4e9aa89ef900", + "https://deno.land/x/mysql@v2.12.1/src/packets/builders/tls.ts": "2abb4a2fa74c47914372b221cb6f178f6015df54421daf0e10e54d80d7156498", + "https://deno.land/x/mysql@v2.12.1/src/packets/packet.ts": "d7800cc142226f7dfd3c5f647f03cd3ef308f9d8551b4edb2e1bfb9c758d33b6", + "https://deno.land/x/mysql@v2.12.1/src/packets/parsers/authswitch.ts": "aa34f21336c4907b3ae968108fcdad8f1c43a303088efd83d972e6c7b258c166", + "https://deno.land/x/mysql@v2.12.1/src/packets/parsers/err.ts": "4110c4ddc2ae8358d6661fa2522f8eda2e603900d1e433e3684765ed50e88ed8", + "https://deno.land/x/mysql@v2.12.1/src/packets/parsers/handshake.ts": "88f7ee34e9e0ef089bc5fdefacaccf256ef002b2f7a8ad684e35327682039e73", + "https://deno.land/x/mysql@v2.12.1/src/packets/parsers/result.ts": "8ab16f1adae67415eefcc17803b0eb828c1f4c6a24c55f25949f418e862d3ec8", + "https://deno.land/x/mysql@v2.12.1/src/pool.ts": "978ba2813b3886d68be007678360ad43c54dab14b1aea1c07fcdb41222fcc432", + "https://deno.land/x/mysql@v2.12.1/src/util.ts": "83d38e87cc3901da00ac44bfcd53c0e8d24525262f5c7647c912dccf3ed2dbb5", "https://deno.land/x/reflect_metadata@v0.1.12/Reflect.ts": "eced7b7e642853c6fb9a209f96ddd1932ab1e0fc3c514705c87d56169c8fe260", - "https://deno.land/x/reflect_metadata@v0.1.12/mod.ts": "8b5e3b20f1e604c118df433e84a409b1d5116e885001047134cc42ecf84fa2cd" + "https://deno.land/x/reflect_metadata@v0.1.12/mod.ts": "8b5e3b20f1e604c118df433e84a409b1d5116e885001047134cc42ecf84fa2cd", + "https://deno.land/x/sql_builder@v1.9.1/util.ts": "b9855dc435972704cf82655019f4ec168ac83550ab4db596c5f6b6d201466384", + "https://deno.land/x/sql_builder@v1.9.2/deps.ts": "58a674cf97708472838e4bd59a58333386052da16d66a6c25f3b574bcf21a8b0", + "https://deno.land/x/sql_builder@v1.9.2/join.ts": "fd7813c57a5c71f90026d5e83f8ec04b320c4097c39bcc3005c22296224cf7d8", + "https://deno.land/x/sql_builder@v1.9.2/mod.ts": "e08fa419c1535502b716cfb6379ec0a7c9902f21ada662d7c3d6fecfecfa7a73", + "https://deno.land/x/sql_builder@v1.9.2/order.ts": "b884c8bb1569ab407205291358248e393c4c39323ecaf699884708c889d25d01", + "https://deno.land/x/sql_builder@v1.9.2/query.ts": "a171b1d6b77bcd2893b6ec9a6af87684b172ef21529be5dce5c7f17760442668", + "https://deno.land/x/sql_builder@v1.9.2/util.ts": "b0c9fb126f7ee0e9e33eae69adbba7744329bdbfd5608080024239eb1169ae8f", + "https://deno.land/x/sql_builder@v1.9.2/where.ts": "2de7a961520070f2e909ca3901cfc432d7c553b43b790f08fc1aaebb4b206827", + "https://deno.land/x/sqlite@v3.9.0/build/sqlite.js": "2afc7875c7b9c85d89730c4a311ab3a304e5d1bf761fbadd8c07bbdf130f5f9b", + "https://deno.land/x/sqlite@v3.9.0/build/vfs.js": "7f7778a9fe499cd10738d6e43867340b50b67d3e39142b0065acd51a84cd2e03", + "https://deno.land/x/sqlite@v3.9.0/mod.ts": "e09fc79d8065fe222578114b109b1fd60077bff1bb75448532077f784f4d6a83", + "https://deno.land/x/sqlite@v3.9.0/src/constants.ts": "90f3be047ec0a89bcb5d6fc30db121685fc82cb00b1c476124ff47a4b0472aa9", + "https://deno.land/x/sqlite@v3.9.0/src/db.ts": "03d0c860957496eadedd86e51a6e650670764630e64f56df0092e86c90752401", + "https://deno.land/x/sqlite@v3.9.0/src/error.ts": "f7a15cb00d7c3797da1aefee3cf86d23e0ae92e73f0ba3165496c3816ab9503a", + "https://deno.land/x/sqlite@v3.9.0/src/function.ts": "e4c83b8ec64bf88bafad2407376b0c6a3b54e777593c70336fb40d43a79865f2", + "https://deno.land/x/sqlite@v3.9.0/src/query.ts": "d58abda928f6582d77bad685ecf551b1be8a15e8e38403e293ec38522e030cad", + "https://deno.land/x/sqlite@v3.9.0/src/wasm.ts": "e79d0baa6e42423257fb3c7cc98091c54399254867e0f34a09b5bdef37bd9487" } } diff --git a/init.ts b/init.ts index bb79955..5492581 100644 --- a/init.ts +++ b/init.ts @@ -74,11 +74,12 @@ export async function init() { }) } else if (client_type == "deno") { - const { DenoKVStorageLocation } = await import("./storage/storage-locations/deno-kv.ts"); - const denoKV = new DenoKVStorageLocation(); - if (denoKV.isSupported()) { - console.log("Using DenoKV as primary storage location (experimental)") - await Storage.addLocation(denoKV, { + if (Deno.env.get("SQLITE_STORAGE") == "1") { + const { SqliteStorageLocation } = await import("./storage/storage-locations/sqlite-db.ts"); + console.log("Using sqlite as primary storage location (experimental feature enabled via SQLITE_STORAGE env variable)") + await Storage.addLocation(new SqliteStorageLocation({ + db: "storage" + }), { modes: [Storage.Mode.SAVE_ON_CHANGE], primary: true }) @@ -86,7 +87,7 @@ export async function init() { else { await Storage.addLocation(new LocalStorageLocation(), { modes: [Storage.Mode.SAVE_ON_EXIT, Storage.Mode.SAVE_ON_CHANGE], - primary: denoKV.isSupported() ? false : true + primary: true }) } } diff --git a/storage/storage-locations/mysql-db.ts b/storage/storage-locations/mysql-db.ts new file mode 100644 index 0000000..0d2477a --- /dev/null +++ b/storage/storage-locations/mysql-db.ts @@ -0,0 +1,65 @@ +import { Client, ExecuteResult } from "https://deno.land/x/mysql@v2.12.1/mod.ts"; +import { SQLDBStorageLocation } from "./sql-db.ts"; +import { Logger } from "../../utils/logger.ts"; +import { Query, Where } from "https://deno.land/x/sql_builder@v1.9.2/mod.ts"; + +const logger = new Logger("MYSQL_DB Storage"); + +export type ConnectionOptions = { + hostname: string + username: string + password:string + port: number + db: string +} + +export class MySQLStorageLocation extends SQLDBStorageLocation { + + name = "MYSQL_DB" + #sqlClient: Client|undefined + + supportsInvisibleColumns = true + + protected async connect(): Promise { + this.#sqlClient = await new Client().connect({poolSize: 20, ...this.options}); + logger.info("Using SQL database " + this.options.db + " on " + this.options.hostname + ":" + this.options.port + " as storage location") + return true + } + + protected executeQuery(query_string: string, query_params?: any[]): Promise { + return this.#sqlClient!.execute(query_string, query_params); + } + + // custom queries + + protected getTableExistsQuery(tableName: string): string { + return new Query() + .table("information_schema.tables") + .select("*") + .where(Where.eq("table_schema", this.options.db)) + .where(Where.eq("table_name", tableName)) + .build() + } + + protected getTableColumnInfoQuery(tableName: string): string { + return new Query() + .table("information_schema.key_column_usage") + .select("COLUMN_NAME", "REFERENCED_TABLE_NAME") + .where(Where.eq("table_schema", this.options.db)) + .where(Where.eq("table_name", tableName)) + .build() + } + + protected getTableConstraintsQuery(tableName: string): string { + return new Query() + .table("information_schema.key_column_usage") + .select("COLUMN_NAME", "REFERENCED_TABLE_NAME") + .where(Where.eq("table_schema", this.options.db)) + .where(Where.eq("table_name", tableName)) + .build() + } + + protected getClearTableQuery(tableName: string): string { + return `TRUNCATE TABLE ${tableName};` + } +} \ No newline at end of file diff --git a/storage/storage-locations/sql-db.ts b/storage/storage-locations/sql-db.ts index 1c8d213..0a92660 100644 --- a/storage/storage-locations/sql-db.ts +++ b/storage/storage-locations/sql-db.ts @@ -3,7 +3,7 @@ import { Query } from "https://deno.land/x/sql_builder@v1.9.2/mod.ts"; import { Where } from "https://deno.land/x/sql_builder@v1.9.2/where.ts"; import { Pointer } from "../../runtime/pointers.ts"; import { AsyncStorageLocation } from "../storage.ts"; -import { ColumnDefinition, ConstraintsDefinition, TableDefinition, dbOptions, mysql_data_type } from "./sql-definitions.ts"; +import { ColumnDefinition, ConstraintsDefinition, TableDefinition, mysql_data_type } from "./sql-definitions.ts"; import { Logger } from "../../utils/logger.ts"; import { Datex } from "../../mod.ts"; import { datex_type_mysql_map } from "./sql-type-map.ts"; @@ -29,7 +29,17 @@ configLogger({level: "WARNING"}) const logger = new Logger("SQL Storage"); -export class SQLDBStorageLocation extends AsyncStorageLocation { +export abstract class SQLDBStorageLocation extends AsyncStorageLocation { + + + protected abstract connect(): boolean|Promise; + protected abstract executeQuery(query_string: string, query_params?: any[]): ExecuteResult|Promise + + protected abstract getTableExistsQuery(tableName: string): string; + protected abstract getTableColumnInfoQuery(tableName: string): string + protected abstract getTableConstraintsQuery(tableName: string): string + protected abstract getClearTableQuery(tableName: string): string; + static #debugMode = false; @@ -48,9 +58,15 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { #connected = false; #initializing = false #initialized = false - #options: dbOptions - #sqlClient: Client|undefined - + protected options: Options + + // use single quotes instead of double quotes in queries + protected useSingleQuotes = false; + // support for INSERT OR REPLACE + protected supportsInsertOrReplace = false; + // support for invisible columns + protected supportsInvisibleColumns = false; + readonly #pointerMysqlType = "varchar(50)" readonly #pointerMysqlColumnName = "_ptr_id" @@ -115,17 +131,14 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { #pointerTables = new Map() #templateMultiQueries = new Map, result: Promise[]>}>() - constructor(options:dbOptions, private log?:(...args:unknown[])=>void) { + constructor(options:Options, private log?:(...args:unknown[])=>void) { super() - this.#options = options + this.options = options } async #connect(){ if (this.#connected) return; - this.#sqlClient = await new Client().connect({poolSize: 20, ...this.#options}); - logger.info("Using SQL database " + this.#options.db + " on " + this.#options.hostname + ":" + this.#options.port + " as storage location") - this.#connected = true; + this.#connected = await this.connect(); } - async #init() { if (this.#initialized) return; this.#initializing = true; @@ -143,30 +156,39 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { new Query() .table(this.#metaTables.typeMapping.name) .select("table_name") - .build() + .build(), + undefined, false, ["table_name"] ) const tableNames = tables.map(({table_name})=>'`'+table_name+'`'); // TODO: better solution to handle drop with foreign constraints // currently just runs multiple drop table queries on failure, which is not ideal - const iterations = 10; - for (let i = 0; i < iterations; i++) { - try { - await this.#query<{table_name:string}>(`DROP TABLE IF EXISTS ${tableNames.join(',')};`) - break; - } - catch (e) { - console.error("Failed to drop some tables due to foreign constraints, repeating", e) + if (tableNames.length) { + const iterations = 10; + for (let i = 0; i < iterations; i++) { + try { + await this.#query<{table_name:string}>(`DROP TABLE IF EXISTS ${tableNames.join(',')};`) + break; + } + catch (e) { + console.error("Failed to drop some tables due to foreign constraints, repeating", e) + } } } // truncate meta tables - await Promise.all(Object.values(this.#metaTables).map(table => this.#query(`TRUNCATE TABLE ${table.name};`))) + await Promise.all(Object.values(this.#metaTables).map(table => this.#query(this.getClearTableQuery(table.name)))) } - async #query(query_string:string, query_params:any[]|undefined, returnRawResult: true): Promise<{rows:row[], result:ExecuteResult}> + + async #query(query_string:string, query_params:any[]|undefined, returnRawResult: true, columnNames?:(keyof row)[]): Promise<{rows:row[], result:ExecuteResult}> + async #query(query_string:string, query_params:any[]|undefined, returnRawResult: false, columnNames?:(keyof row)[]): Promise async #query(query_string:string, query_params?:any[]): Promise - async #query(query_string:string, query_params?:any[], returnRawResult?: boolean): Promise { + async #query(query_string:string, query_params?:any[], returnRawResult?: boolean, columnNames?:string[]): Promise { + + // TODO: only workaround for sqlite, replace all " with ' in queries + if (this.useSingleQuotes) query_string = query_string.replace(/"/g, "'"); + // prevent infinite recursion if calling query from within init() if (!this.#initializing) await this.#init(); @@ -188,7 +210,16 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { if (typeof query_string != "string") {console.error("invalid query:", query_string); throw new Error("invalid query")} if (!query_string) throw new Error("empty query"); try { - const result = await this.#sqlClient!.execute(query_string, query_params); + const result = await this.executeQuery(query_string, query_params); + if (result.rows?.[0] instanceof Array && columnNames) { + result.rows = result.rows.map(row => { + const obj:Record = {} + for (let i = 0; i < columnNames.length; i++) { + obj[columnNames[i]] = row[i] + } + return obj + }) + } if (returnRawResult) return {rows: result.rows ?? [], result}; else return result.rows ?? []; } catch (e) { @@ -209,6 +240,7 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { } } + #stringToBinary(value: string){ return Uint8Array.from(value, x => x.charCodeAt(0)).buffer } @@ -216,18 +248,13 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { return String.fromCharCode.apply(null, new Uint8Array(value) as unknown as number[]) } - async #queryFirst(query_string:string, query_params?:any[]): Promise { - return (await this.#query(query_string, query_params))?.[0] + async #queryFirst(query_string:string, query_params?:any[], column_names?: (keyof row)[]): Promise { + return (await this.#query(query_string, query_params, false, column_names))?.[0] } async #createTableIfNotExists(definition: TableDefinition) { const exists = this.#tableColumns.has(definition.name) || await this.#queryFirst( - new Query() - .table("information_schema.tables") - .select("*") - .where(Where.eq("table_schema", this.#options.db)) - .where(Where.eq("table_name", definition.name)) - .build() + this.getTableExistsQuery(definition.name) ) if (!exists) { await this.#createTableFromDefinition(definition); @@ -250,9 +277,9 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { const primaryKeyDefinition = compositePrimaryKeyColumns.length > 1 ? `, PRIMARY KEY (${compositePrimaryKeyColumns.map(col => `\`${col[0]}\``).join(', ')})` : ''; // create - await this.#queryFirst(`CREATE TABLE IF NOT EXISTS ?? (${definition.columns.map(col => + await this.#queryFirst(`CREATE TABLE IF NOT EXISTS \`${definition.name}\` (${definition.columns.map(col => `\`${col[0]}\` ${col[1]} ${col[2]??''}` - ).join(', ')}${definition.constraints?.length ? ',' + definition.constraints.join(',') : ''}${primaryKeyDefinition});`, [definition.name]) + ).join(', ')}${definition.constraints?.length ? ',' + definition.constraints.join(',') : ''}${primaryKeyDefinition});`) // load column definitions await this.#getTableColumns(definition.name); } @@ -279,12 +306,13 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { const {promise, resolve} = Promise.withResolvers(); this.#tableLoadingTasks.set(type, promise); - const existingTable = (await this.#queryFirst<{table_name: string}|undefined>( + const existingTable = (await this.#queryFirst<{table_name: string}>( new Query() .table(this.#metaTables.typeMapping.name) .select("table_name") .where(Where.eq("type", this.#typeToString(type))) - .build() + .build(), + undefined, ['table_name'] ))?.table_name; const table = existingTable ?? await this.#createTableForType(type); @@ -300,7 +328,8 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { .table(this.#metaTables.typeMapping.name) .select("type") .where(Where.eq("table_name", table)) - .build() + .build(), + undefined, ['type'] ) if (!type) { logger.error("No type found for table " + table); @@ -350,7 +379,7 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { this.#tableCreationTasks.set(type, promise); const columns:ColumnDefinition[] = [ - [this.#pointerMysqlColumnName, this.#pointerMysqlType, 'PRIMARY KEY INVISIBLE DEFAULT "0"'] + [this.#pointerMysqlColumnName, this.#pointerMysqlType, 'PRIMARY KEY'+(this.supportsInvisibleColumns ? ' INVISIBLE' : '')+' DEFAULT "0"'] ] const constraints: ConstraintsDefinition[] = [] @@ -463,22 +492,12 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { this.#tableColumnTasks.set(tableName, promise); const columnData = new Map() - const columns = await this.#query<{COLUMN_NAME:string, COLUMN_KEY:string, DATA_TYPE:string}>( - new Query() - .table("information_schema.columns") - .select("COLUMN_NAME", "COLUMN_KEY", "DATA_TYPE") - .where(Where.eq("table_schema", this.#options.db)) - .where(Where.eq("table_name", tableName)) - .build() + const columns = await this.#query<{_id: string, COLUMN_NAME:string, DATA_TYPE:string}>( + this.getTableColumnInfoQuery(tableName), [], false, ["_id", "COLUMN_NAME", "DATA_TYPE"] ) - const constraints = (await this.#query<{COLUMN_NAME:string, REFERENCED_TABLE_NAME:string}>( - new Query() - .table("information_schema.key_column_usage") - .select("COLUMN_NAME", "REFERENCED_TABLE_NAME") - .where(Where.eq("table_schema", this.#options.db)) - .where(Where.eq("table_name", tableName)) - .build() + const constraints = (await this.#query<{_id:string, COLUMN_NAME:string, REFERENCED_TABLE_NAME:string}>( + this.getTableConstraintsQuery(tableName), [], false, ["_id", "COLUMN_NAME", "REFERENCED_TABLE_NAME"] )); const columnTables = new Map() for (const {COLUMN_NAME, REFERENCED_TABLE_NAME} of constraints) { @@ -486,8 +505,10 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { } for (const col of columns) { - if (col.COLUMN_NAME == this.#pointerMysqlColumnName) continue; - columnData.set(col.COLUMN_NAME, {foreignPtr: columnTables.has(col.COLUMN_NAME), foreignTable: columnTables.get(col.COLUMN_NAME), type: col.DATA_TYPE}) + const columnName = col.COLUMN_NAME + const dataType = col.DATA_TYPE + if (columnName == this.#pointerMysqlColumnName) continue; + columnData.set(columnName, {foreignPtr: columnTables.has(columnName), foreignTable: columnTables.get(columnName), type: dataType}) } this.#tableColumns.set(tableName, columnData) @@ -547,7 +568,13 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { // replace if entry already exists - await this.#query('INSERT INTO ?? ?? VALUES ? ON DUPLICATE KEY UPDATE '+Object.keys(insertData).map((key) => `\`${key}\` = ?`).join(', '), [table, Object.keys(insertData), Object.values(insertData), ...Object.values(insertData)]) + if (this.supportsInsertOrReplace) { + await this.#query(`INSERT OR REPLACE INTO \`${table}\` (${Object.keys(insertData).map(key => `\`${key}\``).join(', ')}) VALUES (${Object.keys(insertData).map(() => '?').join(', ')})`, Object.values(insertData)) + } + else { + await this.#query('INSERT INTO ?? ?? VALUES ? ON DUPLICATE KEY UPDATE '+Object.keys(insertData).map((key) => `\`${key}\` = ?`).join(', '), [table, Object.keys(insertData), Object.values(insertData), ...Object.values(insertData)]) + } + // await this.#query('INSERT INTO ?? ?? VALUES ?;', [table, Object.keys(insertData), Object.values(insertData)]) // add to pointer mapping @@ -704,12 +731,14 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { this.#pointerTables.delete(pointerId); return table; } + return (await this.#queryFirst<{table_name:string}>( new Query() .table(this.#metaTables.pointerMapping.name) .select("table_name") .where(Where.eq(this.#pointerMysqlColumnName, pointerId)) - .build() + .build(), + undefined, ['table_name'] ))?.table_name; } @@ -766,7 +795,8 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { // first delete all existing entries for this pointer (except the default entry) try { await this.#query('SET FOREIGN_KEY_CHECKS=0;'); - await this.#query('DELETE FROM ?? WHERE ?? = ? AND `hash` != "";', [this.#metaTables.sets.name, this.#pointerMysqlColumnName, pointer.id]) + //await this.#query('DELETE FROM ?? WHERE ?? = ? AND `hash` != "";', [this.#metaTables.sets.name, this.#pointerMysqlColumnName, pointer.id]) + await this.#query(`DELETE FROM \`${this.#metaTables.sets.name}\` WHERE \`${this.#pointerMysqlColumnName}\` = ? AND \`hash\` != "";`, [pointer.id]) await this.#query('SET FOREIGN_KEY_CHECKS=1;'); } catch (e) { @@ -781,11 +811,21 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { } async #updatePointerMapping(pointerId: string, tableName: string) { - await this.#query('INSERT INTO ?? ?? VALUES ? ON DUPLICATE KEY UPDATE table_name=?;', [this.#metaTables.pointerMapping.name, [this.#pointerMysqlColumnName, "table_name"], [pointerId, tableName], tableName]) + if (this.supportsInsertOrReplace) { + await this.#query(`INSERT OR REPLACE INTO \`${this.#metaTables.pointerMapping.name}\` (\`${this.#pointerMysqlColumnName}\`, \`table_name\`) VALUES (?, ?);`, [pointerId, tableName]) + } + else { + await this.#query('INSERT INTO ?? ?? VALUES ? ON DUPLICATE KEY UPDATE table_name=?;', [this.#metaTables.pointerMapping.name, [this.#pointerMysqlColumnName, "table_name"], [pointerId, tableName], tableName]) + } } async #setItemPointer(key: string, pointer: Pointer) { - await this.#query('INSERT INTO ?? ?? VALUES ? ON DUPLICATE KEY UPDATE '+this.#pointerMysqlColumnName+'=?;', [this.#metaTables.items.name, ["key", this.#pointerMysqlColumnName], [key, pointer.id], pointer.id]) + if (this.supportsInsertOrReplace) { + await this.#query(`INSERT OR REPLACE INTO \`${this.#metaTables.items.name}\` (\`key\`, \`${this.#pointerMysqlColumnName}\`) VALUES (?, ?);`, [key, pointer.id]) + } + else { + await this.#query('INSERT INTO ?? ?? VALUES ? ON DUPLICATE KEY UPDATE '+this.#pointerMysqlColumnName+'=?;', [this.#metaTables.items.name, ["key", this.#pointerMysqlColumnName], [key, pointer.id], pointer.id]) + } } isSupported() { @@ -909,7 +949,7 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { await this.#getTableForType(type) } - const queryResult = await this.#query<{_ptr_id:string, map_key: string}>(query); + const queryResult = await this.#query<{_ptr_id:string, map_key: string}>(query, undefined, false, ['_ptr_id', 'map_key']); const ptrIds = options.returnKeys ? queryResult.map(({map_key}) => map_key.split(".")[1].slice(1)) : queryResult.map(({_ptr_id}) => _ptr_id) const limitedPtrIds = options.returnPointerIds ? // offset and limit manually after query @@ -918,7 +958,7 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { ptrIds // TODO: atomic operations for multiple queries - const {foundRows} = (options?.returnAdvanced ? await this.#queryFirst<{foundRows: number}>("SELECT FOUND_ROWS() as foundRows") : null) ?? {foundRows: -1} + const {foundRows} = (options?.returnAdvanced ? await this.#queryFirst<{foundRows: number}>("SELECT FOUND_ROWS() as foundRows") : null, undefined, ['foundRows']) ?? {foundRows: -1} // remember pointer table for (const ptrId of ptrIds) { @@ -1297,7 +1337,8 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { .table(this.#metaTables.items.name) .select("COUNT(*) as COUNT") .where(Where.eq("key", key)) - .build() + .build(), + undefined, ['COUNT'] )); const exists = !!count && count.COUNT > 0; if (exists) { @@ -1315,7 +1356,7 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { if (prefix != undefined) builder.where(Where.like("key", prefix + "%")) - const keys = await this.#query<{key:string}>(builder.build()) + const keys = await this.#query<{key:string}>(builder.build(), [], false, ['key']) return function*(){ for (const {key} of keys) { yield key; @@ -1333,7 +1374,8 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { .table(this.#metaTables.items.name) .select("key") .where(Where.eq(this.#pointerMysqlColumnName, ptrId)) - .build() + .build(), + undefined, ['key'] ); return key?.key; } @@ -1344,7 +1386,8 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { .table(this.#metaTables.items.name) .select("key") .where(Where.eq("value", encoded)) - .build() + .build(), + undefined, ['key'] ); return key?.key; @@ -1356,7 +1399,8 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { new Query() .table(this.#metaTables.pointerMapping.name) .select(`${this.#pointerMysqlColumnName} as ptrId`) - .build() + .build(), + undefined, false, ['ptrId'] ) return function*(){ for (const {ptrId} of pointerIds) { @@ -1367,7 +1411,8 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { async removeItem(key: string): Promise { this.#existingItemsCache.delete(key) - await this.#query('DELETE FROM ?? WHERE ??=?;', [this.#metaTables.items.name, "key", key]) + //await this.#query('DELETE FROM ?? WHERE ??=?;', [this.#metaTables.items.name, "key", key]) + await this.#query(`DELETE FROM \`${this.#metaTables.items.name}\` WHERE \`key\`=?;`, [key]) } async getItemValueDXB(key: string): Promise { const encoded = (await this.#queryFirst<{value: string, ptrId: string}>( @@ -1375,7 +1420,8 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { .table(this.#metaTables.items.name) .select("value", `${this.#pointerMysqlColumnName} as ptrId`) .where(Where.eq("key", key)) - .build() + .build(), + undefined, ['value', 'ptrId'] )); if (encoded?.ptrId) return Compiler.compile(`$${encoded.ptrId}`, undefined, {sign: false, encrypt: false, to: Datex.Runtime.endpoint, preemptive_pointer_init: false}, false) as Promise; else if (encoded?.value) return this.#stringToBinary(encoded.value); @@ -1383,7 +1429,12 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { } async setItemValueDXB(key: string, value: ArrayBuffer) { const stringBinary = this.#binaryToString(value) - await this.#query('INSERT INTO ?? ?? VALUES ? ON DUPLICATE KEY UPDATE value=?;', [this.#metaTables.items.name, ["key", "value"], [key, stringBinary], stringBinary]) + if (this.supportsInsertOrReplace) { + await this.#query(`INSERT OR REPLACE INTO \`${this.#metaTables.items.name}\` (\`key\`, \`value\`) VALUES (?, ?);`, [key, stringBinary]) + } + else { + await this.#query('INSERT INTO ?? ?? VALUES ? ON DUPLICATE KEY UPDATE value=?;', [this.#metaTables.items.name, ["key", "value"], [key, stringBinary], stringBinary]) + } } async setPointer(pointer: Pointer, partialUpdateKey: unknown|typeof NOT_EXISTING): Promise>> { @@ -1445,7 +1496,8 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { .table(this.#metaTables.rawPointers.name) .select("value") .where(Where.eq(this.#pointerMysqlColumnName, pointerId)) - .build() + .build(), + undefined, ['value'] ))?.value; if (value) this.#existingPointersCache.add(pointerId); return value ? Runtime.decodeValue(this.#stringToBinary(value), outer_serialized) : NOT_EXISTING; @@ -1459,7 +1511,8 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { .select("value_text", "value_integer", "value_decimal", "value_boolean", "value_time", "value_pointer", "value_dxb") .where(Where.eq(this.#pointerMysqlColumnName, pointerId)) .where(Where.ne("hash", "")) - .build() + .build(), + undefined, false, ['value_text', 'value_integer', 'value_decimal', 'value_boolean', 'value_time', 'value_pointer', 'value_dxb'] ) const result = new Set() @@ -1538,10 +1591,12 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { // get table where pointer is stored const table = await this.#getPointerTable(pointerId); if (table) { - await this.#query('DELETE FROM ?? WHERE ??=?;', [table, this.#pointerMysqlColumnName, pointerId]) + //await this.#query('DELETE FROM ?? WHERE ??=?;', [table, this.#pointerMysqlColumnName, pointerId]) + await this.#query(`DELETE FROM \`${table}\` WHERE \`${this.#pointerMysqlColumnName}\`=?;`, [pointerId]) } // delete from pointer mapping - await this.#query('DELETE FROM ?? WHERE ??=?;', [this.#metaTables.pointerMapping.name, this.#pointerMysqlColumnName, pointerId]) + //await this.#query('DELETE FROM ?? WHERE ??=?;', [this.#metaTables.pointerMapping.name, this.#pointerMysqlColumnName, pointerId]) + await this.#query(`DELETE FROM \`${this.#metaTables.pointerMapping.name}\` WHERE \`${this.#pointerMysqlColumnName}\`=?;`, [pointerId]) } async getPointerValueDXB(pointerId: string): Promise { @@ -1555,7 +1610,8 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { .table(this.#metaTables.rawPointers.name) .select("value") .where(Where.eq(this.#pointerMysqlColumnName, pointerId)) - .build() + .build(), + undefined, ['value'] ))?.value; return value ? this.#stringToBinary(value) : null; } @@ -1568,7 +1624,8 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { .select("value_text", "value_integer", "value_decimal", "value_boolean", "value_time", "value_pointer", "value_dxb") .where(Where.eq(this.#pointerMysqlColumnName, pointerId)) .where(Where.ne("hash", "")) - .build() + .build(), + undefined, false, ['value_text', 'value_integer', 'value_decimal', 'value_boolean', 'value_time', 'value_pointer', 'value_dxb'] ) let setString = ` [` const setEntries:string[] = [] @@ -1606,7 +1663,10 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { async #setPointerInRawTable(pointerId: string, encoded: ArrayBuffer) { const table = this.#metaTables.rawPointers.name; - const {result} = await this.#query('INSERT INTO ?? ?? VALUES ? ON DUPLICATE KEY UPDATE value=?;', [table, [this.#pointerMysqlColumnName, "value"], [pointerId, encoded], encoded], true) + const {result} = + this.supportsInsertOrReplace ? + await this.#query(`INSERT OR REPLACE INTO \`${table}\` (\`${this.#pointerMysqlColumnName}\`, \`value\`) VALUES (?, ?);`, [pointerId, this.#binaryToString(encoded)], true) : + await this.#query('INSERT INTO ?? ?? VALUES ? ON DUPLICATE KEY UPDATE value=?;', [table, [this.#pointerMysqlColumnName, "value"], [pointerId, encoded], encoded], true) // is newly inserted, add to pointer mapping if (result.affectedRows == 1) await this.#updatePointerMapping(pointerId, table) } @@ -1618,7 +1678,8 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { .table(this.#metaTables.pointerMapping.name) .select("COUNT(*) as COUNT") .where(Where.eq(this.#pointerMysqlColumnName, pointerId)) - .build() + .build(), + undefined, ['COUNT'] )); const exists = !!count && count.COUNT > 0; if (exists) { diff --git a/storage/storage-locations/sql-definitions.ts b/storage/storage-locations/sql-definitions.ts index 41cec5b..1880081 100644 --- a/storage/storage-locations/sql-definitions.ts +++ b/storage/storage-locations/sql-definitions.ts @@ -1,13 +1,5 @@ import type { Datex } from "../../mod.ts"; -export type dbOptions = { - hostname: string - username: string - password:string - port: number - db?: string -} - export type Class = (new (...args: any[]) => any); // type for a JS class export enum PropertyMappingType { diff --git a/storage/storage-locations/sqlite-db.ts b/storage/storage-locations/sqlite-db.ts new file mode 100644 index 0000000..622e1dc --- /dev/null +++ b/storage/storage-locations/sqlite-db.ts @@ -0,0 +1,42 @@ +import { ExecuteResult } from "https://deno.land/x/mysql@v2.12.1/mod.ts"; +import { SQLDBStorageLocation } from "./sql-db.ts"; +import { DB } from "https://deno.land/x/sqlite/mod.ts"; +import { logger } from "../../datex_all.ts"; +import { ptr_cache_path } from "../../runtime/cache_path.ts"; + +export class SqliteStorageLocation extends SQLDBStorageLocation<{db: string}> { + + name = "SQLITE_DB" + useSingleQuotes = true + supportsInsertOrReplace = true + + + #db?: DB + + protected connect() { + this.#db = new DB(new URL(this.options.db + ".db", ptr_cache_path).pathname); + logger.info("Using SQLite database " + this.options.db + " as storage location") + return true; + } + + protected executeQuery(query_string: string, query_params?: any[]): ExecuteResult { + return { + rows: this.#db!.query(query_string, query_params) + } + } + + + protected getTableExistsQuery(table: string): string { + return `SELECT * FROM sqlite_master WHERE type = 'table' AND name = '${table}'` + } + protected getTableColumnInfoQuery(tableName: string): string { + return `PRAGMA table_info('${tableName}')` + } + protected getTableConstraintsQuery(tableName: string): string { + return `PRAGMA foreign_key_list('${tableName}')` + } + protected getClearTableQuery(tableName: string): string { + return `DELETE FROM ${tableName}` + // TODO: DELETE FROM sqlite_sequence WHERE name='${tableName}'; VACUUM; + } +} \ No newline at end of file From 30b8662d54810564078f6f2a8b64cc1a5023458e Mon Sep 17 00:00:00 2001 From: benStre Date: Sun, 29 Sep 2024 23:19:04 +0200 Subject: [PATCH 08/27] sqlite improvements --- storage/storage-locations/sql-db.ts | 3 ++- storage/storage-locations/sqlite-db.ts | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/storage/storage-locations/sql-db.ts b/storage/storage-locations/sql-db.ts index 0a92660..93bf511 100644 --- a/storage/storage-locations/sql-db.ts +++ b/storage/storage-locations/sql-db.ts @@ -609,7 +609,8 @@ export abstract class SQLDBStorageLocation extends // normal value ptrVal[key] ) - await this.#query('UPDATE ?? SET ?? = ? WHERE ?? = ?;', [table, key, val, this.#pointerMysqlColumnName, pointer.id]) + //await this.#query('UPDATE ?? SET ?? = ? WHERE ?? = ?;', [table, key, val, this.#pointerMysqlColumnName, pointer.id]) + await this.#query(`UPDATE \`${table}\` SET \`${key}\` = ? WHERE \`${this.#pointerMysqlColumnName}\` = ?`, [val, pointer.id]) } } diff --git a/storage/storage-locations/sqlite-db.ts b/storage/storage-locations/sqlite-db.ts index 622e1dc..1d6ba5b 100644 --- a/storage/storage-locations/sqlite-db.ts +++ b/storage/storage-locations/sqlite-db.ts @@ -10,7 +10,6 @@ export class SqliteStorageLocation extends SQLDBStorageLocation<{db: string}> { useSingleQuotes = true supportsInsertOrReplace = true - #db?: DB protected connect() { @@ -25,6 +24,11 @@ export class SqliteStorageLocation extends SQLDBStorageLocation<{db: string}> { } } + override async clear() { + // delete sqlite file + await Deno.remove(new URL(this.options.db + ".db", ptr_cache_path).pathname) + } + protected getTableExistsQuery(table: string): string { return `SELECT * FROM sqlite_master WHERE type = 'table' AND name = '${table}'` From 15806a49981d1b3390b237888ca7ea1ed6983633 Mon Sep 17 00:00:00 2001 From: benStre Date: Mon, 30 Sep 2024 20:37:04 +0200 Subject: [PATCH 09/27] get sqlite3 to work, improvements for storage sets and maps --- deno.lock | 71 ++++++++++++++++++++ runtime/pointers.ts | 13 +++- storage/storage-locations/mysql-db.ts | 6 +- storage/storage-locations/sql-db.ts | 45 ++++++++----- storage/storage-locations/sql-definitions.ts | 2 +- storage/storage-locations/sqlite-db.ts | 16 +++-- storage/storage.ts | 4 +- types/storage-map.ts | 36 ++++++++++ types/storage-set.ts | 38 ++++++++++- types/type.ts | 12 ++-- 10 files changed, 206 insertions(+), 37 deletions(-) diff --git a/deno.lock b/deno.lock index fd04a5b..d69a3c8 100644 --- a/deno.lock +++ b/deno.lock @@ -1,5 +1,67 @@ { "version": "3", + "packages": { + "specifiers": { + "jsr:@db/sqlite@0.11": "jsr:@db/sqlite@0.11.1", + "jsr:@denosaurs/plug@1": "jsr:@denosaurs/plug@1.0.6", + "jsr:@std/assert@^0.217.0": "jsr:@std/assert@0.217.0", + "jsr:@std/assert@^0.221.0": "jsr:@std/assert@0.221.0", + "jsr:@std/encoding@^0.221.0": "jsr:@std/encoding@0.221.0", + "jsr:@std/fmt@^0.221.0": "jsr:@std/fmt@0.221.0", + "jsr:@std/fs@^0.221.0": "jsr:@std/fs@0.221.0", + "jsr:@std/path@0.217": "jsr:@std/path@0.217.0", + "jsr:@std/path@^0.221.0": "jsr:@std/path@0.221.0" + }, + "jsr": { + "@db/sqlite@0.11.1": { + "integrity": "546434e7ed762db07e6ade0f963540dd5e06723b802937bf260ff855b21ef9c5", + "dependencies": [ + "jsr:@denosaurs/plug@1", + "jsr:@std/path@0.217" + ] + }, + "@denosaurs/plug@1.0.6": { + "integrity": "6cf5b9daba7799837b9ffbe89f3450510f588fafef8115ddab1ff0be9cb7c1a7", + "dependencies": [ + "jsr:@std/encoding@^0.221.0", + "jsr:@std/fmt@^0.221.0", + "jsr:@std/fs@^0.221.0", + "jsr:@std/path@^0.221.0" + ] + }, + "@std/assert@0.217.0": { + "integrity": "c98e279362ca6982d5285c3b89517b757c1e3477ee9f14eb2fdf80a45aaa9642" + }, + "@std/assert@0.221.0": { + "integrity": "a5f1aa6e7909dbea271754fd4ab3f4e687aeff4873b4cef9a320af813adb489a" + }, + "@std/encoding@0.221.0": { + "integrity": "d1dd76ef0dc5d14088411e6dc1dede53bf8308c95d1537df1214c97137208e45" + }, + "@std/fmt@0.221.0": { + "integrity": "379fed69bdd9731110f26b9085aeb740606b20428ce6af31ef6bd45ef8efa62a" + }, + "@std/fs@0.221.0": { + "integrity": "028044450299de8ed5a716ade4e6d524399f035513b85913794f4e81f07da286", + "dependencies": [ + "jsr:@std/assert@^0.221.0", + "jsr:@std/path@^0.221.0" + ] + }, + "@std/path@0.217.0": { + "integrity": "1217cc25534bca9a2f672d7fe7c6f356e4027df400c0e85c0ef3e4343bc67d11", + "dependencies": [ + "jsr:@std/assert@^0.217.0" + ] + }, + "@std/path@0.221.0": { + "integrity": "0a36f6b17314ef653a3a1649740cc8db51b25a133ecfe838f20b79a56ebe0095", + "dependencies": [ + "jsr:@std/assert@^0.221.0" + ] + } + } + }, "redirects": { "https://deno.land/std/async/mod.ts": "https://deno.land/std@0.208.0/async/mod.ts", "https://deno.land/x/sqlite/mod.ts": "https://deno.land/x/sqlite@v3.9.0/mod.ts" @@ -214,6 +276,15 @@ "https://deno.land/x/sql_builder@v1.9.2/query.ts": "a171b1d6b77bcd2893b6ec9a6af87684b172ef21529be5dce5c7f17760442668", "https://deno.land/x/sql_builder@v1.9.2/util.ts": "b0c9fb126f7ee0e9e33eae69adbba7744329bdbfd5608080024239eb1169ae8f", "https://deno.land/x/sql_builder@v1.9.2/where.ts": "2de7a961520070f2e909ca3901cfc432d7c553b43b790f08fc1aaebb4b206827", + "https://deno.land/x/sqlite3@0.12.0/deno.json": "b03d6de05f953886662ea987212539af8456a91352684c84af2188520449d42a", + "https://deno.land/x/sqlite3@0.12.0/deps.ts": "d2f23a4489d27ed7ba1f601b86a85ff488a87603e4be7a15f3ea15154fc288ec", + "https://deno.land/x/sqlite3@0.12.0/mod.ts": "3169f246c0eddd6ed82862758f4109f167b7ba5538236240fbb26a129f1bc16c", + "https://deno.land/x/sqlite3@0.12.0/src/blob.ts": "330886fae9714e4a612786f44d8117d65f91e778cf3f40de59b34879fc7ca9ab", + "https://deno.land/x/sqlite3@0.12.0/src/constants.ts": "85fd27aa6e199093f25f5f437052e16fd0e0870b96ca9b24a98e04ddc8b7d006", + "https://deno.land/x/sqlite3@0.12.0/src/database.ts": "4d380d7f0e5a2cf74635a9fcd2b4e27373533f2816cde5357067e51fd22ad8d0", + "https://deno.land/x/sqlite3@0.12.0/src/ffi.ts": "795b598eeae4d12f182e7bcdab524b74b0f01d6deae7f4d8ce63f25c06a46154", + "https://deno.land/x/sqlite3@0.12.0/src/statement.ts": "e8ccde898aef47c7a2514953aca5359a44a285bc3dc0de5819d66f891f477be1", + "https://deno.land/x/sqlite3@0.12.0/src/util.ts": "c6604183d2ec5fb17fa0a018572ed5f2317b319dbd7bf48d88a5d06ff25b2cc3", "https://deno.land/x/sqlite@v3.9.0/build/sqlite.js": "2afc7875c7b9c85d89730c4a311ab3a304e5d1bf761fbadd8c07bbdf130f5f9b", "https://deno.land/x/sqlite@v3.9.0/build/vfs.js": "7f7778a9fe499cd10738d6e43867340b50b67d3e39142b0065acd51a84cd2e03", "https://deno.land/x/sqlite@v3.9.0/mod.ts": "e09fc79d8065fe222578114b109b1fd60077bff1bb75448532077f784f4d6a83", diff --git a/runtime/pointers.ts b/runtime/pointers.ts index b0a3fcc..261fbda 100644 --- a/runtime/pointers.ts +++ b/runtime/pointers.ts @@ -755,6 +755,7 @@ export class PointerProperty extends ReactiveValue { if (type != Type.std.Any) return type; // TODO: returning Any makes problems else return undefined; } + } @@ -2513,7 +2514,7 @@ export class Pointer extends ReactiveValue { override get val():T { if (this.#garbage_collected) throw new PointerError("Pointer "+this.idString()+" was garbage collected"); else if (!this.#loaded) { - throw new PointerError("Cannot get value of uninitialized pointer") + throw new PointerError("Cannot get value of uninitialized pointer ("+this.idString()+")") } // deref and check if not garbage collected if (!this.is_persistent && !this.is_js_primitive && super.val instanceof WeakRef && this.type !== Type.std.WeakRef) { @@ -2541,7 +2542,7 @@ export class Pointer extends ReactiveValue { override get current_val():T|undefined { if (this.#garbage_collected) throw new PointerError("Pointer "+this.idString()+" was garbage collected"); else if (!this.#loaded) { - throw new PointerError("Cannot get value of uninitialized pointer") + throw new PointerError("Cannot get value of uninitialized pointer ("+this.idString()+")") } // deref and check if not garbage collected if (!this.is_persistent && !this.is_js_primitive && super.current_val instanceof WeakRef && this.type !== Type.std.WeakRef) { @@ -3341,11 +3342,19 @@ export class Pointer extends ReactiveValue { #any_type = false; + /** + * gets the current type of the pointer, or any if pointer is explicitly set to any + */ get type():Type { if (this.#any_type) return Type.std.Any; + return this.current_type; + } + + get current_type():Type { return this.#unwrapped_transform_type ?? this.#type; } + allowAnyType(any_type = true) { this.#any_type = any_type; } diff --git a/storage/storage-locations/mysql-db.ts b/storage/storage-locations/mysql-db.ts index 0d2477a..e60dbcf 100644 --- a/storage/storage-locations/mysql-db.ts +++ b/storage/storage-locations/mysql-db.ts @@ -19,6 +19,8 @@ export class MySQLStorageLocation extends SQLDBStorageLocation { this.#sqlClient = await new Client().connect({poolSize: 20, ...this.options}); @@ -44,7 +46,7 @@ export class MySQLStorageLocation extends SQLDBStorageLocation extends protected abstract getTableColumnInfoQuery(tableName: string): string protected abstract getTableConstraintsQuery(tableName: string): string protected abstract getClearTableQuery(tableName: string): string; - + protected abstract affectedRowsQuery?: string; static #debugMode = false; @@ -54,6 +54,8 @@ export abstract class SQLDBStorageLocation extends supportsPrefixSelection = true; supportsMatchSelection = true; supportsPartialUpdates = true; + supportsBinaryIO = false; + supportsSQLCalcFoundRows = true; #connected = false; #initializing = false @@ -187,7 +189,9 @@ export abstract class SQLDBStorageLocation extends async #query(query_string:string, query_params?:any[], returnRawResult?: boolean, columnNames?:string[]): Promise { // TODO: only workaround for sqlite, replace all " with ' in queries - if (this.useSingleQuotes) query_string = query_string.replace(/"/g, "'"); + if (this.useSingleQuotes) { + query_string = query_string.replace(/"/g, "'"); + } // prevent infinite recursion if calling query from within init() if (!this.#initializing) await this.#init(); @@ -241,10 +245,13 @@ export abstract class SQLDBStorageLocation extends } - #stringToBinary(value: string){ + #stringToBinary(value: string|Uint8Array|ArrayBuffer): ArrayBuffer { + if (value instanceof Uint8Array) return value.buffer; + if (value instanceof ArrayBuffer) return value; return Uint8Array.from(value, x => x.charCodeAt(0)).buffer } #binaryToString(value: ArrayBuffer){ + if (this.supportsBinaryIO) return new Uint8Array(value); return String.fromCharCode.apply(null, new Uint8Array(value) as unknown as number[]) } @@ -492,21 +499,21 @@ export abstract class SQLDBStorageLocation extends this.#tableColumnTasks.set(tableName, promise); const columnData = new Map() - const columns = await this.#query<{_id: string, COLUMN_NAME:string, DATA_TYPE:string}>( - this.getTableColumnInfoQuery(tableName), [], false, ["_id", "COLUMN_NAME", "DATA_TYPE"] + const columns = await this.#query<{_id: string, name:string, type:string}>( + this.getTableColumnInfoQuery(tableName), [], false, ["_id", "name", "type"] ) - const constraints = (await this.#query<{_id:string, COLUMN_NAME:string, REFERENCED_TABLE_NAME:string}>( - this.getTableConstraintsQuery(tableName), [], false, ["_id", "COLUMN_NAME", "REFERENCED_TABLE_NAME"] + const constraints = (await this.#query<{_id:string, name:string, ref_table:string}>( + this.getTableConstraintsQuery(tableName), [], false, ["_id", "name", "ref_table"] )); const columnTables = new Map() - for (const {COLUMN_NAME, REFERENCED_TABLE_NAME} of constraints) { - columnTables.set(COLUMN_NAME, REFERENCED_TABLE_NAME) + for (const {name, ref_table} of constraints) { + columnTables.set(name, ref_table) } for (const col of columns) { - const columnName = col.COLUMN_NAME - const dataType = col.DATA_TYPE + const columnName = col.name + const dataType = col.type if (columnName == this.#pointerMysqlColumnName) continue; columnData.set(columnName, {foreignPtr: columnTables.has(columnName), foreignTable: columnTables.get(columnName), type: dataType}) } @@ -923,7 +930,7 @@ export abstract class SQLDBStorageLocation extends joins.forEach(join => builder.join(join)); const outerBuilder = new Query() - .select(options.returnRaw ? `*` :`DISTINCT SQL_CALC_FOUND_ROWS ${this.#pointerMysqlColumnName} as ${this.#pointerMysqlColumnName}` + ( + .select(options.returnRaw ? `*` :`DISTINCT ${this.supportsSQLCalcFoundRows?'SQL_CALC_FOUND_ROWS ' : ''}${this.#pointerMysqlColumnName} as ${this.#pointerMysqlColumnName}` + ( options.returnKeys ? `, ${this.#metaTables.items.name}.key as map_key` : '' )) .table('__placeholder__'); @@ -937,7 +944,7 @@ export abstract class SQLDBStorageLocation extends // no computed properties else { - builder.select(`DISTINCT SQL_CALC_FOUND_ROWS ${this.#typeToTableName(valueType)}.${this.#pointerMysqlColumnName} as ${this.#pointerMysqlColumnName}` + ( + builder.select(`DISTINCT ${this.supportsSQLCalcFoundRows?'SQL_CALC_FOUND_ROWS ' : ''}\`${this.#typeToTableName(valueType)}\`.${this.#pointerMysqlColumnName} as ${this.#pointerMysqlColumnName}` + ( options.returnKeys ? `, ${this.#metaTables.items.name}.key as map_key` : '' )); this.appendBuilderConditions(builder, options, where) @@ -1429,12 +1436,11 @@ export abstract class SQLDBStorageLocation extends else return null; } async setItemValueDXB(key: string, value: ArrayBuffer) { - const stringBinary = this.#binaryToString(value) if (this.supportsInsertOrReplace) { - await this.#query(`INSERT OR REPLACE INTO \`${this.#metaTables.items.name}\` (\`key\`, \`value\`) VALUES (?, ?);`, [key, stringBinary]) + await this.#query(`INSERT OR REPLACE INTO \`${this.#metaTables.items.name}\` (\`key\`, \`value\`) VALUES (?, ?);`, [key, value]) } else { - await this.#query('INSERT INTO ?? ?? VALUES ? ON DUPLICATE KEY UPDATE value=?;', [this.#metaTables.items.name, ["key", "value"], [key, stringBinary], stringBinary]) + await this.#query('INSERT INTO ?? ?? VALUES ? ON DUPLICATE KEY UPDATE value=?;', [this.#metaTables.items.name, ["key", "value"], [key, value], value]) } } @@ -1666,9 +1672,14 @@ export abstract class SQLDBStorageLocation extends const table = this.#metaTables.rawPointers.name; const {result} = this.supportsInsertOrReplace ? - await this.#query(`INSERT OR REPLACE INTO \`${table}\` (\`${this.#pointerMysqlColumnName}\`, \`value\`) VALUES (?, ?);`, [pointerId, this.#binaryToString(encoded)], true) : + await this.#query(`INSERT OR REPLACE INTO \`${table}\` (\`${this.#pointerMysqlColumnName}\`, \`value\`) VALUES (?, ?);`, [pointerId, encoded], true) : await this.#query('INSERT INTO ?? ?? VALUES ? ON DUPLICATE KEY UPDATE value=?;', [table, [this.#pointerMysqlColumnName, "value"], [pointerId, encoded], encoded], true) // is newly inserted, add to pointer mapping + if (!('affectedRows' in result) && this.affectedRowsQuery) { + // query affected rows + const {affectedRows} = await this.#queryFirst<{affectedRows:number}>(this.affectedRowsQuery, undefined, ['affectedRows']) ?? {}; + result.affectedRows = affectedRows; + } if (result.affectedRows == 1) await this.#updatePointerMapping(pointerId, table) } diff --git a/storage/storage-locations/sql-definitions.ts b/storage/storage-locations/sql-definitions.ts index 1880081..4f7f1f1 100644 --- a/storage/storage-locations/sql-definitions.ts +++ b/storage/storage-locations/sql-definitions.ts @@ -48,7 +48,7 @@ type _mysql_data_type = 'int'|'bigint'|'smallint'|'mediumint'|'tinyint'|'tiny'|' 'timestamp'|'date'|'datetime'| 'time'|'varchar'|'char'|'text'|'tinytext'|'mediumtext'|'longtext'|'enum'| 'set'|'geometry'| - 'tinyblob'|'blob'|'mediumblob'|'longblob'|'binary'|'varbinary'|'bit'| + 'tinyblob'|'BLOB'|'mediumblob'|'longblob'|'binary'|'varbinary'|'bit'| 'boolean'|'json'; export type mysql_data_type = _mysql_data_type | `${_mysql_data_type}(${number})`; diff --git a/storage/storage-locations/sqlite-db.ts b/storage/storage-locations/sqlite-db.ts index 1d6ba5b..a4370e7 100644 --- a/storage/storage-locations/sqlite-db.ts +++ b/storage/storage-locations/sqlite-db.ts @@ -1,6 +1,9 @@ import { ExecuteResult } from "https://deno.land/x/mysql@v2.12.1/mod.ts"; import { SQLDBStorageLocation } from "./sql-db.ts"; -import { DB } from "https://deno.land/x/sqlite/mod.ts"; + +import { Database } from "jsr:@db/sqlite@0.11"; + + import { logger } from "../../datex_all.ts"; import { ptr_cache_path } from "../../runtime/cache_path.ts"; @@ -9,18 +12,23 @@ export class SqliteStorageLocation extends SQLDBStorageLocation<{db: string}> { name = "SQLITE_DB" useSingleQuotes = true supportsInsertOrReplace = true + supportsBinaryIO = true + supportsSQLCalcFoundRows = false + + affectedRowsQuery = "SELECT changes() as affectedRows" - #db?: DB + #db?: Database protected connect() { - this.#db = new DB(new URL(this.options.db + ".db", ptr_cache_path).pathname); + this.#db = new Database(new URL(this.options.db + ".db", ptr_cache_path)); logger.info("Using SQLite database " + this.options.db + " as storage location") return true; } protected executeQuery(query_string: string, query_params?: any[]): ExecuteResult { + // TODO: optimize, don't prepare every time and don't return all rows if not needed return { - rows: this.#db!.query(query_string, query_params) + rows: this.#db!.prepare(query_string).all(query_params) } } diff --git a/storage/storage.ts b/storage/storage.ts index eaef4d6..ecbbaf5 100644 --- a/storage/storage.ts +++ b/storage/storage.ts @@ -901,7 +901,7 @@ export class Storage { /** - * Run a pointer update after 1s on process exit and keep the dirty state until the update is finished + * Run a pointer update after 1s or on process exit and keep the dirty state until the update is finished */ private static scheduleStorageUpdate(update:()=>void|Promise, location: StorageLocation, metadata?: string) { @@ -913,7 +913,7 @@ export class Storage { return res.then(()=>{ this.setDirty(location, false, metadata); this.#scheduledUpdates.delete(updateFn); - }) + }).catch(e => console.error(e)); } else { this.#scheduledUpdates.delete(updateFn); diff --git a/types/storage-map.ts b/types/storage-map.ts index 682ea7b..ce5c1b0 100644 --- a/types/storage-map.ts +++ b/types/storage-map.ts @@ -18,6 +18,27 @@ import { Type } from "./type.ts"; */ export class StorageWeakMap { + /** + * Create a new StorageWeakMap instance with the given key and value types. + * @param keyType Class or DATEX Type of the keys + * @param valueType Class or DATEX Type of the values + * @returns + */ + static of(keyType:Class|Type|undefined|null, valueType: Class|Type) { + const storageMap = new this(); + storageMap.#_type = valueType instanceof Type ? valueType : Type.getClassDatexType(valueType); + storageMap._type = storageMap.#_type.namespace + ":" + storageMap.#_type.name; + return storageMap; + } + + _type?: string + #_type?: Type; + + protected get type() { + if (!this.#_type) this.#_type = Type.get(this._type!); + return this.#_type; + } + #prefix?: string; /** @@ -87,6 +108,11 @@ export class StorageWeakMap { return this._set(storage_key, value); } protected async _set(storage_key:string, value:V) { + // convert to correct type if not already + if (this.type && !(this.type.matches(value))) { + value = this.type.cast(value); + } + // proxify value if (!this.allowNonPointerObjectValues) { value = this.#pointer.proxifyChild("", value); @@ -124,6 +150,16 @@ export class StorageWeakMap { */ export class StorageMap extends StorageWeakMap { + /** + * Create a new StorageMap instance with the given key and value types. + * @param keyType Class or DATEX Type of the keys + * @param valueType Class or DATEX Type of the values + * @returns + */ + static of(keyType:Class|Type|undefined|null, valueType: Class|Type) { + return super.of(keyType, valueType) as StorageMap; + } + static override async from(entries: readonly (readonly [K, V])[]){ const map = $$(new StorageMap()); for (const [key, value] of entries) await map.set(key, value); diff --git a/types/storage-set.ts b/types/storage-set.ts index ff84441..34d3581 100644 --- a/types/storage-set.ts +++ b/types/storage-set.ts @@ -10,8 +10,6 @@ import type { Class } from "../utils/global_types.ts"; import { MatchOptions } from "../utils/match.ts"; import { Type } from "./type.ts"; -const logger = new Logger("StorageSet"); - /** * WeakSet that outsources values to storage. * The API is similar to the JS WeakSet API, but all methods are async. @@ -21,6 +19,26 @@ const logger = new Logger("StorageSet"); */ export class StorageWeakSet { + /** + * Create a new StorageWeakSet instance with the given value type. + * @param type Class or DATEX Type of the values + * @returns + */ + static of(type: Class|Type) { + const storageSet = new this(); + storageSet.#_type = type instanceof Type ? type : Type.getClassDatexType(type); + storageSet._type = storageSet.#_type.namespace + ":" + storageSet.#_type.name; + return storageSet; + } + + _type?: string + #_type?: Type; + + protected get type() { + if (!this.#_type) this.#_type = Type.get(this._type!); + return this.#_type; + } + /** * Time in milliseconds after which a value is removed from the in-memory cache * Default: 5min @@ -66,6 +84,11 @@ export class StorageWeakSet { return this; } protected _add(storage_key:string, value:V|null) { + // convert to correct type if not already + if (this.type && !(this.type.matches(value))) { + value = this.type.cast(value); + } + // proxify value if (!this.allowNonPointerObjectValues) { value = this.#pointer.proxifyChild("", value); @@ -119,6 +142,13 @@ export class StorageWeakSet { */ export class StorageSet extends StorageWeakSet { + /** + * Create a new StorageSet instance with the given value type. + * @param type Class or DATEX Type of the values + */ + static of(type: Class|Type): StorageSet { + return super.of(type) as StorageSet; + } #size?: number; @@ -246,7 +276,9 @@ export class StorageSet extends StorageWeakSet { } } - match(valueType:Class|Type, matchInput: MatchInput, options?: Options): Promise> { + match(matchInput: MatchInput, options?: Options, valueType?:Class|Type): Promise> { + valueType ??= this.type; + if (!valueType) throw new Error("Cannot determine value type. Please provide a valueType parameter to match()"); return match(this as unknown as StorageSet, valueType, matchInput, options) } } \ No newline at end of file diff --git a/types/type.ts b/types/type.ts index 681da67..dfad0f5 100644 --- a/types/type.ts +++ b/types/type.ts @@ -828,7 +828,7 @@ export class Type extends ExtensibleFunction { public static ofValue(value:RefOrValue):Type { if (value instanceof Pointer) { - return value.type ?? Type.std.void; + return value.current_type ?? Type.std.void; } else if (value instanceof PointerProperty && value.type) { return value.type as Type; @@ -855,7 +855,7 @@ export class Type extends ExtensibleFunction { // get type from pointer let type:Type|undefined - if ((type = Pointer.getByValue(value)?.type)) return type; + if ((type = Pointer.getByValue(value)?.current_type)) return type; // get custom type const custom_type = JSInterface.getValueDatexType(value); @@ -1214,28 +1214,28 @@ Type.std.StorageWeakMap.setJSInterface({ class: StorageWeakMap, is_normal_object: true, proxify_children: true, - visible_children: new Set(), + visible_children: new Set(['_type']), }) Type.std.StorageMap.setJSInterface({ class: StorageMap, is_normal_object: true, proxify_children: true, - visible_children: new Set(), + visible_children: new Set(['_type']), }) Type.std.StorageWeakSet.setJSInterface({ class: StorageWeakSet, is_normal_object: true, proxify_children: true, - visible_children: new Set(), + visible_children: new Set(['_type']), }) Type.std.StorageSet.setJSInterface({ class: StorageSet, is_normal_object: true, proxify_children: true, - visible_children: new Set(), + visible_children: new Set(['_type']), }) // proxify_children leads to problems with native types - use plain objects for pointer propagation + don't propagate proxification From a8487cfee4e861bdbbc3722a4dcddb5351850983 Mon Sep 17 00:00:00 2001 From: benStre Date: Mon, 30 Sep 2024 20:46:30 +0200 Subject: [PATCH 10/27] update match methods --- types/storage-map.ts | 8 +++++--- types/storage-set.ts | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/types/storage-map.ts b/types/storage-map.ts index ce5c1b0..da3c78a 100644 --- a/types/storage-map.ts +++ b/types/storage-map.ts @@ -24,7 +24,7 @@ export class StorageWeakMap { * @param valueType Class or DATEX Type of the values * @returns */ - static of(keyType:Class|Type|undefined|null, valueType: Class|Type) { + static of(keyType:Class|Type|undefined|null, valueType: Class|Type): StorageWeakMap { const storageMap = new this(); storageMap.#_type = valueType instanceof Type ? valueType : Type.getClassDatexType(valueType); storageMap._type = storageMap.#_type.namespace + ":" + storageMap.#_type.name; @@ -156,7 +156,7 @@ export class StorageMap extends StorageWeakMap { * @param valueType Class or DATEX Type of the values * @returns */ - static of(keyType:Class|Type|undefined|null, valueType: Class|Type) { + static of(keyType:Class|Type, valueType: Class|Type): StorageMap { return super.of(keyType, valueType) as StorageMap; } @@ -319,7 +319,9 @@ export class StorageMap extends StorageWeakMap { await Promise.all(promises); } - match(valueType:Class|Type, matchInput: MatchInput, options?: Options): Promise> { + match(matchInput: MatchInput, options?: Options, valueType?: Class|Type): Promise> { + valueType ??= this.type; + if (!valueType) throw new Error("Cannot determine value type. Please provide a valueType parameter to match()"); return match(this as unknown as StorageMap, valueType, matchInput, options) as any; } diff --git a/types/storage-set.ts b/types/storage-set.ts index 34d3581..342a5de 100644 --- a/types/storage-set.ts +++ b/types/storage-set.ts @@ -24,7 +24,7 @@ export class StorageWeakSet { * @param type Class or DATEX Type of the values * @returns */ - static of(type: Class|Type) { + static of(type: Class|Type): StorageWeakSet { const storageSet = new this(); storageSet.#_type = type instanceof Type ? type : Type.getClassDatexType(type); storageSet._type = storageSet.#_type.namespace + ":" + storageSet.#_type.name; @@ -88,7 +88,7 @@ export class StorageWeakSet { if (this.type && !(this.type.matches(value))) { value = this.type.cast(value); } - + // proxify value if (!this.allowNonPointerObjectValues) { value = this.#pointer.proxifyChild("", value); From d4483adc6c2ee1716f93e1fb7829bdc6e1476308 Mon Sep 17 00:00:00 2001 From: benStre Date: Mon, 30 Sep 2024 20:53:44 +0200 Subject: [PATCH 11/27] update storage collection docs --- docs/manual/15 Storage Collections.md | 43 ++++++++++----------------- 1 file changed, 15 insertions(+), 28 deletions(-) diff --git a/docs/manual/15 Storage Collections.md b/docs/manual/15 Storage Collections.md index a14cf51..bd09213 100644 --- a/docs/manual/15 Storage Collections.md +++ b/docs/manual/15 Storage Collections.md @@ -66,15 +66,13 @@ const User = struct({ }) type User = inferType -const users = new StorageSet(); +// using StorageSet.of instead of new StorageSet to get a typed StorageSet +const users = StorageSet.of(User); // get all users with age == 18 -const usersAge18 = await users.match( - User, - { - age: 18 - } -); +const usersAge18 = await users.match({ + age: 18 +}); ``` ### Match Conditions @@ -86,26 +84,20 @@ Match between to numbers/dates: import { MatchCondition } from "unyt_core/storage/storage.ts"; // all users where the "created" timestamp is between now and 7 days ago: -const newUsersLastWeek = users.match( - User, - { - created: MatchCondition.between( - new Time().minus(7, "d"), - new Time() - ) - } -) +const newUsersLastWeek = users.match({ + created: MatchCondition.between( + new Time().minus(7, "d"), + new Time() + ) +}) ``` Match not equal: ```ts // all users which do not have the name "John": -const notJohn = users.match( - User, - { - name: MatchCondition.notEqual("John") - } -) +const notJohn = users.match({ + name: MatchCondition.notEqual("John") +}) ``` @@ -118,7 +110,6 @@ You can limit the maximum number of returned entries by setting the `limit` opti ```ts // get all users with name "Josh", limit to 10 const joshes = await users.match( - User, { name: "Josh" }, @@ -133,7 +124,6 @@ You can sort the returned entries by setting the `sortBy` option to a property p ```ts // get all users with age == 18, sorted by their creation timestamp const usersAge18 = await users.match( - User, { age: 18 }, @@ -153,7 +143,6 @@ When the `returnAdvanced` option is set to `true`, the `match` function returns ```ts const {matches, total} = await users.match( - User, { name: "Josh" }, @@ -212,7 +201,6 @@ const distance = ComputedProperty.geographicDistance( ) const nearbyJoshes = await users.match( - User, { name: "Josh", // name = "Josh" distance: MatchCondition.lessThan(1000) // distance < 1000m @@ -235,7 +223,7 @@ const TodoItem = struct({ }) type TodoItem = inferType -const todoItems = new StorageSet() +const todoItems = StorageSet.of(TodoItem) // sum of completedTaskCount and openTaskCount for a given TodoItem const totalTaskCount = ComputedProperty.sum( @@ -245,7 +233,6 @@ const totalTaskCount = ComputedProperty.sum( // match all todo items where the total task count is > 100 const bigTodoItems = await todoItems.match( - User, { totalTaskCount: MatchCondition.greaterThan(100) // totalTaskCount > 100 }, From 3fc0f81a7531545f25b2b71736b7e85aa416e818 Mon Sep 17 00:00:00 2001 From: benStre Date: Wed, 2 Oct 2024 00:52:28 +0200 Subject: [PATCH 12/27] improvements for sqlite storage, fix storage collection setters --- storage/storage-locations/mysql-db.ts | 3 ++ storage/storage-locations/sql-db.ts | 35 ++++++++++++++------ storage/storage-locations/sql-definitions.ts | 2 +- storage/storage-locations/sqlite-db.ts | 16 +++++++-- types/storage-map.ts | 8 ++--- types/storage-set.ts | 8 ++--- 6 files changed, 47 insertions(+), 25 deletions(-) diff --git a/storage/storage-locations/mysql-db.ts b/storage/storage-locations/mysql-db.ts index e60dbcf..80d735f 100644 --- a/storage/storage-locations/mysql-db.ts +++ b/storage/storage-locations/mysql-db.ts @@ -22,6 +22,9 @@ export class MySQLStorageLocation extends SQLDBStorageLocation { this.#sqlClient = await new Client().connect({poolSize: 20, ...this.options}); logger.info("Using SQL database " + this.options.db + " on " + this.options.hostname + ":" + this.options.port + " as storage location") diff --git a/storage/storage-locations/sql-db.ts b/storage/storage-locations/sql-db.ts index cef5b50..57e4b29 100644 --- a/storage/storage-locations/sql-db.ts +++ b/storage/storage-locations/sql-db.ts @@ -40,6 +40,8 @@ export abstract class SQLDBStorageLocation extends protected abstract getTableConstraintsQuery(tableName: string): string protected abstract getClearTableQuery(tableName: string): string; protected abstract affectedRowsQuery?: string; + protected abstract disableForeignKeyChecksQuery: string; + protected abstract enableForeignKeyChecksQuery: string; static #debugMode = false; @@ -56,6 +58,10 @@ export abstract class SQLDBStorageLocation extends supportsPartialUpdates = true; supportsBinaryIO = false; supportsSQLCalcFoundRows = true; + supportsInsertOrIgnore = false; + supportsPartialForeignKeys() { + return true; + } #connected = false; #initializing = false @@ -98,7 +104,7 @@ export abstract class SQLDBStorageLocation extends name: "__datex_sets", columns: [ [this.#pointerMysqlColumnName, this.#pointerMysqlType, "PRIMARY KEY"], - ["hash", "varchar(50)", "PRIMARY KEY"], + this.supportsPartialForeignKeys() ? ["hash", "varchar(50)", "PRIMARY KEY"] : ["hash", "varchar(50)"], ["value_dxb", "blob"], ["value_text", "text"], ["value_integer", "int"], @@ -196,18 +202,24 @@ export abstract class SQLDBStorageLocation extends // prevent infinite recursion if calling query from within init() if (!this.#initializing) await this.#init(); - // handle arraybuffers + // handle arraybuffers, convert undefined to null if (query_params) { for (let i = 0; i < query_params.length; i++) { const param = query_params[i]; if (param instanceof ArrayBuffer) { query_params[i] = this.#binaryToString(param) } + if (param === undefined) query_params[i] = null; if (param instanceof Array) { - query_params[i] = param.map(p => p instanceof ArrayBuffer ? this.#binaryToString(p) : p) + query_params[i] = param.map(p => { + if (p instanceof ArrayBuffer) return this.#binaryToString(p); + else if (p === undefined) return null; + return p; + }) } } } + if (SQLDBStorageLocation.#debugMode) console.log("QUERY: " + query_string, query_params) @@ -503,11 +515,14 @@ export abstract class SQLDBStorageLocation extends this.getTableColumnInfoQuery(tableName), [], false, ["_id", "name", "type"] ) - const constraints = (await this.#query<{_id:string, name:string, ref_table:string}>( + const constraints = (await this.#query<{_id:string, name:string, ref_table:string, table:string, from:string}>( this.getTableConstraintsQuery(tableName), [], false, ["_id", "name", "ref_table"] )); + const columnTables = new Map() - for (const {name, ref_table} of constraints) { + for (const constraint of constraints) { + const name = constraint.name ?? constraint.from; // mysql/sqlite + const ref_table = constraint.ref_table ?? constraint.table; // mysql/sqlite columnTables.set(name, ref_table) } @@ -566,7 +581,7 @@ export abstract class SQLDBStorageLocation extends dependencies.add(propPointer) } } - // is raw dxb value (exception for blob <->A rrayBuffer, TODO: better solution, can lead to issues) + // is raw dxb value (exception for blob <-> ArrayBuffer, TODO: better solution, can lead to issues) else if (type == "blob" && !(value instanceof ArrayBuffer || value instanceof TypedArray)) { insertData[name] = Compiler.encodeValue(value, dependencies, true, false, false); } @@ -802,17 +817,17 @@ export abstract class SQLDBStorageLocation extends // first delete all existing entries for this pointer (except the default entry) try { - await this.#query('SET FOREIGN_KEY_CHECKS=0;'); + await this.#query(this.disableForeignKeyChecksQuery); //await this.#query('DELETE FROM ?? WHERE ?? = ? AND `hash` != "";', [this.#metaTables.sets.name, this.#pointerMysqlColumnName, pointer.id]) await this.#query(`DELETE FROM \`${this.#metaTables.sets.name}\` WHERE \`${this.#pointerMysqlColumnName}\` = ? AND \`hash\` != "";`, [pointer.id]) - await this.#query('SET FOREIGN_KEY_CHECKS=1;'); + await this.#query(this.enableForeignKeyChecksQuery); } catch (e) { console.error("Error deleting old set entries", e) } // replace INSERT with INSERT IGNORE to prevent duplicate key errors - const {result} = await this.#query(builder.build().replace("INSERT", "INSERT IGNORE"), undefined, true) + const {result} = await this.#query(builder.build().replace("INSERT", this.supportsInsertOrIgnore ? "INSERT OR IGNORE" : "INSERT IGNORE"), undefined, true) // add to pointer mapping TODO: better decision if to add to pointer mapping if (result.affectedRows) await this.#updatePointerMapping(pointer.id, this.#metaTables.sets.name) return dependencies; @@ -1324,7 +1339,7 @@ export abstract class SQLDBStorageLocation extends const ptr = Pointer.pointerifyValue(value); if (ptr instanceof Pointer) { dependencies.add(ptr); - this.#setItemPointer(key, ptr) + await this.#setItemPointer(key, ptr) } else { const encoded = Compiler.encodeValue(value, dependencies); diff --git a/storage/storage-locations/sql-definitions.ts b/storage/storage-locations/sql-definitions.ts index 4f7f1f1..1880081 100644 --- a/storage/storage-locations/sql-definitions.ts +++ b/storage/storage-locations/sql-definitions.ts @@ -48,7 +48,7 @@ type _mysql_data_type = 'int'|'bigint'|'smallint'|'mediumint'|'tinyint'|'tiny'|' 'timestamp'|'date'|'datetime'| 'time'|'varchar'|'char'|'text'|'tinytext'|'mediumtext'|'longtext'|'enum'| 'set'|'geometry'| - 'tinyblob'|'BLOB'|'mediumblob'|'longblob'|'binary'|'varbinary'|'bit'| + 'tinyblob'|'blob'|'mediumblob'|'longblob'|'binary'|'varbinary'|'bit'| 'boolean'|'json'; export type mysql_data_type = _mysql_data_type | `${_mysql_data_type}(${number})`; diff --git a/storage/storage-locations/sqlite-db.ts b/storage/storage-locations/sqlite-db.ts index a4370e7..5bb4b2c 100644 --- a/storage/storage-locations/sqlite-db.ts +++ b/storage/storage-locations/sqlite-db.ts @@ -3,9 +3,9 @@ import { SQLDBStorageLocation } from "./sql-db.ts"; import { Database } from "jsr:@db/sqlite@0.11"; - import { logger } from "../../datex_all.ts"; import { ptr_cache_path } from "../../runtime/cache_path.ts"; +import { Path } from "../../utils/path.ts"; export class SqliteStorageLocation extends SQLDBStorageLocation<{db: string}> { @@ -14,13 +14,25 @@ export class SqliteStorageLocation extends SQLDBStorageLocation<{db: string}> { supportsInsertOrReplace = true supportsBinaryIO = true supportsSQLCalcFoundRows = false + supportsInsertOrIgnore = true; + supportsPartialForeignKeys() { + return false; + } affectedRowsQuery = "SELECT changes() as affectedRows" + disableForeignKeyChecksQuery = "PRAGMA foreign_keys = OFF" + enableForeignKeyChecksQuery = "PRAGMA foreign_keys = ON" #db?: Database protected connect() { - this.#db = new Database(new URL(this.options.db + ".db", ptr_cache_path)); + const path = new Path(this.options.db + ".db", ptr_cache_path) + + if (!path.parent_dir.fs_exists) { + Deno.mkdirSync(path.parent_dir.normal_pathname, {recursive:true}); + } + + this.#db = new Database(path); logger.info("Using SQLite database " + this.options.db + " as storage location") return true; } diff --git a/types/storage-map.ts b/types/storage-map.ts index da3c78a..b4afccf 100644 --- a/types/storage-map.ts +++ b/types/storage-map.ts @@ -35,7 +35,8 @@ export class StorageWeakMap { #_type?: Type; protected get type() { - if (!this.#_type) this.#_type = Type.get(this._type!); + if (!this._type) return undefined; + if (!this.#_type) this.#_type = Type.get(this._type); return this.#_type; } @@ -108,11 +109,6 @@ export class StorageWeakMap { return this._set(storage_key, value); } protected async _set(storage_key:string, value:V) { - // convert to correct type if not already - if (this.type && !(this.type.matches(value))) { - value = this.type.cast(value); - } - // proxify value if (!this.allowNonPointerObjectValues) { value = this.#pointer.proxifyChild("", value); diff --git a/types/storage-set.ts b/types/storage-set.ts index 342a5de..356f547 100644 --- a/types/storage-set.ts +++ b/types/storage-set.ts @@ -35,7 +35,8 @@ export class StorageWeakSet { #_type?: Type; protected get type() { - if (!this.#_type) this.#_type = Type.get(this._type!); + if (!this._type) return undefined; + if (!this.#_type) this.#_type = Type.get(this._type); return this.#_type; } @@ -84,11 +85,6 @@ export class StorageWeakSet { return this; } protected _add(storage_key:string, value:V|null) { - // convert to correct type if not already - if (this.type && !(this.type.matches(value))) { - value = this.type.cast(value); - } - // proxify value if (!this.allowNonPointerObjectValues) { value = this.#pointer.proxifyChild("", value); From 5164063bcff40b1245104c665acbe4eb95629f5d Mon Sep 17 00:00:00 2001 From: benStre Date: Mon, 7 Oct 2024 04:20:36 +0200 Subject: [PATCH 13/27] show deno version in init log --- network/unyt.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/network/unyt.ts b/network/unyt.ts index c8f22cc..81d1dae 100644 --- a/network/unyt.ts +++ b/network/unyt.ts @@ -135,6 +135,8 @@ export class Unyt { if (info.datex_version == "0.0.0") content += `${ESCAPE_SEQUENCES.UNYT_GREY}DATEX VERSION${ESCAPE_SEQUENCES.COLOR_DEFAULT}${ESCAPE_SEQUENCES.ITALIC} unmarked${ESCAPE_SEQUENCES.RESET}\n` else if (info.datex_version) content += `${ESCAPE_SEQUENCES.UNYT_GREY}DATEX VERSION${ESCAPE_SEQUENCES.COLOR_DEFAULT} ${info.datex_version.replaceAll('\n','')}\n` + + if (globalThis.Deno) content += `${ESCAPE_SEQUENCES.UNYT_GREY}DENO VERSION${ESCAPE_SEQUENCES.COLOR_DEFAULT} ${Deno.version.deno}\n` content += `\n` // if (info.app?.stage == "dev" && info.app.backend) content += `Worbench Access for this App: https://workbench.unyt.org/\?e=${info.app.backend.toString()}\n` From a527b11aa480712d6091d4cef8863b9229bcf242 Mon Sep 17 00:00:00 2001 From: benStre Date: Mon, 7 Oct 2024 04:20:56 +0200 Subject: [PATCH 14/27] fix mysql-db storage location --- storage/storage-locations/mysql-db.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/storage-locations/mysql-db.ts b/storage/storage-locations/mysql-db.ts index 80d735f..18804ce 100644 --- a/storage/storage-locations/mysql-db.ts +++ b/storage/storage-locations/mysql-db.ts @@ -48,7 +48,7 @@ export class MySQLStorageLocation extends SQLDBStorageLocation Date: Mon, 7 Oct 2024 04:21:05 +0200 Subject: [PATCH 15/27] update types --- types/function-utils.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/types/function-utils.ts b/types/function-utils.ts index b131af9..9413696 100644 --- a/types/function-utils.ts +++ b/types/function-utils.ts @@ -22,7 +22,7 @@ const EXTRACT_USED_VARS = Symbol("EXTRACT_USED_VARS") * ``` * @param variables */ -export function use(noDatex: 'standalone', ...variables: unknown[]): true +export function use(flags: 'standalone'|'silent-errors', ...variables: unknown[]): true /** * Used to declare all variables from the parent scope that are used inside the current function. * This is required for functions that are transferred to a different context or restored from eternal pointers. @@ -69,6 +69,7 @@ function getUsedVars(fn: (...args:unknown[])=>unknown) { let ignoreVarCounter = 0; for (const usedVar of _usedVars) { if (usedVar == `"standalone"` || usedVar == `'standalone'`) flags.push("standalone"); + else if (usedVar == `"silent-errors"` || usedVar == `'silent-errors'`) flags.push("silent-errors"); else if (!usedVar.match(/^[a-zA-Z_$][0-9a-zA-Z_$\u0080-\uFFFF]*$/)) { usedVars.push("#" + (ignoreVarCounter++)); // ignore variables start with # // TODO: only warn if not artifact from minification From ce96504d31408aeeae8c099a0a0974a781d491bd Mon Sep 17 00:00:00 2001 From: benStre Date: Mon, 7 Oct 2024 04:21:28 +0200 Subject: [PATCH 16/27] add note for UIX inferred 'use' dependencies to docs --- docs/manual/11 Types.md | 4 ++++ docs/manual/13 Threads.md | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/docs/manual/11 Types.md b/docs/manual/11 Types.md index f0c7ff3..70d910d 100644 --- a/docs/manual/11 Types.md +++ b/docs/manual/11 Types.md @@ -113,6 +113,10 @@ const transferableFn = JSTransferableFunction.create(() => { await datex `@example :: ${transferableFn}()` ``` +> [!NOTE] +> When you are using transferable functions within a [UIX](https://docs.unyt.org/manual/uix) project, dependencies from the parent scope are automatically detected and transferred. +> You don't need to explicitly declare them with `use()`. + ### js:Object In contrast to `std:Object`, `js:Object` is used for JavaScript object with a prototype other than `Object.prototype` or `null` (e.g. a class instance). diff --git a/docs/manual/13 Threads.md b/docs/manual/13 Threads.md index a6a85e9..adfedaf 100644 --- a/docs/manual/13 Threads.md +++ b/docs/manual/13 Threads.md @@ -52,6 +52,10 @@ disposeThread(thread) Instead of declaring a thread module in a separate file, a task function can be passed to `run` to be executed in a new thread immediately. Values from the parent scope can be passed to the thread by explicitly adding them to the `use()` declaration at the beginning of the function body. +> [!NOTE] +> When you are using transferable functions within a [UIX](https://docs.unyt.org/manual/uix) project, dependencies from the parent scope are automatically detected and transferred. +> You don't need to explicitly declare them with `use()`. + In the following example, a function calculates the nth fibonacci number in a thread. The `n` index variable is accessed from the parent scope. From a9e5fdef7f19a1a58ffa2cc677b841594929552c Mon Sep 17 00:00:00 2001 From: benStre Date: Mon, 7 Oct 2024 04:54:39 +0200 Subject: [PATCH 17/27] handle global vars for use declarations, add 'allow-globals' flag --- types/function-utils.ts | 66 +++++++++++++++++++++++++++++---------- utils/iterable-handler.ts | 4 +-- 2 files changed, 51 insertions(+), 19 deletions(-) diff --git a/types/function-utils.ts b/types/function-utils.ts index 9413696..6e341a5 100644 --- a/types/function-utils.ts +++ b/types/function-utils.ts @@ -1,6 +1,5 @@ import { LazyPointer } from "../runtime/lazy-pointer.ts"; import { callWithMetadata, callWithMetadataAsync, getMeta } from "../utils/caller_metadata.ts"; -import { RuntimeError } from "./errors.ts"; const EXTRACT_USED_VARS = Symbol("EXTRACT_USED_VARS") @@ -20,9 +19,13 @@ const EXTRACT_USED_VARS = Symbol("EXTRACT_USED_VARS") * console.log("x:" + x) * }) * ``` + * @param flags - optional flags: + * - 'standalone': indicates that the function can run standalone without the datex runtime + * - 'silent-errors': suppresses errors when global variables are used + * - 'allow-globals': allows transffering global variables in the function. This only works if the variables are never actually transferred between scopes. * @param variables */ -export function use(flags: 'standalone'|'silent-errors', ...variables: unknown[]): true +export function use(flags: 'standalone'|'silent-errors'|'allow-globals', ...variables: unknown[]): true /** * Used to declare all variables from the parent scope that are used inside the current function. * This is required for functions that are transferred to a different context or restored from eternal pointers. @@ -58,18 +61,22 @@ declare global { const use: _use } +type Flag = 'standalone'|'silent-errors'|'allow-globals' + function getUsedVars(fn: (...args:unknown[])=>unknown) { const source = fn.toString(); const usedVarsSource = source.match(/^(?:(?:[\w\s*])+\(.*?\)\s*{|\(.*?\)\s*=>\s*[{(]?|.*?\s*=>\s*[{(]?)\s*use\s*\(([\s\S]*?)\)/)?.[1] if (!usedVarsSource) return {}; const _usedVars = usedVarsSource.split(",").map(v=>v.trim()).filter(v=>!!v) - const flags = [] + const flags:Flag[] = [] const usedVars = [] let ignoreVarCounter = 0; for (const usedVar of _usedVars) { + // TODO: support multiple flags at once if (usedVar == `"standalone"` || usedVar == `'standalone'`) flags.push("standalone"); else if (usedVar == `"silent-errors"` || usedVar == `'silent-errors'`) flags.push("silent-errors"); + else if (usedVar == `"allow-globals"` || usedVar == `'allow-globals'`) flags.push("allow-globals"); else if (!usedVar.match(/^[a-zA-Z_$][0-9a-zA-Z_$\u0080-\uFFFF]*$/)) { usedVars.push("#" + (ignoreVarCounter++)); // ignore variables start with # // TODO: only warn if not artifact from minification @@ -91,13 +98,7 @@ export function getDeclaredExternalVariables(fn: (...args:unknown[])=>unknown) { callWithMetadata({[EXTRACT_USED_VARS]: true}, fn as any, [{}]) // TODO: provide call arguments that don't lead to a {}/[] destructuring error } catch (e) { - // capture returned variables from use() - if (e instanceof Array && (e as any)[EXTRACT_USED_VARS]) { - if (flags.length) e.splice(0, flags.length); // remove flags - return {vars: Object.fromEntries(usedVars.map((v,i)=>[v, e[i]])), flags} - } - // otherwise, throw normal error - else throw e; + return captureVariables(e, usedVars, flags); } return {vars:{}}; } @@ -111,17 +112,48 @@ export async function getDeclaredExternalVariablesAsync(fn: (...args:unknown[])= await callWithMetadataAsync({[EXTRACT_USED_VARS]: true}, fn as any) } catch (e) { - // capture returned variables from use() - if (e instanceof Array && (e as any)[EXTRACT_USED_VARS]) { - if (flags.length) e.splice(0, flags.length); // remove flags - return {vars: Object.fromEntries(usedVars.map((v,i)=>[v, e[i]])), flags} - } - // otherwise, throw normal error - else throw e; + return captureVariables(e, usedVars, flags); } return {vars:{}}; } +/** + * Whiteliste for global variables that are allowed to be + * transferred to a different context per default. + */ +const allowedGlobalVars = new Set([ + "console", + "alert", + "confirm", + "prompt", +]) + + +function captureVariables(e: unknown, usedVars: string[], flags: Flag[]) { + // capture returned variables from use() + if (e instanceof Array && (e as any)[EXTRACT_USED_VARS]) { + if (flags.length) e.splice(0, flags.length); // remove flags + const vars = Object.fromEntries(usedVars.map((v,i)=>[v, e[i]])); + + // for each variable: remove if global variable + if (!flags.includes("allow-globals")) { + for (const [key, value] of Object.entries(vars)) { + if ((globalThis as any)[key] === value && !allowedGlobalVars.has(key)) { + if (!flags.includes("silent-errors")) { + throw new Error("The global variable '"+key+"' cannot be transferred to a different context. Remove the 'use("+key+")' declaration.") + } + delete vars[key]; + } + } + } + + return {vars, flags} + } + // otherwise, throw normal error + else throw e; +} + + export function getSourceWithoutUsingDeclaration(fn: (...args:unknown[])=>unknown) { let fnSource = fn.toString(); // object methods check if 'this' context is component context; diff --git a/utils/iterable-handler.ts b/utils/iterable-handler.ts index 8b179b1..340d4c0 100644 --- a/utils/iterable-handler.ts +++ b/utils/iterable-handler.ts @@ -53,7 +53,7 @@ export class IterableHandler { weakAction( {self}, ({self}) => { - use (iterableRef, Datex); + use ("allow-globals", iterableRef, Datex); const iterable = iterableRef.deref()! // only here to fix closure scope bug, should always exist at this point const callback = (v:any, k:any, t:any) => { @@ -68,7 +68,7 @@ export class IterableHandler { return callback; }, (callback) => { - use (iterableRef, Datex); + use ("allow-globals", iterableRef, Datex); const deref = iterableRef.deref() if (deref) Datex.ReactiveValue.unobserve(deref, callback); From faee5e4adfa8e2395edc20607eb4b032b5adf2b6 Mon Sep 17 00:00:00 2001 From: benStre Date: Mon, 7 Oct 2024 21:24:05 +0200 Subject: [PATCH 18/27] add support for DX_NOT_TRANSFERABLE, fix async function regex --- runtime/constants.ts | 2 ++ runtime/pointers.ts | 4 +++- types/function-utils.ts | 8 ++++---- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/runtime/constants.ts b/runtime/constants.ts index 9f65bd5..14478dc 100644 --- a/runtime/constants.ts +++ b/runtime/constants.ts @@ -35,6 +35,8 @@ export const DX_SLOTS: unique symbol = Symbol("DX_SLOTS"); export const DX_EXTERNAL_SCOPE_NAME: unique symbol = Symbol("DX_EXTERNAL_SCOPE_NAME"); // string name of the external scope export const DX_EXTERNAL_FUNCTION_NAME: unique symbol = Symbol("DX_EXTERNAL_FUNCTION_NAME"); // string name for an external function +export const DX_NOT_TRANSFERABLE: unique symbol = Symbol("DX_NOT_TRANSFERABLE"); // marks a value that should never be transferred between scopes because it is not serializabe + export const SLOT_WRITE = 0xfef0; export const SLOT_READ = 0xfef1; export const SLOT_EXEC = 0xfef2; diff --git a/runtime/pointers.ts b/runtime/pointers.ts index 261fbda..5e946e6 100644 --- a/runtime/pointers.ts +++ b/runtime/pointers.ts @@ -2,7 +2,7 @@ import { Endpoint, endpoints, endpoint_name, IdEndpoint, Person, Target, target_clause, LOCAL_ENDPOINT, BROADCAST } from "../types/addressing.ts"; import { NetworkError, PermissionError, PointerError, RuntimeError, ValueError } from "../types/errors.ts"; import { Compiler, PrecompiledDXB } from "../compiler/compiler.ts"; -import { DX_PTR, DX_REACTIVE_METHODS, DX_VALUE, INVALID, NOT_EXISTING, SET_PROXY, SHADOW_OBJECT, UNKNOWN_TYPE, VOID } from "./constants.ts"; +import { DX_NOT_TRANSFERABLE, DX_PTR, DX_REACTIVE_METHODS, DX_VALUE, INVALID, NOT_EXISTING, SET_PROXY, SHADOW_OBJECT, UNKNOWN_TYPE, VOID } from "./constants.ts"; import { Runtime, UnresolvedValue } from "./runtime.ts"; import { DEFAULT_HIDDEN_OBJECT_PROPERTIES, logger, TypedArray } from "../utils/global_values.ts"; import type { compile_info, datex_scope, Equals, PointerSource } from "../utils/global_types.ts"; @@ -51,6 +51,8 @@ export type RefLikeOut = PointerWithPrimitive|PointerProperty // root class for pointers and pointer properties, value changes can be observed export abstract class ReactiveValue extends EventTarget { + static [DX_NOT_TRANSFERABLE] = true + #observerCount = 0; #observers?: Map diff --git a/types/function-utils.ts b/types/function-utils.ts index 6e341a5..4e1a9d0 100644 --- a/types/function-utils.ts +++ b/types/function-utils.ts @@ -1,3 +1,4 @@ +import { DX_NOT_TRANSFERABLE } from "../runtime/constants.ts"; import { LazyPointer } from "../runtime/lazy-pointer.ts"; import { callWithMetadata, callWithMetadataAsync, getMeta } from "../utils/caller_metadata.ts"; @@ -138,7 +139,7 @@ function captureVariables(e: unknown, usedVars: string[], flags: Flag[]) { // for each variable: remove if global variable if (!flags.includes("allow-globals")) { for (const [key, value] of Object.entries(vars)) { - if ((globalThis as any)[key] === value && !allowedGlobalVars.has(key)) { + if (((globalThis as any)[key] === value || value?.[DX_NOT_TRANSFERABLE]) && !allowedGlobalVars.has(key)) { if (!flags.includes("silent-errors")) { throw new Error("The global variable '"+key+"' cannot be transferred to a different context. Remove the 'use("+key+")' declaration.") } @@ -173,7 +174,7 @@ const isNormalFunction = (fnSrc:string) => { return !!fnSrc.match(/^(async\s+)?function(\(| |\*)/) } const isArrowFunction = (fnSrc:string) => { - return !!fnSrc.match(/^(async\s+)?(\([^)]*\)|\w+)\s*=>/) + return !!fnSrc.match(/^(async\s*)?(\([^)]*\)|\w+)\s*=>/) } const isNativeFunction = (fnSrc:string) => { @@ -241,7 +242,7 @@ export function createFunctionWithDependencyInjectionsResolveLazyPointers(source * @deprecated use createFunctionWithDependencyInjectionsResolveLazyPointers */ export function createFunctionWithDependencyInjections(source: string, dependencies: Record, allowValueMutations = true): ((...args:unknown[]) => unknown) { - const hasThis = Object.keys(dependencies).includes('this'); + const hasThis = Object.keys(dependencies).includes('this'); let ignoreVarCounter = 0; const renamedVars = Object.keys(dependencies).filter(d => d!=='this').map(k => k.startsWith("#") ? '_ignore_'+(ignoreVarCounter++) : '_'+k); const varMapping = renamedVars.filter(v=>!v.startsWith("_ignore_")).map(k=>`const ${k.slice(1)} = ${allowValueMutations ? 'createStaticObject' : ''}(${k});`).join("\n"); @@ -276,7 +277,6 @@ export function createFunctionWithDependencyInjections(source: string, dependenc } catch (e) { console.error(creatorSource) - console.error(e); throw e; } From 43c0ddd1e2f5201a68e4a7f85ad59af7e3686294 Mon Sep 17 00:00:00 2001 From: benStre Date: Mon, 7 Oct 2024 22:49:54 +0200 Subject: [PATCH 19/27] improve type assertions --- js_adapter/js_class_adapter.ts | 34 ++++++++++++++++++++++++++++------ runtime/pointers.ts | 21 +++++++++++++++++++-- types/type.ts | 2 +- 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/js_adapter/js_class_adapter.ts b/js_adapter/js_class_adapter.ts index 16d7838..1af6101 100644 --- a/js_adapter/js_class_adapter.ts +++ b/js_adapter/js_class_adapter.ts @@ -111,6 +111,10 @@ export class Decorators { } } + public static getMetadata(context:DecoratorContext, key:string|symbol) { + return context.metadata[key]?.public?.[context.name] ?? context.metadata[key]?.constructor + } + /** @endpoint(endpoint?:string|Datex.Endpoint, namespace?:string): declare a class as a #public property */ static endpoint(endpoint:target_clause|endpoint_name, scope_name:string|undefined, value: Class, context: ClassDecoratorContext) { @@ -207,6 +211,20 @@ export class Decorators { } } + static assignType(type:string|Type, context: ClassFieldDecoratorContext|ClassGetterDecoratorContext|ClassMethodDecoratorContext, forceConjunction) { + let newType = this.getMetadata(context, Decorators.FORCE_TYPE); + if (newType instanceof Conjunction) { + newType.add(type) + } + else if (newType) { + newType = new Conjunction(newType, type) + } + else { + newType = forceConjunction ? new Conjunction(type) : type + } + this.setMetadata(context, Decorators.FORCE_TYPE, newType) + } + /** @property: add a field as a template property */ static property(type:string|Type|Class, context: ClassFieldDecoratorContext|ClassGetterDecoratorContext|ClassMethodDecoratorContext) { @@ -221,7 +239,7 @@ export class Decorators { // type if (type) { const normalizedType = normalizeType(type); - this.setMetadata(context, Decorators.FORCE_TYPE, normalizedType) + this.assignType(normalizedType, context) } } @@ -230,8 +248,8 @@ export class Decorators { static assert(assertion: (val:T) => boolean|string|undefined, context: ClassFieldDecoratorContext) { if (context.static) logger.error("Cannot use @assert with static fields"); else { - const assertionType = new Conjunction(Assertion.get(undefined, assertion, false)); - this.setMetadata(context, Decorators.FORCE_TYPE, assertionType) + const assertionType = Assertion.get(undefined, assertion, false); + this.assignType(assertionType, context, true) } } @@ -572,6 +590,10 @@ function getStaticClassData(original_class:Class, staticScope = true, expose = t const templated_classes = new Map() // original class, templated class +function getInferredType(metadataConstructor: any) { + return (metadataConstructor && Type.getClassDatexType(metadataConstructor)) ?? Type.std.Any; +} + export function createTemplateClass(original_class: Class, type:Type, sync = true, add_js_interface = true, callerFile?:string, metadata?:Record){ if (templated_classes.has(original_class)) return templated_classes.get(original_class)!; @@ -621,15 +643,15 @@ export function createTemplateClass(original_class: Class, type:Type, sync = tru if (prototype[DX_ROOT]) break; } - - // iterate over all properties TODO different dx_name? for (const [name, dx_name] of Object.entries(metadata?.[Decorators.PROPERTY]?.public??{})) { let metadataConstructor = MetadataReflect.getMetadata && MetadataReflect.getMetadata("design:type", original_class.prototype, name); // if type is Object -> std:Any if (metadataConstructor == Object) metadataConstructor = null; // set best guess for property type - template[name] = property_types?.[name] ?? (metadataConstructor && Type.getClassDatexType(metadataConstructor)) ?? Type.std.Any; // add type + const existingType = property_types?.[name]; + template[name] = existingType ?? getInferredType(metadataConstructor); // add type + if (allow_filters?.[name]) template[DX_PERMISSIONS][name] = allow_filters[name]; // add filter } diff --git a/runtime/pointers.ts b/runtime/pointers.ts index 5e946e6..cac147c 100644 --- a/runtime/pointers.ts +++ b/runtime/pointers.ts @@ -2236,6 +2236,8 @@ export class Pointer extends ReactiveValue { if (!this.is_js_primitive) throw new PointerError("Assertions are not yet supported for non-primitive pointer") if (!this.#typeAssertions) this.#typeAssertions = new Conjunction(); this.#typeAssertions.add(Assertion.get(undefined, assertion, false)); + + this.validateTypeAssertions(this.val) return this; } @@ -2752,6 +2754,12 @@ export class Pointer extends ReactiveValue { } } + private validateTypeAssertions(val:T, type?:Type) { + type ??= Type.ofValue(val); + return this.#typeAssertions && + !Type.matchesType(type, this.#typeAssertions, val, true) + } + /** * Overrides the current value of the pointer (only if the value has the correct type) * @param v new value @@ -2787,8 +2795,7 @@ export class Pointer extends ReactiveValue { !Type.matchesType(newType, this.type, val, true) || // validate custom type assertions ( - this.#typeAssertions && - !Type.matchesType(newType, this.#typeAssertions, val, true) + this.#typeAssertions && this.validateTypeAssertions(val, newType) ) ) { throw new ValueError("Invalid value type for pointer "+this.idString()+": " + newType + " - must be " + this.type); @@ -3696,6 +3703,16 @@ export class Pointer extends ReactiveValue { for (const name of this.visible_children ?? Object.keys(value)) { const type = Type.ofValue(value[name]) + // run assertions for property if defined in template + const templatePropType = this.type?.template?.[name]; + if (templatePropType instanceof Conjunction) { + Type.matchesType(type, templatePropType, value[name], true) + // for (const type of templatePropType) { + // if (type instanceof) + // } + + } + // non primitive value - proxify always if (!type.is_primitive) { // custom timeout for remote proxy function diff --git a/types/type.ts b/types/type.ts index dfad0f5..da3857d 100644 --- a/types/type.ts +++ b/types/type.ts @@ -830,7 +830,7 @@ export class Type extends ExtensibleFunction { if (value instanceof Pointer) { return value.current_type ?? Type.std.void; } - else if (value instanceof PointerProperty && value.type) { + else if (value instanceof PointerProperty && value.type instanceof Type) { return value.type as Type; } From 4b60f2d29743b24f7466e36035a987aeba5528c1 Mon Sep 17 00:00:00 2001 From: benStre Date: Wed, 9 Oct 2024 22:08:47 +0200 Subject: [PATCH 20/27] Make solutions optional --- utils/error-handling.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/error-handling.ts b/utils/error-handling.ts index 7d4ef60..21705ce 100644 --- a/utils/error-handling.ts +++ b/utils/error-handling.ts @@ -8,7 +8,7 @@ import { logger as defaultLogger } from "./global_values.ts"; * The user might be able to fix it or ask for help. */ export class KnownError extends Error { - constructor(message: string, public solutions: string[], public quickFixes: {description: string, fix: () => void}[] = []) { + constructor(message: string, public solutions: string[] = [], public quickFixes: {description: string, fix: () => void}[] = []) { super(message); } } From 7bb711f961b584f3639093e6971b119d41bfb85b Mon Sep 17 00:00:00 2001 From: benStre Date: Fri, 11 Oct 2024 11:41:48 +0200 Subject: [PATCH 21/27] async function fixes, static transform improvements --- functions.ts | 14 +++++--------- runtime/pointers.ts | 14 +++++++------- types/function-utils.ts | 3 ++- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/functions.ts b/functions.ts index c434399..367effa 100644 --- a/functions.ts +++ b/functions.ts @@ -39,11 +39,11 @@ export function always(script:TemplateStringsArray, ...vars:any[]): P export function always(scriptOrJSTransform:TemplateStringsArray|SmartTransformFunction, ...vars:any[]) { // js function if (typeof scriptOrJSTransform == "function") { + const options: SmartTransformOptions|undefined = typeof vars[0] == "object" ? vars[0] : undefined; // make sure handler is not an async function - if (scriptOrJSTransform.constructor.name == "AsyncFunction") { + if (scriptOrJSTransform.constructor.name == "AsyncFunction" && !options?._allowAsync) { throw new Error("Async functions are not allowed as always transforms") } - const options: SmartTransformOptions|undefined = typeof vars[0] == "object" ? vars[0] : undefined; const ptr = Pointer.createSmartTransform(scriptOrJSTransform, undefined, undefined, undefined, options); if (options?._allowAsync && !ptr.value_initialized && ptr.waiting_for_always_promise) { return ptr.waiting_for_always_promise.then(()=>collapseTransformPointer(ptr, options?._collapseStatic, options?._returnWrapper, options?._allowAnyType)); @@ -69,13 +69,8 @@ export function always(scriptOrJSTransform:TemplateStringsArray|SmartTransformFu function collapseTransformPointer(ptr: Pointer, collapseStatic = false, alwaysReturnWrapper = false, _allowAnyType = false) { - let collapse = false; - if (collapseStatic) { - // check if transform function is static and value is a js primitive - if (ptr.isStaticTransform && ptr.is_js_primitive) { - collapse = true; - } - } + // collapse if transform function is static + const collapse = collapseStatic && ptr.isStaticTransform; if (_allowAnyType) { ptr.allowAnyType(true); @@ -87,6 +82,7 @@ function collapseTransformPointer(ptr: Pointer, collapseStatic = false, alwaysRe const val = ReactiveValue.collapseValue(ptr, false, collapse); if (collapse) ptr.delete(); + // TODO: deproxify static non-primitive objects to garbage-collect pointer and associated data return val; } diff --git a/runtime/pointers.ts b/runtime/pointers.ts index cac147c..961bc01 100644 --- a/runtime/pointers.ts +++ b/runtime/pointers.ts @@ -2654,7 +2654,7 @@ export class Pointer extends ReactiveValue { // TODO: is this required somewhere? // add reference to this DatexPointer to the original value - if (!this.is_anonymous) { + if (!this.is_anonymous && !this.isStaticTransform) { try { Object.defineProperty(val, DX_PTR, {value: this, enumerable: false, writable: true, configurable: true}) } catch(e) {} @@ -2667,9 +2667,9 @@ export class Pointer extends ReactiveValue { Pointer.pointer_value_map.set(val, this); // create proxy - const value = alreadyProxy ? val : this.addObjProxy((val instanceof UnresolvedValue) ? val[DX_VALUE] : val); + const value = alreadyProxy||this.isStaticTransform ? val : this.addObjProxy((val instanceof UnresolvedValue) ? val[DX_VALUE] : val); // add $, $$ - if (!alreadyProxy && typeof value !== "symbol") this.add$Properties(value); + if (!(alreadyProxy||this.isStaticTransform) && typeof value !== "symbol") this.add$Properties(value); this.#loaded = true; // this.value exists (must be set to true before the super.value getter is called) @@ -2685,7 +2685,7 @@ export class Pointer extends ReactiveValue { this.updateGarbageCollection(); // proxify children, if not anonymous - if (this.type.proxify_children) this.proxifyChildren(); + if (this.type.proxify_children && !this.isStaticTransform) this.proxifyChildren(); // save proxy + original value in map to find the right pointer for this value in the future Pointer.pointer_value_map.set(value, this); @@ -3188,9 +3188,6 @@ export class Pointer extends ReactiveValue { this._liveTransform = true } - // update value - if (!ignoreReturnValue) this.setVal(val, true, true); - // remove return value if captured by getters // TODO: this this work as intended? capturedGetters?.delete(val instanceof Pointer ? val : Pointer.getByValue(val)!); @@ -3206,6 +3203,9 @@ export class Pointer extends ReactiveValue { // TODO: cleanup stuff not needed if no reactive transform } + // update value + if (!ignoreReturnValue) this.setVal(val, true, true); + if (state.isLive) { if (capturedGetters) { diff --git a/types/function-utils.ts b/types/function-utils.ts index 4e1a9d0..6166aeb 100644 --- a/types/function-utils.ts +++ b/types/function-utils.ts @@ -1,6 +1,7 @@ import { DX_NOT_TRANSFERABLE } from "../runtime/constants.ts"; import { LazyPointer } from "../runtime/lazy-pointer.ts"; import { callWithMetadata, callWithMetadataAsync, getMeta } from "../utils/caller_metadata.ts"; +import { Type } from "./type.ts"; const EXTRACT_USED_VARS = Symbol("EXTRACT_USED_VARS") @@ -139,7 +140,7 @@ function captureVariables(e: unknown, usedVars: string[], flags: Flag[]) { // for each variable: remove if global variable if (!flags.includes("allow-globals")) { for (const [key, value] of Object.entries(vars)) { - if (((globalThis as any)[key] === value || value?.[DX_NOT_TRANSFERABLE]) && !allowedGlobalVars.has(key)) { + if (((key in globalThis && (globalThis as any)[key] === value) || value?.[DX_NOT_TRANSFERABLE] || value instanceof Type) && !allowedGlobalVars.has(key)) { if (!flags.includes("silent-errors")) { throw new Error("The global variable '"+key+"' cannot be transferred to a different context. Remove the 'use("+key+")' declaration.") } From 39ec987294717ee3b2aabc3c0e8b5d74c45b91aa Mon Sep 17 00:00:00 2001 From: benStre Date: Sun, 13 Oct 2024 18:19:07 +0200 Subject: [PATCH 22/27] fix prop function --- datex_short.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/datex_short.ts b/datex_short.ts index 5620fd9..94f2c55 100644 --- a/datex_short.ts +++ b/datex_short.ts @@ -323,8 +323,13 @@ export function pointer(value:RefOrValue, property?:unknown, callStackInde export function prop>(parent:RefOrValue, propertyKey: T extends Map ? K : unknown): PointerProperty ? V : unknown>|(T extends Map ? V : unknown) export function prop>(parent:RefOrValue, propertyKey: keyof T): PointerProperty|T[keyof T] export function prop(parent:Map|Record, propertyKey: unknown): any { - if (ReactiveValue.isRef(parent)) return PointerProperty.get(parent, propertyKey); - else if (parent instanceof Map) return parent.get(propertyKey); + // try to get pointer property + if (ReactiveValue.isRef(parent)) { + const prop = PointerProperty.getIfExists(parent, propertyKey); + if (prop !== NOT_EXISTING) return prop; + } + + if (parent instanceof Map) return parent.get(propertyKey); else return parent[propertyKey as keyof typeof parent]; } @@ -385,7 +390,7 @@ export const $ = new Proxy(function(){} as unknown as $type, { }, apply(_target, _thisArg, args) { - return pointer(args[0], args[1], 1); + return pointer(args[0], args[1], 1 /* callStackIndex */); }, }) From 6d369750826a4693ffe488c43a031229a9de2a72 Mon Sep 17 00:00:00 2001 From: benStre Date: Sun, 13 Oct 2024 18:20:03 +0200 Subject: [PATCH 23/27] fix transforms --- functions.ts | 5 ++-- init.ts | 3 +- runtime/pointers.ts | 71 ++++++++++++++++++++++++++++++++++++++------- 3 files changed, 65 insertions(+), 14 deletions(-) diff --git a/functions.ts b/functions.ts index 367effa..907bfbb 100644 --- a/functions.ts +++ b/functions.ts @@ -60,8 +60,8 @@ export function always(scriptOrJSTransform:TemplateStringsArray|SmartTransformFu return (async ()=>collapseTransformPointer(await datex(`always (${scriptOrJSTransform.raw.join(INSERT_MARK)})`, vars)))() } else { - handleError(new KnownError("You called 'always' with invalid arguments. It seems like you are not using deno for UIX.", [ - "Install deno for UIX, see https://docs.unyt.org/manual/uix/getting-started#install-deno", + handleError(new KnownError("You called 'always' with invalid arguments. It seems like you are not using Deno for UIX.", [ + "Install Deno for UIX, see https://docs.unyt.org/manual/uix/getting-started#install-deno", "Call 'always' with a function: always(() => ...)" ])) } @@ -81,6 +81,7 @@ function collapseTransformPointer(ptr: Pointer, collapseStatic = false, alwaysRe } const val = ReactiveValue.collapseValue(ptr, false, collapse); + if (collapse) ptr.delete(); // TODO: deproxify static non-primitive objects to garbage-collect pointer and associated data return val; diff --git a/init.ts b/init.ts index 5492581..1d3df99 100644 --- a/init.ts +++ b/init.ts @@ -75,8 +75,7 @@ export async function init() { } else if (client_type == "deno") { if (Deno.env.get("SQLITE_STORAGE") == "1") { - const { SqliteStorageLocation } = await import("./storage/storage-locations/sqlite-db.ts"); - console.log("Using sqlite as primary storage location (experimental feature enabled via SQLITE_STORAGE env variable)") + const { SqliteStorageLocation } = await import("./storage/storage-locations/sqlite-db.ts" /* lazy */); await Storage.addLocation(new SqliteStorageLocation({ db: "storage" }), { diff --git a/runtime/pointers.ts b/runtime/pointers.ts index 961bc01..b3648dc 100644 --- a/runtime/pointers.ts +++ b/runtime/pointers.ts @@ -336,7 +336,17 @@ export abstract class ReactiveValue extends EventTarget { !(value instanceof PointerProperty || value.indirectReference) ) ) { - return value.val + // is a static transform + if (value instanceof Pointer && value.isStaticTransform) { + return value.staticTransformValue; + } + // unwrap previously wrapped value + if (value instanceof Pointer && value.current_type?.interface_config?.unwrap_transform) { + return value.current_type.interface_config.unwrap_transform(value.val); + } + else { + return value.val + } } else return > value; } @@ -458,6 +468,7 @@ export abstract class ReactiveValue extends EventTarget { #forceLiveTransform = false; #transformSource?: TransformSource #isStaticTransform = false; + #staticTransformValue?: unknown /** * if true, there are no dependencies and the value is never updated @@ -466,8 +477,14 @@ export abstract class ReactiveValue extends EventTarget { get isStaticTransform() { return this.#isStaticTransform; } - protected set _isStaticTransform(val: boolean) { - this.#isStaticTransform = val; + + get staticTransformValue() { + return this.#staticTransformValue; + } + + protected set _staticTransformValue(val: unknown) { + this.#isStaticTransform = true; + this.#staticTransformValue = val; } @@ -663,6 +680,22 @@ export class PointerProperty extends ReactiveValue { return new PointerProperty(pointer, key, leak_js_properties); } + public static getIfExists>(parent: Parent|Pointer|LazyPointer, key: Key, leak_js_properties = false): PointerProperty ? MV : Parent[Key&keyof Parent]>|typeof NOT_EXISTING { + + parent = Pointer.pointerifyValue(parent); + if (parent instanceof Pointer && parent.type.template){ + if ((typeof key == "string" || typeof key == "symbol" || typeof key == "number") && !(key in parent.type.template)) return NOT_EXISTING; + } + // normal html node without custom template, does not support pointer properties + else if (parent instanceof Pointer && parent.val instanceof Node && !parent.type.template) { + return NOT_EXISTING; + } + + const prop = this.get(parent, key, leak_js_properties); + if (prop.current_val === NOT_EXISTING) return NOT_EXISTING; + else return prop; + } + // get current pointer property public override get val():T { if (this.lazy_pointer) return undefined as T @@ -1738,7 +1771,9 @@ export class Pointer extends ReactiveValue { return ptr; } // create new pointer - else return >Pointer.create(id, value, sealed, undefined, persistant, anonymous, false, allowed_access); + else { + return >Pointer.create(id, value, sealed, undefined, persistant, anonymous, false, allowed_access); + } } // same as createOrGet, but also return lazy pointer if it exists @@ -2620,7 +2655,10 @@ export class Pointer extends ReactiveValue { if (is_transform) val = this.getInitialTransformValue(val) // Get type from initial value, keep as if initial value is null/undefined or indirect reference - if (val!==undefined && val !== null && !this.#indirectReference) this.#type = Type.ofValue(val); + if (val!==undefined && val !== null && + // allow false (workaround for UIX DOM elements) + !(val === false) + && !this.#indirectReference) this.#type = Type.ofValue(val); // console.log("loaded : "+ this.id + " - " + this.#type, val) @@ -2657,7 +2695,9 @@ export class Pointer extends ReactiveValue { if (!this.is_anonymous && !this.isStaticTransform) { try { Object.defineProperty(val, DX_PTR, {value: this, enumerable: false, writable: true, configurable: true}) - } catch(e) {} + } catch { + logger.error("Cannot set DX_PTR for " + this.idString()) + } } if (this.sealed) this.visible_children = new Set(Object.keys(val)); // get current keys and don't allow any other children @@ -2750,6 +2790,8 @@ export class Pointer extends ReactiveValue { }) } catch(e) { + console.error(e); + console.log(val.$); logger.error("Cannot set $ properties for " + this.idString()) } } @@ -3198,9 +3240,10 @@ export class Pointer extends ReactiveValue { // no dependencies, will never change, this is not the intention of the transform if (!ignoreReturnValue && hasGetters && !gettersCount) { - this._isStaticTransform = true + this._staticTransformValue = val; if (!options?.allowStatic) logger.warn("The transform value for " + this.idString() + " is a static value:", val); - // TODO: cleanup stuff not needed if no reactive transform + // cleanup stuff not needed if no reactive transform + if (options?.allowStatic) return; } // update value @@ -3776,6 +3819,11 @@ export class Pointer extends ReactiveValue { return obj; } + // don't proxyify Text nodes + if (globalThis.Text && obj instanceof Text) { + return obj; + } + // convert date to time if (obj instanceof Date) { return new Time(obj); @@ -4559,8 +4607,11 @@ export class Pointer extends ReactiveValue { // observe specific property else { // make sure the array index is a number - if (this.current_val instanceof Array) key = Number(key); - + if (this.current_val instanceof Array) { + if (!(typeof key == "number" || typeof key == "bigint" || typeof key == "string")) return; + key = Number(key); + } + if (bound_object) { if (!this.bound_change_observers.has(bound_object)) { this.bound_change_observers.set(bound_object, new Map()); From 6b45ace79174172e7d012a99931451b4670b04df Mon Sep 17 00:00:00 2001 From: benStre Date: Sun, 13 Oct 2024 18:20:11 +0200 Subject: [PATCH 24/27] fix getMetadata --- js_adapter/js_class_adapter.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/js_adapter/js_class_adapter.ts b/js_adapter/js_class_adapter.ts index 1af6101..7f57c03 100644 --- a/js_adapter/js_class_adapter.ts +++ b/js_adapter/js_class_adapter.ts @@ -112,7 +112,8 @@ export class Decorators { } public static getMetadata(context:DecoratorContext, key:string|symbol) { - return context.metadata[key]?.public?.[context.name] ?? context.metadata[key]?.constructor + const data = context.metadata[key]?.public?.[context.name] ?? context.metadata[key]?.constructor; + if (data !== Object) return data; } @@ -211,8 +212,9 @@ export class Decorators { } } - static assignType(type:string|Type, context: ClassFieldDecoratorContext|ClassGetterDecoratorContext|ClassMethodDecoratorContext, forceConjunction) { + static assignType(type:string|Type, context: ClassFieldDecoratorContext|ClassGetterDecoratorContext|ClassMethodDecoratorContext, forceConjunction = false) { let newType = this.getMetadata(context, Decorators.FORCE_TYPE); + if (newType instanceof Conjunction) { newType.add(type) } From 9a574e79e1403276b4c1a471de848197e5fcca44 Mon Sep 17 00:00:00 2001 From: benStre Date: Sun, 13 Oct 2024 18:20:41 +0200 Subject: [PATCH 25/27] fix regex for use() statements --- runtime/js_interface.ts | 1 + storage/storage.ts | 1 - types/function-utils.ts | 4 ++-- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime/js_interface.ts b/runtime/js_interface.ts index f6600f6..259fae0 100644 --- a/runtime/js_interface.ts +++ b/runtime/js_interface.ts @@ -23,6 +23,7 @@ export type js_interface_configuration = { override_silently?: (ref:T, value:T)=>void, // reset the reference, copy the value to the ref silently wrap_transform?: (value: T) => any, // returns a special wrapper value to use instead of the initial value of a transform pointer, + unwrap_transform?: (value: any) => T, // returns the original value from a wrapped value handle_transform?: (value: T, pointer: Pointer) => void, // gets called when a transform function produces a new value, default override behaviour is ignored allow_transform_value?: (type: Type, pointer: Pointer) => string|true, // returns true if the value can be wrapped with wrap transform, allows pointer union types class?: Class, // the corresponding JS class or a prototype diff --git a/storage/storage.ts b/storage/storage.ts index ecbbaf5..55ffe1f 100644 --- a/storage/storage.ts +++ b/storage/storage.ts @@ -8,7 +8,6 @@ import { NOT_EXISTING } from "../runtime/constants.ts"; import { Pointer, type MinimalJSRef, ReactiveValue } from "../runtime/pointers.ts"; import { localStorage } from "./storage-locations/local-storage-compat.ts"; import { MessageLogger } from "../utils/message_logger.ts"; -import { displayFatalError } from "../runtime/display.ts" import { Type } from "../types/type.ts"; import { addPersistentListener } from "../utils/persistent-listeners.ts"; import { Endpoint, LOCAL_ENDPOINT } from "../types/addressing.ts"; diff --git a/types/function-utils.ts b/types/function-utils.ts index 6166aeb..2aecea0 100644 --- a/types/function-utils.ts +++ b/types/function-utils.ts @@ -67,7 +67,7 @@ type Flag = 'standalone'|'silent-errors'|'allow-globals' function getUsedVars(fn: (...args:unknown[])=>unknown) { const source = fn.toString(); - const usedVarsSource = source.match(/^(?:(?:[\w\s*])+\(.*?\)\s*{|\(.*?\)\s*=>\s*[{(]?|.*?\s*=>\s*[{(]?)\s*use\s*\(([\s\S]*?)\)/)?.[1] + const usedVarsSource = source.match(/^(?:(?:[\w\s*])+\(.*?\)\s*{|\(.*?\)\s*=>\s*[{(]?|.*?\s*=>\s*[{(]?)\s*(?:return *)?use\s*\(([\s\S]*?)\)/)?.[1] if (!usedVarsSource) return {}; const _usedVars = usedVarsSource.split(",").map(v=>v.trim()).filter(v=>!!v) @@ -165,7 +165,7 @@ export function getSourceWithoutUsingDeclaration(fn: (...args:unknown[])=>unknow } return fnSource - .replace(/(?<=(?:(?:[\w\s*])+\(.*\)\s*{|\(.*\)\s*=>\s*{?|.*\s*=>\s*{?)\s*)(use\s*\((?:[\s\S]*?)\))/, 'true /*$1*/') + .replace(/(?<=(?:(?:[\w\s*])+\(.*\)\s*{|\(.*\)\s*=>\s*{?|.*\s*=>\s*{?)\s*)(?:return *)?(use\s*\((?:[\s\S]*?)\))/, 'true /*$1*/') } const isObjectMethod = (fnSrc:string) => { From ac421ebc3d571f052063a283ef4f8e5d7ee30faa Mon Sep 17 00:00:00 2001 From: benStre Date: Sun, 13 Oct 2024 18:22:08 +0200 Subject: [PATCH 26/27] error handling log improvements --- types/type.ts | 1 - utils/error-handling.ts | 15 ++++++++------- utils/interface-generator.ts | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/types/type.ts b/types/type.ts index da3857d..f25dfa8 100644 --- a/types/type.ts +++ b/types/type.ts @@ -209,7 +209,6 @@ export class Type extends ExtensibleFunction { for (const key of Object.keys(this.#template)) { // @ts-ignore this.#template is always a Tuple const required_type = this.#template[key]; - // check if can set property (has setter of value) const desc = Object.getOwnPropertyDescriptor(assign_to_object, key); diff --git a/utils/error-handling.ts b/utils/error-handling.ts index 21705ce..0f479f4 100644 --- a/utils/error-handling.ts +++ b/utils/error-handling.ts @@ -1,7 +1,8 @@ -import { ESCAPE_SEQUENCES } from "./logger.ts"; +import { ESCAPE_SEQUENCES, Logger } from "./logger.ts"; import DATEX_VERSION from "../VERSION.ts" import { sendReport } from "./error-reporting.ts"; -import { logger as defaultLogger } from "./global_values.ts"; + +const errorLogger = new Logger("Error"); /** * Represents an error that the UIX developer community knows about. @@ -21,13 +22,13 @@ export class KnownError extends Error { * @param [exit=true] Specifies whether the process should exit, defaults to `true` * @param [exitCode=1] Code to exit with if `exit` is set to true, defaults to `1` */ -export async function handleError(error: Error|string, logger = defaultLogger, exit = true, exitCode = 1) { +export async function handleError(error: Error|string, logger = errorLogger, exit = true, exitCode = 1) { + console.log(); if (typeof error === "string" || error instanceof String) { logger.error(error); } else if (error instanceof KnownError) { logger.error(error.message); if (error.solutions.length > 0) { - console.log(); logger.info(`Suggested Problem Solutions:\n${error.solutions.map(s => `- ${s}`).join("\n")}\n`); } if (error.quickFixes) { @@ -45,7 +46,7 @@ export async function handleError(error: Error|string, logger = defaultLogger, e details = stack.join("\n"); } else details = error.toString(); - logger.error(`An unexpected error occured.\n${ESCAPE_SEQUENCES.BOLD}DATEX${ESCAPE_SEQUENCES.DEFAULT} Version: ${DATEX_VERSION}\n${ESCAPE_SEQUENCES.BOLD}Deno${ESCAPE_SEQUENCES.DEFAULT} Version: ${Deno.version.deno}\n\n${details}`); + logger.error(`An unexpected error occured.\n${ESCAPE_SEQUENCES.BOLD}DATEX${ESCAPE_SEQUENCES.DEFAULT} Version: ${DATEX_VERSION}\n${ESCAPE_SEQUENCES.BOLD}Deno${ESCAPE_SEQUENCES.DEFAULT} Version: ${Deno.version.deno}${(Deno as any).uix ? ' (Deno for UIX)' : ''}\n\n${details}`); await sendReport("unexpectederror", { name: error.name, message: error.message, @@ -53,7 +54,7 @@ export async function handleError(error: Error|string, logger = defaultLogger, e }); } - if (exit) Deno.exit(exitCode); + if (exit && globalThis.Deno) Deno.exit(exitCode); } @@ -64,7 +65,7 @@ let unhandledRejectionHandlerEnabled = false; * and prevents the program from crashing. * @param customLogger a custom logger to use for logging unhandled rejections */ -export function enableUnhandledRejectionHandler(customLogger = defaultLogger) { +export function enableUnhandledRejectionHandler(customLogger = errorLogger) { if (unhandledRejectionHandlerEnabled) return; unhandledRejectionHandlerEnabled = true; // clean error presentation diff --git a/utils/interface-generator.ts b/utils/interface-generator.ts index 28db8d2..0b4a3de 100644 --- a/utils/interface-generator.ts +++ b/utils/interface-generator.ts @@ -1,7 +1,7 @@ // generates typescript code for @namespace JS classes with static @expose methods // (matching code to call the methods on another endpoint) import { Pointer } from "../datex_all.ts"; -import { $$, Datex } from "../mod.ts"; +import { $, Datex } from "../mod.ts"; import { DX_PTR, DX_SOURCE } from "../runtime/constants.ts"; import { Storage } from "../storage/storage.ts"; import { indent } from "./indent.ts"; @@ -147,14 +147,14 @@ async function getValueTSCode(module_name:string, name:string, value: any, no_po // log warning for primitive non-pointer values (cannot be converted to pointer) if (type.is_primitive && (!is_pointer || implicitly_converted_primitives.get(module_name)?.has(name))) { - if (!is_datex_module) code += name ? `logger.warn('The export "${name}" cannot be converted to a shared value. Consider explicitly converting it to a primitive pointer using $$().');\n` : `logger.warn('The default export cannot be converted to a shared value. Consider explicitly converting it to a primitive pointer using $$().');\n` + if (!is_datex_module) code += name ? `logger.warn('The export "${name}" cannot be converted to a shared value. Consider explicitly converting it to a primitive pointer using $().');\n` : `logger.warn('The default export cannot be converted to a shared value. Consider explicitly converting it to a primitive pointer using $().');\n` implicitly_converted_primitives.getAuto(module_name).add(name); } // other value -> create pointers else { if (implicitly_converted.get(module_name)?.has(name)) { - if (!is_datex_module) code += name ? `logger.warn('The export "${name}" was implicitly converted to a shared pointer value. This might have unintended side effects. Consider explicitly converting it to a ${type} pointer using $$().');\n` : `logger.warn('The default export was implicitly converted to a shared pointer value. This might have unintended side effects. Consider explicitly converting it to a ${type} pointer using $$().');\n` + if (!is_datex_module) code += name ? `logger.warn('The export "${name}" was implicitly converted to a shared pointer value. This might have unintended side effects. Consider explicitly converting it to a ${type} pointer using $().');\n` : `logger.warn('The default export was implicitly converted to a shared pointer value. This might have unintended side effects. Consider explicitly converting it to a ${type} pointer using $().');\n` } // special convertions for non-pointer values @@ -166,7 +166,7 @@ async function getValueTSCode(module_name:string, name:string, value: any, no_po value = {} for (const prop of Object.getOwnPropertyNames(original_value)) { if (prop != "length" && prop != "name" && prop != "prototype") { - value[prop] = typeof original_value[prop] == "function" ? $$(Datex.Function.createFromJSFunction(original_value[prop], original_value)) : $$(original_value[prop]); + value[prop] = typeof original_value[prop] == "function" ? $(Datex.Function.createFromJSFunction(original_value[prop], original_value)) : $(original_value[prop]); value[prop][BACKEND_EXPORT] = true; if (original_value[prop]!=undefined) original_value[prop][BACKEND_EXPORT] = true; } @@ -181,7 +181,7 @@ async function getValueTSCode(module_name:string, name:string, value: any, no_po // log warning for non-pointer arrays and object (ignore defaults aka 'no_pointer') else if ((type == Datex.Type.std.Array || type == Datex.Type.std.Object || type == Datex.Type.js.NativeObject) && !no_pointer) { - if (!is_datex_module) code += name ? `logger.warn('The export "${name}" was implicitly converted to a shared pointer value. This might have unintended side effects. Consider explicitly converting it to a ${type} pointer using $$().');\n` : `logger.warn('The default export was implicitly converted to a shared pointer value. This might have unintended side effects. Consider explicitly converting it to a ${type} pointer using $$().');\n` + if (!is_datex_module) code += name ? `logger.warn('The export "${name}" was implicitly converted to a shared pointer value. This might have unintended side effects. Consider explicitly converting it to a ${type} pointer using $().');\n` : `logger.warn('The default export was implicitly converted to a shared pointer value. This might have unintended side effects. Consider explicitly converting it to a ${type} pointer using $().');\n` implicitly_converted.getAuto(module_name).add(name); } } From 3ff1ee3543c7f21eb96df2da9f1c46b65b3553b3 Mon Sep 17 00:00:00 2001 From: benStre Date: Sun, 13 Oct 2024 18:39:50 +0200 Subject: [PATCH 27/27] sql storage, sqlite db improvements --- storage/storage-locations/mysql-db.ts | 2 +- storage/storage-locations/sql-db.ts | 48 ++++++++++++++------------ storage/storage-locations/sqlite-db.ts | 8 +++++ 3 files changed, 35 insertions(+), 23 deletions(-) diff --git a/storage/storage-locations/mysql-db.ts b/storage/storage-locations/mysql-db.ts index 18804ce..abdc6a0 100644 --- a/storage/storage-locations/mysql-db.ts +++ b/storage/storage-locations/mysql-db.ts @@ -49,7 +49,7 @@ export class MySQLStorageLocation extends SQLDBStorageLocation extends } - async #query(query_string:string, query_params:any[]|undefined, returnRawResult: true, columnNames?:(keyof row)[]): Promise<{rows:row[], result:ExecuteResult}> - async #query(query_string:string, query_params:any[]|undefined, returnRawResult: false, columnNames?:(keyof row)[]): Promise + async #query(query_string:string, query_params:any[]|undefined, returnRawResult: true): Promise<{rows:row[], result:ExecuteResult}> + async #query(query_string:string, query_params:any[]|undefined, returnRawResult: false): Promise async #query(query_string:string, query_params?:any[]): Promise - async #query(query_string:string, query_params?:any[], returnRawResult?: boolean, columnNames?:string[]): Promise { + async #query(query_string:string, query_params?:any[], returnRawResult?: boolean): Promise { // TODO: only workaround for sqlite, replace all " with ' in queries if (this.useSingleQuotes) { @@ -227,15 +227,6 @@ export abstract class SQLDBStorageLocation extends if (!query_string) throw new Error("empty query"); try { const result = await this.executeQuery(query_string, query_params); - if (result.rows?.[0] instanceof Array && columnNames) { - result.rows = result.rows.map(row => { - const obj:Record = {} - for (let i = 0; i < columnNames.length; i++) { - obj[columnNames[i]] = row[i] - } - return obj - }) - } if (returnRawResult) return {rows: result.rows ?? [], result}; else return result.rows ?? []; } catch (e) { @@ -267,8 +258,8 @@ export abstract class SQLDBStorageLocation extends return String.fromCharCode.apply(null, new Uint8Array(value) as unknown as number[]) } - async #queryFirst(query_string:string, query_params?:any[], column_names?: (keyof row)[]): Promise { - return (await this.#query(query_string, query_params, false, column_names))?.[0] + async #queryFirst(query_string:string, query_params?:any[]): Promise { + return (await this.#query(query_string, query_params, false))?.[0] } async #createTableIfNotExists(definition: TableDefinition) { @@ -945,7 +936,7 @@ export abstract class SQLDBStorageLocation extends joins.forEach(join => builder.join(join)); const outerBuilder = new Query() - .select(options.returnRaw ? `*` :`DISTINCT ${this.supportsSQLCalcFoundRows?'SQL_CALC_FOUND_ROWS ' : ''}${this.#pointerMysqlColumnName} as ${this.#pointerMysqlColumnName}` + ( + .select(options.returnRaw ? `*` :`DISTINCT ${this.supportsSQLCalcFoundRows?'SQL_CALC_FOUND_ROWS ' : 'COUNT(*) OVER () AS foundRows, '}${this.#pointerMysqlColumnName} as ${this.#pointerMysqlColumnName}` + ( options.returnKeys ? `, ${this.#metaTables.items.name}.key as map_key` : '' )) .table('__placeholder__'); @@ -959,7 +950,7 @@ export abstract class SQLDBStorageLocation extends // no computed properties else { - builder.select(`DISTINCT ${this.supportsSQLCalcFoundRows?'SQL_CALC_FOUND_ROWS ' : ''}\`${this.#typeToTableName(valueType)}\`.${this.#pointerMysqlColumnName} as ${this.#pointerMysqlColumnName}` + ( + builder.select(`DISTINCT ${this.supportsSQLCalcFoundRows?'SQL_CALC_FOUND_ROWS ' : 'COUNT(*) OVER () AS foundRows, '}\`${this.#typeToTableName(valueType)}\`.${this.#pointerMysqlColumnName} as ${this.#pointerMysqlColumnName}` + ( options.returnKeys ? `, ${this.#metaTables.items.name}.key as map_key` : '' )); this.appendBuilderConditions(builder, options, where) @@ -972,7 +963,10 @@ export abstract class SQLDBStorageLocation extends await this.#getTableForType(type) } - const queryResult = await this.#query<{_ptr_id:string, map_key: string}>(query, undefined, false, ['_ptr_id', 'map_key']); + SQLDBStorageLocation.debug(true); + const queryResult = await this.#query<{_ptr_id:string, map_key: string}>(query); + SQLDBStorageLocation.debug(false); + const ptrIds = options.returnKeys ? queryResult.map(({map_key}) => map_key.split(".")[1].slice(1)) : queryResult.map(({_ptr_id}) => _ptr_id) const limitedPtrIds = options.returnPointerIds ? // offset and limit manually after query @@ -980,8 +974,17 @@ export abstract class SQLDBStorageLocation extends // use ptrIds returned from query (already limited) ptrIds - // TODO: atomic operations for multiple queries - const {foundRows} = (options?.returnAdvanced ? await this.#queryFirst<{foundRows: number}>("SELECT FOUND_ROWS() as foundRows") : null, undefined, ['foundRows']) ?? {foundRows: -1} + let foundRows = -1; + + if (options?.returnAdvanced) { + if (this.supportsSQLCalcFoundRows) { + const res = await this.#queryFirst<{foundRows: number}>("SELECT FOUND_ROWS() as foundRows"); + if (res?.foundRows != undefined) foundRows = res.foundRows; + } + else { + foundRows = (queryResult[0] as any)?.foundRows ?? 0; + } + } // remember pointer table for (const ptrId of ptrIds) { @@ -1571,18 +1574,19 @@ export abstract class SQLDBStorageLocation extends ) this.#existingPointersCache.add(pointerId); - return type.cast(object, undefined, undefined, false); + return type.cast(object, undefined, undefined, false, undefined, undefined, true); } } private async assignPointerProperty(object:Record, colName:string, type:string, foreignPtr:boolean, foreignTable?:string) { + // custom conversions: // convert blob strings to ArrayBuffer if (type == "blob" && typeof object[colName] == "string") { object[colName] = this.#stringToBinary(object[colName] as string) } // convert Date ot Time - else if (object[colName] instanceof Date) { + else if (object[colName] instanceof Date || type == "datetime") { object[colName] = new Time(object[colName] as Date) } @@ -1590,7 +1594,7 @@ export abstract class SQLDBStorageLocation extends else if (typeof object[colName] == "number" && (type == "tinyint" || type == "boolean")) { object[colName] = Boolean(object[colName]) } - + // is an object type with a template if (foreignPtr) { if (typeof object[colName] == "string") { diff --git a/storage/storage-locations/sqlite-db.ts b/storage/storage-locations/sqlite-db.ts index 5bb4b2c..f9be451 100644 --- a/storage/storage-locations/sqlite-db.ts +++ b/storage/storage-locations/sqlite-db.ts @@ -7,6 +7,12 @@ import { logger } from "../../datex_all.ts"; import { ptr_cache_path } from "../../runtime/cache_path.ts"; import { Path } from "../../utils/path.ts"; +function regexp(pattern: string, value: string): boolean { + console.log("REGEXP", pattern, value) + const regex = new RegExp(pattern, "i"); + return regex.test(value); +} + export class SqliteStorageLocation extends SQLDBStorageLocation<{db: string}> { name = "SQLITE_DB" @@ -33,6 +39,8 @@ export class SqliteStorageLocation extends SQLDBStorageLocation<{db: string}> { } this.#db = new Database(path); + this.#db.function("REGEXP", regexp); + logger.info("Using SQLite database " + this.options.db + " as storage location") return true; }