diff --git a/package-lock.json b/package-lock.json index 2be2d17833..66e0be5a6e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -314,7 +314,6 @@ "resolved": "https://registry.npmjs.org/@aws-sdk/abort-controller/-/abort-controller-3.370.0.tgz", "integrity": "sha512-/W4arzC/+yVW/cvEXbuwvG0uly4yFSZnnIA+gkqgAm+0HVfacwcPpNf4BjyxjnvIdh03l7w2DriF6MlKUfiQ3A==", "license": "Apache-2.0", - "peer": true, "dependencies": { "@aws-sdk/types": "3.370.0", "tslib": "^2.5.0" @@ -328,7 +327,6 @@ "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.370.0.tgz", "integrity": "sha512-8PGMKklSkRKjunFhzM2y5Jm0H2TBu7YRNISdYzXLUHKSP9zlMEYagseKVdmox0zKHf1LXVNuSlUV2b6SRrieCQ==", "license": "Apache-2.0", - "peer": true, "dependencies": { "@smithy/types": "^1.1.0", "tslib": "^2.5.0" @@ -342,7 +340,6 @@ "resolved": "https://registry.npmjs.org/@smithy/types/-/types-1.2.0.tgz", "integrity": "sha512-z1r00TvBqF3dh4aHhya7nz1HhvCg4TRmw51fjMrh5do3h+ngSstt/yKlNbHeb9QxJmFbmN8KEVSWgb1bRvfEoA==", "license": "Apache-2.0", - "peer": true, "dependencies": { "tslib": "^2.5.0" }, @@ -3622,6 +3619,10 @@ "resolved": "packages/type-gen", "link": true }, + "node_modules/@based/types": { + "resolved": "packages/types", + "link": true + }, "node_modules/@based/ui": { "version": "8.2.0-alpha.4", "resolved": "https://registry.npmjs.org/@based/ui/-/ui-8.2.0-alpha.4.tgz", @@ -3869,6 +3870,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=18" }, @@ -3891,6 +3893,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=18" } @@ -6582,6 +6585,7 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.0.tgz", "integrity": "sha512-xpr/lmLPQEj+TUnHmR+Ab91/glhJvsqcjB+yY0Ix9GO70H6Lb4FHH5GeqdOE5btAx7eIMwuHkp4H2MSkLcqWbA==", "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~6.21.0" } @@ -6623,6 +6627,7 @@ "integrity": "sha512-RFA/bURkcKzx/X9oumPG9Vp3D3JUgus/d0b67KB0t5S/raciymilkOa66olh78MUI92QLbEJevO7rvqU/kjwKA==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@types/prop-types": "*", "csstype": "^3.0.2" @@ -6662,6 +6667,13 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/validator": { + "version": "13.15.10", + "resolved": "https://registry.npmjs.org/@types/validator/-/validator-13.15.10.tgz", + "integrity": "sha512-T8L6i7wCuyoK8A/ZeLYt1+q0ty3Zb9+qbSSvrIVitzT3YjZqkTZ40IbRsPanlB4h1QB3JVL1SYCdR6ngtFYcuA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/ws": { "version": "8.18.1", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz", @@ -8553,6 +8565,7 @@ "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", "hasInstallScript": true, "license": "MIT", + "peer": true, "bin": { "esbuild": "bin/esbuild" }, @@ -8963,6 +8976,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/flat": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/flat/-/flat-6.0.1.tgz", + "integrity": "sha512-/3FfIa8mbrg3xE7+wAhWeV+bd7L2Mof+xtZb5dRDKZ+wDvYJK4WDYeIOuOhre5Yv5aQObZrlbRmk3RTSiuQBtw==", + "license": "BSD-3-Clause", + "bin": { + "flat": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/follow-redirects": { "version": "1.15.11", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", @@ -9475,6 +9500,7 @@ "resolved": "https://registry.npmjs.org/ink/-/ink-5.2.1.tgz", "integrity": "sha512-BqcUyWrG9zq5HIwW6JcfFHsIYebJkWWb4fczNah1goUO0vv5vneIlfwuS85twyJ5hYR/y18FlAYUxrO9ChIWVg==", "license": "MIT", + "peer": true, "dependencies": { "@alcalzone/ansi-tokenize": "^0.1.3", "ansi-escapes": "^7.0.0", @@ -9879,7 +9905,6 @@ "integrity": "sha512-PIeMbHqMt4DnUP3MA/Flc0HElYjMXArsw1qwJZcm9sqR8mq3l8NYizFMty0pWwE/tzIGH3EKK5+jes5mAr85yw==", "dev": true, "license": "MIT", - "peer": true, "funding": { "type": "GitHub Sponsors ❤", "url": "https://github.com/sponsors/dmonad" @@ -10038,7 +10063,6 @@ "integrity": "sha512-gcxmNFzA4hv8UYi8j43uPlQ7CGcyMJ2KQb5kZASw6SnAKAf10hK12i2fjrS3Cl/ugZa5Ui6WwIu1/6MIXiHttQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "isomorphic.js": "^0.2.4" }, @@ -10911,7 +10935,8 @@ "resolved": "https://registry.npmjs.org/overlayscrollbars/-/overlayscrollbars-2.12.0.tgz", "integrity": "sha512-mWJ5MOkcZ/ljHwfLw8+bN0V9ziGCoNoqULcp994j5DTGNQvnkWKWkA7rnO29Kyew5AoHxUnJ4Ndqfcl0HSQjXg==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/overlayscrollbars-react": { "version": "0.5.6", @@ -11254,6 +11279,7 @@ "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", "dev": true, "license": "MIT", + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -11491,6 +11517,7 @@ "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", "license": "MIT", + "peer": true, "dependencies": { "loose-envify": "^1.1.0" }, @@ -11503,6 +11530,7 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", "license": "MIT", + "peer": true, "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" @@ -12806,6 +12834,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -13061,6 +13090,7 @@ "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -13247,6 +13277,20 @@ "dev": true, "license": "MIT" }, + "node_modules/valibot": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/valibot/-/valibot-1.1.0.tgz", + "integrity": "sha512-Nk8lX30Qhu+9txPYTwM0cFlWLdPFsFr6LblzqIySfbZph9+BFsAHsNvHOymEviUepeIW6KFHzpX8TKhbptBXXw==", + "license": "MIT", + "peerDependencies": { + "typescript": ">=5" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, "node_modules/validate-npm-package-license": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", @@ -13281,6 +13325,7 @@ "integrity": "sha512-BxAKBWmIbrDgrokdGZH1IgkIk/5mMHDreLDmCJ0qpyJaAteP8NvMhkwr/ZCQNqNH97bw/dANTE9PDzqwJghfMQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.5.0", @@ -13397,6 +13442,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -13750,6 +13796,7 @@ "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz", "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==", "license": "MIT", + "peer": true, "engines": { "node": ">=10.0.0" }, @@ -14366,14 +14413,18 @@ "name": "@based/schema", "version": "5.1.3", "dependencies": { - "@based/utils": "1.2.0", + "@based/hash": "^1.1.0", + "@based/utils": "^1.2.0", + "flat": "^6.0.1", "picocolors": "^1.1.0", + "valibot": "^1.1.0", "validator": "^13.15.20" }, "devDependencies": { "@saulx/prettier-config": "2.0.0", "@saulx/tsconfig": "^1.1.0", "@types/node": "^22.5.3", + "@types/validator": "^13.15.10", "tsx": "^4.19.0", "typescript": "^5.6.3" } @@ -14419,6 +14470,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -14587,6 +14639,10 @@ "@types/node": "*" } }, + "packages/types": { + "name": "@based/types", + "version": "1.0.0" + }, "packages/utils": { "name": "@based/utils", "version": "1.2.0", diff --git a/packages/db/native/modify/modify.zig b/packages/db/native/modify/modify.zig index 6daea4090b..e2fccfc4d3 100644 --- a/packages/db/native/modify/modify.zig +++ b/packages/db/native/modify/modify.zig @@ -337,8 +337,10 @@ fn modifyInternal(env: napi.c.napi_env, info: napi.c.napi_callback_info, resCoun db.expire(&ctx); const newDirtyRanges = ctx.dirtyRanges.values(); + std.debug.print("newDirtyRanges.len {d} - {d}\n", .{ newDirtyRanges.len, dirtyRanges.len }); assert(newDirtyRanges.len < dirtyRanges.len); _ = napi.c.memcpy(dirtyRanges.ptr, newDirtyRanges.ptr, newDirtyRanges.len * 8); + std.debug.print("newDirtyRanges.len {d} - {d}\n", .{ newDirtyRanges.len, dirtyRanges.len }); dirtyRanges[newDirtyRanges.len] = 0.0; writeoutPrevNodeId(&ctx, resCount, ctx.id); } diff --git a/packages/db/src/client/index.ts b/packages/db/src/client/index.ts index 3c3694e1b2..69d4c4e083 100644 --- a/packages/db/src/client/index.ts +++ b/packages/db/src/client/index.ts @@ -1,9 +1,9 @@ import { - MigrateFns, parse, - Schema, - StrictSchema, - SchemaChecksum, + type Schema, + type SchemaIn, + type SchemaMigrateFns, + type SchemaOut, } from '@based/schema' import { BasedDbQuery, QueryByAliasObj } from './query/BasedDbQuery.js' import { debugMode } from '../utils.js' @@ -38,7 +38,7 @@ export class DbClient extends DbShared { this.hooks = hooks this.maxModifySize = maxModifySize this.modifyCtx = new Ctx( - 0, + { locales: {}, types: {}, hash: 0 }, new Uint8Array( new ArrayBuffer(Math.min(1e3, maxModifySize), { maxByteLength: maxModifySize, @@ -75,21 +75,21 @@ export class DbClient extends DbShared { } async setSchema( - schema: Schema, - transformFns?: MigrateFns, - ): Promise { + schema: SchemaIn, + transformFns?: SchemaMigrateFns, + ): Promise { const strictSchema = parse(schema).schema await this.drain() const schemaChecksum = await this.hooks.setSchema( - strictSchema as StrictSchema, + strictSchema as Schema, transformFns, ) if (this.stopped) { - return this.schema.hash + return this.schema?.hash ?? 0 } if (schemaChecksum !== this.schema?.hash) { await this.once('schema') - return this.schema.hash + return this.schema?.hash ?? 0 } return schemaChecksum } @@ -261,7 +261,7 @@ export class DbClient extends DbShared { destroy() { this.stop() - delete this.listeners + this.listeners = {} } stop() { diff --git a/packages/db/src/client/modify/Ctx.ts b/packages/db/src/client/modify/Ctx.ts index 07e54e2e11..21890a9ee1 100644 --- a/packages/db/src/client/modify/Ctx.ts +++ b/packages/db/src/client/modify/Ctx.ts @@ -1,27 +1,28 @@ import { writeUint64 } from '@based/utils' -import type { SchemaTypeDef, PropDef } from '@based/schema/def' -import type { LangCode } from '@based/schema' +import type { LangCode, MainDef, SchemaOut, TypeDef } from '@based/schema' import { type ModifyOp } from './types.js' import type { Tmp } from './Tmp.js' export class Ctx { - constructor(schemaChecksum: number, array: Uint8Array) { + constructor(schema: SchemaOut, array: Uint8Array) { this.array = array this.max = array.buffer.maxByteLength - 4 // dataLen this.size = array.buffer.byteLength - 4 - writeUint64(array, schemaChecksum, 0) + this.schema = schema + writeUint64(array, schema.hash, 0) } + schema: SchemaOut start: number index: number = 8 - schema: SchemaTypeDef + typeDef: TypeDef array: Uint8Array max: number size: number unsafe?: boolean operation: ModifyOp - main: Map = new Map() + main: Map = new Map() draining: Promise - scheduled: Promise + scheduled?: Promise locale: LangCode sort: number = 0 sortText: number = 0 @@ -29,10 +30,12 @@ export class Ctx { cursor: { type?: number prop?: number - main?: number + main: number operation?: ModifyOp upserting?: boolean - } = {} + } = { + main: 0, + } batch: { count?: number promises?: Tmp[] diff --git a/packages/db/src/client/modify/Tmp.ts b/packages/db/src/client/modify/Tmp.ts index cf908b4c90..cdc44b7e17 100644 --- a/packages/db/src/client/modify/Tmp.ts +++ b/packages/db/src/client/modify/Tmp.ts @@ -1,7 +1,7 @@ import { readUint32 } from '@based/utils' import { Ctx } from './Ctx.js' -import { errorMap, errors } from './error.js' -import { SchemaTypeDef } from '@based/schema/def' +import { errorMap } from './error.js' +import type { TypeDef } from '@based/schema' const promisify = (tmp: Tmp) => { if (!tmp.promise) { @@ -39,15 +39,15 @@ export const rejectTmp = (tmp: Tmp) => { export class Tmp implements Promise { constructor(ctx: Ctx) { ctx.batch.count ??= 0 - this.#schema = ctx.schema + this.#schema = ctx.typeDef this.batch = ctx.batch this.tmpId = ctx.batch.count++ } [Symbol.toStringTag]: 'ModifyPromise' - #schema: SchemaTypeDef + #schema: TypeDef #id: number #err: number - get error(): Error { + get error(): Error | void { if (this.batch.ready && !this.id) { if (this.#err in errorMap) { return new errorMap[this.#err](this.#id, this.#schema) @@ -55,7 +55,7 @@ export class Tmp implements Promise { return this.batch.error || Error('Modify error') } } - get id(): number { + get id(): number | void { if (this.batch.res) { this.#err ??= this.batch.res[this.tmpId * 5 + 4] this.#id ??= readUint32(this.batch.res, this.tmpId * 5) @@ -65,8 +65,8 @@ export class Tmp implements Promise { tmpId: number batch: Ctx['batch'] promise?: Promise - resolve?: (value: number | PromiseLike) => void - reject?: (reason?: any) => void + resolve: (value: number | PromiseLike) => void + reject: (reason?: any) => void then( onfulfilled?: ((value: number) => Res1 | PromiseLike) | null, onrejected?: ((reason: any) => Res2 | PromiseLike) | null, diff --git a/packages/db/src/client/modify/create/index.ts b/packages/db/src/client/modify/create/index.ts index 3e7591c19f..3184acad02 100644 --- a/packages/db/src/client/modify/create/index.ts +++ b/packages/db/src/client/modify/create/index.ts @@ -1,4 +1,3 @@ -import { SchemaTypeDef, TEXT } from '@based/schema/def' import { Ctx } from '../Ctx.js' import { writeObject } from '../props/object.js' import { reserve } from '../resize.js' @@ -23,129 +22,133 @@ import { SWITCH_ID_CREATE_RING, SWITCH_ID_CREATE_UNSAFE, } from '../types.js' -import { inverseLangMap, LangCode, langCodesMap } from '@based/schema' +import { + inverseLangMap, + LangCode, + langCodesMap, + type TypeDef, +} from '@based/schema' import { writeSeparate } from '../props/separate.js' import { writeString } from '../props/string.js' import { writeU32, writeU8 } from '../uint.js' import { getValidSchema, validatePayload } from '../validate.js' import { handleError } from '../error.js' -const writeDefaults = (ctx: Ctx) => { - if (!ctx.schema.hasSeperateDefaults) { - return - } - if (ctx.defaults !== ctx.schema.separateDefaults.props.size) { - for (const def of ctx.schema.separateDefaults.props.values()) { - const type = def.typeIndex - if (ctx.schema.separateDefaults.bufferTmp[def.prop] === 0) { - writeSeparate(ctx, def, def.default) - continue - } - - if (type !== TEXT) { - continue - } - - const buf = ctx.schema.separateTextSort.bufferTmp - const amount = ctx.schema.localeSize + 1 - const len = amount * ctx.schema.separateTextSort.props.length - for (const sortDef of ctx.schema.separateTextSort.props) { - const index = sortDef.prop * amount - if (buf[index] === 0) { - continue - } - for (let i = index + 1; i < len + index; i++) { - const lang = buf[i] as LangCode - if (lang === 0) { - continue - } - const val = def.default[inverseLangMap.get(lang)] - if (val !== undefined) { - writeString(ctx, def, val, lang) - } - } - } - } - } -} - -const writeSortable = (ctx: Ctx) => { - if (!ctx.schema.hasSeperateSort) { - return - } - if (ctx.sort !== ctx.schema.separateSort.size) { - reserve(ctx, 3) - writeU8(ctx, ADD_EMPTY_SORT) - const index = ctx.index - ctx.index += 2 - const start = ctx.index - for (const def of ctx.schema.separateSort.props) { - if (ctx.schema.separateSort.bufferTmp[def.prop] === 0) { - reserve(ctx, 1) - writeU8(ctx, def.prop) - } - } - writeUint16(ctx.array, ctx.index - start, index) - } -} - -const writeSortableText = (ctx: Ctx) => { - if (!ctx.schema.hasSeperateTextSort) { - return - } - - if (ctx.sortText !== ctx.schema.separateTextSort.size) { - reserve(ctx, 3) - writeU8(ctx, ADD_EMPTY_SORT_TEXT) - const index = ctx.index - ctx.index += 2 - const start = ctx.index - const amount = ctx.schema.localeSize + 1 - const len = amount * ctx.schema.separateTextSort.props.length - const buf = ctx.schema.separateTextSort.bufferTmp - for (const def of ctx.schema.separateTextSort.props) { - const index = def.prop * amount - if (buf[index] === 0) { - continue - } - reserve(ctx, 2) - writeU8(ctx, def.prop) - writeU8(ctx, buf[index]) - for (let i = index + 1; i < len + index; i++) { - const lang = buf[i] - if (lang === 0) { - continue - } - reserve(ctx, 1) - writeU8(ctx, lang) - } - } - writeUint16(ctx.array, ctx.index - start, index) - if (ctx.sortText) { - buf.set(ctx.schema.separateTextSort.buffer, 0) - } - } -} - -const writeCreateTs = (ctx: Ctx, payload: any) => { - if (!ctx.schema.createTs) { - return - } - let createTs: number - for (const prop of ctx.schema.createTs) { - if (getByPath(payload, prop.path) !== undefined) { - continue - } - createTs ??= Date.now() - writeMainValue(ctx, prop, createTs) - } -} +// const writeDefaults = (ctx: Ctx) => { +// if (!ctx.typeDef.hasSeperateDefaults) { +// return +// } +// if (ctx.defaults !== ctx.typeDef.separateDefaults.props.size) { +// for (const def of ctx.typeDef.separateDefaults.props.values()) { +// if (ctx.typeDef.separateDefaults.bufferTmp[def.prop] === 0) { +// writeSeparate(ctx, def, def.default) +// continue +// } + +// if (def.type !== 'text') { +// continue +// } + +// const buf = ctx.typeDef.separateTextSort.bufferTmp +// const amount = ctx.typeDef.localeSize + 1 +// const len = amount * ctx.typeDef.separateTextSort.props.length +// for (const sortDef of ctx.typeDef.separateTextSort.props) { +// const index = sortDef.prop * amount +// if (buf[index] === 0) { +// continue +// } +// for (let i = index + 1; i < len + index; i++) { +// const lang = buf[i] as LangCode +// if (lang === 0) { +// continue +// } +// const val = def.default[inverseLangMap.get(lang)] +// if (val !== undefined) { +// writeString(ctx, def, val, lang) +// } +// } +// } +// } +// } +// } + +// const writeSortable = (ctx: Ctx) => { +// if (!ctx.typeDef.hasSeperateSort) { +// return +// } +// if (ctx.sort !== ctx.typeDef.separateSort.size) { +// reserve(ctx, 3) +// writeU8(ctx, ADD_EMPTY_SORT) +// const index = ctx.index +// ctx.index += 2 +// const start = ctx.index +// for (const def of ctx.typeDef.separateSort.props) { +// if (ctx.typeDef.separateSort.bufferTmp[def.prop] === 0) { +// reserve(ctx, 1) +// writeU8(ctx, def.prop) +// } +// } +// writeUint16(ctx.array, ctx.index - start, index) +// } +// } + +// const writeSortableText = (ctx: Ctx) => { +// if (!ctx.typeDef.hasSeperateTextSort) { +// return +// } + +// if (ctx.sortText !== ctx.typeDef.separateTextSort.size) { +// reserve(ctx, 3) +// writeU8(ctx, ADD_EMPTY_SORT_TEXT) +// const index = ctx.index +// ctx.index += 2 +// const start = ctx.index +// const amount = ctx.typeDef.localeSize + 1 +// const len = amount * ctx.typeDef.separateTextSort.props.length +// const buf = ctx.typeDef.separateTextSort.bufferTmp +// for (const def of ctx.typeDef.separateTextSort.props) { +// const index = def.prop * amount +// if (buf[index] === 0) { +// continue +// } +// reserve(ctx, 2) +// writeU8(ctx, def.prop) +// writeU8(ctx, buf[index]) +// for (let i = index + 1; i < len + index; i++) { +// const lang = buf[i] +// if (lang === 0) { +// continue +// } +// reserve(ctx, 1) +// writeU8(ctx, lang) +// } +// } +// writeUint16(ctx.array, ctx.index - start, index) +// if (ctx.sortText) { +// buf.set(ctx.typeDef.separateTextSort.buffer, 0) +// } +// } +// } + +// const writeCreateTs = (ctx: Ctx, payload: any) => { +// if (!ctx.typeDef.createTs) { +// return +// } +// let createTs: number +// for (const prop of ctx.typeDef.createTs) { +// if (getByPath(payload, prop.path) !== undefined) { +// continue +// } +// createTs ??= Date.now() +// writeMainValue(ctx, prop, createTs) +// } +// } export const writeCreate = ( ctx: Ctx, - schema: SchemaTypeDef, + schema: TypeDef, payload: any, - opts: ModifyOpts, + opts?: ModifyOpts, ) => { validatePayload(payload) @@ -159,6 +162,7 @@ export const writeCreate = ( val = val?.[key] } if (val !== undefined) { + // @ts-ignore obj[key] = def.hooks.create(val, obj) } } @@ -168,27 +172,27 @@ export const writeCreate = ( payload = schema.hooks.create(payload) || payload } - if (ctx.defaults) { - ctx.defaults = 0 - schema.separateDefaults?.bufferTmp.fill(0) - } + // if (ctx.defaults) { + // ctx.defaults = 0 + // schema.separateDefaults?.bufferTmp.fill(0) + // } - if (ctx.sort) { - ctx.sort = 0 - schema.separateSort.bufferTmp.set(schema.separateSort.buffer) - } + // if (ctx.sort) { + // ctx.sort = 0 + // schema.separateSort.bufferTmp.set(schema.separateSort.buffer) + // } - if (ctx.sortText) { - ctx.sortText = 0 - schema.separateTextSort.bufferTmp.set(schema.separateTextSort.buffer) - } + // if (ctx.sortText) { + // ctx.sortText = 0 + // schema.separateTextSort.bufferTmp.set(schema.separateTextSort.buffer) + // } - ctx.schema = schema + ctx.typeDef = schema ctx.operation = CREATE ctx.unsafe = opts?.unsafe - ctx.locale = opts?.locale && langCodesMap.get(opts.locale) + ctx.locale = (opts?.locale && langCodesMap.get(opts.locale)) || 0 // TODO: can we remove this (and just init main buffer here?) - ctx.cursor.main = null + ctx.cursor.main = 0 reserve(ctx, FULL_CURSOR_SIZE) writeTypeCursor(ctx) @@ -206,15 +210,15 @@ export const writeCreate = ( writeU8(ctx, SWITCH_ID_CREATE) } const index = ctx.index - writeObject(ctx, ctx.schema.tree, payload) - if (ctx.index === index || ctx.schema.mainLen === 0) { + writeObject(ctx, ctx.typeDef, payload) + if (ctx.index === index || ctx.typeDef.size === 0) { reserve(ctx, PROP_CURSOR_SIZE) writeMainCursor(ctx) } - writeCreateTs(ctx, payload) - writeDefaults(ctx) - writeSortable(ctx) - writeSortableText(ctx) + // writeCreateTs(ctx, payload) + // writeDefaults(ctx) + // writeSortable(ctx) + // writeSortableText(ctx) while (ctx.index < ctx.start + 5) { writeU8(ctx, PADDING) } @@ -224,7 +228,7 @@ export function create( db: DbClient, type: string, payload: any, - opts: ModifyOpts, + opts?: ModifyOpts, ): Promise { const schema = getValidSchema(db, type) const ctx = db.modifyCtx diff --git a/packages/db/src/client/modify/create/mark.ts b/packages/db/src/client/modify/create/mark.ts index d8366cc86c..fe22d558f2 100644 --- a/packages/db/src/client/modify/create/mark.ts +++ b/packages/db/src/client/modify/create/mark.ts @@ -1,55 +1,58 @@ -import { PropDef } from '@based/schema/def' import { Ctx } from '../Ctx.js' import { CREATE } from '../types.js' -import { LangCode } from '@based/schema' +import { type LangCode, type LeafDef } from '@based/schema' -export const markString = (ctx: Ctx, def: PropDef) => { +export const markString = (ctx: Ctx, def: LeafDef) => { if (ctx.operation === CREATE) { - ctx.schema.separateSort.bufferTmp[def.prop] = 2 - ctx.sort++ - if (ctx.schema.hasSeperateDefaults) { - ctx.schema.separateDefaults.bufferTmp[def.prop] = 1 - ctx.defaults++ - } + console.warn('TODO: markString') + // ctx.typeDef.separateSort.bufferTmp[def.id] = 2 + // ctx.sort++ + // if (ctx.typeDef.hasSeperateDefaults) { + // ctx.typeDef.separateDefaults.bufferTmp[def.id] = 1 + // ctx.defaults++ + // } } } -export const markDefaults = (ctx: Ctx, def: PropDef, val: any) => { - if ( - ctx.operation === CREATE && - ctx.schema.hasSeperateDefaults && - val !== null - ) { - if (!ctx.schema.separateDefaults.bufferTmp[def.prop]) { - ctx.schema.separateDefaults.bufferTmp[def.prop] = 1 - ctx.defaults++ - } +export const markDefaults = (ctx: Ctx, def: LeafDef, val: any) => { + if (ctx.operation === CREATE) { + console.warn('TODO: markDefaults') + // if (ctx.typeDef.hasSeperateDefaults && val !== null) { + // if (!ctx.typeDef.separateDefaults.bufferTmp[def.id]) { + // ctx.typeDef.separateDefaults.bufferTmp[def.id] = 1 + // ctx.defaults++ + // } + // } } } export const markTextObj = (ctx: Ctx) => { - if (ctx.operation === CREATE && ctx.schema.hasSeperateDefaults) { - ctx.defaults++ + if (ctx.operation === CREATE) { + console.warn('TODO: markTextObj') + // if (ctx.typeDef.hasSeperateDefaults) { + // ctx.defaults++ + // } } } export const markTextValue = ( ctx: Ctx, - def: PropDef, + def: LeafDef, locale: LangCode, textStringValue: boolean, ) => { if (ctx.operation === CREATE) { - const index = def.prop * (1 + ctx.schema.localeSize) - const langIndex = ctx.schema.separateTextSort.localeToIndex.get(locale) - ctx.schema.separateTextSort.bufferTmp[index] -= 1 - ctx.schema.separateTextSort.bufferTmp[index + langIndex] = 0 - ctx.sortText++ - if (ctx.schema.hasSeperateDefaults) { - ctx.schema.separateDefaults.bufferTmp[def.prop]++ - if (textStringValue) { - ctx.defaults++ - } - } + console.warn('TODO: markTextValue') + // const index = def.id * (1 + ctx.typeDef.localeSize) + // const langIndex = ctx.typeDef.separateTextSort.localeToIndex.get(locale) + // ctx.typeDef.separateTextSort.bufferTmp[index] -= 1 + // ctx.typeDef.separateTextSort.bufferTmp[index + langIndex] = 0 + // ctx.sortText++ + // if (ctx.typeDef.hasSeperateDefaults) { + // ctx.typeDef.separateDefaults.bufferTmp[def.id]++ + // if (textStringValue) { + // ctx.defaults++ + // } + // } } } diff --git a/packages/db/src/client/modify/cursor.ts b/packages/db/src/client/modify/cursor.ts index e1d84045bd..e4554b0242 100644 --- a/packages/db/src/client/modify/cursor.ts +++ b/packages/db/src/client/modify/cursor.ts @@ -1,10 +1,7 @@ -import { MICRO_BUFFER, PropDef } from '@based/schema/def' import { Ctx } from './Ctx.js' -import { - SWITCH_TYPE, - SWITCH_FIELD -} from './types.js' -import { writeU8, writeU8Array } from './uint.js' +import { writeU16, writeU8 } from './uint.js' +import { SWITCH_TYPE, SWITCH_FIELD } from './types.js' +import { typeIndexMap, type LeafDef } from '@based/schema' export const TYPE_CURSOR_SIZE = 3 export const PROP_CURSOR_SIZE = 3 @@ -13,24 +10,24 @@ export const FULL_CURSOR_SIZE = TYPE_CURSOR_SIZE + NODE_CURSOR_SIZE + PROP_CURSOR_SIZE export const writeTypeCursor = (ctx: Ctx) => { - if (ctx.schema.id !== ctx.cursor.type) { + if (ctx.typeDef.id !== ctx.cursor.type) { writeU8(ctx, SWITCH_TYPE) - writeU8Array(ctx, ctx.schema.idUint8) - ctx.cursor.type = ctx.schema.id - ctx.cursor.prop = null + writeU16(ctx, ctx.typeDef.id) // is this correct? + ctx.cursor.type = ctx.typeDef.id + ctx.cursor.prop = undefined } } export const writePropCursor = ( ctx: Ctx, - def: PropDef, + def: LeafDef, typeIndex = def.typeIndex, ) => { - if (def.prop !== ctx.cursor.prop) { + if (def.id !== ctx.cursor.prop) { writeU8(ctx, SWITCH_FIELD) - writeU8(ctx, def.prop) + writeU8(ctx, def.id) writeU8(ctx, typeIndex) - ctx.cursor.prop = def.prop + ctx.cursor.prop = def.id } } @@ -38,7 +35,7 @@ export const writeMainCursor = (ctx: Ctx) => { if (ctx.cursor.prop !== 0) { writeU8(ctx, SWITCH_FIELD) writeU8(ctx, 0) - writeU8(ctx, MICRO_BUFFER) + writeU8(ctx, typeIndexMap.microbuffer) ctx.cursor.prop = 0 } -} \ No newline at end of file +} diff --git a/packages/db/src/client/modify/delete/index.ts b/packages/db/src/client/modify/delete/index.ts index 31100e8685..3d3398198e 100644 --- a/packages/db/src/client/modify/delete/index.ts +++ b/packages/db/src/client/modify/delete/index.ts @@ -22,12 +22,12 @@ export function del(db: DbClient, type: string, id: number) { const schema = getValidSchema(db, type) const ctx = db.modifyCtx try { - if (schema.insertOnly) { - throw `This type is insertOnly` - } + // if (schema.insertOnly) { + // throw `This type is insertOnly` + // } ctx.start = ctx.index - ctx.schema = schema + ctx.typeDef = schema ctx.operation = UPDATE validateId(id) diff --git a/packages/db/src/client/modify/drain.ts b/packages/db/src/client/modify/drain.ts index 84eab42834..da4911e87c 100644 --- a/packages/db/src/client/modify/drain.ts +++ b/packages/db/src/client/modify/drain.ts @@ -6,7 +6,7 @@ export const reset = (ctx: Ctx) => { ctx.index = 8 ctx.max = ctx.array.buffer.maxByteLength - 4 ctx.size = ctx.array.buffer.byteLength - 4 - ctx.cursor = {} + ctx.cursor = { main: 0 } ctx.batch = {} } @@ -48,7 +48,7 @@ export const drain = (db: DbClient, ctx: Ctx) => { if (batch.promises) { start = ctx.index batch.promises.forEach(batch.error ? rejectTmp : resolveTmp) - batch.promises = null + batch.promises = undefined } }) .then(() => { @@ -70,12 +70,12 @@ export const schedule = (db: DbClient, ctx: Ctx) => { ctx.scheduled = new Promise((resolve) => { if (db.flushTime === 0) { process.nextTick(() => { - ctx.scheduled = null + ctx.scheduled = undefined resolve(drain(db, ctx)) }) } else { setTimeout(() => { - ctx.scheduled = null + ctx.scheduled = undefined resolve(drain(db, ctx)) }, db.flushTime) } diff --git a/packages/db/src/client/modify/edges/binary.ts b/packages/db/src/client/modify/edges/binary.ts deleted file mode 100644 index ebef741fd9..0000000000 --- a/packages/db/src/client/modify/edges/binary.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { PropDefEdge, STRING } from '@based/schema/def' -import { Ctx } from '../Ctx.js' -import { reserve } from '../resize.js' -import { getBuffer, writeBinaryRaw } from '../props/binary.js' -import { writeU32 } from '../uint.js' -import { writeEdgeHeader } from './header.js' -import { PROP_CURSOR_SIZE } from '../cursor.js' -import { validate } from '../validate.js' - -export const writeBinaryEdge = (ctx: Ctx, edge: PropDefEdge, val: any) => { - let size = 0 - if (val !== null) { - const buf = getBuffer(val) - if (!buf) throw [edge, val] - validate(buf, edge) - size = buf.byteLength - val = buf - } - - if (size) { - reserve(ctx, PROP_CURSOR_SIZE + size + 10) - writeEdgeHeader(ctx, edge, STRING) - writeBinaryRaw(ctx, val) - } else { - reserve(ctx, 3 + 4) - writeEdgeHeader(ctx, edge, STRING) - writeU32(ctx, 0) - } -} diff --git a/packages/db/src/client/modify/edges/cardinality.ts b/packages/db/src/client/modify/edges/cardinality.ts deleted file mode 100644 index b3e331963d..0000000000 --- a/packages/db/src/client/modify/edges/cardinality.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { PropDefEdge, CARDINALITY } from '@based/schema/def' -import { Ctx } from '../Ctx.js' -import { reserve } from '../resize.js' -import { writeU32 } from '../uint.js' -import { writeEdgeHeader } from './header.js' -import { writeCardinalityRaw } from '../props/cardinality.js' -import { PROP_CURSOR_SIZE } from '../cursor.js' - -export const writeCardinalityEdge = (ctx: Ctx, edge: PropDefEdge, val: any) => { - if (!Array.isArray(val)) { - val = [val] - } - const size = 4 + val.length * 8 - reserve(ctx, PROP_CURSOR_SIZE + size) - writeEdgeHeader(ctx, edge, CARDINALITY) - writeCardinalityRaw(ctx, edge, val, size) -} diff --git a/packages/db/src/client/modify/edges/header.ts b/packages/db/src/client/modify/edges/header.ts deleted file mode 100644 index 078fd3b750..0000000000 --- a/packages/db/src/client/modify/edges/header.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { MICRO_BUFFER, PropDefEdge } from '@based/schema/def' -import { Ctx } from '../Ctx.js' -import { UPDATE, UPDATE_PARTIAL } from '../types.js' - -export const writeEdgeHeader = (ctx: Ctx, edge: PropDefEdge, type: number) => { - ctx.array[ctx.index] = UPDATE - ctx.array[ctx.index + 1] = edge.prop - ctx.array[ctx.index + 2] = type - ctx.index += 3 -} - -export const writeEdgeHeaderMain = (ctx: Ctx) => { - ctx.array[ctx.index] = UPDATE - ctx.array[ctx.index + 1] = 0 - ctx.array[ctx.index + 2] = MICRO_BUFFER - ctx.index += 3 -} - -export const writeEdgeHeaderPartial = (ctx: Ctx) => { - ctx.array[ctx.index] = UPDATE_PARTIAL - ctx.array[ctx.index + 1] = 0 - ctx.array[ctx.index + 2] = MICRO_BUFFER - ctx.index += 3 -} diff --git a/packages/db/src/client/modify/edges/index.ts b/packages/db/src/client/modify/edges/index.ts deleted file mode 100644 index 12a27530b5..0000000000 --- a/packages/db/src/client/modify/edges/index.ts +++ /dev/null @@ -1,141 +0,0 @@ -import { PropDef, PropDefEdge } from '@based/schema/def' -import { Ctx } from '../Ctx.js' -import { writeSeparateEdge } from './separate.js' -import { DECREMENT, INCREMENT, UPDATE } from '../types.js' -import { writeEdgeHeaderMain, writeEdgeHeaderPartial } from './header.js' -import { writeU16, writeU32 } from '../uint.js' -import { writeFixed } from '../props/fixed.js' -import { writeUint16, writeUint32 } from '@based/utils' -import { reserve } from '../resize.js' -import { PROP_CURSOR_SIZE } from '../cursor.js' - -const setDefaultEdges = (def: PropDef, val: Record) => { - if (def.hasDefaultEdges) { - for (const key in def.edges) { - const edge = def.edges[key] - if (edge.separate && val[key] === undefined) { - val[key] = edge.default - } - } - } -} - -type EdgeOperation = typeof UPDATE | typeof INCREMENT | typeof DECREMENT - -export const writeEdges = ( - ctx: Ctx, - def: PropDef, - obj: Record, - isSingleRefFix: boolean, -) => { - const index = ctx.index - ctx.index += 4 - const start = ctx.index - - let hasIncr = false - let mainSize = 0 - let mainFields: (PropDefEdge | any | EdgeOperation)[] - let operation: EdgeOperation - - setDefaultEdges(def, obj) - - for (const key in obj) { - let val = obj[key] - if (key === 'id' || key === '$index' || val === undefined) { - continue - } - const edge = def.edges[key] - if (!edge) { - throw [def, obj] - } - if (edge.separate) { - writeSeparateEdge(ctx, edge, val) - continue - } - - if (typeof val !== 'object' || val === null) { - operation = UPDATE - } else if (val.increment > 0) { - operation = INCREMENT - hasIncr = true - val = val.increment - } else if (val.increment < 0) { - operation = DECREMENT - hasIncr = true - val = val.increment - } else { - throw [edge, val] - } - - if (!hasIncr && def.edgeMainLen === edge.len) { - reserve(ctx, 3 + 4 + edge.len) - writeEdgeHeaderMain(ctx) - writeU32(ctx, edge.len) - writeFixed(ctx, edge, val) - } else { - mainSize += edge.len - if (mainFields) { - const len = mainFields.length - for (let i = 0; i < len; i += 3) { - if (edge.start < mainFields[i].start) { - mainFields.splice(i, 0, edge, val, operation) - break - } else if (mainFields[len - i - 3].start < edge.start) { - mainFields.splice(len - i, 0, edge, val, operation) - break - } - } - } else { - mainFields = [edge, val, operation] - } - } - } - - if (mainFields || def.hasDefaultEdges) { - if (!hasIncr && mainSize === def.edgeMainLen) { - reserve(ctx, 3 + 4 + mainSize) - writeEdgeHeaderMain(ctx) - writeU32(ctx, mainSize) - for (let i = 0; i < mainFields.length; i += 3) { - const edge: PropDefEdge = mainFields[i] - const val = mainFields[i + 1] - writeFixed(ctx, edge, val) - } - } else { - mainFields ??= [] - const mainFieldsStartSize = mainFields.length * 2 - reserve( - ctx, - PROP_CURSOR_SIZE + 4 + 2 + mainFieldsStartSize + def.edgeMainLen, - ) - writeEdgeHeaderPartial(ctx) - writeU32(ctx, mainFieldsStartSize + def.edgeMainLen) - writeU16(ctx, def.edgeMainLen) - // Index of start of fields - const sIndex = ctx.index - ctx.index += mainFieldsStartSize - // Add zeroes - ctx.array.set(def.edgeMainEmpty, ctx.index) - // Keep track of written bytes from append fixed - let startMain = ctx.index - for (let i = 0; i < mainFields.length; i += 3) { - const edge: PropDefEdge = mainFields[i] - const value = mainFields[i + 1] - const operation = mainFields[i + 2] - const sIndexI = i * 2 + sIndex - writeUint16(ctx.array, edge.start, sIndexI) - writeUint16(ctx.array, edge.len, sIndexI + 2) - ctx.array[sIndexI + 4] = operation - ctx.array[sIndexI + 5] = edge.typeIndex - ctx.index = startMain + edge.start - // Add null support (defaults) - writeFixed(ctx, edge, value) - } - // Correction append fixed value writes the len - ctx.index = startMain + def.edgeMainLen - } - } - - const size = ctx.index - start + (isSingleRefFix ? 4 : 0) - writeUint32(ctx.array, size, index) -} diff --git a/packages/db/src/client/modify/edges/reference.ts b/packages/db/src/client/modify/edges/reference.ts deleted file mode 100644 index 89403725b7..0000000000 --- a/packages/db/src/client/modify/edges/reference.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { PropDefEdge, REFERENCE } from '@based/schema/def' -import { Ctx } from '../Ctx.js' -import { reserve } from '../resize.js' -import { writeU32 } from '../uint.js' -import { writeEdgeHeader } from './header.js' -import { validate } from '../validate.js' - -export const writeReferenceEdge = (ctx: Ctx, edge: PropDefEdge, val: any) => { - if (val === null) { - reserve(ctx, 3 + 4) - writeEdgeHeader(ctx, edge, REFERENCE) - writeU32(ctx, 0) - return - } - if (typeof val === 'object') { - if (val.id) { - val = val.id - } else if (typeof val.then === 'function') { - throw val - } - } - if (typeof val === 'number') { - validate(val, edge) - reserve(ctx, 3 + 4) - writeEdgeHeader(ctx, edge, REFERENCE) - writeU32(ctx, val) - return - } - throw [edge, val] -} diff --git a/packages/db/src/client/modify/edges/references.ts b/packages/db/src/client/modify/edges/references.ts deleted file mode 100644 index 4565c21abb..0000000000 --- a/packages/db/src/client/modify/edges/references.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { PropDefEdge, REFERENCES } from '@based/schema/def' -import { Ctx } from '../Ctx.js' -import { reserve } from '../resize.js' -import { writePadding, writeU32 } from '../uint.js' -import { writeEdgeHeader } from './header.js' -import { validate } from '../validate.js' - -export const writeReferencesEdge = (ctx: Ctx, edge: PropDefEdge, vals: any) => { - if (vals === null) { - reserve(ctx, 3 + 4) - writeEdgeHeader(ctx, edge, REFERENCES) - writeU32(ctx, 0) - return - } - if (!Array.isArray(vals)) { - throw [edge, vals] - } - - const size = vals.length * 4 + 3 // add 3 padding - reserve(ctx, 3 + 4 + size) - writeEdgeHeader(ctx, edge, REFERENCES) - writeU32(ctx, size) - for (let val of vals) { - if (typeof val === 'object') { - if (val.id) { - val = val.id - } else if (typeof val.then === 'function') { - throw val - } - } - if (typeof val === 'number') { - validate(val, edge) - writeU32(ctx, val) - continue - } - throw [edge, vals] - } - - writePadding(ctx, 3) -} diff --git a/packages/db/src/client/modify/edges/separate.ts b/packages/db/src/client/modify/edges/separate.ts deleted file mode 100644 index 8579b02127..0000000000 --- a/packages/db/src/client/modify/edges/separate.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { - PropDefEdge, - BINARY, - STRING, - REFERENCE, - REFERENCES, - CARDINALITY, - JSON as JSONProp, -} from '@based/schema/def' -import { Ctx } from '../Ctx.js' -import { writeBinaryEdge } from './binary.js' -import { writeStringEdge } from './string.js' -import { writeReferenceEdge } from './reference.js' -import { writeReferencesEdge } from './references.js' -import { writeCardinalityEdge } from './cardinality.js' - -export const writeSeparateEdge = (ctx: Ctx, edge: PropDefEdge, val: any) => { - if (edge.typeIndex === BINARY) { - writeBinaryEdge(ctx, edge, val) - } else if (edge.typeIndex == JSONProp) { - writeBinaryEdge(ctx, edge, val === null ? null : JSON.stringify(val)) - } else if (edge.typeIndex === STRING) { - writeStringEdge(ctx, edge, val) - } else if (edge.typeIndex === REFERENCE) { - writeReferenceEdge(ctx, edge, val) - } else if (edge.typeIndex === REFERENCES) { - writeReferencesEdge(ctx, edge, val) - } else if (edge.typeIndex === CARDINALITY) { - writeCardinalityEdge(ctx, edge, val) - } -} diff --git a/packages/db/src/client/modify/edges/string.ts b/packages/db/src/client/modify/edges/string.ts deleted file mode 100644 index ca8071ca22..0000000000 --- a/packages/db/src/client/modify/edges/string.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { PropDefEdge, STRING } from '@based/schema/def' -import { ENCODER } from '@based/utils' -import { write } from '../../string.js' -import { Ctx } from '../Ctx.js' -import { reserve } from '../resize.js' -import { RANGE_ERR } from '../types.js' -import { writeU32 } from '../uint.js' -import { writeEdgeHeader } from './header.js' -import { validate } from '../validate.js' - -export const writeStringEdge = (ctx: Ctx, edge: PropDefEdge, val: any) => { - if (val === null) { - reserve(ctx, 3 + 4) - writeEdgeHeader(ctx, edge, STRING) - writeU32(ctx, 0) - return - } - validate(val, edge) - const maxSize = - val instanceof Uint8Array - ? val.byteLength - : ENCODER.encode(val).byteLength + 6 - reserve(ctx, 3 + maxSize + 4) - writeEdgeHeader(ctx, edge, STRING) - const realSize = write(ctx, val, ctx.index + 4, !edge.compression) - if (realSize === null) { - throw RANGE_ERR - } - writeU32(ctx, realSize) - ctx.index += realSize -} diff --git a/packages/db/src/client/modify/error.ts b/packages/db/src/client/modify/error.ts index 065f4d8d06..c56b80af3b 100644 --- a/packages/db/src/client/modify/error.ts +++ b/packages/db/src/client/modify/error.ts @@ -1,11 +1,4 @@ -import { - isPropDef, - PropDef, - PropDefEdge, - REVERSE_TYPE_INDEX_MAP, - SchemaPropTree, - SchemaTypeDef, -} from '@based/schema/def' +import type { PropDef, TypeDef } from '@based/schema' import { DbClient } from '../../index.js' import { create } from './create/index.js' import { Ctx } from './Ctx.js' @@ -44,16 +37,12 @@ const parseVal = (val) => { return val } -const parseErrorArr = ( - prop: PropDef | PropDefEdge | SchemaPropTree, - val: any, - msg?: string, -) => { - if (isPropDef(prop)) { +const parseErrorArr = (prop: PropDef, val: any, msg?: string) => { + if ('path' in prop) { if (msg) { return `Invalid value at '${prop.path.join('.')}'. Expected ${msg} received '${parseVal(val)}'` } - return `Invalid value at '${prop.path.join('.')}'. Expected ${REVERSE_TYPE_INDEX_MAP[prop.typeIndex]}, received '${parseVal(val)}'` + return `Invalid value at '${prop.path.join('.')}'. Expected ${prop.type}, received '${parseVal(val)}'` } return `Unknown property '${val}'. Expected one of: ${Object.keys(prop).join(', ')}` } @@ -71,7 +60,7 @@ export const handleError = ( e: any, ): Promise => { ctx.index = ctx.start - ctx.cursor = {} + ctx.cursor = { main: 0 } if (e === RANGE_ERR) { if (ctx.start === 8) { @@ -104,8 +93,8 @@ export const handleError = ( export const errors = { NotExists: class extends Error { - constructor(id: number, schema: SchemaTypeDef) { - super(`Target ${schema.type}:${id} does not exist`) + constructor(id: number, schema: TypeDef) { + super(`Target ${schema.name}:${id} does not exist`) } }, } as const diff --git a/packages/db/src/client/modify/expire/index.ts b/packages/db/src/client/modify/expire/index.ts index f21a2d572c..b9c12edfc7 100644 --- a/packages/db/src/client/modify/expire/index.ts +++ b/packages/db/src/client/modify/expire/index.ts @@ -22,7 +22,7 @@ export function expire( const schema = getValidSchema(db, type) try { ctx.start = ctx.index - ctx.schema = schema + ctx.typeDef = schema validateId(id) reserve(ctx, TYPE_CURSOR_SIZE + NODE_CURSOR_SIZE + 5) writeTypeCursor(ctx) diff --git a/packages/db/src/client/modify/props/alias.ts b/packages/db/src/client/modify/props/alias.ts index fe68b3a309..7ed2571232 100644 --- a/packages/db/src/client/modify/props/alias.ts +++ b/packages/db/src/client/modify/props/alias.ts @@ -1,4 +1,3 @@ -import { PropDef } from '@based/schema/def' import { Ctx } from '../Ctx.js' import { deleteProp } from './delete.js' import { validate } from '../validate.js' @@ -7,8 +6,9 @@ import { reserve } from '../resize.js' import { PROP_CURSOR_SIZE, writePropCursor } from '../cursor.js' import { writeU32, writeU8, writeU8Array } from '../uint.js' import { markString } from '../create/mark.js' +import type { LeafDef, SchemaAlias } from '@based/schema' -export const writeAlias = (ctx: Ctx, def: PropDef, val: any) => { +export const writeAlias = (ctx: Ctx, def: SchemaAlias & LeafDef, val: any) => { if (val === null) { deleteProp(ctx, def) return diff --git a/packages/db/src/client/modify/props/binary.ts b/packages/db/src/client/modify/props/binary.ts index a7f6d7f33d..3a5e7f13f3 100644 --- a/packages/db/src/client/modify/props/binary.ts +++ b/packages/db/src/client/modify/props/binary.ts @@ -1,15 +1,14 @@ import { writeU16, writeU32, writeU8, writeU8Array } from '../uint.js' import { PROP_CURSOR_SIZE, writePropCursor } from '../cursor.js' import { validate } from '../validate.js' -import { BINARY, PropDef } from '@based/schema/def' import native from '../../../native.js' import { reserve } from '../resize.js' -import { markDefaults } from '../create/mark.js' import { deleteProp } from './delete.js' import { ENCODER } from '@based/utils' import { Ctx } from '../Ctx.js' +import { typeIndexMap, type LeafDef } from '@based/schema' -export const getBuffer = (val: any): Uint8Array => { +export const getBuffer = (val: any): Uint8Array | undefined => { if (typeof val === 'string') { return ENCODER.encode(val) } @@ -35,7 +34,7 @@ export const writeBinaryRaw = (ctx: Ctx, val: Uint8Array): void => { export const writeBinary = ( ctx: Ctx, - def: PropDef, + def: LeafDef, val: any, validated?: boolean, ) => { @@ -57,7 +56,7 @@ export const writeBinary = ( } const size = buf.byteLength + 6 reserve(ctx, PROP_CURSOR_SIZE + size + 11) - writePropCursor(ctx, def, BINARY) + writePropCursor(ctx, def, typeIndexMap.enum) writeU8(ctx, ctx.operation) writeBinaryRaw(ctx, buf) } diff --git a/packages/db/src/client/modify/props/cardinality.ts b/packages/db/src/client/modify/props/cardinality.ts index 8cb30a1787..cd3f08e073 100644 --- a/packages/db/src/client/modify/props/cardinality.ts +++ b/packages/db/src/client/modify/props/cardinality.ts @@ -1,4 +1,3 @@ -import { PropDef, PropDefEdge } from '@based/schema/def' import { Ctx } from '../Ctx.js' import { deleteProp } from './delete.js' import { writeU32, writeU8, writeU8Array } from '../uint.js' @@ -7,12 +6,12 @@ import { xxHash64 } from '../../xxHash64.js' import { ENCODER } from '@based/utils' import { reserve } from '../resize.js' import { PROP_CURSOR_SIZE, writePropCursor } from '../cursor.js' -import { CREATE } from '../types.js' import { writeBinary } from './binary.js' +import type { LeafDef, SchemaCardinality } from '@based/schema' export const writeCardinalityRaw = ( ctx: Ctx, - def: PropDef | PropDefEdge, + def: LeafDef, val: any[], sizeFixBecauseEdgeIsDifferent = val.length, ) => { @@ -33,7 +32,11 @@ export const writeCardinalityRaw = ( } } -export const writeCardinality = (ctx: Ctx, def: PropDef, val: any) => { +export const writeCardinality = ( + ctx: Ctx, + def: SchemaCardinality & LeafDef, + val: any, +) => { if (val === null) { deleteProp(ctx, def) return @@ -56,11 +59,12 @@ export const writeCardinality = (ctx: Ctx, def: PropDef, val: any) => { reserve(ctx, PROP_CURSOR_SIZE + size + 1) writePropCursor(ctx, def) writeU8(ctx, ctx.operation) - writeU8(ctx, def.cardinalityMode) - writeU8(ctx, def.cardinalityPrecision) + writeU8(ctx, def.mode === 'dense' ? 1 : 0) + writeU8(ctx, def.precision ?? 8) writeCardinalityRaw(ctx, def, val) - if (ctx.operation === CREATE) { - ctx.schema.separateSort.bufferTmp[def.prop] = 2 - ctx.sort++ - } + // if (ctx.operation === CREATE) { + // TODO SORT + // ctx.typeDef.separateSort.bufferTmp[def.prop] = 2 + // ctx.sort++ + // } } diff --git a/packages/db/src/client/modify/props/delete.ts b/packages/db/src/client/modify/props/delete.ts index 5c191d4bde..1aff72968e 100644 --- a/packages/db/src/client/modify/props/delete.ts +++ b/packages/db/src/client/modify/props/delete.ts @@ -1,11 +1,11 @@ import { Ctx } from '../Ctx.js' -import { PropDef } from '@based/schema/def' import { DELETE, UPDATE } from '../types.js' import { reserve } from '../resize.js' import { PROP_CURSOR_SIZE, writePropCursor } from '../cursor.js' import { writeU8 } from '../uint.js' +import type { LeafDef } from '@based/schema' -export const deleteProp = (ctx: Ctx, def: PropDef) => { +export const deleteProp = (ctx: Ctx, def: LeafDef) => { if (ctx.operation !== UPDATE) { return } diff --git a/packages/db/src/client/modify/props/fixed.ts b/packages/db/src/client/modify/props/fixed.ts index 47d3686974..e44266cafa 100644 --- a/packages/db/src/client/modify/props/fixed.ts +++ b/packages/db/src/client/modify/props/fixed.ts @@ -1,138 +1,114 @@ import { Ctx } from '../Ctx.js' -import { - BINARY, - BOOLEAN, - ENUM, - INT16, - INT32, - INT8, - NUMBER, - PropDef, - PropDefEdge, - STRING, - TIMESTAMP, - UINT16, - UINT32, - UINT8, -} from '@based/schema/def' import { convertToTimestamp, ENCODER, writeDoubleLE } from '@based/utils' import { getBuffer } from './binary.js' import { reserve } from '../resize.js' import { writeU16, writeU32, writeU64, writeU8, writeU8Array } from '../uint.js' import { validate } from '../validate.js' +import { + type mainSizeMap, + type MainDef, + type SchemaBoolean, + type SchemaEnum, + type SchemaNumber, + type SchemaTimestamp, + typeIndexMap, +} from '@based/schema' -const map: Record< - number, - (ctx: Ctx, val: any, def: PropDef | PropDefEdge) => void -> = {} - -map[BINARY] = (ctx, val, def) => { - val = getBuffer(val) - validate(val, def) - reserve(ctx, val.byteLength + 1) - writeU8(ctx, val.byteLength) - writeU8Array(ctx, val) -} - -map[STRING] = (ctx, val, def) => { - const valBuf = ENCODER.encode(val) - const size = valBuf.byteLength - if (size + 1 > def.len) { - throw [def, val, `max length of ${def.len - 1},`] - } - validate(val, def) - reserve(ctx, size + 1) - const fullSize = def.len - 1 - ctx.array[ctx.index] = size - ctx.array.set(valBuf, ctx.index + 1) - ctx.index += fullSize + 1 - if (fullSize !== size) { - ctx.array.fill(0, ctx.index - (fullSize - size), ctx.index) +type WriteFixed = (ctx: Ctx, val: any, def: MainDef) => void +const getWriteInt = + (size: number, writeU: typeof writeU8) => + (ctx: Ctx, val: any, def: SchemaNumber & MainDef) => { + val ??= def.default + validate(val, def) + reserve(ctx, size) + writeU(ctx, val) } -} -map[BOOLEAN] = (ctx, val, def) => { - val ??= def.default - validate(val, def) - reserve(ctx, 1) - writeU8(ctx, val ? 1 : 0) -} +const write8 = getWriteInt(1, writeU8) +const write16 = getWriteInt(2, writeU16) +const write32 = getWriteInt(4, writeU32) -map[ENUM] = (ctx, val, def) => { - validate(val, def) - if (val === null) { - reserve(ctx, 1) - writeU8(ctx, def.default) - } else if (val in def.reverseEnum) { - reserve(ctx, 1) - writeU8(ctx, def.reverseEnum[val] + 1) - } else { - throw [def, val] - } -} +type Fixed = (typeof typeIndexMap)[keyof typeof mainSizeMap] +const map: Record = { + [typeIndexMap.binary](ctx, val, def) { + val = getBuffer(val) + validate(val, def) + reserve(ctx, val.byteLength + 1) + writeU8(ctx, val.byteLength) + writeU8Array(ctx, val) + }, -map[NUMBER] = (ctx, val, def) => { - val ??= def.default - validate(val, def) - reserve(ctx, 8) - writeDoubleLE(ctx.array, val, ctx.array.byteOffset + ctx.index) - ctx.index += 8 -} - -map[TIMESTAMP] = (ctx, val, def) => { - val ??= def.default - const parsedValue = convertToTimestamp(val) - validate(parsedValue, def) - reserve(ctx, 8) - writeU64(ctx, parsedValue) -} + [typeIndexMap.string](ctx, val, def) { + const valBuf = ENCODER.encode(val) + const size = valBuf.byteLength + if (size + 1 > def.main.size) { + throw [def, val, `max length of ${def.main.size - 1},`] + } + validate(val, def) + reserve(ctx, size + 1) + const fullSize = def.main.size - 1 + ctx.array[ctx.index] = size + ctx.array.set(valBuf, ctx.index + 1) + ctx.index += fullSize + 1 + if (fullSize !== size) { + ctx.array.fill(0, ctx.index - (fullSize - size), ctx.index) + } + }, -map[UINT32] = (ctx, val, def) => { - val ??= def.default - validate(val, def) - reserve(ctx, 4) - writeU32(ctx, val) -} - -map[UINT16] = (ctx, val, def) => { - val ??= def.default - validate(val, def) - reserve(ctx, 2) - writeU16(ctx, val) -} + [typeIndexMap.boolean](ctx, val, def: SchemaBoolean & MainDef) { + val ??= def.default + validate(val, def) + reserve(ctx, 1) + writeU8(ctx, val ? 1 : 0) + }, -map[UINT8] = map[INT8] = (ctx, val, def) => { - val ??= def.default - validate(val, def) - reserve(ctx, 1) - writeU8(ctx, val) -} + [typeIndexMap.enum](ctx, val, def: SchemaEnum & MainDef) { + validate(val, def) + if (val === null) { + reserve(ctx, 1) + writeU8(ctx, def.default ? def.enumMap[def.default as string] : 0) + } else if (val in def.enumMap) { + reserve(ctx, 1) + writeU8(ctx, def.enumMap[val]) + } else { + throw [def, val] + } + }, -map[INT32] = (ctx, val, def) => { - val ??= def.default - validate(val, def) - reserve(ctx, 4) - writeU32(ctx, val) -} + [typeIndexMap.number](ctx, val, def: SchemaNumber & MainDef) { + val ??= def.default + console.log('??', val) + validate(val, def) + console.log('??xxx') + reserve(ctx, 8) + writeDoubleLE(ctx.array, val, ctx.array.byteOffset + ctx.index) + ctx.index += 8 + }, -map[INT16] = (ctx, val, def) => { - val ??= def.default - validate(val, def) - reserve(ctx, 2) - writeU16(ctx, val) + [typeIndexMap.timestamp](ctx, val, def: SchemaTimestamp & MainDef) { + val ??= def.default + const parsedValue = convertToTimestamp(val) + validate(parsedValue, def) + reserve(ctx, 8) + writeU64(ctx, parsedValue) + }, + [typeIndexMap.uint32]: write32, + [typeIndexMap.uint16]: write16, + [typeIndexMap.uint8]: write8, + [typeIndexMap.int32]: write32, + [typeIndexMap.int16]: write16, + [typeIndexMap.int8]: write8, } export const writeFixed = ( ctx: Ctx, - def: PropDef | PropDefEdge, + def: MainDef, val: string | boolean | number, -) => { - return map[def.typeIndex](ctx, val, def) -} +) => map[def.typeIndex](ctx, val, def) export const writeFixedAtOffset = ( ctx: Ctx, - def: PropDef, + def: MainDef, val: string | boolean | number, offset: number, ) => { diff --git a/packages/db/src/client/modify/props/increment.ts b/packages/db/src/client/modify/props/increment.ts index 0953c28a6c..b0b5ba8794 100644 --- a/packages/db/src/client/modify/props/increment.ts +++ b/packages/db/src/client/modify/props/increment.ts @@ -1,29 +1,29 @@ import { PROP_CURSOR_SIZE, writeMainCursor } from '../cursor.js' import { writeU16, writeU8 } from '../uint.js' -import { PropDef } from '@based/schema/def' import { reserve } from '../resize.js' import { writeFixed } from './fixed.js' import { Ctx } from '../Ctx.js' import { DECREMENT, INCREMENT } from '../types.js' +import type { MainDef } from '@based/schema' -export const writeIncrement = (ctx: Ctx, def: PropDef, val: any) => { +export const writeIncrement = (ctx: Ctx, def: MainDef, val: any) => { if (typeof val.increment !== 'number') { throw [def, val] } if (val.increment === 0) { return } - reserve(ctx, PROP_CURSOR_SIZE + 4 + def.len) + reserve(ctx, PROP_CURSOR_SIZE + 4 + def.main.size) writeMainCursor(ctx) if (val.increment > 0) { writeU8(ctx, INCREMENT) writeU8(ctx, def.typeIndex) - writeU16(ctx, def.start) + writeU16(ctx, def.main.start) writeFixed(ctx, def, val.increment) } else { writeU8(ctx, DECREMENT) writeU8(ctx, def.typeIndex) - writeU16(ctx, def.start) + writeU16(ctx, def.main.start) writeFixed(ctx, def, -val.increment) } } diff --git a/packages/db/src/client/modify/props/json.ts b/packages/db/src/client/modify/props/json.ts index 5ffc8dd3c8..9006aae0f5 100644 --- a/packages/db/src/client/modify/props/json.ts +++ b/packages/db/src/client/modify/props/json.ts @@ -1,7 +1,7 @@ +import type { LeafDef } from '@based/schema' import { Ctx } from '../Ctx.js' -import { PropDef } from '@based/schema/def' import { writeBinary } from './binary.js' -export const writeJson = (ctx: Ctx, def: PropDef, val: any) => { +export const writeJson = (ctx: Ctx, def: LeafDef, val: any) => { writeBinary(ctx, def, val === null ? null : JSON.stringify(val), true) } diff --git a/packages/db/src/client/modify/props/main.ts b/packages/db/src/client/modify/props/main.ts index fd210e412c..c022d33fe6 100644 --- a/packages/db/src/client/modify/props/main.ts +++ b/packages/db/src/client/modify/props/main.ts @@ -1,23 +1,24 @@ -import { PropDef } from '@based/schema/def' import { Ctx } from '../Ctx.js' import { writeFixedAtOffset } from './fixed.js' import { reserve } from '../resize.js' import { PROP_CURSOR_SIZE, writeMainCursor } from '../cursor.js' -import { writeU32, writeU8, writeU8Array } from '../uint.js' +import { writePadding, writeU32, writeU8 } from '../uint.js' +import type { MainDef } from '@based/schema' export const writeMainBuffer = (ctx: Ctx) => { if (ctx.cursor.main === null) { - reserve(ctx, PROP_CURSOR_SIZE + 5 + ctx.schema.mainLen) + reserve(ctx, PROP_CURSOR_SIZE + 5 + ctx.typeDef.size) writeMainCursor(ctx) writeU8(ctx, ctx.operation) - writeU32(ctx, ctx.schema.mainLen) + writeU32(ctx, ctx.typeDef.size) ctx.cursor.main = ctx.index - writeU8Array(ctx, ctx.schema.mainEmpty) + writePadding(ctx, ctx.typeDef.size) + // writeU8Array(ctx, ctx.typeDef.mainEmpty) } } -export const writeMainValue = (ctx: Ctx, def: PropDef, val: any) => { - reserve(ctx, PROP_CURSOR_SIZE + 5 + ctx.schema.mainLen) +export const writeMainValue = (ctx: Ctx, def: MainDef, val: any) => { + reserve(ctx, PROP_CURSOR_SIZE + 5 + ctx.typeDef.size) writeMainBuffer(ctx) writeFixedAtOffset( ctx, @@ -25,6 +26,6 @@ export const writeMainValue = (ctx: Ctx, def: PropDef, val: any) => { typeof val === 'object' && val !== null && 'increment' in val ? val.increment : val, - ctx.cursor.main + def.start, + ctx.cursor.main + def.main.start, ) } diff --git a/packages/db/src/client/modify/props/object.ts b/packages/db/src/client/modify/props/object.ts index 9ae4d66f48..d6c02a13ed 100644 --- a/packages/db/src/client/modify/props/object.ts +++ b/packages/db/src/client/modify/props/object.ts @@ -1,25 +1,27 @@ -import { PropDef, SchemaTypeDef, isPropDef } from '@based/schema/def' import { Ctx } from '../Ctx.js' import { writeSeparate } from './separate.js' import { writeMainValue } from './main.js' import { writeIncrement } from './increment.js' import { CREATE } from '../types.js' +import type { LeafDef, BranchDef } from '@based/schema' -const writeProp = (ctx: Ctx, def: PropDef, val: any) => { - if (def.separate) { - writeSeparate(ctx, def, val) - } else if (ctx.operation === CREATE) { - writeMainValue(ctx, def, val) - } else if (typeof val === 'object' && val !== null) { - writeIncrement(ctx, def, val) +const writeProp = (ctx: Ctx, def: LeafDef, val: any) => { + if ('main' in def) { + if (ctx.operation === CREATE) { + writeMainValue(ctx, def, val) + } else if (typeof val === 'object' && val !== null) { + writeIncrement(ctx, def, val) + } else { + ctx.main.set(def, val) + } } else { - ctx.main.set(def, val) + writeSeparate(ctx, def, val) } } export const writeObjectSafe = ( ctx: Ctx, - tree: SchemaTypeDef['tree'], + tree: BranchDef, obj: Record, ) => { for (const key in obj) { @@ -27,11 +29,12 @@ export const writeObjectSafe = ( if (val === undefined) { continue } - const def = tree[key] + const def = tree.props[key] if (def === undefined) { throw [tree, val] } - if (isPropDef(def)) { + + if ('id' in def) { writeProp(ctx, def, val) } else { writeObjectSafe(ctx, def, val) @@ -41,16 +44,16 @@ export const writeObjectSafe = ( export const writeObjectUnsafe = ( ctx: Ctx, - tree: SchemaTypeDef['tree'], + tree: BranchDef, obj: Record, ) => { for (const key in obj) { - const def = tree[key] + const def = tree.props[key] const val = obj[key] if (def === undefined || val === undefined) { continue } - if (isPropDef(def)) { + if ('id' in def) { const index = ctx.index try { writeProp(ctx, def, val) @@ -69,12 +72,12 @@ export const writeObjectUnsafe = ( export const writeObject = ( ctx: Ctx, - tree: SchemaTypeDef['tree'], + def: BranchDef, obj: Record, ) => { if (ctx.unsafe) { - writeObjectUnsafe(ctx, tree, obj) + writeObjectUnsafe(ctx, def, obj) } else { - writeObjectSafe(ctx, tree, obj) + writeObjectSafe(ctx, def, obj) } } diff --git a/packages/db/src/client/modify/props/reference.ts b/packages/db/src/client/modify/props/reference.ts index 915050a48d..9b7d96502a 100644 --- a/packages/db/src/client/modify/props/reference.ts +++ b/packages/db/src/client/modify/props/reference.ts @@ -1,4 +1,3 @@ -import { PropDef } from '@based/schema/def' import { Ctx } from '../Ctx.js' import { Tmp } from '../Tmp.js' import { reserve } from '../resize.js' @@ -9,14 +8,15 @@ import { NOEDGE_NOINDEX_REALID, NOEDGE_NOINDEX_TMPID, } from '../types.js' -import { writeEdges } from '../edges/index.js' +// import { writeEdges } from '../edges/index.js' import { deleteProp } from './delete.js' import { writeU32, writeU8 } from '../uint.js' import { validate } from '../validate.js' +import type { LeafDef } from '@based/schema' const writeReferenceId = ( ctx: Ctx, - def: PropDef, + def: LeafDef, val: number, refOp: | typeof NOEDGE_NOINDEX_REALID @@ -33,7 +33,7 @@ const writeReferenceId = ( export const writeReference = ( ctx: Ctx, - def: PropDef, + def: LeafDef, val: number | Tmp | { id: number | Tmp | Promise }, ) => { if (val === null) { @@ -43,12 +43,13 @@ export const writeReference = ( if (typeof val === 'number') { validate(val, def) - if (def.hasDefaultEdges) { - writeReferenceId(ctx, def, val, EDGE_NOINDEX_REALID) - writeEdges(ctx, def, {}, true) - } else { - writeReferenceId(ctx, def, val, NOEDGE_NOINDEX_REALID) - } + console.warn('DO EDGES') + // if (def.hasDefaultEdges) { + // writeReferenceId(ctx, def, val, EDGE_NOINDEX_REALID) + // writeEdges(ctx, def, {}, true) + // } else { + writeReferenceId(ctx, def, val, NOEDGE_NOINDEX_REALID) + // } return } @@ -69,12 +70,13 @@ export const writeReference = ( if (typeof val.id === 'number') { validate(val.id, def) - if (!def.edges || val instanceof Tmp || val instanceof Promise) { - writeReferenceId(ctx, def, val.id, NOEDGE_NOINDEX_REALID) - } else { - writeReferenceId(ctx, def, val.id, EDGE_NOINDEX_REALID) - writeEdges(ctx, def, val, true) - } + console.warn('DO EDGES') + // if (!def.edges || val instanceof Tmp || val instanceof Promise) { + writeReferenceId(ctx, def, val.id, NOEDGE_NOINDEX_REALID) + // } else { + // writeReferenceId(ctx, def, val.id, EDGE_NOINDEX_REALID) + // writeEdges(ctx, def, val, true) + // } return } @@ -82,12 +84,13 @@ export const writeReference = ( if (val.id.batch !== ctx.batch) { throw val.id } - if (!def.edges) { - writeReferenceId(ctx, def, val.id.tmpId, NOEDGE_NOINDEX_TMPID) - } else { - writeReferenceId(ctx, def, val.id.tmpId, EDGE_NOINDEX_TMPID) - writeEdges(ctx, def, val, true) - } + // if (!def.edges) { + console.warn('DO EDGES') + writeReferenceId(ctx, def, val.id.tmpId, NOEDGE_NOINDEX_TMPID) + // } else { + // writeReferenceId(ctx, def, val.id.tmpId, EDGE_NOINDEX_TMPID) + // writeEdges(ctx, def, val, true) + // } return } @@ -95,12 +98,13 @@ export const writeReference = ( if (val.batch !== ctx.batch) { throw val } - if (def.hasDefaultEdges) { - writeReferenceId(ctx, def, val.tmpId, EDGE_NOINDEX_TMPID) - writeEdges(ctx, def, {}, true) - } else { - writeReferenceId(ctx, def, val.tmpId, NOEDGE_NOINDEX_TMPID) - } + // if (def.hasDefaultEdges) { + // writeReferenceId(ctx, def, val.tmpId, EDGE_NOINDEX_TMPID) + // writeEdges(ctx, def, {}, true) + // } else { + console.warn('DO EDGES') + writeReferenceId(ctx, def, val.tmpId, NOEDGE_NOINDEX_TMPID) + // } return } diff --git a/packages/db/src/client/modify/props/references.ts b/packages/db/src/client/modify/props/references.ts index f609706e28..7a63bb9704 100644 --- a/packages/db/src/client/modify/props/references.ts +++ b/packages/db/src/client/modify/props/references.ts @@ -1,11 +1,10 @@ import { Ctx } from '../Ctx.js' -import { PropDef } from '@based/schema/def' import { reserve } from '../resize.js' import { PROP_CURSOR_SIZE, writePropCursor } from '../cursor.js' import { writeU32, writeU8 } from '../uint.js' import { Tmp } from '../Tmp.js' import { validate } from '../validate.js' -import { writeEdges } from '../edges/index.js' +// import { writeEdges } from '../edges/index.js' import { writeUint32 } from '@based/utils' import { DELETE, @@ -24,15 +23,22 @@ import { REF_OP_UPDATE, RefOp, } from '../types.js' +import type { LeafDef, RefPropDef } from '@based/schema' -const clearReferences = (ctx: Ctx, def: PropDef) => { +const clearReferences = (ctx: Ctx, def: RefPropDef) => { reserve(ctx, PROP_CURSOR_SIZE + 1) writePropCursor(ctx, def) writeU8(ctx, DELETE) } -const hasEdgeOrIndex = (def: PropDef, obj: Record): boolean => { - if (def.edges) { +const hasEdgeOrIndex = ( + def: RefPropDef, + obj: Record, +): true | void => { + if ('$index' in obj) { + return true + } + if (def.edgesDef) { for (const key in obj) { if (key[0] === '$') { return true @@ -41,11 +47,11 @@ const hasEdgeOrIndex = (def: PropDef, obj: Record): boolean => { } } -const hasAnEdge = (def: PropDef, obj: Record): boolean => { - if (def.hasDefaultEdges) { - return true - } - if (def.edges) { +const hasAnEdge = (def: RefPropDef, obj: Record): true | void => { + // if (def.hasDefaultEdges) { + // return true + // } + if (def.edgesDef) { for (const key in obj) { if (key[0] === '$' && key !== '$index') { return true @@ -56,7 +62,7 @@ const hasAnEdge = (def: PropDef, obj: Record): boolean => { const putReferences = ( ctx: Ctx, - def: PropDef, + def: RefPropDef, val: any, refOp: RefOp, ): number => { @@ -92,7 +98,7 @@ const putReferences = ( const updateReferences = ( ctx: Ctx, - def: PropDef, + def: RefPropDef, val: any[], index: number, length: number, @@ -108,15 +114,16 @@ const updateReferences = ( writeU32(ctx, length) while (length--) { let id = val[index++] - if (def.hasDefaultEdges) { - if ( - typeof id === 'number' || - id instanceof Tmp || - id instanceof Promise - ) { - id = { id: id } - } - } + console.warn('DO EDGES') + // if (def.hasDefaultEdges) { + // if ( + // typeof id === 'number' || + // id instanceof Tmp || + // id instanceof Promise + // ) { + // id = { id: id } + // } + // } if (typeof id === 'number') { validate(id, def) @@ -172,7 +179,7 @@ const updateReferences = ( const putOrUpdateReferences = ( ctx: Ctx, - def: PropDef, + def: RefPropDef, val: any, refOp: RefOp, ) => { @@ -181,10 +188,11 @@ const putOrUpdateReferences = ( return } - if (def.hasDefaultEdges) { - updateReferences(ctx, def, val, 0, val.length, refOp) - return - } + console.warn('DO DEFAULT EDGES') + // if (def.hasDefaultEdges) { + // updateReferences(ctx, def, val, 0, val.length, refOp) + // return + // } const start = ctx.index const index = putReferences(ctx, def, val, refOp) @@ -195,18 +203,18 @@ const putOrUpdateReferences = ( if (index === 0) { // did nothing ctx.index = start - ctx.cursor.prop = null + ctx.cursor.prop = undefined updateReferences(ctx, def, val, 0, val.length, refOp) } else { // did partial - ctx.cursor.prop = null + ctx.cursor.prop = undefined updateReferences(ctx, def, val, index, val.length - index, refOp) } } const writeReferenceObj = ( ctx: Ctx, - def: PropDef, + def: RefPropDef, id: number, obj: Record, isTmp: boolean, @@ -218,7 +226,8 @@ const writeReferenceObj = ( writeU8(ctx, isTmp ? EDGE_INDEX_TMPID : EDGE_INDEX_REALID) writeU32(ctx, id) writeU32(ctx, obj.$index) - writeEdges(ctx, def, obj, false) + console.warn('DO EDGES') + // writeEdges(ctx, def, obj, false) } else { writeU8(ctx, isTmp ? NOEDGE_INDEX_TMPID : NOEDGE_INDEX_REALID) writeU32(ctx, id) @@ -227,14 +236,15 @@ const writeReferenceObj = ( } else if (hasEdges) { writeU8(ctx, isTmp ? EDGE_NOINDEX_TMPID : EDGE_NOINDEX_REALID) writeU32(ctx, id) - writeEdges(ctx, def, obj, false) + console.warn('DO EDGES') + // writeEdges(ctx, def, obj, false) } else { writeU8(ctx, isTmp ? NOEDGE_NOINDEX_TMPID : NOEDGE_NOINDEX_REALID) writeU32(ctx, id) } } -const deleteReferences = (ctx: Ctx, def: PropDef, val: any[]) => { +const deleteReferences = (ctx: Ctx, def: LeafDef, val: any[]) => { const size = 4 * val.length + 1 reserve(ctx, PROP_CURSOR_SIZE + 6 + size) writePropCursor(ctx, def) @@ -265,7 +275,7 @@ const deleteReferences = (ctx: Ctx, def: PropDef, val: any[]) => { } } -export const writeReferences = (ctx: Ctx, def: PropDef, val: any) => { +export const writeReferences = (ctx: Ctx, def: RefPropDef, val: any) => { if (typeof val !== 'object') { throw [def, val] } diff --git a/packages/db/src/client/modify/props/separate.ts b/packages/db/src/client/modify/props/separate.ts index 0d1ce539d9..a73c905fb5 100644 --- a/packages/db/src/client/modify/props/separate.ts +++ b/packages/db/src/client/modify/props/separate.ts @@ -1,16 +1,3 @@ -import { - PropDef, - STRING, - TEXT, - REFERENCE, - REFERENCES, - BINARY, - ALIAS, - CARDINALITY, - VECTOR, - COLVEC, - JSON, -} from '@based/schema/def' import { Ctx } from '../Ctx.js' import { writeReference } from './reference.js' import { writeString } from './string.js' @@ -22,26 +9,26 @@ import { writeCardinality } from './cardinality.js' import { writeVector } from './vector.js' import { writeJson } from './json.js' import { markDefaults } from '../create/mark.js' +import type { LeafDef } from '@based/schema' -export const writeSeparate = (ctx: Ctx, def: PropDef, val: any) => { - const type = def.typeIndex - if (type === STRING) { +export const writeSeparate = (ctx: Ctx, def: LeafDef, val: any) => { + if (def.type === 'string') { writeString(ctx, def, val, 0) - } else if (type === TEXT) { + } else if (def.type === 'text') { writeText(ctx, def, val) - } else if (type === REFERENCE) { + } else if (def.type === 'reference') { writeReference(ctx, def, val) - } else if (type === REFERENCES) { + } else if (def.type === 'references') { writeReferences(ctx, def, val) - } else if (type === BINARY) { + } else if (def.type === 'binary') { writeBinary(ctx, def, val) - } else if (type === ALIAS) { + } else if (def.type === 'alias') { writeAlias(ctx, def, val) - } else if (type === CARDINALITY) { + } else if (def.type === 'cardinality') { writeCardinality(ctx, def, val) - } else if (type === VECTOR || type === COLVEC) { + } else if (def.type === 'vector' || def.type === 'colvec') { writeVector(ctx, def, val) - } else if (type === JSON) { + } else if (def.type === 'json') { writeJson(ctx, def, val) } markDefaults(ctx, def, val) diff --git a/packages/db/src/client/modify/props/string.ts b/packages/db/src/client/modify/props/string.ts index ab7a30f5bb..9f1372fa1d 100644 --- a/packages/db/src/client/modify/props/string.ts +++ b/packages/db/src/client/modify/props/string.ts @@ -1,13 +1,11 @@ -import { PropDef } from '@based/schema/def' import { Ctx } from '../Ctx.js' -import { LangCode } from '@based/schema' import { - CREATE, - DELETE, - DELETE_TEXT_FIELD, - RANGE_ERR, - UPDATE, -} from '../types.js' + LangCode, + type LeafDef, + type PropDef, + type SchemaString, +} from '@based/schema' +import { DELETE, DELETE_TEXT_FIELD, RANGE_ERR, UPDATE } from '../types.js' import { FULL_CURSOR_SIZE, PROP_CURSOR_SIZE, @@ -20,7 +18,7 @@ import { writeU8, writeU8Array } from '../uint.js' import { markString } from '../create/mark.js' import { validate } from '../validate.js' -export const deleteString = (ctx: Ctx, def: PropDef, lang: LangCode): void => { +export const deleteString = (ctx: Ctx, def: LeafDef, lang: LangCode): void => { if (ctx.operation !== UPDATE) { return } @@ -38,7 +36,7 @@ export const deleteString = (ctx: Ctx, def: PropDef, lang: LangCode): void => { export const writeString = ( ctx: Ctx, - def: PropDef, + def: LeafDef & { compression?: SchemaString['compression'] }, val: any, lang: LangCode, ): void => { @@ -49,7 +47,9 @@ export const writeString = ( } validate(val, def) - let size = isUint8 ? val.byteLength : ENCODER.encode(val).byteLength + 6 + let size: number | null = isUint8 + ? val.byteLength + : ENCODER.encode(val).byteLength + 6 reserve(ctx, FULL_CURSOR_SIZE + 11 + size) writePropCursor(ctx, def) writeU8(ctx, ctx.operation) @@ -58,7 +58,7 @@ export const writeString = ( if (isUint8) { writeU8Array(ctx, val) } else { - size = write(ctx, val, ctx.index, def.compression === 0, lang) + size = write(ctx, val, ctx.index, def.compression === 'none', lang) if (size === null) { throw RANGE_ERR } diff --git a/packages/db/src/client/modify/props/text.ts b/packages/db/src/client/modify/props/text.ts index 412cf45fe0..a4a3466cd7 100644 --- a/packages/db/src/client/modify/props/text.ts +++ b/packages/db/src/client/modify/props/text.ts @@ -1,10 +1,10 @@ -import { PropDef } from '@based/schema/def' +// import { PropDef } from '@based/schema/def' import { Ctx } from '../Ctx.js' import { deleteString, writeString } from './string.js' -import { LangCode } from '@based/schema' +import { LangCode, langCodesMap, type LeafDef } from '@based/schema' import { markTextValue, markTextObj } from '../create/mark.js' -export const writeText = (ctx: Ctx, def: PropDef, val: any): void => { +export const writeText = (ctx: Ctx, def: LeafDef, val: any): void => { if (val === null) { deleteString(ctx, def, ctx.locale) return @@ -13,9 +13,9 @@ export const writeText = (ctx: Ctx, def: PropDef, val: any): void => { if (!ctx.locale) { throw [def, val] } - if (!ctx.schema.separateTextSort.localeToIndex.has(ctx.locale)) { - throw [def, val, 'Invalid locale'] - } + // if (!ctx.typeDef.separateTextSort.localeToIndex.has(ctx.locale)) { + // throw [def, val, 'Invalid locale'] + // } writeString(ctx, def, val, ctx.locale) markTextValue(ctx, def, ctx.locale, true) @@ -26,12 +26,16 @@ export const writeText = (ctx: Ctx, def: PropDef, val: any): void => { markTextObj(ctx) for (const lang in val) { - const langU8 = ctx.schema.separateTextSort.localeStringToIndex.get(lang) - if (!langU8) { + // const langU8 = ctx.typeDef.separateTextSort.localeStringToIndex.get(lang) + // if (!langU8) { + // throw [def, val, 'Invalid locale'] + // } + + if (!(lang in ctx.schema.locales)) { throw [def, val, 'Invalid locale'] } const text = val[lang] - const locale = langU8[1] as LangCode + const locale = langCodesMap.get(lang) as LangCode writeString(ctx, def, text, locale) markTextValue(ctx, def, locale, false) } diff --git a/packages/db/src/client/modify/props/vector.ts b/packages/db/src/client/modify/props/vector.ts index 1a3526ff64..863cab331f 100644 --- a/packages/db/src/client/modify/props/vector.ts +++ b/packages/db/src/client/modify/props/vector.ts @@ -1,12 +1,16 @@ import { Ctx } from '../Ctx.js' -import { PropDef } from '@based/schema/def' import { deleteProp } from './delete.js' import { validate } from '../validate.js' import { PROP_CURSOR_SIZE, writePropCursor } from '../cursor.js' import { reserve } from '../resize.js' import { writeU8 } from '../uint.js' +import type { LeafDef, SchemaVector } from '@based/schema' -export const writeVector = (ctx: Ctx, def: PropDef, val: any) => { +export const writeVector = ( + ctx: Ctx, + def: SchemaVector & LeafDef, + val: any, +) => { if (val === null) { deleteProp(ctx, def) return @@ -23,7 +27,7 @@ export const writeVector = (ctx: Ctx, def: PropDef, val: any) => { writePropCursor(ctx, def) writeU8(ctx, ctx.operation) - let size = Math.min(val.byteLength, def.len) + let size = Math.min(val.byteLength, def.size * def.baseSize) let padding = 0 if (ctx.index % 4 != 0) { padding = ctx.index % 4 diff --git a/packages/db/src/client/modify/uint.ts b/packages/db/src/client/modify/uint.ts index ffe26842a0..5330823ae6 100644 --- a/packages/db/src/client/modify/uint.ts +++ b/packages/db/src/client/modify/uint.ts @@ -12,9 +12,8 @@ export const writeU32 = (ctx: Ctx, val: number) => { } export const writePadding = (ctx: Ctx, padding: number) => { - while (padding--) { - ctx.array[ctx.index++] = 0 - } + ctx.array.fill(0, ctx.index, ctx.index + padding) + ctx.index += padding } export const writeU16 = (ctx: Ctx, val: number) => { diff --git a/packages/db/src/client/modify/update/index.ts b/packages/db/src/client/modify/update/index.ts index f6a45c4534..356f40c9a7 100644 --- a/packages/db/src/client/modify/update/index.ts +++ b/packages/db/src/client/modify/update/index.ts @@ -7,7 +7,7 @@ import { } from '../types.js' import { DbClient } from '../../../index.js' import { getValidSchema, validateId, validatePayload } from '../validate.js' -import { langCodesMap } from '@based/schema' +import { langCodesMap, type TypeDef } from '@based/schema' import { handleError } from '../error.js' import { Tmp } from '../Tmp.js' import { writeObject } from '../props/object.js' @@ -22,19 +22,19 @@ import { getByPath, writeUint32 } from '@based/utils' import { writeU16, writeU32, writeU8 } from '../uint.js' import { writeFixed } from '../props/fixed.js' import { schedule } from '../drain.js' -import { SchemaTypeDef } from '@based/schema/def' -const writeUpdateTs = (ctx: Ctx, payload: any) => { - if (ctx.schema.updateTs) { - const updateTs = Date.now() - for (const def of ctx.schema.updateTs) { - if (getByPath(payload, def.path) !== undefined) { - continue - } - ctx.main.set(def, updateTs) - } - } -} +// remove this stuff +// const writeUpdateTs = (ctx: Ctx, payload: any) => { +// if (ctx.typeDef.updateTs) { +// const updateTs = Date.now() +// for (const def of ctx.typeDef.updateTs) { +// if (getByPath(payload, def.path) !== undefined) { +// continue +// } +// ctx.main.set(def, updateTs) +// } +// } +// } const writeMergeMain = (ctx: Ctx) => { if (ctx.main.size) { @@ -45,8 +45,8 @@ const writeMergeMain = (ctx: Ctx) => { ctx.index += 4 const start = ctx.index for (const [def, val] of ctx.main) { - writeU16(ctx, def.start) - writeU16(ctx, def.len) + writeU16(ctx, def.main.start) + writeU16(ctx, def.main.size) writeFixed(ctx, def, val) } writeUint32(ctx.array, ctx.index - start, index) @@ -55,10 +55,10 @@ const writeMergeMain = (ctx: Ctx) => { export const writeUpdate = ( ctx: Ctx, - schema: SchemaTypeDef, + schema: TypeDef, id: number, payload: any, - opts: ModifyOpts, + opts?: ModifyOpts, ) => { validatePayload(payload) @@ -72,6 +72,7 @@ export const writeUpdate = ( val = val?.[key] } if (val !== undefined) { + // @ts-ignore key is assigned and hooks is always defined in this case obj[key] = def.hooks.update(val, obj) } } @@ -81,9 +82,9 @@ export const writeUpdate = ( payload = schema.hooks.update(payload) || payload } - ctx.schema = schema + ctx.typeDef = schema ctx.operation = UPDATE - ctx.locale = opts?.locale && langCodesMap.get(opts.locale) + ctx.locale = (opts?.locale && langCodesMap.get(opts.locale)) || 0 if (ctx.main.size) { ctx.main.clear() @@ -93,8 +94,8 @@ export const writeUpdate = ( writeTypeCursor(ctx) writeU8(ctx, SWITCH_ID_UPDATE) writeU32(ctx, id) - writeObject(ctx, ctx.schema.tree, payload) - writeUpdateTs(ctx, payload) + writeObject(ctx, ctx.typeDef, payload) + // writeUpdateTs(ctx, payload) writeMergeMain(ctx) } @@ -103,7 +104,7 @@ export function update( type: string, id: number, payload: any, - opts: ModifyOpts, + opts?: ModifyOpts, ): Promise { const schema = getValidSchema(db, type) const ctx = db.modifyCtx diff --git a/packages/db/src/client/modify/upsert/index.ts b/packages/db/src/client/modify/upsert/index.ts index b2992ca507..0690720fad 100644 --- a/packages/db/src/client/modify/upsert/index.ts +++ b/packages/db/src/client/modify/upsert/index.ts @@ -1,31 +1,32 @@ -import { ALIAS, isPropDef, SchemaPropTree } from '@based/schema/def' -import { DbClient } from '../../../index.js' -import { INSERT, ModifyOpts, UPSERT } from '../types.js' -import { getValidSchema } from '../validate.js' +import { TYPE_CURSOR_SIZE, writeTypeCursor } from '../cursor.js' import { writeU32, writeU8, writeU8Array } from '../uint.js' -import { reserve } from '../resize.js' -import { Ctx } from '../Ctx.js' +import { INSERT, ModifyOpts, UPSERT } from '../types.js' import { ENCODER, writeUint32 } from '@based/utils' import { writeCreate } from '../create/index.js' -import { handleError } from '../error.js' import { writeUpdate } from '../update/index.js' +import { getValidSchema } from '../validate.js' +import { DbClient } from '../../../index.js' +import { handleError } from '../error.js' +import { reserve } from '../resize.js' import { schedule } from '../drain.js' -import { TYPE_CURSOR_SIZE, writeTypeCursor } from '../cursor.js' +import { Ctx } from '../Ctx.js' import { Tmp } from '../Tmp.js' -const writeAliases = (ctx: Ctx, tree: SchemaPropTree, obj: any) => { +import type { BranchDef } from '@based/schema' + +const writeAliases = (ctx: Ctx, tree: BranchDef, obj: any) => { for (const key in obj) { - const def = tree[key] + const def = tree.props[key] const val = obj[key] if (def === undefined || val === undefined) { continue } - if (!isPropDef(def)) { + if ('props' in def) { writeAliases(ctx, def, val) - } else if (def.typeIndex === ALIAS) { + } else if (def.type === 'alias') { const buf = ENCODER.encode(val) reserve(ctx, 1 + 4 + buf.byteLength) - writeU8(ctx, def.prop) + writeU8(ctx, def.id) writeU32(ctx, buf.byteLength) writeU8Array(ctx, buf) } @@ -36,12 +37,12 @@ export function upsert( db: DbClient, type: string, payload: any, - opts: ModifyOpts, + opts?: ModifyOpts, ): Promise { - const schema = getValidSchema(db, type) + const typeDef = getValidSchema(db, type) const ctx = db.modifyCtx ctx.start = ctx.index - ctx.schema = schema + ctx.typeDef = typeDef try { reserve(ctx, TYPE_CURSOR_SIZE + 1 + 4 + 4) @@ -49,11 +50,11 @@ export function upsert( writeU8(ctx, UPSERT) const start = ctx.index ctx.index += 8 - writeAliases(ctx, schema.tree, payload) + writeAliases(ctx, typeDef, payload) writeUint32(ctx.array, ctx.index - start, start) - writeCreate(ctx, schema, {}, opts) + writeCreate(ctx, typeDef, {}, opts) writeUint32(ctx.array, ctx.index - start, start + 4) - writeUpdate(ctx, schema, 0, payload, opts) + writeUpdate(ctx, typeDef, 0, payload, opts) schedule(db, ctx) return new Tmp(ctx) } catch (e) { @@ -65,12 +66,12 @@ export function insert( db: DbClient, type: string, payload: any, - opts: ModifyOpts, + opts?: ModifyOpts, ) { - const schema = getValidSchema(db, type) + const typeDef = getValidSchema(db, type) const ctx = db.modifyCtx ctx.start = ctx.index - ctx.schema = schema + ctx.typeDef = typeDef try { reserve(ctx, TYPE_CURSOR_SIZE + 1 + 4 + 4) @@ -78,9 +79,9 @@ export function insert( writeU8(ctx, INSERT) const start = ctx.index ctx.index += 8 - writeAliases(ctx, schema.tree, payload) + writeAliases(ctx, typeDef, payload) writeUint32(ctx.array, ctx.index - start, start) - writeCreate(ctx, schema, payload, opts) + writeCreate(ctx, typeDef, payload, opts) writeUint32(ctx.array, ctx.index - start, start + 4) schedule(db, ctx) return new Tmp(ctx) diff --git a/packages/db/src/client/modify/validate.ts b/packages/db/src/client/modify/validate.ts index be43718ba5..bde5fba0c5 100644 --- a/packages/db/src/client/modify/validate.ts +++ b/packages/db/src/client/modify/validate.ts @@ -1,29 +1,25 @@ -import { isValidId, PropDef, PropDefEdge } from '@based/schema/def' +import { isValidId, type PropDef, type TypeDef } from '@based/schema' import { DbClient } from '../../index.js' -export const validate = (val: any, def: PropDef | PropDefEdge) => { - const msg = def.validation(val, def.schema) +export const validate = (val: any, def: PropDef) => { + const msg = def.validation(val, def) if (msg !== true) { throw [def, val, msg] } } - +export const validateId = (v: number) => { + if (!isValidId(v)) { + throw 'Invalid id' + } +} export const validatePayload = (payload: any) => { if (typeof payload !== 'object' || payload === null) { throw 'Invalid payload' } } -export const validateId = (id: number) => { - if (!isValidId(id)) { - throw 'Invalid id' - } -} - -export const getValidSchema = (db: DbClient, type: string) => { - const schema = db.schemaTypesParsed[type] - if (schema) { - return schema - } - throw `Unknown type: ${type}. Did you mean on of: ${Object.keys(db.schemaTypesParsed).join(', ')}` +export const getValidSchema = (db: DbClient, type: string): TypeDef => { + const schema = db.defs.byName[type] + if (schema) return schema + throw `Unknown type: ${type}. Did you mean on of: ${Object.keys(db.defs.byName).filter(Number.isNaN).join(', ')}` } diff --git a/packages/db/src/client/query/BasedDbQuery.ts b/packages/db/src/client/query/BasedDbQuery.ts index 95e8cbbfa3..189d262a19 100644 --- a/packages/db/src/client/query/BasedDbQuery.ts +++ b/packages/db/src/client/query/BasedDbQuery.ts @@ -43,7 +43,7 @@ export type QueryCommand = { export class QueryBranch { db: DbClient - def: QueryDef + def?: QueryDef queryCommands: QueryCommand[] constructor(db: DbClient, def?: QueryDef) { @@ -58,7 +58,7 @@ export class QueryBranch { args: [field, order], }) } else { - sort(this.def, field, order) + sort(this.def as QueryDef, field, order) } // @ts-ignore return this @@ -81,7 +81,7 @@ export class QueryBranch { // @ts-ignore return this } - filter(this.db, this.def, f, this.def.filter) + filter(this.db, this.def as QueryDef, f, (this.def as QueryDef).filter) } // @ts-ignore return this @@ -367,18 +367,19 @@ export class QueryBranch { args: [field, operator, value, opts], }) } else { + const def = this.def as QueryDef if (typeof field === 'function') { const f = new FilterBranch( this.db, - filterOr(this.db, this.def, [], this.def.filter), - this.def, + filterOr(this.db, def, [], def.filter), + def, ) field(f) - this.def.filter.size += f.filterBranch.size + def.filter.size += f.filterBranch.size } else { const f = convertFilter(this, field, operator, value, opts) if (f) { - filterOr(this.db, this.def, f, this.def.filter) + filterOr(this.db, def, f, def.filter) } } } @@ -392,14 +393,15 @@ export class QueryBranch { } else { const offset = start const limit = end - start - if (validateRange(this.def, offset, limit)) { - this.def.range.offset = 0 - this.def.range.limit = DEF_RANGE_PROP_LIMIT + const def = this.def as QueryDef + if (validateRange(def, offset, limit)) { + def.range.offset = 0 + def.range.limit = DEF_RANGE_PROP_LIMIT // @ts-ignore return this } - this.def.range.offset = offset - this.def.range.limit = limit + def.range.offset = offset + def.range.limit = limit } // @ts-ignore return this @@ -430,22 +432,23 @@ export class QueryBranch { this.queryCommands.push({ method: 'include', args: fields }) } else { include(this, fields) - if (this.def.schema.propHooks?.include) { - for (const field of this.def.include.stringFields.keys()) { - const hooks = this.def.schema.props[field]?.hooks - const includeHook = hooks?.include - if (includeHook) { - hooks.include = null - includeHook(this, this.def.include.stringFields) - hooks.include = includeHook + const def = this.def as QueryDef + if (def.schema.propHooks?.include) { + for (const field of def.include.stringFields.keys()) { + const hooks = def.schema.props[field]?.hooks + if (hooks?.include) { + const hook = hooks.include + hooks.include = undefined + hook(this, def.include.stringFields) + hooks.include = hook } } } - const includeHook = this.def.schema.hooks?.include - if (includeHook) { - this.def.schema.hooks.include = null - includeHook(this, this.def.include.stringFields) - this.def.schema.hooks.include = includeHook + if (def.schema.hooks?.include) { + const hook = def.schema.hooks.include + def.schema.hooks.include = undefined + hook(this, def.include.stringFields) + def.schema.hooks.include = hook } } // @ts-ignore @@ -515,7 +518,7 @@ export class BasedDbQuery extends QueryBranch { super(db) this.db = db - this.skipValidation = skipValidation + this.skipValidation = skipValidation ?? false this.queryCommands = [] this.target = target } @@ -527,9 +530,10 @@ export class BasedDbQuery extends QueryBranch { args: [index], }) } else { - this.def.selectFirstResult = true - this.def.range.limit = 1 - this.def.range.offset = index + const def = this.def as QueryDef + def.selectFirstResult = true + def.range.limit = 1 + def.range.offset = index } return this } @@ -556,8 +560,8 @@ export class BasedDbQuery extends QueryBranch { } await this.db.isModified() - - if (this.db.schema?.hash !== this.def.schemaChecksum) { + const def = this.def as QueryDef + if (this.db.schema?.hash !== def.schemaChecksum) { this.reset() return this.#getInternal(resolve, reject) } @@ -591,7 +595,7 @@ export class BasedDbQuery extends QueryBranch { } else if (res instanceof Error) { reject(res) } else { - resolve(new BasedQueryResponse(this.def, res, performance.now() - d)) + resolve(new BasedQueryResponse(def, res, performance.now() - d)) } } @@ -613,21 +617,24 @@ export class BasedDbQuery extends QueryBranch { args: [locale], }) } else { - if (fallBack === undefined) { - // Uses fallback from schema if available - const localeDescriptor = this.def.schema.locales[locale] - fallBack = - typeof localeDescriptor === 'object' - ? localeDescriptor.fallback || false - : false - } - validateLocale(this.def, locale) - const fallBackCode: LangCode[] = - fallBack === false ? [] : [langCodesMap.get(fallBack)] - this.def.lang = { - lang: langCodesMap.get(locale) ?? 0, - fallback: fallBackCode, - } + console.warn('TODO: implement locale in query') + // const def = this.def as QueryDef + // if (fallBack === undefined) { + + // // Uses fallback from schema if available + // const localeDescriptor = def.schema.locales[locale] + // fallBack = + // typeof localeDescriptor === 'object' + // ? localeDescriptor.fallback || false + // : false + // } + // validateLocale(def, locale) + // const fallBackCode: LangCode[] = + // fallBack === false ? [] : [langCodesMap.get(fallBack)] + // def.lang = { + // lang: langCodesMap.get(locale) ?? 0, + // fallback: fallBackCode, + // } } return this } @@ -639,7 +646,7 @@ export class BasedDbQuery extends QueryBranch { try { onData(res) } catch (err) { - const def = this.def + const def = this.def as QueryDef let name = picocolors.red(`QueryError[${displayTarget(def)}]\n`) name += ` Error executing onData handler in subscription\n` name += ` ${err.message}\n` @@ -658,8 +665,8 @@ export class BasedDbQuery extends QueryBranch { const d = performance.now() const res = native.getQueryBuf(buf, dbCtxExternal) return new BasedQueryResponse( - this.def, - new Uint8Array(res), + this.def as QueryDef, + res ? new Uint8Array(res) : new Uint8Array(), performance.now() - d, ) } diff --git a/packages/db/src/client/query/BasedQueryResponse.ts b/packages/db/src/client/query/BasedQueryResponse.ts index 783b9632c5..229e1c62ef 100644 --- a/packages/db/src/client/query/BasedQueryResponse.ts +++ b/packages/db/src/client/query/BasedQueryResponse.ts @@ -11,6 +11,7 @@ import { readId, readChecksum, readVersion, + type ReaderSchema, } from '@based/protocol/db-read' export { time, size, inspectData } @@ -34,7 +35,7 @@ export class BasedQueryResponse { } get id() { - return readId(this.def.readSchema, this.result) + return readId(this.def?.readSchema as ReaderSchema, this.result) } get version() { @@ -104,7 +105,7 @@ export class BasedQueryResponse { i += 4 } const l = readProps( - this.def.readSchema, + this.def?.readSchema as ReaderSchema, result, i, result.byteLength - 4, @@ -150,7 +151,12 @@ export class BasedQueryResponse { } toObject(): any { - return resultToObject(this.def.readSchema, this.result, this.end - 4, 0) + return resultToObject( + this.def?.readSchema as ReaderSchema, + this.result, + this.end - 4, + 0, + ) } toJSON( diff --git a/packages/db/src/client/query/aggregates/aggregation.ts b/packages/db/src/client/query/aggregates/aggregation.ts index ee88154e8d..ad728940ed 100644 --- a/packages/db/src/client/query/aggregates/aggregation.ts +++ b/packages/db/src/client/query/aggregates/aggregation.ts @@ -1,15 +1,6 @@ import { writeUint16, writeInt16, writeUint32 } from '@based/utils' import { QueryDef, QueryDefAggregation, QueryDefType } from '../types.js' import { GroupBy, StepInput, aggFnOptions, setMode } from './types.js' -import { - PropDef, - UINT32, - SchemaPropTree, - REFERENCE, - REFERENCES, - PropDefEdge, - isPropDef, -} from '@based/schema/def' import { aggregationFieldDoesNotExist, validateStepRange, @@ -22,6 +13,12 @@ import { } from '../aggregates/types.js' import { QueryBranch } from '../BasedDbQuery.js' import { AggregateType } from '@based/protocol/db-read' +import { + typeIndexMap, + type QueryPropDef, + type PropDef, + type TypeDef, +} from '@based/schema' export const aggregateToBuffer = ( aggregates: QueryDefAggregation, @@ -31,13 +28,19 @@ export const aggregateToBuffer = ( if (aggregates.groupBy) { aggBuffer[i] = GroupBy.HAS_GROUP i += 1 - aggBuffer[i] = aggregates.groupBy.prop + aggBuffer[i] = aggregates.groupBy.id i += 1 aggBuffer[i] = aggregates.groupBy.typeIndex i += 1 - writeUint16(aggBuffer, aggregates.groupBy.start, i) - i += 2 - writeUint16(aggBuffer, aggregates.groupBy.len, i) + if ('main' in aggregates.groupBy) { + writeUint16(aggBuffer, aggregates.groupBy.main.start || 0, i) + i += 2 + writeUint16(aggBuffer, aggregates.groupBy.main.size || 0, i) + } else { + writeUint16(aggBuffer, 0, i) + i += 2 + writeUint16(aggBuffer, 0, i) + } i += 2 aggBuffer[i] = aggregates.groupBy.stepType || 0 i += 1 @@ -53,7 +56,8 @@ export const aggregateToBuffer = ( i += 2 writeUint16(aggBuffer, aggregates.totalAccumulatorSize, i) i += 2 - aggBuffer[i] = setMode[aggregates?.option?.mode] || 0 + const mode = aggregates?.option?.mode + aggBuffer[i] = (mode && setMode[mode]) || 0 i += 1 for (const [prop, aggregatesArray] of aggregates.aggregates.entries()) { aggBuffer[i] = prop @@ -67,13 +71,17 @@ export const aggregateToBuffer = ( i += 1 aggBuffer[i] = agg.propDef.typeIndex i += 1 - writeUint16(aggBuffer, agg.propDef.start, i) + writeUint16( + aggBuffer, + 'main' in agg.propDef ? agg.propDef.main.start : 0, + i, + ) i += 2 writeUint16(aggBuffer, agg.resultPos, i) i += 2 writeUint16(aggBuffer, agg.accumulatorPos, i) i += 2 - aggBuffer[i] = agg.propDef.__isEdge ? 1 : 0 + aggBuffer[i] = agg.propDef.typeDef.edge ? 1 : 0 i += 1 size += i - startI } @@ -83,7 +91,9 @@ export const aggregateToBuffer = ( return aggBuffer } -const ensureAggregate = (def: QueryDef) => { +function ensureAggregate(def: QueryDef): asserts def is QueryDef & { + aggregate: Exclude +} { if (!def.aggregate) { def.aggregate = { size: 6, // groupBy + resultSize, accumulatorSize, mode, @@ -97,33 +107,36 @@ const ensureAggregate = (def: QueryDef) => { export const groupBy = ( q: QueryBranch, field: string, - StepInput: StepInput, + StepInput?: StepInput, ) => { - const def = q.def - const fieldDef = def.schema.props[field] - if (!fieldDef) { + const def = q.def as QueryDef + const fieldDef = def.schema?.props[field] + + if (!fieldDef || fieldDef.type === 'object') { aggregationFieldDoesNotExist(def, field) + return } - const groupByPropHook = fieldDef.hooks?.groupBy - if (groupByPropHook) { - fieldDef.hooks.groupBy = null - groupByPropHook(q, field) - fieldDef.hooks.groupBy = groupByPropHook - } - const groupByHook = def.schema.hooks?.groupBy - if (groupByHook) { - def.schema.hooks.groupBy = null - groupByHook(q, field) - def.schema.hooks.groupBy = groupByHook + + if (fieldDef.hooks?.groupBy) { + const hook = fieldDef.hooks.groupBy + fieldDef.hooks.groupBy = undefined + hook(q, field) + fieldDef.hooks.groupBy = hook } - if (!def.aggregate) { - ensureAggregate(def) + if (def.schema?.hooks?.groupBy) { + const hook = def.schema.hooks.groupBy + def.schema.hooks.groupBy = undefined + hook(q, field) + def.schema.hooks.groupBy = hook } + ensureAggregate(def) + if (!def.aggregate.groupBy) { def.aggregate.size += 13 // field, srcPropType, start, len, stepType, stepRange, timezone } + def.aggregate.groupBy = fieldDef def.aggregate.groupBy.stepRange = undefined def.aggregate.groupBy.stepType = undefined @@ -159,17 +172,18 @@ export const groupBy = ( const updateAggregateDefs = ( def: QueryDef, - propDef: PropDef | PropDefEdge, + propDef: QueryPropDef, aggType: AggregateType, ) => { + ensureAggregate(def) const aggregates = def.aggregate.aggregates - if (!aggregates.get(propDef.prop)) { - aggregates.set(propDef.prop, []) + if (!aggregates.get(propDef.id)) { + aggregates.set(propDef.id, []) def.aggregate.size += 3 // field + fieldAggSize } - const aggregateField = aggregates.get(propDef.prop) - aggregateField.push({ + const aggregateField = aggregates.get(propDef.id) + aggregateField?.push({ type: aggType, propDef: propDef, resultPos: def.aggregate.totalResultsSize, @@ -198,30 +212,39 @@ const isEdge = (propString) => { } const isReferenceOrReferences = (typeIndex) => { - return typeIndex === REFERENCE || typeIndex === REFERENCES + return ( + typeIndex === typeIndexMap.reference || + typeIndex === typeIndexMap.references + ) } const getPropDefinition = ( def: QueryDef, propName: string, type: AggregateType, - resolvedPropDef: PropDef | PropDefEdge | undefined, -): PropDef | PropDefEdge | undefined => { - if (isCount(propName, type)) { - return { - schema: null, - prop: 255, - path: [propName], - __isPropDef: true, - len: 4, - start: 0, - typeIndex: 1, - separate: true, - validation: () => true, - default: 0, - } as PropDef - } - return resolvedPropDef || def.schema.props[propName] + resolvedPropDef: QueryPropDef, +): QueryPropDef | void => { + console.warn('TODO: aggregates getPropDefinition') + // if (isCount(propName, type)) { + + // return { + + // id: 255, + // typeIndex: 1, + // path: [propName], + // // schema: null, + // // prop: 255, + // // path: [propName], + // // __isPropDef: true, + // // len: 4, + // // start: 0, + // // typeIndex: 1, + // // separate: true, + // // validation: () => true, + // // default: 0, + // } + // } + // return resolvedPropDef || def.schema?.props[propName] } const IN_RECURSION = Symbol('IN_RECURSION') @@ -233,30 +256,38 @@ const processPropPath = ( path: string[], originalPropName: string, type: AggregateType, -): PropDef | PropDefEdge | undefined | typeof IN_RECURSION => { - let t: PropDef | SchemaPropTree = query.def.schema.tree +): QueryPropDef | undefined | typeof IN_RECURSION => { + const def = query.def as QueryDef + let t: PropDef | TypeDef | null = def.schema for (let i = 0; i < path.length; i++) { const propName = path[i] if (isEdge(propName)) { + // TODO what to do here? // @ts-ignore - const edgePropDef = query.def.target?.propDef?.edges[propName] + const edgePropDef = def.target?.propDef?.edges[propName] if (edgePropDef) { return edgePropDef } else { - edgeNotImplemented(query.def, propName) + edgeNotImplemented(def, propName) return undefined } } + if (!t) { - return undefined // end + return undefined // path nor exist } - t = t[propName] + + if ('props' in t) { + t = t.props[propName] + } + if (!t) { return undefined // path nor exist } - if (isPropDef(t) && isReferenceOrReferences(t.typeIndex)) { + + if ('target' in t) { const remainingPath = path.slice(i + 1).join('.') if (isEdge(remainingPath)) { @@ -264,13 +295,14 @@ const processPropPath = ( } if (remainingPath) { - addAggregate(query, type, [remainingPath], query.def.aggregate.option) + addAggregate(query, type, [remainingPath], def.aggregate?.option) } return IN_RECURSION } } - return isPropDef(t) ? (t as PropDef) : undefined + + return t && 'typeIndex' in t ? t : undefined } export const addAggregate = ( @@ -279,8 +311,8 @@ export const addAggregate = ( propNames: string[], option?: aggFnOptions, ) => { - const def = query.def - let hookPropNames: Set + const def = query.def as QueryDef + let hookPropNames: Set | undefined ensureAggregate(def) @@ -295,7 +327,7 @@ export const addAggregate = ( const path = propName.split('.') let resolvedPropDef = processPropPath(query, path, propName, type) - if (resolvedPropDef === IN_RECURSION) { + if (resolvedPropDef === IN_RECURSION || !resolvedPropDef) { continue } let propDef = getPropDefinition(def, propName, type, resolvedPropDef) @@ -311,7 +343,7 @@ export const addAggregate = ( } updateAggregateDefs(def, propDef, type) - if (def.schema.hooks?.aggregate) { + if (def.schema?.hooks?.aggregate) { def.schema.hooks.aggregate(query, hookPropNames || new Set(propNames)) } } @@ -325,6 +357,7 @@ export const isRootCountOnly = (def: QueryDef, filterSize: number) => { if (def.type !== QueryDefType.Root) { return false } + ensureAggregate(def) if (def.aggregate.groupBy) { return false } @@ -335,7 +368,7 @@ export const isRootCountOnly = (def: QueryDef, filterSize: number) => { return false } const aggs = def.aggregate.aggregates.get(255) - if (aggs.length !== 1) { + if (aggs?.length !== 1) { return false } if (aggs[0].type !== AggregateType.COUNT) { diff --git a/packages/db/src/client/query/debug.ts b/packages/db/src/client/query/debug.ts index be931ff987..194a7b633d 100644 --- a/packages/db/src/client/query/debug.ts +++ b/packages/db/src/client/query/debug.ts @@ -1,5 +1,4 @@ import picocolors from 'picocolors' -import { isPropDef, REVERSE_TYPE_INDEX_MAP } from '@based/schema/def' import { QueryDef, QueryDefType } from './types.js' import { concatUint8Arr } from '@based/utils' @@ -27,8 +26,8 @@ export const debugQueryDef = (q: QueryDef, returnIt?: boolean) => { if (a.type && a.include && a.filter && a.range) { return debugQueryDef(a, true) } - if (isPropDef(a)) { - return `${a.path.join('.')}: ${a.prop} ${REVERSE_TYPE_INDEX_MAP[a.typeIndex]}` + if ('id' in a) { + return `${a.path.join('.')}: ${a.prop} ${a.type}` } else { const b = Array.isArray(a) ? [] : {} walk(a, b) diff --git a/packages/db/src/client/query/display.ts b/packages/db/src/client/query/display.ts index 0324900227..c5273c9165 100644 --- a/packages/db/src/client/query/display.ts +++ b/packages/db/src/client/query/display.ts @@ -1,22 +1,25 @@ import picocolors from 'picocolors' import { QueryDef } from './types.js' -import { - ALIAS, - BINARY, - CARDINALITY, - NUMBER, - PropDef, - PropDefEdge, - REFERENCE, - REFERENCES, - STRING, - TEXT, - TIMESTAMP, - TypeIndex, -} from '@based/schema/def' +// import { +// typeIndexMap.alias, +// typeIndexMap.binary, +// typeIndexMap.cardinality, +// typeIndexMap.number, +// PropDef, +// PropDefEdge, +// typeIndexMap.reference, +// typeIndexMap.references, +// typeIndexMap.string, +// typeIndexMap.text, +// typeIndexMap.timestamp, +// TypeIndex, +// } from '@based/schema/def' + +import { typeIndexMap, type TypeIndex } from '@based/schema' import { BasedQueryResponse } from './BasedQueryResponse.js' import { ENCODER } from '@based/utils' import { AggregateType } from '@based/protocol/db-read' +import type { PropDef, QueryPropDef } from '@based/schema' const decimals = (v: number) => ~~(v * 100) / 100 @@ -63,7 +66,7 @@ export const printNumber = (nr: number) => { } export const prettyPrintVal = (v: any, type: TypeIndex): string => { - if (type === BINARY) { + if (type === typeIndexMap.binary) { const nr = 12 const isLarger = v.length > nr // RFE Doesn't slice make a new alloc? subarray would be probably sufficient here. @@ -79,7 +82,11 @@ export const prettyPrintVal = (v: any, type: TypeIndex): string => { ) } - if (type === STRING || type === TEXT || type === ALIAS) { + if ( + type === typeIndexMap.string || + type === typeIndexMap.text || + type === typeIndexMap.alias + ) { if (v.length > 50) { const byteLength = ENCODER.encode(v).byteLength const chars = picocolors.italic( @@ -92,18 +99,18 @@ export const prettyPrintVal = (v: any, type: TypeIndex): string => { chars } - if (type === ALIAS) { + if (type === typeIndexMap.alias) { return `"${v}" ${picocolors.italic(picocolors.dim('alias'))}` } return `"${v}"` } - if (type === CARDINALITY) { + if (type === typeIndexMap.cardinality) { return `${picocolors.blue(v)} ${picocolors.italic(picocolors.dim('unique'))}` } - if (type === TIMESTAMP) { + if (type === typeIndexMap.timestamp) { if (v === 0) { return `0 ${picocolors.italic(picocolors.dim('No date'))}` } else { @@ -119,6 +126,7 @@ export const parseUint8Array = (p: any) => { const x = [] // @ts-ignore for (let i = 0; i < p.length; i++) { + // @ts-ignore x[i] = p[i] } p = x @@ -168,15 +176,18 @@ const inspectObject = ( // use reader schema for (const k in object) { const key = path ? path + '.' + k : k - let def: PropDef | PropDefEdge + let def: QueryPropDef def = q.props[key] let v = object[k] let isEdge = k[0] === '$' if (k === '$searchScore') { - edges.push({ k, v, def: { typeIndex: NUMBER } }) + // @ts-ignore + edges.push({ k, v, def: { typeIndex: typeIndexMap.number } }) } else if (isEdge) { + // @ts-ignore if (q.edges?.props?.[k]) { + // @ts-ignore edges.push({ k, v, def: q.edges?.props?.[k] }) } else { str += prefixBody + `${k}: ` @@ -207,21 +218,22 @@ const inspectObject = ( str += inspectObject(v, q, key, level + 2, false, false, true, depth) + '' } - } else if ('__isPropDef' in def) { - if (def.typeIndex === REFERENCES) { + } else if ('typeIndex' in def) { + if (def.typeIndex === typeIndexMap.references) { if (q.aggregate) { str += printNumber(v) str += picocolors.italic(picocolors.dim(` ${k.toLowerCase()}`)) } else { str += inspectData( v, - q.references.get(def.prop), + // @ts-ignore + q.references.get(def.id), level + 2, false, depth, ) } - } else if (def.typeIndex === REFERENCE) { + } else if (def.typeIndex === typeIndexMap.reference) { if (!v || !v.id) { str += 'null,\n' } else { @@ -231,7 +243,8 @@ const inspectObject = ( } else { str += inspectObject( v, - q.references.get(def.prop), + // @ts-ignore + q.references.get(def.id), '', level + 2, false, @@ -241,12 +254,12 @@ const inspectObject = ( ) } } - } else if (def.typeIndex === BINARY) { + } else if (def.typeIndex === typeIndexMap.binary) { if (v === undefined) { return '' } str += prettyPrintVal(v, def.typeIndex) - } else if (def.typeIndex === TEXT) { + } else if (def.typeIndex === typeIndexMap.text) { if (typeof v === 'object') { str += '{\n' for (const lang in v) { @@ -259,19 +272,22 @@ const inspectObject = ( } str += prettyPrintVal(v, def.typeIndex) } - } else if (def.typeIndex === STRING || def.typeIndex === ALIAS) { + } else if ( + def.typeIndex === typeIndexMap.string || + def.typeIndex === typeIndexMap.alias + ) { if (v === undefined) { return '' } str += prettyPrintVal(v, def.typeIndex) - } else if (def.typeIndex === CARDINALITY) { + } else if (def.typeIndex === typeIndexMap.cardinality) { if (typeof v === 'object' && v !== null) { str += inspectObject(v, q, key, level + 2, false, false, true, depth) + '' } else { str += prettyPrintVal(v, def.typeIndex) } - } else if (def.typeIndex === TIMESTAMP) { + } else if (def.typeIndex === typeIndexMap.timestamp) { str += prettyPrintVal(v, def.typeIndex) } else { if (typeof v === 'number') { @@ -289,8 +305,8 @@ const inspectObject = ( } } if ( - def?.typeIndex !== REFERENCE && - def?.typeIndex !== REFERENCES && + def?.typeIndex !== typeIndexMap.reference && + def?.typeIndex !== typeIndexMap.references && typeof v !== 'object' ) { str += ',\n' @@ -300,37 +316,38 @@ const inspectObject = ( } } - for (const edge of edges) { - if (edge.def.typeIndex === REFERENCE) { - str += prefixBody + picocolors.bold(`${edge.k}: `) - str += inspectObject( - edge.v, - q.edges.references.get(edge.def.prop), - '', - level + 2, - false, - false, - true, - depth, - ) - } else if (edge.def.typeIndex === REFERENCES) { - str += prefixBody + picocolors.bold(`${edge.k}: `) - str += - inspectData( - edge.v, - q.edges.references.get(edge.def.prop), - level + 3, - false, - depth + 2, - ) + '\n' - } else { - str += - prefixBody + - picocolors.bold(`${edge.k}: `) + - prettyPrintVal(edge.v, edge.def.typeIndex) + - ',\n' - } - } + // TODO fix edges display + // for (const edge of edges) { + // if (edge.def.typeIndex === typeIndexMap.reference) { + // str += prefixBody + picocolors.bold(`${edge.k}: `) + // str += inspectObject( + // edge.v, + // q.edges.references.get(edge.def.id), + // '', + // level + 2, + // false, + // false, + // true, + // depth, + // ) + // } else if (edge.def.typeIndex === typeIndexMap.references) { + // str += prefixBody + picocolors.bold(`${edge.k}: `) + // str += + // inspectData( + // edge.v, + // q.edges.references.get(edge.def.id), + // level + 3, + // false, + // depth + 2, + // ) + '\n' + // } else { + // str += + // prefixBody + + // picocolors.bold(`${edge.k}: `) + + // prettyPrintVal(edge.v, edge.def.typeIndex) + + // ',\n' + // } + // } if (isObject) { str += prefix + ' },\n' @@ -435,7 +452,7 @@ export const displayTarget = (def: QueryDef) => { const target = hasId || hasIds - ? def.schema.type + + ? def.schema.name + ':' + (hasIds ? // @ts-ignore @@ -444,6 +461,6 @@ export const displayTarget = (def: QueryDef) => { ? safeStringify(def.target.alias, 30) : // @ts-ignore def.target.id) - : def.schema.type + : def.schema.name return target } diff --git a/packages/db/src/client/query/filter/convertFilter.ts b/packages/db/src/client/query/filter/convertFilter.ts index 91249934c0..40465c4a80 100644 --- a/packages/db/src/client/query/filter/convertFilter.ts +++ b/packages/db/src/client/query/filter/convertFilter.ts @@ -1,4 +1,5 @@ import { QueryBranch } from '../BasedDbQuery.js' +import type { QueryDef } from '../types.js' import { Operator } from './filter.js' import { FilterBranch } from './FilterBranch.js' import { FilterOpts, FilterAst, toFilterCtx } from './types.js' @@ -17,30 +18,10 @@ export const convertFilter = ( operator?: Operator | boolean, value?: any, opts?: FilterOpts | undefined, -): FilterAst => { - const def = query.def - const propHooks = def.schema.props[field]?.hooks - const hooks = def.schema.hooks - const propFilterHook = propHooks?.filter - const filterHook = hooks?.filter - if (propFilterHook) { - propHooks.filter = null - if (typeof operator === 'boolean') { - propFilterHook(query, field, '=', operator) - } else { - propFilterHook(query, field, operator, value) - } - propHooks.filter = propFilterHook - } - if (filterHook) { - hooks.filter = null - if (typeof operator === 'boolean') { - filterHook(query, field, '=', operator) - } else { - filterHook(query, field, operator, value) - } - hooks.filter = filterHook - } +): FilterAst | void => { + const def = query.def as QueryDef + const propHooks = def.schema?.props[field]?.hooks + const hooks = def.schema?.hooks if (operator === undefined) { operator = '=' @@ -49,14 +30,30 @@ export const convertFilter = ( value = operator operator = '=' } + + if (propHooks?.filter) { + const hook = propHooks.filter + propHooks.filter = undefined + hook(query, field, operator, value) + propHooks.filter = hook + } + if (hooks?.filter) { + const hook = hooks.filter + hooks.filter = undefined + hook(query, field, operator, value) + hooks.filter = hook + } + if ( !(operator === 'exists' || operator === '!exists') && (value === '' || value === undefined) ) { if (value === '' && operator === '=') { - return [[field, toFilterCtx(def, '!exists', opts), undefined]] + const f = toFilterCtx(def, '!exists', opts) + return f && [[field, f, undefined]] } else if (value === '' && operator === '!=') { - return [[field, toFilterCtx(def, 'exists', opts), undefined]] + const f = toFilterCtx(def, 'exists', opts) + return f && [[field, f, undefined]] } return diff --git a/packages/db/src/client/query/filter/createFixedFilterBuffer.ts b/packages/db/src/client/query/filter/createFixedFilterBuffer.ts index f4bc517a85..3d09d17059 100644 --- a/packages/db/src/client/query/filter/createFixedFilterBuffer.ts +++ b/packages/db/src/client/query/filter/createFixedFilterBuffer.ts @@ -1,11 +1,3 @@ -import { - PropDef, - PropDefEdge, - BINARY, - STRING, - REFERENCES, - TIMESTAMP, -} from '@based/schema/def' import { ALIGNMENT_NOT_SET, EQUAL, @@ -24,14 +16,11 @@ import { writeUint32, } from '@based/utils' import { FilterCondition, FilterMetaNow } from '../types.js' +import { type QueryPropDef } from '@based/schema' -const isNowQuery = ( - prop: PropDef | PropDefEdge, - value: any, - ctx: FilterCtx, -) => { +const isNowQuery = (prop: QueryPropDef, value: any, ctx: FilterCtx) => { return ( - prop.typeIndex === TIMESTAMP && + prop.type === 'timestamp' && typeof value === 'string' && value.includes('now') && isNumerical(ctx.operation) @@ -39,7 +28,7 @@ const isNowQuery = ( } const createNowMeta = ( - prop: PropDef | PropDefEdge, + prop: QueryPropDef, parsedValue: number, ctx: FilterCtx, ): FilterMetaNow => { @@ -53,13 +42,13 @@ const createNowMeta = ( } export const writeFixed = ( - prop: PropDef | PropDefEdge, + prop: QueryPropDef, buf: Uint8Array, value: any, size: number, offset: number, ) => { - if (prop.typeIndex === BINARY || prop.typeIndex === STRING) { + if (prop.type === 'binary' || prop.type === 'string') { if (typeof value === 'string') { const { written } = ENCODER.encodeInto(value, buf.subarray(offset + 1)) buf[offset] = written @@ -72,7 +61,7 @@ export const writeFixed = ( buf[offset] = value } else { if (size === 8) { - if (prop.typeIndex === TIMESTAMP) { + if (prop.type === 'timestamp') { writeInt64(buf, value, offset) } else { writeDoubleLE(buf, value, offset) @@ -88,13 +77,13 @@ export const writeFixed = ( } export const createFixedFilterBuffer = ( - prop: PropDef | PropDefEdge, + prop: QueryPropDef, size: number, ctx: FilterCtx, value: any, sort: boolean, ): FilterCondition => { - const start = prop.start + const start = 'main' in prop ? prop.main.start : 0 if (Array.isArray(value)) { const len = value.length // Add 8 extra bytes for alignment @@ -102,7 +91,7 @@ export const createFixedFilterBuffer = ( const result: FilterCondition = { buffer, propDef: prop } buffer[0] = ctx.type buffer[1] = - prop.typeIndex === REFERENCES && ctx.operation === EQUAL + prop.type === 'references' && ctx.operation === EQUAL ? MODE_AND_FIXED : MODE_OR_FIXED buffer[2] = prop.typeIndex @@ -121,12 +110,8 @@ export const createFixedFilterBuffer = ( for (let i = 0; i < len; i++) { const parsedValue = parseFilterValue(prop, value[i]) if (isNowQuery(prop, value, ctx)) { - if (!result.subscriptionMeta) { - result.subscriptionMeta = {} - } - if (!result.subscriptionMeta.now) { - result.subscriptionMeta = { now: [] } - } + result.subscriptionMeta ??= {} + result.subscriptionMeta.now ??= [] result.subscriptionMeta.now.push( createNowMeta(prop, parsedValue, ctx), ) diff --git a/packages/db/src/client/query/filter/createReferenceFilter.ts b/packages/db/src/client/query/filter/createReferenceFilter.ts index 023535e6e1..1c78ad1a5f 100644 --- a/packages/db/src/client/query/filter/createReferenceFilter.ts +++ b/packages/db/src/client/query/filter/createReferenceFilter.ts @@ -1,10 +1,10 @@ -import { PropDef, PropDefEdge } from '@based/schema/def' import { ALIGNMENT_NOT_SET, FilterCtx, MODE_REFERENCE } from './types.js' import { FilterCondition } from '../types.js' import { writeUint16, writeUint32 } from '@based/utils' +import type { RefPropDef } from '@based/schema' export const createReferenceFilter = ( - prop: PropDef | PropDefEdge, + prop: RefPropDef, ctx: FilterCtx, value: any, ): FilterCondition => { @@ -18,7 +18,7 @@ export const createReferenceFilter = ( writeUint16(buffer, len, 5) buffer[7] = ctx.operation buffer[8] = 0 - writeUint16(buffer, prop.inverseTypeId, 9) + writeUint16(buffer, prop.target.typeDef.id, 9) if (isArray) { buffer[11] = ALIGNMENT_NOT_SET for (let i = 0; i < len; i++) { diff --git a/packages/db/src/client/query/filter/createVariableFilterBuffer.ts b/packages/db/src/client/query/filter/createVariableFilterBuffer.ts index 87a0151775..c4a011793d 100644 --- a/packages/db/src/client/query/filter/createVariableFilterBuffer.ts +++ b/packages/db/src/client/query/filter/createVariableFilterBuffer.ts @@ -1,4 +1,3 @@ -import { ALIAS, PropDef, PropDefEdge, TEXT, VECTOR } from '@based/schema/def' import { EQUAL, EQUAL_CRC32, @@ -15,12 +14,13 @@ import { createFixedFilterBuffer } from './createFixedFilterBuffer.js' import { crc32 } from '../../crc32.js' import { ENCODER, concatUint8Arr, writeUint16, writeUint32 } from '@based/utils' import { FilterCondition, QueryDef } from '../types.js' +import { type QueryPropDef } from '@based/schema' const DEFAULT_SCORE = new Uint8Array(new Float32Array([0.5]).buffer) const parseValue = ( value: any, - prop: PropDef | PropDefEdge, + prop: QueryPropDef, ctx: FilterCtx, lang: QueryDef['lang'], ): Uint8Array => { @@ -28,7 +28,7 @@ const parseValue = ( value = value.toLowerCase() } - if (ctx.operation === LIKE && prop.typeIndex === VECTOR) { + if (ctx.operation === LIKE && prop.type === 'vector') { if (!(value instanceof ArrayBuffer)) { throw new Error('Vector should be an arrayBuffer') } @@ -50,13 +50,13 @@ const parseValue = ( if ( value instanceof Uint8Array || typeof value === 'string' || - !prop.separate || + 'main' in prop || ctx.operation !== EQUAL ) { if (typeof value === 'string') { value = ENCODER.encode(value.normalize('NFKD')) } - if (prop.typeIndex === TEXT) { + if (prop.type === 'text') { // 1 + size const fallbacksSize = lang.lang === 0 ? 0 : lang.fallback.length const tmp = new Uint8Array(value.byteLength + 2 + fallbacksSize) @@ -75,7 +75,7 @@ const parseValue = ( if (!(value instanceof Uint8Array || value instanceof ArrayBuffer)) { throw new Error(`Incorrect value for filter: ${prop.path}`) } - if (ctx.operation === LIKE && prop.typeIndex !== VECTOR) { + if (ctx.operation === LIKE && prop.type !== 'vector') { const tmp = new Uint8Array(value.byteLength + 1) tmp.set(value instanceof ArrayBuffer ? new Uint8Array(value) : value) tmp[tmp.byteLength - 1] = ctx.opts.score ?? 2 @@ -86,17 +86,16 @@ const parseValue = ( export const createVariableFilterBuffer = ( value: any, - prop: PropDef | PropDefEdge, + prop: QueryPropDef, ctx: FilterCtx, lang: QueryDef['lang'], -): FilterCondition => { +): FilterCondition | void => { let mode: FILTER_MODE = MODE_DEFAULT_VAR let val: any - let parsedCondition: FilterCondition if (Array.isArray(value)) { - if (ctx.operation !== EQUAL || !prop.separate) { + if (ctx.operation !== EQUAL || 'main' in prop) { mode = MODE_OR_VAR - const values = [] + const values: any[] = [] for (const v of value) { const parsedValue = parseValue(v, prop, ctx, lang) const size = new Uint8Array(2) @@ -105,7 +104,7 @@ export const createVariableFilterBuffer = ( } val = concatUint8Arr(values) } else { - const x = [] + const x: any[] = [] for (const v of value) { x.push(parseValue(v, prop, ctx, lang)) } @@ -116,69 +115,76 @@ export const createVariableFilterBuffer = ( } if ( - ctx.operation === EQUAL || - ctx.operation === INCLUDES || - ctx.operation === LIKE || - ctx.operation === INCLUDES_TO_LOWER_CASE + ctx.operation !== EQUAL && + ctx.operation !== INCLUDES && + ctx.operation !== LIKE && + ctx.operation !== INCLUDES_TO_LOWER_CASE ) { - if (prop.separate) { - if ( - ctx.operation === EQUAL && - prop.typeIndex !== ALIAS && - prop.typeIndex !== VECTOR - ) { - if (prop.typeIndex === TEXT) { - const fbLen = 2 + val[val.byteLength - 1] - const crc = crc32(val.slice(0, -fbLen)) - const len = val.byteLength - fbLen - const v = new Uint8Array(8 + fbLen) - writeUint32(v, crc, 0) - writeUint32(v, len, 4) - for (let i = 0; i < fbLen; i++) { - v[v.byteLength - (i + 1)] = val[val.byteLength - (i + 1)] - } - parsedCondition = { - buffer: writeVarFilter(mode, v, ctx, prop, 0, 0), - propDef: prop, - } - } else { - parsedCondition = createFixedFilterBuffer( - prop, - 8, - { - operation: EQUAL_CRC32, - type: ctx.type, - opts: ctx.opts, - typeId: ctx.typeId, - }, - val, - false, - ) - } - } else { - if (val instanceof ArrayBuffer) { - val = new Uint8Array(val) - } - parsedCondition = { - buffer: writeVarFilter(mode, val, ctx, prop, 0, 0), - propDef: prop, - } + return + } + + if ('main' in prop) { + return { + buffer: writeVarFilter( + mode, + val, + ctx, + prop, + prop.main.start, + prop.main.size, + ), + propDef: prop, + } + } + + if ( + ctx.operation === EQUAL && + prop.type !== 'alias' && + prop.type !== 'vector' + ) { + if (prop.type === 'text') { + const fbLen = 2 + val[val.byteLength - 1] + const crc = crc32(val.slice(0, -fbLen)) + const len = val.byteLength - fbLen + const v = new Uint8Array(8 + fbLen) + writeUint32(v, crc, 0) + writeUint32(v, len, 4) + for (let i = 0; i < fbLen; i++) { + v[v.byteLength - (i + 1)] = val[val.byteLength - (i + 1)] } - } else { - parsedCondition = { - buffer: writeVarFilter(mode, val, ctx, prop, prop.start, prop.len), + return { + buffer: writeVarFilter(mode, v, ctx, prop, 0, 0), propDef: prop, } } + + return createFixedFilterBuffer( + prop, + 8, + { + operation: EQUAL_CRC32, + type: ctx.type, + opts: ctx.opts, + typeId: ctx.typeId, + }, + val, + false, + ) + } + if (val instanceof ArrayBuffer) { + val = new Uint8Array(val) + } + return { + buffer: writeVarFilter(mode, val, ctx, prop, 0, 0), + propDef: prop, } - return parsedCondition } function writeVarFilter( mode: FILTER_MODE, val: Uint8Array, ctx: FilterCtx, - prop: PropDef | PropDefEdge, + prop: QueryPropDef, start: number, len: number, ): Uint8Array { diff --git a/packages/db/src/client/query/filter/filter.ts b/packages/db/src/client/query/filter/filter.ts index 0307440a4e..62c4080509 100644 --- a/packages/db/src/client/query/filter/filter.ts +++ b/packages/db/src/client/query/filter/filter.ts @@ -5,22 +5,17 @@ import { ReferenceSelect, ReferenceSelectValue, } from '../types.js' -import { - isPropDef, - SchemaTypeDef, - SchemaPropTree, - PropDef, - ID_FIELD_DEF, - TEXT, - REFERENCE, - REFERENCES, - PropDefEdge, -} from '@based/schema/def' import { primitiveFilter } from './primitiveFilter.js' import { Operator } from './types.js' import { Filter, FilterAst, IsFilter } from './types.js' import { DbClient } from '../../index.js' -import { langCodesMap } from '@based/schema' +import { + langCodesMap, + type BranchDef, + type QueryPropDef, + type PropDef, + type TypeDef, +} from '@based/schema' import { filterFieldDoesNotExist, filterInvalidLang } from '../validation.js' export { Operator, Filter } @@ -28,14 +23,16 @@ export { Operator, Filter } const referencesFilter = ( db: DbClient, filter: Filter, - schema: SchemaTypeDef, + schema: TypeDef, conditions: QueryDefFilter, def: QueryDef, -): number => { +): number | void => { + console.warn('TODO: referencesFilter') + /* const [fieldStr, ctx, value] = filter var size = 0 const path = fieldStr.split('.') - let t: PropDef | PropDefEdge | SchemaPropTree = schema.tree + let t: BranchDef | QueryPropDef = schema let referencesSelect: ReferenceSelectValue | void for (let i = 0; i < path.length; i++) { const p = path[i] @@ -50,12 +47,12 @@ const referencesFilter = ( edges = def.target.propDef.edges } if (edges) { - const edgeDef = edges[p] - if (edgeDef) { + const edgesDef = edges[p] + if (edgesDef) { conditions.edges ??= new Map() size += 3 + - primitiveFilter(def, edgeDef, filter, conditions, { + primitiveFilter(def, edgesDef, filter, conditions, { lang: def.lang.lang, fallback: [], }) @@ -78,7 +75,7 @@ const referencesFilter = ( conditions.references ??= new Map() let refConditions = conditions.references.get(t.prop) if (!refConditions) { - const schema = db.schemaTypesParsed[t.inverseTypeName] + const schema = db.defs.byName[t.inverseTypeName] size += t.typeIndex === REFERENCES ? 11 : 6 refConditions = { conditions: { @@ -114,15 +111,19 @@ const referencesFilter = ( filterFieldDoesNotExist(def, fieldStr) return 0 } + */ } export const filterRaw = ( db: DbClient, filter: Filter, - schema: SchemaTypeDef, + schema: TypeDef, conditions: QueryDefFilter, def: QueryDef, ): number => { + console.warn('TODO: filterRaw') + return 0 + /* const field = filter[0] let fieldDef = schema.props[field] @@ -155,6 +156,7 @@ export const filterRaw = ( } return primitiveFilter(def, fieldDef, filter, conditions, def.lang) + */ } export const filter = ( @@ -165,7 +167,9 @@ export const filter = ( ) => { for (const f of filterAst) { if (IsFilter(f)) { - conditions.size += filterRaw(db, f, def.schema, conditions, def) + if (def.schema) { + conditions.size += filterRaw(db, f, def.schema, conditions, def) + } } else { filterOr(db, def, f, conditions) } diff --git a/packages/db/src/client/query/filter/parseFilterValue.ts b/packages/db/src/client/query/filter/parseFilterValue.ts index 5794fac2cc..dd6290edaf 100644 --- a/packages/db/src/client/query/filter/parseFilterValue.ts +++ b/packages/db/src/client/query/filter/parseFilterValue.ts @@ -1,13 +1,4 @@ -import { - PropDef, - PropDefEdge, - TIMESTAMP, - ENUM, - BOOLEAN, - STRING, - BINARY, - TEXT, -} from '@based/schema/def' +import type { QueryPropDef } from '@based/schema' import { crc32 } from '../../crc32.js' import { convertToTimestamp, ENCODER, writeUint32 } from '@based/utils' // ------------------------------------------- @@ -24,25 +15,22 @@ import { convertToTimestamp, ENCODER, writeUint32 } from '@based/utils' // [or = 2] [size 2] [start 2], [op], [size 2], value[size], [size 2], value[size] // ------------------------------------------- -export const parseFilterValue = ( - prop: PropDef | PropDefEdge, - value: any, -): any => { +export const parseFilterValue = (prop: QueryPropDef, value: any): any => { if ( - prop.typeIndex === BINARY || - prop.typeIndex === STRING || - prop.typeIndex === TEXT + prop.type === 'binary' || + prop.type === 'string' || + prop.type === 'text' ) { const b = value instanceof Uint8Array ? value : ENCODER.encode(value) const buf = new Uint8Array(8) writeUint32(buf, crc32(b), 0) writeUint32(buf, b.byteLength, 4) return buf - } else if (prop.typeIndex === BOOLEAN) { + } else if (prop.type === 'boolean') { return value ? 1 : 0 - } else if (prop.typeIndex === ENUM) { - return prop.reverseEnum[value] + 1 - } else if (prop.typeIndex === TIMESTAMP) { + } else if (prop.type === 'enum') { + return prop.enumMap[value] // + 1 // we now just have it plus one already + } else if (prop.type === 'timestamp') { const v = convertToTimestamp(value) if (typeof v !== 'number') { throw new Error(`Incorrect value for timestamp ${prop.path.join('.')}`) diff --git a/packages/db/src/client/query/filter/primitiveFilter.ts b/packages/db/src/client/query/filter/primitiveFilter.ts index e05d5b5c0a..0546b4a52f 100644 --- a/packages/db/src/client/query/filter/primitiveFilter.ts +++ b/packages/db/src/client/query/filter/primitiveFilter.ts @@ -1,13 +1,3 @@ -import { - CARDINALITY, - PropDef, - PropDefEdge, - REFERENCE, - REFERENCES, - REVERSE_SIZE_MAP, - REVERSE_TYPE_INDEX_MAP, - STRING, -} from '@based/schema/def' import { FilterCondition, QueryDef, QueryDefFilter } from '../types.js' import { EQUAL, @@ -21,10 +11,11 @@ import { createVariableFilterBuffer } from './createVariableFilterBuffer.js' import { createFixedFilterBuffer } from './createFixedFilterBuffer.js' import { createReferenceFilter } from './createReferenceFilter.js' import { validateFilter } from '../validation.js' +import type { QueryPropDef } from '@based/schema' export const primitiveFilter = ( def: QueryDef, - prop: PropDef | PropDefEdge, + prop: QueryPropDef, filter: Filter, conditions: QueryDefFilter, lang: QueryDef['lang'], @@ -33,27 +24,21 @@ export const primitiveFilter = ( return 0 } let [, ctx, value] = filter - let parsedCondition: FilterCondition - const fieldIndexChar = prop.prop - const bufferMap = prop.__isEdge ? conditions.edges : conditions.conditions + let parsedCondition: FilterCondition | void + const fieldIndexChar = prop.id + const bufferMap = prop.typeDef.edge ? conditions.edges : conditions.conditions if (ctx.operation === EXISTS) { - if (!prop.separate) { - if (prop.typeIndex === STRING) { + if ('main' in prop) { + if (prop.type === 'string') { ctx.operation = EQUAL ctx.type = ctx.type === TYPE_NEGATE ? TYPE_DEFAULT : TYPE_NEGATE value = '' } else { - console.error( - 'MISSING EXIST / !EXIST FILTER FOR', - prop.path, - REVERSE_TYPE_INDEX_MAP[prop.typeIndex], - ) + console.error('MISSING EXIST / !EXIST FILTER FOR', prop.path, prop.type) } } else { - if (!conditions.exists) { - conditions.exists = [] - } + conditions.exists ??= [] conditions.exists.push({ prop: prop, negate: filter[1].type === TYPE_NEGATE, @@ -67,10 +52,10 @@ export const primitiveFilter = ( if (isArray && value.length === 1) { value = value[0] } - const propSize = REVERSE_SIZE_MAP[prop.typeIndex] - if (prop.typeIndex === REFERENCE) { + const propSize = 'main' in prop ? prop.main.size : 0 + if (prop.type === 'reference') { parsedCondition = createReferenceFilter(prop, ctx, value) - } else if (prop.typeIndex === REFERENCES) { + } else if (prop.type === 'references') { if (ctx.operation === EQUAL && !isArray) { value = [value] } @@ -81,25 +66,34 @@ export const primitiveFilter = ( value, !isNumerical(ctx.operation), ) - } else if (prop.typeIndex === CARDINALITY) { + } else if (prop.type === 'cardinality') { parsedCondition = createFixedFilterBuffer(prop, 4, ctx, value, false) } else if (propSize) { parsedCondition = createFixedFilterBuffer(prop, propSize, ctx, value, false) } else { parsedCondition = createVariableFilterBuffer(value, prop, ctx, lang) } - // ADD OR if array for value - let arr = bufferMap.get(fieldIndexChar) - if (!arr) { - size += 3 // [field] [size 2] - arr = [] - bufferMap.set(fieldIndexChar, arr) + + let arr + if (bufferMap) { + // ADD OR if array for value + arr = bufferMap.get(fieldIndexChar) + if (!arr) { + size += 3 // [field] [size 2] + arr = [] + bufferMap.set(fieldIndexChar, arr) + } } - size += parsedCondition.buffer.byteLength - if ('subscriptionMeta' in parsedCondition) { - conditions.hasSubMeta = true + if (parsedCondition) { + size += parsedCondition.buffer.byteLength + + if ('subscriptionMeta' in parsedCondition) { + conditions.hasSubMeta = true + } + + arr.push(parsedCondition) } - arr.push(parsedCondition) + return size } diff --git a/packages/db/src/client/query/filter/toByteCode.ts b/packages/db/src/client/query/filter/toByteCode.ts index 6278504770..8b9deec113 100644 --- a/packages/db/src/client/query/filter/toByteCode.ts +++ b/packages/db/src/client/query/filter/toByteCode.ts @@ -9,7 +9,6 @@ import { TYPE_DEFAULT, TYPE_NEGATE, } from './types.js' -import { REFERENCES } from '@based/schema/prop-types' const writeConditions = ( result: Uint8Array, @@ -27,13 +26,17 @@ const writeConditions = ( for (const condition of conditions) { conditionSize += condition.buffer.byteLength result.set(condition.buffer, lastWritten) - if ('subscriptionMeta' in condition) { - if ('now' in condition.subscriptionMeta) { - for (const n of condition.subscriptionMeta.now) { - n.resolvedByteIndex = n.byteIndex + lastWritten + metaOffset - } + if (condition?.subscriptionMeta?.now) { + for (const n of condition.subscriptionMeta.now) { + n.resolvedByteIndex = n.byteIndex + lastWritten + metaOffset } } + // if ('now' in condition.subscriptionMeta) { + // for (const n of condition.subscriptionMeta.now) { + // n.resolvedByteIndex = n.byteIndex + lastWritten + metaOffset + // } + // } + // } lastWritten += condition.buffer.byteLength } writeUint16(result, conditionSize, sizeIndex) @@ -63,12 +66,12 @@ export const fillConditionsBuffer = ( if (conditions.references) { for (const [refField, refConditions] of conditions.references) { - const isReferences = refConditions.select.prop.typeIndex === REFERENCES + const isReferences = refConditions.select.prop.type === 'references' result[lastWritten] = isReferences ? META_REFERENCES : META_REFERENCE lastWritten += 1 result[lastWritten] = refField lastWritten += 1 - writeUint16(result, refConditions.conditions.schema.id, lastWritten) + writeUint16(result, refConditions.conditions.schema?.id || 0, lastWritten) lastWritten += 2 if (isReferences) { result[lastWritten] = refConditions.select.type @@ -116,14 +119,10 @@ export const fillConditionsBuffer = ( if (conditions.exists) { for (const exists of conditions.exists) { - result[lastWritten] = META_EXISTS - lastWritten++ - result[lastWritten] = exists.prop.prop - lastWritten++ - result[lastWritten] = exists.negate ? TYPE_NEGATE : TYPE_DEFAULT - lastWritten++ - result[lastWritten] = exists.prop.typeIndex - lastWritten++ + result[lastWritten++] = META_EXISTS + result[lastWritten++] = exists.prop.id + result[lastWritten++] = exists.negate ? TYPE_NEGATE : TYPE_DEFAULT + result[lastWritten++] = exists.prop.typeIndex } } diff --git a/packages/db/src/client/query/filter/types.ts b/packages/db/src/client/query/filter/types.ts index 7f8d428b57..cc191627c9 100644 --- a/packages/db/src/client/query/filter/types.ts +++ b/packages/db/src/client/query/filter/types.ts @@ -157,29 +157,30 @@ export const VECTOR_COSTINE_SIMILARITY = 2 export const VECTOR_EUCLIDEAN_DIST = 3 export const getVectorFn = (optsFn?: FilterOpts['fn']) => { - if (!optsFn) { - return VECTOR_COSTINE_SIMILARITY - } - if (optsFn === 'dotProduct') { - return VECTOR_DOT_PRODUCT - } else if (optsFn === 'euclideanDistance') { - return VECTOR_EUCLIDEAN_DIST - } else if (optsFn === 'manhattanDistance') { - return VECTOR_MANHATTAN_DIST + if (optsFn) { + if (optsFn === 'dotProduct') { + return VECTOR_DOT_PRODUCT + } else if (optsFn === 'euclideanDistance') { + return VECTOR_EUCLIDEAN_DIST + } else if (optsFn === 'manhattanDistance') { + return VECTOR_MANHATTAN_DIST + } } + return VECTOR_COSTINE_SIMILARITY } export const toFilterCtx = ( def: QueryDef, op: Operator, opts: FilterOpts = {}, -): FilterCtx => { +): FilterCtx | void => { + const id = def.schema?.id || 0 if (op === '=' || op === '!=') { return { operation: EQUAL, type: op === '!=' ? TYPE_NEGATE : TYPE_DEFAULT, opts, - typeId: def.schema.id, + typeId: id, } } @@ -188,7 +189,7 @@ export const toFilterCtx = ( operation: EXISTS, type: op === '!exists' ? TYPE_NEGATE : TYPE_DEFAULT, opts, - typeId: def.schema.id, + typeId: id, } } @@ -197,7 +198,7 @@ export const toFilterCtx = ( operation: opts.lowerCase ? INCLUDES_TO_LOWER_CASE : INCLUDES, type: op === '!includes' ? TYPE_NEGATE : TYPE_DEFAULT, opts, - typeId: def.schema.id, + typeId: id, } } @@ -206,7 +207,7 @@ export const toFilterCtx = ( operation: GREATER_THAN, opts, type: TYPE_DEFAULT, - typeId: def.schema.id, + typeId: id, } } @@ -215,7 +216,7 @@ export const toFilterCtx = ( operation: SMALLER_THAN, opts, type: TYPE_DEFAULT, - typeId: def.schema.id, + typeId: id, } } @@ -224,7 +225,7 @@ export const toFilterCtx = ( operation: GREATER_THAN_INCLUSIVE, opts, type: TYPE_DEFAULT, - typeId: def.schema.id, + typeId: id, } } @@ -233,7 +234,7 @@ export const toFilterCtx = ( operation: SMALLER_THAN_INCLUSIVE, opts, type: TYPE_DEFAULT, - typeId: def.schema.id, + typeId: id, } } @@ -242,7 +243,7 @@ export const toFilterCtx = ( operation: LIKE, opts, type: TYPE_DEFAULT, - typeId: def.schema.id, + typeId: id, } } diff --git a/packages/db/src/client/query/include/include.ts b/packages/db/src/client/query/include/include.ts index 79eb87d5ee..ea21d1d85d 100644 --- a/packages/db/src/client/query/include/include.ts +++ b/packages/db/src/client/query/include/include.ts @@ -1,8 +1,7 @@ -import { IncludeOpts } from '../types.js' +import { IncludeOpts, type QueryDef } from '../types.js' import { BranchInclude, QueryBranch } from '../BasedDbQuery.js' import { includeField } from './props.js' -import { REFERENCE, REFERENCES } from '@based/schema/def' -import { createOrGetEdgeRefQueryDef, createOrGetRefQueryDef } from './utils.js' +import { createOrGetRefQueryDef } from './utils.js' export const include = ( query: QueryBranch, @@ -10,41 +9,38 @@ export const include = ( ) => { for (let i = 0; i < fields.length; i++) { const f = fields[i] - const opts = - typeof fields[i + 1] === 'object' && typeof fields[i + 1] !== 'function' - ? (fields[i + 1] as IncludeOpts) - : undefined - + const next = fields[i + 1] + const opts = typeof next === 'object' ? (next as IncludeOpts) : undefined + const def = query.def as QueryDef if (opts) { i++ } if (typeof f === 'string') { - includeField(query.def, { field: f, opts }) + includeField(def, { field: f, opts }) } else if (typeof f === 'function') { f((field: string) => { if (field[0] === '$') { - // @ts-ignore - const prop = query.def.target?.propDef?.edges[field] - if ( - prop && - (prop.typeIndex === REFERENCE || prop.typeIndex === REFERENCES) - ) { - const refDef = createOrGetEdgeRefQueryDef(query.db, query.def, prop) - // @ts-ignore - return new QueryBranch(query.db, refDef) - } - throw new Error( - `No edge reference or edge references field named "${field}"`, - ) + console.warn('TODO edge stuff here') + throw Error('not implemented') + // const prop = def.target?.propDef?.edges[field] + // if ( + // prop && + // (prop.type === 'reference' || prop.type === 'references') + // ) { + // const refDef = createOrGetEdgeRefQueryDef(query.db, def, prop) + // return new QueryBranch(query.db, refDef) + // } + // throw new Error( + // `No edge reference or edge references field named "${field}"`, + // ) } else { - const prop = query.def.props[field] + const prop = def.props[field] if ( prop && - (prop.typeIndex === REFERENCE || prop.typeIndex === REFERENCES) + (prop.type === 'reference' || prop.type === 'references') ) { - const refDef = createOrGetRefQueryDef(query.db, query.def, prop) - // @ts-ignore + const refDef = createOrGetRefQueryDef(query.db, def, prop) return new QueryBranch(query.db, refDef) } throw new Error(`No reference or references field named "${field}"`) @@ -52,7 +48,7 @@ export const include = ( }) } else if (Array.isArray(f)) { if (f.length === 0) { - includeField(query.def, { field: 'id', opts }) + includeField(def, { field: 'id', opts }) } else { include(query, f) } diff --git a/packages/db/src/client/query/include/props.ts b/packages/db/src/client/query/include/props.ts index db4ecafdbf..92a5a2a323 100644 --- a/packages/db/src/client/query/include/props.ts +++ b/packages/db/src/client/query/include/props.ts @@ -1,10 +1,4 @@ -import { - PropDef, - PropDefEdge, - REFERENCE, - REFERENCES, - TEXT, -} from '@based/schema/def' +import { getAllProps, type QueryPropDef } from '@based/schema' import { IncludeField, IncludeOpts, QueryDef, QueryDefType } from '../types.js' export const getAll = ( @@ -14,7 +8,7 @@ export const getAll = ( const fields: IncludeField[] = [] for (const key in props) { const prop = props[key] - if (prop.typeIndex !== REFERENCE && prop.typeIndex !== REFERENCES) { + if (prop.type !== 'reference' && prop.type !== 'references') { fields.push({ field: prop.path.join('.'), opts }) } } @@ -29,15 +23,15 @@ export const getAllRefs = ( const fields: IncludeField[] = [] for (const key in props) { const prop = props[key] - if (prop.typeIndex === REFERENCE || prop.typeIndex === REFERENCES) { + if (prop.type === 'reference' || prop.type === 'references') { const refPath = prop.path.join('.') + affix fields.push({ field: refPath, opts }) - - if (prop.edges) { - for (const edge in prop.edges) { - fields.push({ field: refPath + '.' + edge, opts }) - } - } + console.warn('TODO: edges getAllRefs') + // if (prop.edges) { + // for (const edge in prop.edges) { + // fields.push({ field: refPath + '.' + edge, opts }) + // } + // } } } return fields @@ -51,11 +45,12 @@ export const includeField = (def: QueryDef, include: IncludeField) => { def.type === QueryDefType.References ) { const fields: IncludeField[] = [] - if (def.target.propDef.edges) { - for (const edge in def.target.propDef.edges) { - fields.push({ field: edge, opts }) - } - } + console.warn('TODO: edges includeField') + // if ('target' in def.target.propDef && def.target.propDef.edges) { + // for (const edge in def.target.propDef.edges) { + // fields.push({ field: edge, opts }) + // } + // } includeFields(def, fields) } includeFields(def, getAll(def.props, opts)) @@ -75,26 +70,23 @@ export const includeFields = (def: QueryDef, fields: IncludeField[]) => { } export const includeAllProps = (def: QueryDef, opts?: IncludeOpts) => { - for (const key in def.props) { - const prop = def.props[key] - if (prop.typeIndex !== REFERENCE && prop.typeIndex !== REFERENCES) { - includeProp(def, prop, opts) - } + for (const prop of getAllProps(def.schema)) { + includeProp(def, prop, opts) } } export const includeProp = ( def: QueryDef, - prop: PropDef | PropDefEdge, + prop: QueryPropDef, opts?: IncludeOpts, ) => { - if (!prop || prop.typeIndex === REFERENCE || prop.typeIndex === REFERENCES) { + if (!prop || prop.type === 'reference' || prop.type === 'references') { return false } - if (prop.typeIndex === TEXT) { - if (!def.include.props.has(prop.prop)) { - def.include.props.set(prop.prop, { + if (prop.type === 'text') { + if (!def.include.props.has(prop.id)) { + def.include.props.set(prop.id, { def: prop, opts: { codes: new Set(), @@ -105,27 +97,31 @@ export const includeProp = ( }) } - const langs = def.include.props.get(prop.prop).opts - if (def.lang.fallback.length > 0) { - for (const fallback of def.lang.fallback) { - if (!langs.fallBacks.includes(fallback)) { - langs.fallBacks.push(fallback) + const langs = def.include.props.get(prop.id)?.opts + if (langs?.codes && langs.fallBacks) { + if (def.lang.fallback.length > 0) { + for (const fallback of def.lang.fallback) { + if (!langs.fallBacks.includes(fallback)) { + langs.fallBacks.push(fallback) + } } } + const langCode = def.lang.lang ?? 0 + langs.codes.add(langCode) + if (langCode === 0 || langs.codes.size > 1) { + langs.fallBacks = [] + } } - const langCode = def.lang.lang ?? 0 - langs.codes.add(langCode) - if (langCode === 0 || langs.codes.size > 1) { - langs.fallBacks = [] - } + } else if ('main' in prop) { + def.include.main.len += prop.main.size + def.include.main.include.set(prop.main.start, [ + 0, + prop, + opts as IncludeOpts, + ]) + return true } else { - if (prop.separate) { - def.include.props.set(prop.prop, { def: prop, opts }) - } else { - def.include.main.len += prop.len - def.include.main.include.set(prop.start, [0, prop as PropDef, opts]) - return true - } + def.include.props.set(prop.id, { def: prop, opts }) } return false } diff --git a/packages/db/src/client/query/include/toByteCode.ts b/packages/db/src/client/query/include/toByteCode.ts index c1c4908cb2..352081b54c 100644 --- a/packages/db/src/client/query/include/toByteCode.ts +++ b/packages/db/src/client/query/include/toByteCode.ts @@ -1,4 +1,3 @@ -import { MICRO_BUFFER, STRING, TEXT, JSON, BINARY } from '@based/schema/def' import { DbClient } from '../../index.js' import { IntermediateByteCode, @@ -7,7 +6,7 @@ import { includeOp, } from '../types.js' import { walkDefs } from './walk.js' -import { langCodesMap } from '@based/schema' +import { langCodesMap, typeIndexMap } from '@based/schema' import { writeUint16, writeUint32 } from '@based/utils' import { getEnd } from './utils.js' @@ -28,7 +27,7 @@ export const includeToBuffer = ( return result } - let mainBuffer: Uint8Array + let mainBuffer: Uint8Array | undefined if (def.include.stringFields) { for (const [field, include] of def.include.stringFields.entries()) { @@ -37,17 +36,19 @@ export const includeToBuffer = ( } if (def.include.main.len > 0) { - const len = - def.type === QueryDefType.Edge - ? def.target.ref.edgeMainLen - : def.schema.mainLen + // const len = + // def.type === QueryDefType.Edge + // ? def.target.ref.edgeMainLen + // : def.schema.mainLen + console.warn('TODO: includeToBuffer handle edges here?') + const len = def.schema.size if (def.include.main.len === len) { // Get all main fields mainBuffer = EMPTY_BUFFER let i = 2 for (const value of def.include.main.include.values()) { - value[0] = value[1].start + value[0] = value[1].main.start i += 4 } } else { @@ -58,11 +59,11 @@ export const includeToBuffer = ( let m = 0 for (const value of def.include.main.include.values()) { const propDef = value[1] - writeUint16(mainBuffer, propDef.start, i) - writeUint16(mainBuffer, propDef.len, i + 2) + writeUint16(mainBuffer, propDef.main.start, i) + writeUint16(mainBuffer, propDef.main.size, i + 2) value[0] = m i += 4 - m += propDef.len + m += propDef.main.size } } } @@ -74,14 +75,14 @@ export const includeToBuffer = ( const buf = new Uint8Array(5) buf[0] = includeOp.PARTIAL buf[1] = 0 // field name 0 - buf[2] = MICRO_BUFFER + buf[2] = typeIndexMap.microbuffer writeUint16(buf, mainBuffer.byteLength, 3) result.push(buf, mainBuffer) } else { const buf = new Uint8Array(4) buf[0] = includeOp.DEFAULT buf[1] = 0 // field name 0 - buf[2] = MICRO_BUFFER + buf[2] = typeIndexMap.microbuffer buf[3] = 0 // opts len result.push(buf) } @@ -89,19 +90,25 @@ export const includeToBuffer = ( if (propSize) { for (const [prop, propDef] of def.include.props.entries()) { - const typeIndex = propDef.opts?.raw ? BINARY : propDef.def.typeIndex - if (propDef.opts?.meta) { + const typeIndex = propDef.opts?.raw + ? typeIndexMap.binary + : propDef.def.typeIndex + if (!propDef.opts?.fallBacks) { + continue + } + if (propDef.opts.meta) { if (propDef.opts.codes) { if (propDef.opts.codes.has(0)) { + console.warn('TODO: handle locales!') // TODO use locales for 0 make this NICE - for (const code in def.schema.locales) { - const buf = new Uint8Array(4) - buf[0] = includeOp.META - buf[1] = prop - buf[2] = typeIndex - buf[3] = langCodesMap.get(code) - result.push(buf) - } + // for (const code in def.schema.locales) { + // const buf = new Uint8Array(4) + // buf[0] = includeOp.META + // buf[1] = prop + // buf[2] = typeIndex + // buf[3] = langCodesMap.get(code) ?? 0 + // result.push(buf) + // } } else { for (const code of propDef.opts.codes) { const buf = new Uint8Array(4) @@ -122,10 +129,11 @@ export const includeToBuffer = ( } } - if (propDef.opts?.meta !== 'only') { + if (propDef.opts.meta !== 'only') { const hasEnd = propDef.opts?.end - if (typeIndex === TEXT) { - const codes = propDef.opts.codes + if (typeIndex === typeIndexMap.text) { + const codes = propDef.opts?.codes + if (!codes) continue if (codes.has(0)) { const b = new Uint8Array(hasEnd ? 12 : 4) b[0] = includeOp.DEFAULT @@ -182,7 +190,8 @@ export const includeToBuffer = ( buf[3] = 5 // opts len buf[4] = propDef.opts?.bytes || - (typeIndex !== JSON && typeIndex !== STRING) + (typeIndex !== typeIndexMap.json && + typeIndex !== typeIndexMap.string) ? 0 : 1 writeUint32(buf, getEnd(propDef.opts), 5) diff --git a/packages/db/src/client/query/include/utils.ts b/packages/db/src/client/query/include/utils.ts index 4adf798776..3a87b9006f 100644 --- a/packages/db/src/client/query/include/utils.ts +++ b/packages/db/src/client/query/include/utils.ts @@ -1,9 +1,3 @@ -import { - PropDef, - PropDefEdge, - REFERENCE, - SchemaPropTree, -} from '@based/schema/def' import { DbClient } from '../../index.js' import { createQueryDef } from '../queryDef.js' import { @@ -12,16 +6,20 @@ import { QueryDefType, ReferenceSelectValue, } from '../types.js' -import { inverseLangMap, LangCode, langCodesMap } from '@based/schema' -import { ref } from 'node:process' +import { + inverseLangMap, + LangCode, + langCodesMap, + type BranchDef, + type QueryPropDef, + type PropDef, + type RefPropDef, +} from '@based/schema' -export const getAllFieldFromObject = ( - tree: SchemaPropTree | PropDef, - arr: string[] = [], -) => { - for (const key in tree) { - const leaf = tree[key] - if (!leaf.typeIndex && !leaf.__isPropDef) { +export const getAllFieldFromObject = (tree: BranchDef, arr: string[] = []) => { + for (const key in tree.props) { + const leaf = tree.props[key] + if ('props' in leaf) { getAllFieldFromObject(leaf, arr) } else { arr.push(leaf.path.join('.')) @@ -33,56 +31,54 @@ export const getAllFieldFromObject = ( const createRefQueryDef = ( db: DbClient, def: QueryDef, - t: PropDef | PropDefEdge, + t: RefPropDef, refSelect?: ReferenceSelectValue, ) => { const defRef = createQueryDef( db, - t.typeIndex === REFERENCE - ? QueryDefType.Reference - : QueryDefType.References, + t.type === 'reference' ? QueryDefType.Reference : QueryDefType.References, { - type: t.inverseTypeName, + type: t.target.typeDef.name, //t.inverseTypeName, propDef: t, }, def.skipValidation, ) defRef.lang = def.lang - def.references.set(t.prop, defRef) + def.references.set(t.id, defRef) return defRef } export const createOrGetRefQueryDef = ( db: DbClient, def: QueryDef, - t: PropDef | PropDefEdge, + t: RefPropDef, refSelect?: ReferenceSelectValue, ) => { - let refDef = def.references.get(t.prop) + let refDef = def.references.get(t.id) if (!refDef) { refDef = createRefQueryDef(db, def, t, refSelect) } return refDef } -export const createOrGetEdgeRefQueryDef = ( - db: DbClient, - def: QueryDef, - t: PropDefEdge, -) => { - def.edges ??= createQueryDef( - db, - QueryDefType.Edge, - { - ref: t, - }, - def.skipValidation, - ) - def.edges.props ??= {} - def.edges.props[t.name] = t - const refDef = createOrGetRefQueryDef(db, def.edges, t) - return refDef -} +// export const createOrGetEdgeRefQueryDef = ( +// db: DbClient, +// def: QueryDef, +// t: PropDefEdge, +// ) => { +// def.edges ??= createQueryDef( +// db, +// QueryDefType.Edge, +// { +// ref: t, +// }, +// def.skipValidation, +// ) +// def.edges.props ??= {} +// def.edges.props[t.name] = t +// const refDef = createOrGetRefQueryDef(db, def.edges, t) +// return refDef +// } export const getEnd = (opts?: IncludeOpts, lang?: LangCode): number => { if (!opts || !opts.end) { diff --git a/packages/db/src/client/query/include/walk.ts b/packages/db/src/client/query/include/walk.ts index fa50e1fb2b..648c2f4d1e 100644 --- a/packages/db/src/client/query/include/walk.ts +++ b/packages/db/src/client/query/include/walk.ts @@ -1,11 +1,3 @@ -import { - isPropDef, - PropDef, - REFERENCE, - REFERENCES, - SchemaPropTree, - TEXT, -} from '@based/schema/def' import { createQueryDef } from '../queryDef.js' import { getReferenceSelect, @@ -29,7 +21,8 @@ export const walkDefs = ( const prop = def.props[include.field] const path = include.field.split('.') let referencesSelect: ReferenceSelectValue | void - + console.warn('TODO: walkDefs') + /* if (!prop) { let t: PropDef | SchemaPropTree = def.schema.tree for (let i = 0; i < path.length; i++) { @@ -154,4 +147,5 @@ export const walkDefs = ( } else { includeProp(def, prop, include.opts) } + */ } diff --git a/packages/db/src/client/query/queryDef.ts b/packages/db/src/client/query/queryDef.ts index 16f7de4b09..62fbd88b5f 100644 --- a/packages/db/src/client/query/queryDef.ts +++ b/packages/db/src/client/query/queryDef.ts @@ -4,7 +4,7 @@ import { DEF_RANGE_PROP_LIMIT, DEF_RANGE_REF_LIMIT } from './thresholds.js' import { EdgeTarget, QueryDef, - QueryDefEdges, + // QueryDefEdges, QueryDefRest, QueryDefShared, QueryDefType, @@ -25,7 +25,7 @@ const createEmptySharedDef = (skipValidation: boolean) => { filter: { conditions: new Map(), size: 0, hasSubMeta: false }, range: { offset: 0, limit: 0 }, lang: { - lang: langCodesMap.get('none'), + lang: langCodesMap.get('none') ?? 0, fallback: [], }, include: { @@ -43,14 +43,21 @@ const createEmptySharedDef = (skipValidation: boolean) => { return q } -type CreateQueryDefReturn = T extends QueryDefType.Edge - ? QueryDefEdges - : T extends - | QueryDefType.Root - | QueryDefType.Reference - | QueryDefType.References - ? QueryDefRest - : QueryDef +// type CreateQueryDefReturn = T extends QueryDefType.Edge +// ? QueryDefEdges +// : T extends +// | QueryDefType.Root +// | QueryDefType.Reference +// | QueryDefType.References +// ? QueryDefRest +// : QueryDef + +type CreateQueryDefReturn = T extends + | QueryDefType.Root + | QueryDefType.Reference + | QueryDefType.References + ? QueryDefRest + : QueryDef export function createQueryDef( db: DbClient, @@ -60,17 +67,19 @@ export function createQueryDef( ): CreateQueryDefReturn { const queryDef = createEmptySharedDef(skipValidation) if (type === QueryDefType.Edge) { - const t = target as EdgeTarget - const q = queryDef as QueryDefEdges - q.props = t.ref.edges - q.type = type - q.target = t - return q as CreateQueryDefReturn + console.warn('TODO: edges createQueryDef') + return queryDef as CreateQueryDefReturn + // const t = target as EdgeTarget + // const q = queryDef as QueryDefEdges + // q.props = t.ref.edges + // q.type = type + // q.target = t + // return q as CreateQueryDefReturn } else { const t = target as Target const q = queryDef as QueryDefRest q.schema = validateType(db, q, t.type) - q.props = q.schema.props + q.props = q.schema.queryProps q.type = type q.target = t if (type === QueryDefType.Root) { diff --git a/packages/db/src/client/query/queryDefToReadSchema.ts b/packages/db/src/client/query/queryDefToReadSchema.ts index d80d907e53..4f27548f89 100644 --- a/packages/db/src/client/query/queryDefToReadSchema.ts +++ b/packages/db/src/client/query/queryDefToReadSchema.ts @@ -1,56 +1,84 @@ // import type { IncludeOpts, QueryDef, Target } from '@based/db' -import { inverseLangMap, langCodesMap } from '@based/schema' import { - PropDef, - PropDefEdge, - COLVEC, - ENUM, - TEXT, - VECTOR, - BINARY, - CARDINALITY, -} from '@based/schema/def' + inverseLangMap, + langCodesMap, + typeIndexMap, + type LangCode, + type QueryPropDef, + type PropDef, + type SchemaVector, +} from '@based/schema' import { ReaderLocales, ReaderMeta, ReaderPropDef, ReaderSchema, ReaderSchemaEnum, + VectorBaseType, } from '@based/protocol/db-read' import { IncludeOpts, QueryDef, Target } from './types.js' +const schemaVectorBaseTypeToEnum = ( + vector: SchemaVector['baseType'], +): VectorBaseType => { + switch (vector) { + case 'int8': + return VectorBaseType.Int8 + case 'uint8': + return VectorBaseType.Uint8 + case 'int16': + return VectorBaseType.Int16 + case 'uint16': + return VectorBaseType.Uint16 + case 'int32': + return VectorBaseType.Int32 + case 'uint32': + return VectorBaseType.Uint32 + case 'float32': + return VectorBaseType.Float32 + case 'float64': + return VectorBaseType.Float64 + case 'number': + return VectorBaseType.Float64 + } +} + const createReaderPropDef = ( - p: PropDef | PropDefEdge, + p: QueryPropDef, locales: ReaderLocales, opts?: IncludeOpts, ): ReaderPropDef => { + console.warn('TODO: handle edges in createReaderPropDef') const readerPropDef: ReaderPropDef = { - path: p.__isEdge ? p.path.slice(1) : p.path, - typeIndex: opts?.raw ? BINARY : p.typeIndex, + path: p.path, //p.__isEdge ? p.path.slice(1) : p.path, + typeIndex: opts?.raw ? typeIndexMap.binary : p.typeIndex, readBy: 0, } if (opts?.meta) { readerPropDef.meta = opts?.meta === 'only' ? ReaderMeta.only : ReaderMeta.combined } - if (p.typeIndex === ENUM) { + if (p.type === 'enum') { readerPropDef.enum = p.enum } - if (p.typeIndex === VECTOR || p.typeIndex === COLVEC) { - readerPropDef.vectorBaseType = p.vectorBaseType - readerPropDef.len = p.len + if (p.type === 'vector' || p.type === 'colvec') { + readerPropDef.vectorBaseType = schemaVectorBaseTypeToEnum(p.baseType) + readerPropDef.len = p.size } - if (p.typeIndex === CARDINALITY) { - readerPropDef.cardinalityMode = p.cardinalityMode - readerPropDef.cardinalityPrecision = p.cardinalityPrecision + if (p.type === 'cardinality') { + readerPropDef.cardinalityMode = p.mode === 'dense' ? 1 : 0 + readerPropDef.cardinalityPrecision = p.precision ?? 8 } - if (p.typeIndex === TEXT) { - if (opts.codes.has(0)) { + if (p.type === 'text') { + if (opts?.codes?.has(0)) { readerPropDef.locales = locales } else { - if (opts.codes.size === 1 && opts.codes.has(opts.localeFromDef)) { + if ( + opts?.codes?.size === 1 && + opts.codes.has(opts.localeFromDef as LangCode) + ) { // dont add locales - interpets it as a normal prop - } else { + } else if (opts?.codes) { readerPropDef.locales = {} for (const code of opts.codes) { readerPropDef.locales[code] = inverseLangMap.get(code) @@ -75,14 +103,16 @@ export const convertToReaderSchema = ( ): ReaderSchema => { if (!locales) { locales = {} - for (const lang in q.schema.locales) { - locales[langCodesMap.get(lang)] = lang - } + console.warn('TODO: locales convertToReaderSchema') + // for (const lang in q.schema.locales) { + // locales[langCodesMap.get(lang)] = lang + // } } const t = q.type const isRoot = t === 4 // QueryDefType.Root (cant import type enum ofc) const isSingle = (isRoot && ('id' in q.target || 'alias' in q.target)) || q.selectFirstResult + // @ts-ignore const isEdge = t === 1 // QueryDefType.Edge (cant import type enum ofc) const readerSchema: ReaderSchema = { readId: 0, @@ -124,7 +154,7 @@ export const convertToReaderSchema = ( if (q.aggregate.groupBy.display) { a.groupBy.display = q.aggregate.groupBy.display } - if (q.aggregate.groupBy.enum) { + if ('enum' in q.aggregate.groupBy) { a.groupBy.enum = q.aggregate.groupBy.enum } if (q.aggregate.groupBy.stepType) { @@ -136,7 +166,7 @@ export const convertToReaderSchema = ( let body = '' for (const def of q.schema.propHooks.read) { const target = `r.${def.path.join('.')}` - body += `if(r.${def.path.join('?.')}!=null)${target}=(${normalizeHookFn(def.hooks.read)})(${target},r);` + body += `if(r.${def.path.join('?.')}!=null)${target}=(${normalizeHookFn(def.hooks?.read as any)})(${target},r);` } if (q.schema?.hooks?.read) { @@ -160,15 +190,16 @@ export const convertToReaderSchema = ( } for (const [k, v] of q.references.entries()) { const target = v.target as Target - const propDef = target.propDef + const propDef = target.propDef as QueryPropDef readerSchema.refs[k] = { schema: convertToReaderSchema(v, locales), prop: createReaderPropDef(propDef, locales), } } - if (q.edges) { - readerSchema.edges = convertToReaderSchema(q.edges, locales) - } + console.warn('TODO: handle edges here convertToReaderSchema') + // if (q.edges) { + // readerSchema.edges = convertToReaderSchema(q.edges, locales) + // } } return readerSchema } diff --git a/packages/db/src/client/query/registerQuery.ts b/packages/db/src/client/query/registerQuery.ts index 8afb28bf58..4b5fcd51a9 100644 --- a/packages/db/src/client/query/registerQuery.ts +++ b/packages/db/src/client/query/registerQuery.ts @@ -2,13 +2,14 @@ import { BasedDbQuery } from './BasedDbQuery.js' import { queryToBuffer } from './toByteCode/toByteCode.js' import { handleErrors } from './validation.js' import { createQueryDef } from './queryDef.js' -import { QueryDefType } from './types.js' +import { QueryDefType, type QueryDef } from './types.js' import { includeField } from './query.js' import { convertToReaderSchema } from './queryDefToReadSchema.js' export const registerQuery = (q: BasedDbQuery): Uint8Array => { if (!q.buffer) { const commands = q.queryCommands + // @ts-ignore q.queryCommands = null const def = createQueryDef( q.db, @@ -45,6 +46,6 @@ export const registerQuery = (q: BasedDbQuery): Uint8Array => { handleErrors(q.def) return buf } - handleErrors(q.def) + handleErrors(q.def as QueryDef) return q.buffer } diff --git a/packages/db/src/client/query/search/index.ts b/packages/db/src/client/query/search/index.ts index 80fe5e46c8..7219289ade 100644 --- a/packages/db/src/client/query/search/index.ts +++ b/packages/db/src/client/query/search/index.ts @@ -1,5 +1,9 @@ -import { langCodesMap } from '@based/schema' -import { STRING, TEXT, VECTOR } from '@based/schema/def' +import { + langCodesMap, + typeIndexMap, + type LangCode, + type LeafDef, +} from '@based/schema' import { QueryDefSearch, QueryDef } from '../types.js' import { FilterOpts, getVectorFn } from '../filter/types.js' import { @@ -24,19 +28,16 @@ export const vectorSearch = ( field: string, opts: Omit, ) => { - let prop = def.props[field] - if (!prop) { - prop = searchDoesNotExist(def, field, true) - } - if (prop.typeIndex !== VECTOR) { - searchIncorrectType(def, prop) + const prop = def.props[field] || searchDoesNotExist(def, field, true) + if (prop.type !== 'vector') { + searchIncorrectType(def, prop as any) } let size = 17 const vec = new Uint8Array(q.buffer, 0, q.byteLength) size += vec.byteLength def.search = { size: size, - prop: prop.prop, + prop: prop.id, query: vec, isVector: true, opts, @@ -48,7 +49,7 @@ export const search = ( q: string, s?: Search, ) => { - const def = queryBranch.def + const def = queryBranch.def as QueryDef const bufs: Uint8Array[] = [] let nrBlocks = 0 let totalByteLength = 1 @@ -93,7 +94,7 @@ export const search = ( for (const k in def.props) { const prop = def.props[k] // if title / name / headline add ROLE: - if (prop.typeIndex === STRING || prop.typeIndex === TEXT) { + if (prop.type === 'string' || prop.type === 'text') { s[k] = k === 'title' || k === 'name' || k === 'headline' ? 0 : 2 } } @@ -105,27 +106,27 @@ export const search = ( s = x } - let hookFields: Set + let hookFields: Set | undefined for (const key in s) { let prop = def.props[key] - let lang = def.lang.lang + let lang: LangCode = def.lang.lang let fallback = def.lang.fallback if (!prop) { if (key.includes('.')) { const k = key.split('.') prop = def.props[k.slice(0, -1).join('.')] - if (prop && prop.typeIndex === TEXT) { - lang = langCodesMap.get(k[k.length - 1]) + if (prop && prop.type === 'text') { + lang = langCodesMap.get(k[k.length - 1]) as LangCode fallback = [] // handle incorrect LANG } else { - prop = searchDoesNotExist(def, key, false) + prop = searchDoesNotExist(def, key, false) as any } } else { - prop = searchDoesNotExist(def, key, false) + prop = searchDoesNotExist(def, key, false) as any } } - if (prop.typeIndex !== STRING && prop.typeIndex !== TEXT) { + if (prop.type !== 'string' && prop.type !== 'text') { searchIncorrectType(def, prop) } @@ -141,24 +142,24 @@ export const search = ( typeIndex: prop.typeIndex, weight: s[key], lang: { lang, fallback }, - field: prop.prop, - start: prop.start ?? 0, // also need lang ofc if you have start + field: prop.id, + start: 'main' in prop ? prop.main.start : 0, // also need lang ofc if you have start }) - const searchHook = prop.hooks?.search - if (searchHook) { + if (prop.hooks?.search) { + const hook = prop.hooks.search hookFields ??= new Set(Object.keys(s)) - prop.hooks.search = null - searchHook(queryBranch, hookFields) - prop.hooks.search = searchHook + prop.hooks.search = undefined + hook(queryBranch, hookFields) + prop.hooks.search = hook } } - const searchHook = def.schema.hooks?.search - if (searchHook) { - def.schema.hooks.search = null - searchHook(queryBranch, hookFields || new Set(Object.keys(s))) - def.schema.hooks.search = searchHook + if (def.schema.hooks?.search) { + const hook = def.schema.hooks.search + def.schema.hooks.search = undefined + hook(queryBranch, hookFields || new Set(Object.keys(s))) + def.schema.hooks.search = hook } } diff --git a/packages/db/src/client/query/sort.ts b/packages/db/src/client/query/sort.ts index 12bbf91e62..2c0bd9be36 100644 --- a/packages/db/src/client/query/sort.ts +++ b/packages/db/src/client/query/sort.ts @@ -5,12 +5,19 @@ export const createSortBuffer = (sort: QueryDefSort) => { const buf = new Uint8Array(8) // [order] [propType] [start] [start] [len] [len] [lang] buf[0] = sort.order - buf[1] = sort.prop.prop + buf[1] = sort.prop.id buf[2] = sort.prop.typeIndex - buf[3] = sort.prop.start - buf[4] = sort.prop.start >>> 8 - buf[5] = sort.prop.len - buf[6] = sort.prop.len >>> 8 + if ('main' in sort.prop) { + buf[3] = sort.prop.main.start + buf[4] = sort.prop.main.start >>> 8 + buf[5] = sort.prop.main.size + buf[6] = sort.prop.main.size >>> 8 + } else { + buf[3] = 0 + buf[4] = 0 + buf[5] = 0 + buf[6] = 0 + } buf[7] = sort.lang return buf } diff --git a/packages/db/src/client/query/subscription/index.ts b/packages/db/src/client/query/subscription/index.ts index 61574a0c42..7b8cc73b43 100644 --- a/packages/db/src/client/query/subscription/index.ts +++ b/packages/db/src/client/query/subscription/index.ts @@ -1,6 +1,7 @@ import { BasedDbQuery } from '../BasedDbQuery.js' import { BasedQueryResponse } from '../BasedQueryResponse.js' import { registerQuery } from '../registerQuery.js' +import type { QueryDef } from '../types.js' import { registerSubscription } from './toByteCode.js' import { OnData, OnError, OnClose } from './types.js' @@ -13,7 +14,7 @@ export class SubStore { subscribe(q: BasedDbQuery) { const onData = (res: Uint8Array) => { if (!this.response) { - this.response = new BasedQueryResponse(q.def, res, 0) + this.response = new BasedQueryResponse(q.def as QueryDef, res, 0) } else { this.response.result = res this.response.end = res.byteLength @@ -70,7 +71,7 @@ export class SubStore { resubscribe(q: BasedDbQuery) { this.onClose() q.reset() - this.response = null + this.response = undefined this.subscribe(q) } } @@ -87,7 +88,7 @@ export const subscribe = ( q.db.subs.set(q, store) } else { const store = q.db.subs.get(q) - store.listeners.set(onData, onError) + store?.listeners.set(onData, onError) } return () => { const store = q.db.subs.get(q) diff --git a/packages/db/src/client/query/subscription/toByteCode.ts b/packages/db/src/client/query/subscription/toByteCode.ts index 684253a554..c73b84f373 100644 --- a/packages/db/src/client/query/subscription/toByteCode.ts +++ b/packages/db/src/client/query/subscription/toByteCode.ts @@ -4,6 +4,7 @@ import { BasedDbQuery } from '../BasedDbQuery.js' import { ID } from '../toByteCode/offsets.js' import { FilterMetaNow, QueryDef, QueryDefFilter, QueryType } from '../types.js' import { SubscriptionType } from './types.js' +import type { MainDef } from '@based/schema' type Fields = { separate: Set; main: Set } @@ -29,7 +30,7 @@ export const collectFilters = ( fields.separate.add(prop) if (prop === 0) { for (const condition of conditions) { - fields.main.add(condition.propDef.start) + fields.main.add((condition.propDef as MainDef).main.start) } } } @@ -45,7 +46,7 @@ export const collectFilters = ( if (filter.references) { for (const ref of filter.references.values()) { for (const prop of filter.conditions.keys()) { - fields.separate.add(prop) + fields?.separate.add(prop) } collectFilters(ref.conditions, undefined, nowQueries) } @@ -60,13 +61,13 @@ export const collectFields = (def: QueryDef) => { } if (def.include.main.len > 0) { for (const [, propDef] of def.include.main.include.values()) { - fields.main.add(propDef.start) + fields.main.add(propDef.main.start) } // Add 0 fields.separate.add(0) } for (const prop of def.include.props.values()) { - fields.separate.add(prop.def.prop) + fields.separate.add(prop.def.id) } for (const prop of def.references.keys()) { fields.separate.add(prop) @@ -78,20 +79,20 @@ export const collectTypes = ( def: QueryDef | QueryDefFilter, types: Set = new Set(), ) => { - if ('references' in def) { + if (def.references) { for (const ref of def.references.values()) { if ('schema' in ref) { types.add(ref.schema.id) collectTypes(ref, types) } else { - types.add(ref.conditions.schema.id) + types.add(ref.conditions.schema?.id ?? 0) collectTypes(ref.conditions, types) } } } - if ('filter' in def && 'references' in def.filter) { + if ('filter' in def && def.filter.references) { for (const ref of def.filter.references.values()) { - types.add(ref.conditions.schema.id) + types.add(ref.conditions.schema?.id ?? 0) collectTypes(ref.conditions) } } @@ -99,17 +100,18 @@ export const collectTypes = ( } export const registerSubscription = (query: BasedDbQuery) => { - if (query.def.queryType === QueryType.id) { + const def = query.def as QueryDef + + if (def.queryType === QueryType.id) { + const buf = query.buffer as Uint8Array // @ts-ignore - const id = query.def.target.id - const fields = collectFields(query.def) - const typeId = query.def.schema.id - const subId = native.crc32( - query.buffer.subarray(ID.id + 4, query.buffer.byteLength - 4), - ) + const id = def.target.id as number + const fields = collectFields(def) + const typeId = def.schema.id + const subId = native.crc32(buf.subarray(ID.id + 4, buf.byteLength - 4)) const headerLen = 18 - const types = collectTypes(query.def) - const nowQueries = collectFilters(query.def.filter, fields) + const types = collectTypes(def) + const nowQueries = collectFilters(def.filter, fields) const buffer = new Uint8Array( headerLen + fields.separate.size + @@ -139,7 +141,7 @@ export const registerSubscription = (query: BasedDbQuery) => { i += 2 } for (const now of nowQueries) { - buffer[i] = now.prop.prop + buffer[i] = now.prop.id buffer[i + 1] = now.ctx.operation writeUint16(buffer, now.ctx.typeId, i + 2) writeInt64(buffer, now.offset, i + 4) @@ -148,9 +150,9 @@ export const registerSubscription = (query: BasedDbQuery) => { } query.subscriptionBuffer = buffer } else { - const typeId = query.def.schema.id - const types = collectTypes(query.def) - const nowQueries = collectFilters(query.def.filter, { + const typeId = def.schema.id + const types = collectTypes(def) + const nowQueries = collectFilters(def.filter, { separate: new Set(), main: new Set(), }) @@ -171,7 +173,7 @@ export const registerSubscription = (query: BasedDbQuery) => { } } for (const now of nowQueries) { - buffer[i] = now.prop.prop + buffer[i] = now.prop.id buffer[i + 1] = now.ctx.operation writeUint16(buffer, now.ctx.typeId, i + 2) writeInt64(buffer, now.offset, i + 4) diff --git a/packages/db/src/client/query/toByteCode/aggregates.ts b/packages/db/src/client/query/toByteCode/aggregates.ts index 4b23dd76a9..d99c3cbc18 100644 --- a/packages/db/src/client/query/toByteCode/aggregates.ts +++ b/packages/db/src/client/query/toByteCode/aggregates.ts @@ -5,14 +5,17 @@ import { QueryType, includeOp, IntermediateByteCode, + type QueryDefAggregation, } from '../types.js' import { aggregateToBuffer, isRootCountOnly, } from '../aggregates/aggregation.js' +import { writeUint16 } from '@based/utils' export const aggregatesQuery = (def: QueryDef): IntermediateByteCode => { - const aggregateSize = def.aggregate.size || 0 + const aggregate = def.aggregate as QueryDefAggregation + const aggregateSize = aggregate.size || 0 if (aggregateSize === 0) { throw new Error('Wrong aggregate size (0)') } @@ -37,10 +40,12 @@ export const aggregatesQuery = (def: QueryDef): IntermediateByteCode => { } // required to get typeEntry and fieldSchema - buf[9 + filterSize] = def.schema.idUint8[0] // typeId - buf[9 + 1 + filterSize] = def.schema.idUint8[1] // typeId - buf[9 + 2 + filterSize] = def.target.propDef.prop // refField - const aggregateBuffer = aggregateToBuffer(def.aggregate) + writeUint16(buf, def.schema.id, 9 + filterSize) + + // buf[9 + filterSize] = def.schema.idUint8[0] // typeId + // buf[9 + 1 + filterSize] = def.schema.idUint8[1] // typeId + buf[9 + 2 + filterSize] = def.target.propDef?.id ?? 0 // refField + const aggregateBuffer = aggregateToBuffer(aggregate) buf.set(aggregateBuffer, 9 + 3 + filterSize) return { buffer: buf, def, needsMetaResolve: def.filter.hasSubMeta } } else { @@ -48,8 +53,10 @@ export const aggregatesQuery = (def: QueryDef): IntermediateByteCode => { buf[0] = isRootCountOnly(def, filterSize) ? QueryType.aggregatesCountType : QueryType.aggregates - buf[1] = def.schema.idUint8[0] - buf[2] = def.schema.idUint8[1] + + writeUint16(buf, def.schema.id, 1) + // buf[1] = def.schema.idUint8[0] + // buf[2] = def.schema.idUint8[1] buf[3] = def.range.offset buf[4] = def.range.offset >>> 8 buf[5] = def.range.offset >>> 16 @@ -63,7 +70,7 @@ export const aggregatesQuery = (def: QueryDef): IntermediateByteCode => { if (filterSize) { buf.set(filterToBuffer(def.filter, 13), 13) } - const aggregateBuffer = aggregateToBuffer(def.aggregate) + const aggregateBuffer = aggregateToBuffer(aggregate) buf[14 + filterSize] = aggregateSize buf[15 + filterSize] = aggregateSize >>> 8 buf.set(aggregateBuffer, 16 + filterSize) diff --git a/packages/db/src/client/query/toByteCode/ids.ts b/packages/db/src/client/query/toByteCode/ids.ts index 508b6ab73d..fc117b3974 100644 --- a/packages/db/src/client/query/toByteCode/ids.ts +++ b/packages/db/src/client/query/toByteCode/ids.ts @@ -8,14 +8,14 @@ import { IDS } from './offsets.js' export const idsQuery = (def: QueryDef): IntermediateByteCode => { const filterSize = def.filter.size || 0 - let sort: Uint8Array + let sort: Uint8Array | undefined let sortSize = 0 if (def.sort) { sort = createSortBuffer(def.sort) sortSize = sort.byteLength } - let search: Uint8Array + let search: Uint8Array | undefined let searchSize = 0 if (def.search) { search = searchToBuffer(def.search) @@ -49,14 +49,14 @@ export const idsQuery = (def: QueryDef): IntermediateByteCode => { writeUint16(buffer, sortSize, index) index += 2 if (sortSize) { - buffer.set(sort, index) + buffer.set(sort as Uint8Array, index) index += sortSize } writeUint16(buffer, searchSize, index) index += 2 if (searchSize) { - buffer.set(search, index) + buffer.set(search as Uint8Array, index) } return { buffer, def, needsMetaResolve: def.filter.hasSubMeta } diff --git a/packages/db/src/client/query/toByteCode/references.ts b/packages/db/src/client/query/toByteCode/references.ts index fe35d3fa38..5a3c254ba3 100644 --- a/packages/db/src/client/query/toByteCode/references.ts +++ b/packages/db/src/client/query/toByteCode/references.ts @@ -10,7 +10,7 @@ export const referencesQuery = ( ): IntermediateByteCode => { const filterSize = def.filter.size || 0 - let sort: Uint8Array + let sort: Uint8Array | undefined let sortSize = 0 if (def.sort) { sort = createSortBuffer(def.sort) diff --git a/packages/db/src/client/query/toByteCode/toByteCode.ts b/packages/db/src/client/query/toByteCode/toByteCode.ts index d811f3ecf9..f8ea07e415 100644 --- a/packages/db/src/client/query/toByteCode/toByteCode.ts +++ b/packages/db/src/client/query/toByteCode/toByteCode.ts @@ -4,6 +4,7 @@ import { QueryDef, QueryDefType, includeOp, + type QueryDefRest, } from '../types.js' import { includeToBuffer } from '../include/toByteCode.js' import { searchToBuffer } from '../search/index.js' @@ -45,21 +46,24 @@ export function defToBuffer( } }) - let edges: IntermediateByteCode[] - let edgesSize = 0 + // let edges: IntermediateByteCode[] + // let edgesSize = 0 - if (def.edges) { - edges = includeToBuffer(db, def.edges) - def.edges.references.forEach((ref) => { - edges.push(...defToBuffer(db, ref)) - if (ref.errors) { - def.errors.push(...ref.errors) - } - }) - edgesSize = byteSize(edges) - } + // if (def.edges) { + // edges = includeToBuffer(db, def.edges) + // def.edges.references.forEach((ref) => { + // edges.push(...defToBuffer(db, ref)) + // if (ref.errors) { + // def.errors.push(...ref.errors) + // } + // }) + // edgesSize = byteSize(edges) + // } - const size = (edges ? edgesSize + 3 : 0) + byteSize(include) + // const size = (edges ? edgesSize + 3 : 0) + byteSize(include) + + console.warn('TODO: handle edges defToBuffer') + const size = byteSize(include) if (def.aggregate) { result.push(aggregatesQuery(def)) @@ -90,14 +94,14 @@ export function defToBuffer( } result.push(idsQuery(def)) } else { - let search: Uint8Array + let search: Uint8Array | undefined let searchSize = 0 if (def.search) { search = searchToBuffer(def.search) searchSize = def.search.size } - let sort: Uint8Array + let sort: Uint8Array | undefined let sortSize = 0 if (def.sort) { sort = createSortBuffer(def.sort) @@ -106,7 +110,14 @@ export function defToBuffer( const filterSize = def.filter.size || 0 result.push( - defaultQuery(def, filterSize, sortSize, searchSize, sort, search), + defaultQuery( + def, + filterSize, + sortSize, + searchSize, + sort as Uint8Array, + search as Uint8Array, + ), ) } } @@ -118,13 +129,13 @@ export function defToBuffer( result.push(...include) - if (edges) { - const metaEdgeBuffer = new Uint8Array(3) - metaEdgeBuffer[0] = includeOp.EDGE - metaEdgeBuffer[1] = edgesSize - metaEdgeBuffer[2] = edgesSize >>> 8 - result.push(metaEdgeBuffer, ...edges) - } + // if (edges) { + // const metaEdgeBuffer = new Uint8Array(3) + // metaEdgeBuffer[0] = includeOp.EDGE + // metaEdgeBuffer[1] = edgesSize + // metaEdgeBuffer[2] = edgesSize >>> 8 + // result.push(metaEdgeBuffer, ...edges) + // } if (def.type === QueryDefType.Root) { result.push(schemaChecksum(def)) @@ -134,7 +145,7 @@ export function defToBuffer( } export const queryToBuffer = (query: BasedDbQuery) => { - const bufs = defToBuffer(query.db, query.def) + const bufs = defToBuffer(query.db, query.def as QueryDef) // allow both uint8 and def let totalByteLength = bufs.reduce( (acc, cur) => acc + cur.buffer.byteLength, diff --git a/packages/db/src/client/query/types.ts b/packages/db/src/client/query/types.ts index 4c0bd93f46..73c193bbb5 100644 --- a/packages/db/src/client/query/types.ts +++ b/packages/db/src/client/query/types.ts @@ -1,5 +1,10 @@ -import { LangCode, LangName } from '@based/schema' -import { PropDef, PropDefEdge, SchemaTypeDef } from '@based/schema/def' +import type { + LangCode, + LangName, + TypeDef, + MainDef, + QueryPropDef, +} from '@based/schema' import { FilterCtx, FilterOpts } from './filter/types.js' import { QueryError } from './validation.js' import { Interval, aggFnOptions } from './aggregates/types.js' @@ -20,9 +25,9 @@ export type IncludeField = { opts?: IncludeOpts } -export type MainIncludes = Map +export type MainIncludes = Map -export type IncludeTreeArr = (string | PropDef | IncludeTreeArr)[] +export type IncludeTreeArr = (string | QueryPropDef | IncludeTreeArr)[] export enum QueryType { id = 0, @@ -42,7 +47,7 @@ export enum ReferenceSelect { export type ReferenceSelectValue = { type: ReferenceSelect index?: number - prop: PropDef | PropDefEdge + prop: QueryPropDef } export type ReferenceSelectOperator = '*' | '*?' | number @@ -54,7 +59,10 @@ export const getReferenceSelect = ( if (p[p.length - 1] === ']') { const [refsField, indexNotation] = p.split('[') const index = indexNotation.slice(0, -1) - const ref = def.schema.props[refsField] + const ref = def.schema?.props[refsField] + if (!ref || 'props' in ref) { + return + } if (index === '*') { return { type: ReferenceSelect.All, prop: ref } } @@ -76,17 +84,17 @@ enum QueryDefType { } export type EdgeTarget = { - ref: PropDef | PropDefEdge | null + ref: QueryPropDef | null } export type Target = { type: string id?: number | void | Promise ids?: Uint32Array | void - propDef?: PropDef | PropDefEdge + propDef?: QueryPropDef alias?: QueryByAliasObj // This can just instantly be added - resolvedAlias?: { def: PropDef; value: string } + resolvedAlias?: { def: QueryPropDef; value: string } } export const isRefDef = (def: QueryDef): def is QueryDefRest => { @@ -100,12 +108,12 @@ export type FilterMetaNow = { resolvedByteIndex: number offset: number ctx: FilterCtx - prop: PropDef | PropDefEdge + prop: QueryPropDef } export type FilterCondition = { buffer: Uint8Array - propDef: PropDef | PropDefEdge + propDef: QueryPropDef subscriptionMeta?: { now?: FilterMetaNow[] } @@ -114,7 +122,7 @@ export type FilterCondition = { export type QueryDefFilter = { size: number conditions: Map - exists?: { prop: PropDef | PropDefEdge; negate: boolean }[] + exists?: { prop: QueryPropDef; negate: boolean }[] references?: Map< number, { @@ -122,8 +130,8 @@ export type QueryDefFilter = { select: ReferenceSelectValue } > - fromRef?: PropDef - schema?: SchemaTypeDef + fromRef?: QueryPropDef + schema?: TypeDef edges?: Map or?: QueryDefFilter // Make this work @@ -153,14 +161,14 @@ export type QueryDefSearch = } export type QueryDefSort = { - prop: PropDefEdge | PropDef + prop: QueryPropDef order: 0 | 1 lang: LangCode } export type Aggregation = { type: AggregateType - propDef: PropDef | PropDefEdge + propDef: QueryPropDef resultPos: number accumulatorPos: number isEdge: boolean @@ -168,7 +176,7 @@ export type Aggregation = { export type QueryDefAggregation = { size: number - groupBy?: aggPropDef + groupBy?: AggQueryPropDef // only field 0 to start aggregates: Map option?: aggFnOptions @@ -176,7 +184,7 @@ export type QueryDefAggregation = { totalAccumulatorSize: number } -export interface aggPropDef extends PropDef { +export type AggQueryPropDef = QueryPropDef & { stepType?: Interval stepRange?: number tz?: number @@ -202,32 +210,33 @@ export type QueryDefShared = { } include: { stringFields: Map - props: Map + props: Map main: { include: MainIncludes len: number } } references: Map - edges?: QueryDefEdges + // edges?: QueryDefEdges readSchema?: ReaderSchema } -export type QueryDefEdges = { - type: QueryDefType.Edge - target: EdgeTarget - schema: null - props: PropDef['edges'] -} & QueryDefShared +// export type QueryDefEdges = { +// type: QueryDefType.Edge +// target: EdgeTarget +// schema: null +// props: Record +// } & QueryDefShared export type QueryDefRest = { type: QueryDefType.References | QueryDefType.Reference | QueryDefType.Root target: Target - schema: SchemaTypeDef | null - props: SchemaTypeDef['props'] | PropDef['edges'] + schema: TypeDef // | null + props: Record } & QueryDefShared -export type QueryDef = QueryDefEdges | QueryDefRest +// export type QueryDef = QueryDefEdges | QueryDefRest +export type QueryDef = QueryDefRest export type QueryTarget = EdgeTarget | Target diff --git a/packages/db/src/client/query/validation.ts b/packages/db/src/client/query/validation.ts index 19d5811964..cfa1ce0962 100644 --- a/packages/db/src/client/query/validation.ts +++ b/packages/db/src/client/query/validation.ts @@ -1,23 +1,4 @@ import picocolors from 'picocolors' -import { - ALIAS, - BINARY, - BOOLEAN, - PropDef, - PropDefEdge, - REFERENCE, - REFERENCES, - REVERSE_TYPE_INDEX_MAP, - SchemaTypeDef, - STRING, - TEXT, - TIMESTAMP, - VECTOR, - propIsNumerical, - createEmptyDef, - DEFAULT_MAP, - ID_FIELD_DEF, -} from '@based/schema/def' import { DbClient } from '../index.js' import { EQUAL, @@ -29,16 +10,18 @@ import { VECTOR_FNS, } from './filter/types.js' import { Filter } from './query.js' -import { MAX_IDS_PER_QUERY, MIN_ID_VALUE } from './thresholds.js' +import { MAX_IDS_PER_QUERY } from './thresholds.js' import { QueryByAliasObj, QueryDef } from './types.js' import { displayTarget, safeStringify } from './display.js' import { + createErrorProp, + createIdProp, isValidId, - isValidString, LangCode, langCodesMap, - MAX_ID, Validation, + type QueryPropDef, + type TypeDef, } from '@based/schema' import { StepInput } from './aggregates/types.js' @@ -109,7 +92,7 @@ const messages = { [ERR_SORT_ORDER]: (p) => `Sort: incorrect order option "${safeStringify(p.order)}" passed to sort "${p.field}"`, [ERR_SORT_TYPE]: (p) => - `Sort: cannot sort on type "${REVERSE_TYPE_INDEX_MAP[p.typeIndex]}" on field "${p.path.join('.')}"`, + `Sort: cannot sort on type "${p.type}" on field "${p.path.join('.')}"`, [ERR_RANGE_INVALID_OFFSET]: (p) => `Range: incorrect start "${safeStringify(p)}"`, [ERR_RANGE_INVALID_LIMIT]: (p) => @@ -130,16 +113,38 @@ const messages = { `Aggregate: Can't aggregate, feature not implemented yet. Prop: "${p}".`, } +const isValidString = (v: any) => { + const isVal = + typeof v === 'string' || + (v as any) instanceof Uint8Array || + ArrayBuffer.isView(v) + return isVal +} + +const propIsNumerical = (prop: QueryPropDef) => { + const t = prop.type + if ( + t === 'int16' || + t === 'int32' || + t === 'int8' || + t === 'uint16' || + t === 'uint32' || + t === 'uint8' || + t === 'number' || + t === 'timestamp' + ) { + return true + } + return false +} + export type ErrorCode = keyof typeof messages export const searchIncorrecQueryValue = (def: QueryDef, payload: any) => { def.errors.push({ code: ERR_SEARCH_INCORRECT_VALUE, payload }) } -export const searchIncorrectType = ( - def: QueryDef, - payload: PropDef | PropDefEdge, -) => { +export const searchIncorrectType = (def: QueryDef, payload: QueryPropDef) => { def.errors.push({ code: ERR_SEARCH_TYPE, payload }) } @@ -149,12 +154,15 @@ export const searchDoesNotExist = ( isVector: boolean, ) => { def.errors.push({ code: ERR_SEARCH_ENOENT, payload: field }) - if (isVector) { - return ERROR_VECTOR - } - return ERROR_STRING + return createErrorProp(def.schema) + // if (isVector) { + // return ERROR_VECTOR + // } + // return ERROR_STRING } +const MAX_ID = 4_294_967_295 + export const validateRange = (def: QueryDef, offset: number, limit: number) => { var r = false if (typeof offset !== 'number' || offset > MAX_ID || offset < 0) { @@ -208,13 +216,12 @@ export const validateVal = ( export const validateFilter = ( def: QueryDef, - prop: PropDef | PropDefEdge, + prop: QueryPropDef, f: Filter, ) => { if (def.skipValidation) { return false } - const t = prop.typeIndex const op = f[1].operation if (op == EXISTS) { @@ -226,12 +233,12 @@ export const validateFilter = ( }) return true } - } else if (t === REFERENCES || t === REFERENCE) { + } else if (prop.type === 'references' || prop.type === 'reference') { if (op == LIKE) { def.errors.push({ code: ERR_FILTER_OP_FIELD, payload: f }) return true } - if (t === REFERENCE && op != EQUAL) { + if (prop.type === 'reference' && op != EQUAL) { def.errors.push({ code: ERR_FILTER_OP_FIELD, payload: f, @@ -264,7 +271,7 @@ export const validateFilter = ( if (validateVal(def, f, prop.validation)) { return true } - } else if (t === VECTOR) { + } else if (prop.type === 'vector') { if (isNumerical(op) || op === INCLUDES) { def.errors.push({ code: ERR_FILTER_OP_FIELD, payload: f }) return true @@ -288,7 +295,11 @@ export const validateFilter = ( ) { return true } - } else if (t === TEXT || t === STRING || t === BINARY) { + } else if ( + prop.type === 'text' || + prop.type === 'string' || + prop.type === 'binary' + ) { if (isNumerical(op)) { def.errors.push({ code: ERR_FILTER_OP_FIELD, payload: f }) return true @@ -311,25 +322,48 @@ export const validateFilter = ( def.errors.push({ code: ERR_FILTER_OP_FIELD, payload: f }) return true } - if (validateVal(def, f, (v) => t == TIMESTAMP || typeof v === 'number')) { + if ( + validateVal( + def, + f, + (v) => prop.type === 'timestamp' || typeof v === 'number', + ) + ) { return true } - } else if (t === BOOLEAN && op !== EQUAL) { + } else if (prop.type === 'boolean' && op !== EQUAL) { def.errors.push({ code: ERR_FILTER_OP_FIELD, payload: f }) return true } return false } -export const validateType = (db: DbClient, def: QueryDef, type: string) => { - const r = db.schemaTypesParsed[type] +const emptyTypeDef = { + id: -1, + name: 'error', + size: 0, + props: {}, + hooks: {}, + locales: {}, + propHooks: {}, +} as TypeDef + +emptyTypeDef.queryProps = { id: createIdProp(emptyTypeDef) } + +export const validateType = ( + db: DbClient, + def: QueryDef, + type: string, +): TypeDef => { + const r = db.defs.byName[type] if (!r) { def.errors.push({ code: ERR_TARGET_INVAL_TYPE, payload: type, }) - EMPTY_SCHEMA_DEF.locales = db.schema.locales - return EMPTY_SCHEMA_DEF + console.warn('TODO: validateType locales') + // EMPTY_SCHEMA_DEF.locales = db.schema.locales + return emptyTypeDef } return r } @@ -384,7 +418,8 @@ export const validateSort = ( field: string, orderInput?: 'asc' | 'desc', ): QueryDef['sort'] => { - let propDef = field === 'id' ? ID_FIELD_DEF : def.props[field] + // let propDef = field === 'id' ? ID_FIELD_DEF : def.props[field] + let propDef = def.props[field] if (orderInput && orderInput !== 'asc' && orderInput !== 'desc') { def.errors.push({ code: ERR_SORT_ORDER, @@ -401,9 +436,9 @@ export const validateSort = ( const path = field.split('.') const x = path.slice(0, -1).join('.') propDef = def.props[x] - if (propDef && propDef.typeIndex === TEXT) { + if (propDef && propDef.type === 'text') { const k = path[path.length - 1] - lang = langCodesMap.get(k) + lang = langCodesMap.get(k) ?? 0 isText = true } } @@ -413,19 +448,22 @@ export const validateSort = ( payload: field, }) return { - prop: EMPTY_ALIAS_PROP_DEF, + prop: createErrorProp(def.schema) as any, order, lang: def.lang?.lang, } } } - const type = propDef.typeIndex - if (type === REFERENCES || type === REFERENCE || type === VECTOR) { + if ( + propDef.type === 'references' || + propDef.type === 'reference' || + propDef.type === 'vector' + ) { def.errors.push({ code: ERR_SORT_TYPE, payload: propDef, }) - } else if (type === TEXT) { + } else if (propDef.type === 'text') { if (lang === 0) { lang = def.lang?.lang ?? 0 if (lang === 0) { @@ -456,7 +494,7 @@ export const validateAlias = ( def: QueryDef, alias: QueryByAliasObj, path?: string, -): { def: PropDef; value: string } => { +): { def: QueryPropDef; value: string } => { const schema = def.schema for (const k in alias) { if (typeof alias[k] === 'string') { @@ -464,7 +502,7 @@ export const validateAlias = ( const prop = schema.props[p] if (!prop) { // def.errors.push({ code: ERR_TARGET_INVAL_ALIAS, payload: def }) - } else if (prop.typeIndex === ALIAS) { + } else if (prop.type === 'alias') { return { def: prop, value: alias[k] } } } else if (typeof alias[k] === 'object') { @@ -479,7 +517,8 @@ export const validateAlias = ( code: ERR_TARGET_INVAL_ALIAS, payload: alias, }) - return { value: '', def: EMPTY_ALIAS_PROP_DEF } + + return { value: '', def: createErrorProp(def.schema) as any } } export const validateId = (def: QueryDef, id: any): number => { @@ -562,53 +601,53 @@ export const handleErrors = (def: QueryDef) => { } } -export const EMPTY_ALIAS_PROP_DEF: PropDef = { - schema: null, - prop: 1, - typeIndex: ALIAS, - __isPropDef: true, - separate: true, - validation: () => true, - len: 0, - start: 0, - default: DEFAULT_MAP[ALIAS], - path: ['ERROR_ALIAS'], -} - -export const ERROR_STRING: PropDef = { - schema: null, - prop: 1, - typeIndex: STRING, - __isPropDef: true, - separate: true, - validation: () => true, - len: 0, - start: 0, - default: DEFAULT_MAP[STRING], - path: ['ERROR_STRING'], -} - -export const ERROR_VECTOR: PropDef = { - schema: null, - prop: 1, - typeIndex: VECTOR, - __isPropDef: true, - separate: true, - validation: () => true, - len: 0, - start: 0, - default: DEFAULT_MAP[VECTOR], - path: ['ERROR_VECTOR'], -} - -export const EMPTY_SCHEMA_DEF: SchemaTypeDef = { - ...createEmptyDef('_error', { props: {} }, {}), - buf: new Uint8Array([]), - propNames: new Uint8Array([]), - idUint8: new Uint8Array([0, 0]), - mainEmptyAllZeroes: true, - hasSeperateDefaults: false, -} +// export const EMPTY_ALIAS_PROP_DEF: QueryPropDef = { +// schema: null, +// prop: 1, +// typeIndex: ALIAS, +// __isQueryPropDef: true, +// separate: true, +// validation: () => true, +// len: 0, +// start: 0, +// default: DEFAULT_MAP[ALIAS], +// path: ['ERROR_ALIAS'], +// } + +// export const ERROR_STRING: QueryPropDef = { +// schema: null, +// prop: 1, +// typeIndex: STRING, +// __isQueryPropDef: true, +// separate: true, +// validation: () => true, +// len: 0, +// start: 0, +// default: DEFAULT_MAP[STRING], +// path: ['ERROR_STRING'], +// } + +// export const ERROR_VECTOR: QueryPropDef = { +// schema: null, +// prop: 1, +// typeIndex: VECTOR, +// __isQueryPropDef: true, +// separate: true, +// validation: () => true, +// len: 0, +// start: 0, +// default: DEFAULT_MAP[VECTOR], +// path: ['ERROR_VECTOR'], +// } + +// export const EMPTY_SCHEMA_DEF: SchemaTypeDef = { +// ...createEmptyDef('_error', { props: {} }, {}), +// buf: new Uint8Array([]), +// propNames: new Uint8Array([]), +// idUint8: new Uint8Array([0, 0]), +// mainEmptyAllZeroes: true, +// hasSeperateDefaults: false, +// } export const aggregationFieldDoesNotExist = (def: QueryDef, field: string) => { def.errors.push({ @@ -625,7 +664,10 @@ export const aggregationFieldNotNumber = (def: QueryDef, field: string) => { handleErrors(def) } -export const validateStepRange = (def: QueryDef, step: StepInput) => { +export const validateStepRange = ( + def: QueryDef, + step: StepInput | undefined, +) => { if (typeof step !== 'number' || step >= 4294967296) { def.errors.push({ code: ERR_AGG_INVALID_STEP_RANGE, diff --git a/packages/db/src/client/setLocalClientSchema.ts b/packages/db/src/client/setLocalClientSchema.ts index d838562bb6..0e1bb17f86 100644 --- a/packages/db/src/client/setLocalClientSchema.ts +++ b/packages/db/src/client/setLocalClientSchema.ts @@ -1,24 +1,21 @@ -import { updateTypeDefs } from '@based/schema/def' -import { DbSchema } from '@based/schema' import { DbClient } from '../index.js' import { cancel } from './modify/drain.js' import { Ctx } from './modify/Ctx.js' +import { schemaToTypeDefs, type SchemaOut } from '@based/schema' -export const setLocalClientSchema = (client: DbClient, schema: DbSchema) => { +export const setLocalClientSchema = (client: DbClient, schema: SchemaOut) => { if (client.schema && client.schema.hash === schema.hash) { return client.schema } - const { schemaTypesParsed, schemaTypesParsedById } = updateTypeDefs(schema) client.schema = schema - client.schemaTypesParsed = schemaTypesParsed - client.schemaTypesParsedById = schemaTypesParsedById + client.defs = schemaToTypeDefs(schema) if (client.modifyCtx.index > 8) { console.info('Modify cancelled - schema updated') } cancel(client.modifyCtx, Error('Schema changed - in-flight modify cancelled')) - client.modifyCtx = new Ctx(schema.hash, client.modifyCtx.array) + client.modifyCtx = new Ctx(schema, client.modifyCtx.array) // resubscribe for (const [q, store] of client.subs) { diff --git a/packages/db/src/client/string.ts b/packages/db/src/client/string.ts index 95dd9d125b..07ade5e2c4 100644 --- a/packages/db/src/client/string.ts +++ b/packages/db/src/client/string.ts @@ -56,7 +56,7 @@ export const stringCompress = (str: string): Uint8Array => { const len = ENCODER.encode(str).byteLength const tmpCompressBlock = getTmpBuffer(len * 3) const l = write({ array: tmpCompressBlock } as Ctx, str, 0, false) - const nBuffer = new Uint8Array(l) - nBuffer.set(tmpCompressBlock.subarray(0, l)) + const nBuffer = l ? new Uint8Array(l) : new Uint8Array() + nBuffer.set(tmpCompressBlock.subarray(0, l ?? 0)) return nBuffer } diff --git a/packages/db/src/client/xxHash64.ts b/packages/db/src/client/xxHash64.ts index 88050ac0a0..871c8803b4 100644 --- a/packages/db/src/client/xxHash64.ts +++ b/packages/db/src/client/xxHash64.ts @@ -9,6 +9,6 @@ export const xxHash64 = ( target = new Uint8Array(8) index = 0 } - native.xxHash64(buf, target, index) + native.xxHash64(buf, target, index ?? 0) return target } diff --git a/packages/db/src/hooks.ts b/packages/db/src/hooks.ts index a6ebe21c2b..7e0348ad78 100644 --- a/packages/db/src/hooks.ts +++ b/packages/db/src/hooks.ts @@ -1,9 +1,4 @@ -import { - StrictSchema, - MigrateFns, - DbSchema, - SchemaChecksum, -} from '@based/schema' +import type { SchemaMigrateFns, SchemaOut } from '@based/schema' import type { BasedDbQuery } from './client/query/BasedDbQuery.js' import { OnClose, OnData, OnError } from './client/query/subscription/types.js' import { DbServer } from './server/index.js' @@ -11,9 +6,9 @@ import { registerSubscription } from './server/subscription.js' export type DbClientHooks = { setSchema( - schema: StrictSchema, - transformFns?: MigrateFns, - ): Promise + schema: SchemaOut, + transformFns?: SchemaMigrateFns, + ): Promise flushModify(buf: Uint8Array): Promise getQueryBuf(buf: Uint8Array): ReturnType subscribe( @@ -21,7 +16,7 @@ export type DbClientHooks = { onData: (buf: Uint8Array) => ReturnType, onError?: OnError, ): OnClose - subscribeSchema(cb: (schema: DbSchema) => void): void + subscribeSchema(cb: (schema: SchemaOut) => void): void } export const getDefaultHooks = ( @@ -43,7 +38,7 @@ export const getDefaultHooks = ( subInterval, ) }, - setSchema(schema: StrictSchema, transformFns) { + setSchema(schema: SchemaOut, transformFns) { return server.setSchema(schema, transformFns) }, subscribeSchema(setSchema) { diff --git a/packages/db/src/server/blocks.ts b/packages/db/src/server/blocks.ts index b574e4f360..1c8546cb58 100644 --- a/packages/db/src/server/blocks.ts +++ b/packages/db/src/server/blocks.ts @@ -1,10 +1,10 @@ import native from '../native.js' import { join } from 'node:path' -import { SchemaTypeDef } from '@based/schema/def' import { bufToHex, equals, readInt32 } from '@based/utils' import { VerifTree, destructureTreeKey, makeTreeKey } from './tree.js' import { DbServer } from './index.js' import { IoJobSave } from './workers/io_worker_types.js' +import type { TypeDef } from '@based/schema' const SELVA_ENOENT = -8 @@ -69,11 +69,7 @@ export async function saveBlocks( /** * Load an existing block (typically of a partial type) back to memory. */ -export async function loadBlock( - db: DbServer, - def: SchemaTypeDef, - start: number, -) { +export async function loadBlock(db: DbServer, def: TypeDef, start: number) { const key = makeTreeKey(def.id, start) const block = db.verifTree.getBlock(key) if (!block) { @@ -114,11 +110,7 @@ export async function loadBlock( /** * Save a block and remove it from memory. */ -export async function unloadBlock( - db: DbServer, - def: SchemaTypeDef, - start: number, -) { +export async function unloadBlock(db: DbServer, def: TypeDef, start: number) { const typeId = def.id const end = start + def.blockCapacity - 1 const key = makeTreeKey(typeId, start) @@ -148,7 +140,7 @@ export async function unloadBlock( */ export function foreachBlock( db: DbServer, - def: SchemaTypeDef, + def: TypeDef, cb: (start: number, end: number, hash: Uint8Array) => void, includeEmptyBlocks: boolean = false, ) { @@ -181,9 +173,9 @@ export function foreachDirtyBlock( db: DbServer, cb: (mtKey: number, typeId: number, start: number, end: number) => void, ) { - const typeIdMap: { [key: number]: SchemaTypeDef } = {} - for (const typeName in db.schemaTypesParsed) { - const type = db.schemaTypesParsed[typeName] + const typeIdMap: { [key: number]: TypeDef } = {} + for (const typeName in db.defs.byName) { + const type = db.defs.byName[typeName] const typeId = type.id typeIdMap[typeId] = type } diff --git a/packages/db/src/server/index.ts b/packages/db/src/server/index.ts index cee56139e9..de02a93e90 100644 --- a/packages/db/src/server/index.ts +++ b/packages/db/src/server/index.ts @@ -1,14 +1,5 @@ import native from '../native.js' import { rm } from 'node:fs/promises' -import { - StrictSchema, - langCodesMap, - LangName, - MigrateFns, - SchemaChecksum, - strictSchemaToDbSchema, -} from '@based/schema' -import { ID_FIELD_DEF, PropDef, SchemaTypeDef } from '@based/schema/def' import { start, StartOpts } from './start.js' import { VerifTree, destructureTreeKey, makeTreeKeyFromNodeId } from './tree.js' import { save } from './save.js' @@ -28,6 +19,14 @@ import { import { resizeModifyDirtyRanges } from './resizeModifyDirtyRanges.js' import { loadBlock, unloadBlock } from './blocks.js' import { Subscriptions } from './subscription.js' +import { + langCodesMap, + type LangName, + type QueryPropDef, + type SchemaMigrateFns, + type SchemaOut, + type TypeDef, +} from '@based/schema' const emptyUint8Array = new Uint8Array(0) @@ -104,7 +103,7 @@ export class DbServer extends DbShared { } async loadBlock(typeName: string, nodeId: number) { - const def = this.schemaTypesParsed[typeName] + const def = this.defs.byName[typeName] if (!def) { throw new Error('Type not found') } @@ -117,7 +116,7 @@ export class DbServer extends DbShared { } async unloadBlock(typeName: string, nodeId: number) { - const def = this.schemaTypesParsed[typeName] + const def = this.defs.byName[typeName] if (!def) { throw new Error('Type not found') } @@ -183,8 +182,8 @@ export class DbServer extends DbShared { field: string, lang: LangName = 'none', ): SortIndex { - const t = this.schemaTypesParsed[type] - const prop = t.props[field] + const t = this.defs.byName[type] + const prop = t.queryProps[field] const langCode = langCodesMap.get(lang ?? Object.keys(this.schema?.locales ?? 'en')[0]) ?? 0 @@ -193,13 +192,15 @@ export class DbServer extends DbShared { if (!types) { types = this.sortIndexes[t.id] = {} } - let f = types[prop.prop] + let f = types[prop.id] if (!f) { - f = types[prop.prop] = {} + f = types[prop.id] = {} } - let fields = f[prop.start] + const start = 'main' in prop ? prop.main.start : 0 + const size = 'main' in prop ? prop.main.size : 0 + let fields = f[start] if (!fields) { - fields = f[prop.start] = {} + fields = f[start] = {} } let sortIndex = fields[langCode] if (sortIndex) { @@ -210,11 +211,11 @@ export class DbServer extends DbShared { // call createSortBuf here buf[0] = t.id buf[1] = t.id >>> 8 - buf[2] = prop.prop - buf[3] = prop.start - buf[4] = prop.start >>> 8 - buf[5] = prop.len - buf[6] = prop.len >>> 8 + buf[2] = prop.id + buf[3] = start + buf[4] = start >>> 8 + buf[5] = size + buf[6] = size >>> 8 buf[7] = prop.typeIndex buf[8] = langCode sortIndex = new SortIndex(buf, this.dbCtxExternal) @@ -223,31 +224,32 @@ export class DbServer extends DbShared { } destroySortIndex(type: string, field: string, lang: LangName = 'none'): any { - const t = this.schemaTypesParsed[type] - const prop = t.props[field] + const t = this.defs.byName[type] + const prop = t.queryProps[field] let types = this.sortIndexes[t.id] if (!type) { return } - let fields = types[prop.prop] + let fields = types[prop.id] if (!fields) { - fields = types[prop.prop] = {} + fields = types[prop.id] = {} } - let sortIndex = fields[prop.start] + const start = 'main' in prop ? prop.main.start : 0 + let sortIndex = fields[start] if (sortIndex) { const buf = new Uint8Array(6) buf[0] = t.id buf[1] = t.id >>> 8 - buf[2] = prop.prop - buf[3] = prop.start - buf[4] = prop.start >>> 8 + buf[2] = prop.id + buf[3] = start + buf[4] = start >>> 8 buf[5] = langCodesMap.get( lang ?? Object.keys(this.schema?.locales ?? 'en')[0], ) ?? 0 native.destroySortIndex(buf, this.dbCtxExternal) - delete fields[prop.start] + delete fields[start] } } @@ -284,22 +286,25 @@ export class DbServer extends DbShared { buf[2] = field buf[3] = start buf[4] = start >>> 8 - let typeDef: SchemaTypeDef - let prop: PropDef - - if (field === 255) { - prop = ID_FIELD_DEF - typeDef = this.schemaTypesParsedById[typeId] - } else { - typeDef = this.schemaTypesParsedById[typeId] - for (const p in typeDef.props) { - const propDef = typeDef.props[p] - if (propDef.prop == field && propDef.start == start) { - prop = propDef - break + let typeDef: TypeDef + let prop: QueryPropDef + + // if (field === 255) { + // prop = ID_FIELD_DEF + // typeDef = this.defsById[typeId] + // } else { + typeDef = this.defs.byName[typeId] + for (const p in typeDef.queryProps) { + const propDef = typeDef.queryProps[p] + if (propDef.id == field) { + if ('main' in propDef && propDef.main.start !== start) { + continue } + prop = propDef + break } } + // } if (!typeDef) { throw new Error(`Cannot find type id on db from query for sort ${typeId}`) @@ -309,14 +314,24 @@ export class DbServer extends DbShared { throw new Error(`Cannot find prop on db from query for sort ${field}`) } - buf[5] = prop.len - buf[6] = prop.len >>> 8 + if ('main' in prop) { + buf[5] = prop.main.size + buf[6] = prop.main.size >>> 8 + } else { + buf[5] = 0 + buf[6] = 0 + } + buf[7] = prop.typeIndex buf[8] = lang // put in modify stuff const sortIndex = - this.getSortIndex(typeId, prop.prop, prop.start, lang) ?? - new SortIndex(buf, this.dbCtxExternal) + this.getSortIndex( + typeId, + prop.id, + 'main' in prop ? prop.main.start : 0, + lang, + ) ?? new SortIndex(buf, this.dbCtxExternal) const types = this.sortIndexes[typeId] const fields = types[field] fields[start][lang] = sortIndex @@ -324,15 +339,13 @@ export class DbServer extends DbShared { } async setSchema( - strictSchema: StrictSchema, - transformFns?: MigrateFns, - ): Promise { + schema: SchemaOut, + transformFns?: SchemaMigrateFns, + ): Promise { if (this.stopped || !this.dbCtxExternal) { throw new Error('Db is stopped') } - const schema = strictSchemaToDbSchema(strictSchema) - if (schema.hash === this.schema?.hash) { // Todo something for sending back to actual client return schema.hash @@ -357,7 +370,7 @@ export class DbServer extends DbShared { } setSchemaOnServer(this, schema) - setNativeSchema(this, schema) + setNativeSchema(this) await writeSchemaFile(this, schema) process.nextTick(() => { diff --git a/packages/db/src/server/migrate/index.ts b/packages/db/src/server/migrate/index.ts index 31cab266d9..d61e656573 100644 --- a/packages/db/src/server/migrate/index.ts +++ b/packages/db/src/server/migrate/index.ts @@ -12,8 +12,12 @@ import { writeSchemaFile, } from '../schema.js' import { setToAwake, waitUntilSleeping } from './utils.js' -import { DbSchema, MigrateFns, serialize } from '@based/schema' -import { semver } from '@based/schema' +import { + semver, + serialize, + type SchemaMigrateFns, + type SchemaOut, +} from '@based/schema' const { satisfies, parseRange, parse } = semver export type MigrateRange = { typeId: number; start: number; end: number } @@ -22,7 +26,7 @@ const __filename = fileURLToPath(import.meta.url) const __dirname = dirname(__filename) const workerPath = join(__dirname, 'worker.js') -const parseTransform = (transform?: MigrateFns) => { +const parseTransform = (transform?: SchemaMigrateFns) => { const res = {} if (typeof transform === 'object' && transform !== null) { for (const type in transform) { @@ -40,7 +44,7 @@ const parseTransform = (transform?: MigrateFns) => { return res } -const stripHooks = (schema: DbSchema): DbSchema => { +const stripHooks = (schema: SchemaOut): SchemaOut => { const res = {} for (const i in schema) { if (i === 'types') { @@ -53,14 +57,14 @@ const stripHooks = (schema: DbSchema): DbSchema => { res[i] = schema[i] } } - return res as DbSchema + return res as SchemaOut } export const migrate = async ( server: DbServer, - fromSchema: DbSchema, - toSchema: DbSchema, - transform?: MigrateFns, + fromSchema: SchemaOut, + toSchema: SchemaOut, + transform?: SchemaMigrateFns, ): Promise => { const migrationId = toSchema.hash @@ -121,7 +125,7 @@ export const migrate = async ( } setSchemaOnServer(tmpDb.server, toSchema) - setNativeSchema(tmpDb.server, toSchema) + setNativeSchema(tmpDb.server) if (abort()) { await tmpDb.destroy() @@ -173,7 +177,7 @@ export const migrate = async ( server.verifTree.foreachBlock((block) => { const [typeId, start] = destructureTreeKey(block.key) - const def = server.schemaTypesParsedById[typeId] + const def = server.defs.byName[typeId] const end = start + def.blockCapacity - 1 rangesToMigrate.push({ typeId, start, end }) }) diff --git a/packages/db/src/server/migrate/worker.ts b/packages/db/src/server/migrate/worker.ts index a174a21422..cee1ceefaf 100644 --- a/packages/db/src/server/migrate/worker.ts +++ b/packages/db/src/server/migrate/worker.ts @@ -5,12 +5,11 @@ import { } from 'node:worker_threads' import native from '../../native.js' import { BasedDb } from '../../index.js' -import { CARDINALITY, REFERENCE, REFERENCES } from '@based/schema/def' import { setSchemaOnServer } from '../schema.js' import { setToSleep } from './utils.js' import { setLocalClientSchema } from '../../client/setLocalClientSchema.js' import { MigrateRange } from './index.js' -import { DbSchema, deSerialize } from '@based/schema' +import { deSerialize } from '@based/schema' if (isMainThread) { console.warn('running worker.ts in mainthread') @@ -37,8 +36,8 @@ if (isMainThread) { fromDb.server.dbCtxExternal = fromCtx toDb.server.dbCtxExternal = toCtx - setSchemaOnServer(fromDb.server, deSerialize(fromSchema) as DbSchema) - setSchemaOnServer(toDb.server, deSerialize(toSchema) as DbSchema) + setSchemaOnServer(fromDb.server, deSerialize(fromSchema)) + setSchemaOnServer(toDb.server, deSerialize(toSchema)) setLocalClientSchema(fromDb.client, fromDb.server.schema) setLocalClientSchema(toDb.client, toDb.server.schema) @@ -47,31 +46,32 @@ if (isMainThread) { number, { type: string; include: string[]; includeRaw: string[] } > = {} - for (const type in fromDb.server.schemaTypesParsed) { - const { id, props } = fromDb.server.schemaTypesParsed[type] + for (const type in fromDb.server.defs.byName) { + const { id, props } = fromDb.server.defs.byName[type] const include = [] const includeRaw = [] for (const path in props) { const prop = props[path] - if (prop.typeIndex === REFERENCE || prop.typeIndex === REFERENCES) { + if (prop.type === 'reference' || prop.type === 'references') { include.push(`${path}.id`) - if (prop.edges) { - for (const key in prop.edges) { - const edge = prop.edges[key] - if ( - edge.typeIndex === REFERENCE || - edge.typeIndex === REFERENCES - ) { - include.push(`${path}.${key}.id`) - } else if (edge.typeIndex === CARDINALITY) { - includeRaw.push(`${path}.${key}`) - } else { - include.push(`${path}.${key}`) - } - } - } - } else if (prop.typeIndex === CARDINALITY) { + // TODO edges? + // if (prop.edges) { + // for (const key in prop.edges) { + // const edge = prop.edges[key] + // if ( + // edge.typeIndex === REFERENCE || + // edge.typeIndex === REFERENCES + // ) { + // include.push(`${path}.${key}.id`) + // } else if (edge.typeIndex === CARDINALITY) { + // includeRaw.push(`${path}.${key}`) + // } else { + // include.push(`${path}.${key}`) + // } + // } + // } + } else if (prop.type === 'cardinality') { includeRaw.push(path) } else { include.push(path) @@ -114,7 +114,7 @@ if (isMainThread) { toDb.create(type, res || node, { unsafe: true }) } } - } else if (type in toDb.server.schemaTypesParsed) { + } else if (type in toDb.server.defs.byName) { for (const node of nodes) { toDb.create(type, node, { unsafe: true }) } diff --git a/packages/db/src/server/resizeModifyDirtyRanges.ts b/packages/db/src/server/resizeModifyDirtyRanges.ts index 914d8e19f1..c0c12d7fb3 100644 --- a/packages/db/src/server/resizeModifyDirtyRanges.ts +++ b/packages/db/src/server/resizeModifyDirtyRanges.ts @@ -3,8 +3,8 @@ import { DbServer } from './index.js' export const resizeModifyDirtyRanges = (server: DbServer) => { let maxNrChanges = 0 - for (const typeId in server.schemaTypesParsedById) { - const def = server.schemaTypesParsedById[typeId] + for (const typeId in server.defs.byId) { + const def = server.defs.byId[typeId] const lastId = server.ids[def.id - 1] const blockCapacity = def.blockCapacity const tmp = lastId - +!(lastId % def.blockCapacity) diff --git a/packages/db/src/server/save.ts b/packages/db/src/server/save.ts index f860cf445d..a90a9d0e30 100644 --- a/packages/db/src/server/save.ts +++ b/packages/db/src/server/save.ts @@ -32,8 +32,8 @@ export type Writelog = { } function hasPartialTypes(db: DbServer): boolean { - for (const id in db.schemaTypesParsedById) { - if (db.schemaTypesParsedById[id].partial) { + for (const id in db.defs.byId) { + if (db.defs.byId[id].partial) { return true } } @@ -78,15 +78,15 @@ function makeWritelog(db: DbServer, ts: number): Writelog { const types: Writelog['types'] = {} const rangeDumps: Writelog['rangeDumps'] = {} - for (const key in db.schemaTypesParsed) { - const { id, blockCapacity } = db.schemaTypesParsed[key] + for (const key in db.defs.byName) { + const { id, blockCapacity } = db.defs.byName[key] types[id] = { blockCapacity } rangeDumps[id] = [] } db.verifTree.foreachBlock((block) => { const [typeId, start] = destructureTreeKey(block.key) - const def = db.schemaTypesParsedById[typeId] + const def = db.defs.byId[typeId] const end = start + def.blockCapacity - 1 const data: RangeDump = { file: db.verifTree.getBlockFile(block), @@ -126,12 +126,12 @@ export function saveSync(db: DbServer, opts: SaveOpts = {}): void { if (opts.forceFullDump) { // reset the state just in case - db.verifTree = new VerifTree(db.schemaTypesParsed) + db.verifTree = new VerifTree(db.defs.byName) - // We use db.verifTree.types instead of db.schemaTypesParsed because it's + // We use db.verifTree.types instead of db.defs because it's // ordered. for (const { typeId } of db.verifTree.types()) { - const def = db.schemaTypesParsedById[typeId] + const def = db.defs.byId[typeId] foreachBlock(db, def, (start: number, end: number, _hash: Uint8Array) => saveBlock(db, def.id, start, end), ) @@ -184,13 +184,13 @@ export async function save(db: DbServer, opts: SaveOpts = {}): Promise { if (opts.forceFullDump) { // reset the state just in case - db.verifTree = new VerifTree(db.schemaTypesParsed) + db.verifTree = new VerifTree(db.defs.byName) - // We use db.verifTree.types instead of db.schemaTypesParsed because it's + // We use db.verifTree.types instead of db.defs because it's // ordered. for (const { typeId } of db.verifTree.types()) { - const def = db.schemaTypesParsedById[typeId] + const def = db.defs.byId[typeId] foreachBlock( db, def, diff --git a/packages/db/src/server/schema.ts b/packages/db/src/server/schema.ts index 3c617e884d..6529877fb1 100644 --- a/packages/db/src/server/schema.ts +++ b/packages/db/src/server/schema.ts @@ -1,25 +1,19 @@ -import { updateTypeDefs } from '@based/schema/def' -import { DbSchema, serialize } from '@based/schema' import { DbServer } from './index.js' import { join } from 'node:path' import { writeFile } from 'node:fs/promises' import native from '../native.js' import { SCHEMA_FILE } from '../types.js' import { saveSync } from './save.js' -import { writeCreate } from '../client/modify/create/index.js' -import { Ctx } from '../client/modify/Ctx.js' -import { consume } from '../client/modify/drain.js' import { schemaToSelvaBuffer } from './schemaSelvaBuffer.js' +import { schemaToTypeDefs, serialize, type SchemaOut } from '@based/schema' -export const setSchemaOnServer = (server: DbServer, schema: DbSchema) => { - const { schemaTypesParsed, schemaTypesParsedById } = updateTypeDefs(schema) +export const setSchemaOnServer = (server: DbServer, schema: SchemaOut) => { server.schema = schema - server.schemaTypesParsed = schemaTypesParsed - server.schemaTypesParsedById = schemaTypesParsedById + server.defs = schemaToTypeDefs(schema) server.ids = native.getSchemaIds(server.dbCtxExternal) } -export const writeSchemaFile = async (server: DbServer, schema: DbSchema) => { +export const writeSchemaFile = async (server: DbServer, schema: SchemaOut) => { if (server.fileSystemPath) { const schemaFilePath = join(server.fileSystemPath, SCHEMA_FILE) try { @@ -36,18 +30,20 @@ export const writeSchemaFile = async (server: DbServer, schema: DbSchema) => { * instance. If a `common.sdb` file is loaded then calling this function isn't * necessary because `common.sdb` already contains the required schema. */ -export const setNativeSchema = (server: DbServer, schema: DbSchema) => { - const types = Object.keys(server.schemaTypesParsed) - const s = schemaToSelvaBuffer(server.schemaTypesParsed) +export const setNativeSchema = (server: DbServer) => { + const types = Object.keys(server.defs.byName) + const s = schemaToSelvaBuffer(server.defs.byName) + console.log(s) let maxTid = 0 for (let i = 0; i < s.length; i++) { - const type = server.schemaTypesParsed[types[i]] + const type = server.defs.byName[types[i]] maxTid = Math.max(maxTid, type.id) + console.log(type.id, new Uint8Array(s[i])) try { native.setSchemaType(server.dbCtxExternal, type.id, new Uint8Array(s[i])) } catch (err) { throw new Error( - `Cannot update schema on selva (native) ${type.type} ${err.message}`, + `Cannot update schema on selva (native) ${type.name} ${err.message}`, ) } } @@ -55,16 +51,7 @@ export const setNativeSchema = (server: DbServer, schema: DbSchema) => { // Init the last ids native.setSchemaIds(new Uint32Array(maxTid), server.dbCtxExternal) - // Insert a root node - if (schema.types._root) { - const tmpArr = new Uint8Array(new ArrayBuffer(1e3, { maxByteLength: 10e3 })) - const tmpCtx = new Ctx(schema.hash, tmpArr) - writeCreate(tmpCtx, server.schemaTypesParsed._root, {}, null) - const buf = consume(tmpCtx) - server.modify(buf) - } - - server.verifTree.updateTypes(server.schemaTypesParsed) + server.verifTree.updateTypes(server.defs.byName) if (server.fileSystemPath) { saveSync(server, { skipDirtyCheck: true }) } diff --git a/packages/db/src/server/schemaSelvaBuffer.ts b/packages/db/src/server/schemaSelvaBuffer.ts index 6439234562..7e4ed8a1ac 100644 --- a/packages/db/src/server/schemaSelvaBuffer.ts +++ b/packages/db/src/server/schemaSelvaBuffer.ts @@ -1,35 +1,15 @@ -import { convertToTimestamp, ENCODER, writeDoubleLE, writeFloatLE, writeUint16, writeUint32, writeUint64 } from '@based/utils' import { - SchemaTypeDef, - PropDef, - PropDefEdge, - ALIAS, - ALIASES, - BINARY, - EMPTY_MICRO_BUFFER, - CARDINALITY, - MICRO_BUFFER, - REFERENCE, - REFERENCES, - STRING, - TEXT, - VECTOR, - JSON, - COLVEC, - VECTOR_BASE_TYPE_SIZE_MAP, - INT8, - UINT8, - BOOLEAN, - INT16, - UINT16, - INT32, - UINT32, - NUMBER, - TIMESTAMP, - ENUM, -} from '@based/schema/def' + convertToTimestamp, + ENCODER, + writeDoubleLE, + writeUint16, + writeUint32, + writeUint64, +} from '@based/utils' import { NOT_COMPRESSED } from '@based/protocol' import native from '../native.js' +import { typeIndexMap, type LeafDef, type TypeDef } from '@based/schema' +import { BLOCK_CAPACITY_DEFAULT } from '../types.js' const selvaFieldType: Readonly> = { NULL: 0, @@ -39,23 +19,23 @@ const selvaFieldType: Readonly> = { REFERENCE: 4, REFERENCES: 5, ALIAS: 8, - ALIASES: 9, + // ALIASES: 9, COLVEC: 10, } const selvaTypeMap = new Uint8Array(32) // 1.2x faster than JS array -selvaTypeMap[MICRO_BUFFER] = selvaFieldType.MICRO_BUFFER -selvaTypeMap[VECTOR] = selvaFieldType.MICRO_BUFFER -selvaTypeMap[BINARY] = selvaFieldType.STRING -selvaTypeMap[CARDINALITY] = selvaFieldType.STRING -selvaTypeMap[JSON] = selvaFieldType.STRING -selvaTypeMap[STRING] = selvaFieldType.STRING -selvaTypeMap[TEXT] = selvaFieldType.TEXT -selvaTypeMap[REFERENCE] = selvaFieldType.REFERENCE -selvaTypeMap[REFERENCES] = selvaFieldType.REFERENCES -selvaTypeMap[ALIAS] = selvaFieldType.ALIAS -selvaTypeMap[ALIASES] = selvaFieldType.ALIASES -selvaTypeMap[COLVEC] = selvaFieldType.COLVEC +selvaTypeMap[typeIndexMap.microbuffer] = selvaFieldType.MICRO_BUFFER +selvaTypeMap[typeIndexMap.vector] = selvaFieldType.MICRO_BUFFER +selvaTypeMap[typeIndexMap.binary] = selvaFieldType.STRING +selvaTypeMap[typeIndexMap.cardinality] = selvaFieldType.STRING +selvaTypeMap[typeIndexMap.json] = selvaFieldType.STRING +selvaTypeMap[typeIndexMap.string] = selvaFieldType.STRING +selvaTypeMap[typeIndexMap.text] = selvaFieldType.TEXT +selvaTypeMap[typeIndexMap.reference] = selvaFieldType.REFERENCE +selvaTypeMap[typeIndexMap.references] = selvaFieldType.REFERENCES +selvaTypeMap[typeIndexMap.alias] = selvaFieldType.ALIAS +// selvaTypeMap[ALIASES] = selvaFieldType.ALIASES +selvaTypeMap[typeIndexMap.colvec] = selvaFieldType.COLVEC const EDGE_FIELD_CONSTRAINT_FLAG_DEPENDENT = 0x01 @@ -66,164 +46,170 @@ function blockCapacity(blockCapacity: number): Uint8Array { return buf } -function sepPropCount(props: Array): number { - return props.filter((prop) => prop.separate).length +function sepPropCount(props: LeafDef[]): number { + return props.filter((prop) => !('main' in prop)).length } -function makeEdgeConstraintFlags( - prop: PropDef, -): number { +function makeEdgeConstraintFlags(prop: LeafDef): number { let flags = 0 - flags |= prop.dependent ? EDGE_FIELD_CONSTRAINT_FLAG_DEPENDENT : 0x00 + flags |= + 'dependent' in prop && prop.dependent + ? EDGE_FIELD_CONSTRAINT_FLAG_DEPENDENT + : 0x00 return flags } -const propDefBuffer = ( - schema: { [key: string]: SchemaTypeDef }, - prop: PropDef, -): number[] => { - const type = prop.typeIndex - const selvaType = selvaTypeMap[type] +const microBuffer = (mainBuf?: Uint8Array) => { + const buf = new Uint8Array(4) + const view = new DataView(buf.buffer) + buf[0] = selvaFieldType.MICRO_BUFFER + view.setUint16(1, mainBuf.byteLength, true) + buf[3] = 1 // has default + return [...buf, ...mainBuf] +} - if (prop.len && (type === MICRO_BUFFER || type === VECTOR)) { +const propDefBuffer = (prop: LeafDef): number[] => { + const selvaType = selvaTypeMap[prop.typeIndex] + + if (prop.type === 'vector') { const buf = new Uint8Array(4) const view = new DataView(buf.buffer) - buf[0] = selvaType - view.setUint16(1, prop.len, true) - if (prop.default) { - buf[3] = 1 // has default - return [...buf, ...prop.default] - } else { - buf[3] = 0 // has default - return [...buf] - } - } else if (prop.len && type === COLVEC) { + view.setUint16(1, prop.size * prop.baseSize, true) + buf[3] = 0 // has no default + return [...buf] + } + + if (prop.type === 'colvec') { const buf = new Uint8Array(5) const view = new DataView(buf.buffer) - buf[0] = selvaType - const baseSize = VECTOR_BASE_TYPE_SIZE_MAP[prop.vectorBaseType] - - view.setUint16(1, prop.len / baseSize, true) // elements - view.setUint16(3, baseSize, true) // element size + view.setUint16(1, prop.size, true) // elements + view.setUint16(3, prop.baseSize, true) // element size return [...buf] - } else if (type === REFERENCE || type === REFERENCES) { + } + + if ('target' in prop) { const buf = new Uint8Array(11) const view = new DataView(buf.buffer) - const dstType: SchemaTypeDef = schema[prop.inverseTypeName] + const dstType = prop.target.typeDef buf[0] = selvaType // field type buf[1] = makeEdgeConstraintFlags(prop) // flags view.setUint16(2, dstType.id, true) // dst_node_type - buf[4] = prop.inversePropNumber // inverse_field - view.setUint16(5, prop.edgeNodeTypeId ?? 0, true) // edge_node_type - view.setUint32(7, prop.referencesCapped ?? 0, true) + buf[4] = prop.target.id // inverse_field + view.setUint16(5, prop.edgesDef?.id ?? 0, true) // edge_node_type + view.setUint32(7, prop.typeDef.capped ?? 0, true) return [...buf] - } else if ( - type === STRING || - type === BINARY || - type === CARDINALITY || - type === JSON - ) { - return [selvaType, prop.len < 50 ? prop.len : 0] } - { - return [selvaType] + + if ( + prop.type === 'string' || + prop.type === 'binary' || + prop.type === 'cardinality' || + prop.type === 'json' + ) { + if ('main' in prop && prop.main.size < 50) { + return [selvaType, prop.main.size] + } + return [selvaType, 0] } + + return [selvaType] } // TODO rewrite export function schemaToSelvaBuffer(schema: { - [key: string]: SchemaTypeDef + [key: string]: TypeDef }): ArrayBuffer[] { return Object.values(schema).map((t) => { - const props = Object.values(t.props) - const rest: PropDef[] = [] + const props = Object.values(t.dbProps) + const rest: LeafDef[] = [] const nrFields = 1 + sepPropCount(props) let refFields = 0 let virtualFields = 0 - if (nrFields >= 250) { + if (nrFields >= 251) { throw new Error('Too many fields') } - const main = { - ...EMPTY_MICRO_BUFFER, - len: t.mainLen === 0 ? 1 : t.mainLen, - } - - for (const f of props) { - if (f.separate) { - if (f.typeIndex === REFERENCE || f.typeIndex === REFERENCES) { - refFields++ - } else if ( - f.typeIndex === ALIAS || - f.typeIndex === ALIASES || - f.typeIndex === COLVEC - ) { - // We assume that these are always the last props! - virtualFields++ - } - rest.push(f) - } else { - if (f.default) { - if (!main.default) { - main.default = new Uint8Array(main.len) - } - const buf = main.default as Uint8Array - - switch (f.typeIndex) { - case INT8: - case UINT8: - case BOOLEAN: - case ENUM: - main.default[f.start] = f.default + const mainBuf = new Uint8Array(Math.min(1, t.size)) + for (const prop of props) { + if ('main' in prop) { + if ('default' in prop && prop.default) { + switch (prop.type) { + case 'int8': + case 'uint8': + case 'boolean': + mainBuf[prop.main.start] = Number(prop.default) + break + case 'enum': + mainBuf[prop.main.start] = prop.enumMap[prop.default] break - case INT16: - case UINT16: - writeUint16(buf, f.default, f.start) + case 'int16': + case 'uint16': + writeUint16(mainBuf, prop.default, prop.main.start) break - case INT32: - case UINT32: - writeUint32(buf, f.default, f.start) + case 'int32': + case 'uint32': + writeUint32(mainBuf, prop.default, prop.main.start) break - case NUMBER: - writeDoubleLE(buf, f.default, f.start) + case 'number': + writeDoubleLE(mainBuf, prop.default, prop.main.start) break - case TIMESTAMP: - writeUint64(buf, f.default, f.start) + case 'timestamp': + writeUint64( + mainBuf, + convertToTimestamp(prop.default), + prop.main.start, + ) break - case BINARY: - case STRING: - if (f.default instanceof Uint8Array) { - buf.set(f.default, f.start) + case 'binary': + case 'string': + if (prop.default instanceof Uint8Array) { + mainBuf.set(prop.default, prop.main.start) } else { - const value = f.default.normalize('NFKD') - buf[f.start] = 0 // lang - buf[f.start + 1] = NOT_COMPRESSED - const { written: l } = ENCODER.encodeInto(value, buf.subarray(f.start + 2)) - let crc = native.crc32(buf.subarray(f.start + 2, f.start + 2 + l)) - writeUint32(buf, crc, f.start + 2 + l) + const value = prop.default.normalize('NFKD') + mainBuf[prop.main.start] = 0 // lang + mainBuf[prop.main.start + 1] = NOT_COMPRESSED + const { written: l } = ENCODER.encodeInto( + value, + mainBuf.subarray(prop.main.start + 2), + ) + let crc = native.crc32( + mainBuf.subarray( + prop.main.start + 2, + prop.main.start + 2 + l, + ), + ) + writeUint32(mainBuf, crc, prop.main.start + 2 + l) } break } } + } else { + if (prop.type === 'reference' || prop.type === 'references') { + refFields++ + } else if (prop.type === 'alias' || prop.type === 'colvec') { + // We assume that these are always the last props! + virtualFields++ + } + rest.push(prop) } } - rest.sort((a, b) => a.prop - b.prop) + rest.sort((a, b) => a.id - b.id) return Uint8Array.from([ - ...blockCapacity(t.blockCapacity), // u32 blockCapacity + ...blockCapacity(t.blockCapacity || BLOCK_CAPACITY_DEFAULT), // u32 blockCapacity nrFields, // u8 nrFields 1 + refFields, // u8 nrFixedFields virtualFields, // u8 nrVirtualFields 7, // u8 version (generally follows the sdb version) - ...propDefBuffer(schema, main), - ...rest.map((f) => propDefBuffer(schema, f)).flat(1), + ...microBuffer(mainBuf), + ...rest.map((f) => propDefBuffer(f)).flat(1), ]).buffer }) } diff --git a/packages/db/src/server/start.ts b/packages/db/src/server/start.ts index 0221b92b7c..4db285552c 100644 --- a/packages/db/src/server/start.ts +++ b/packages/db/src/server/start.ts @@ -9,10 +9,14 @@ import { VerifTree, makeTreeKey } from './tree.js' import { foreachBlock } from './blocks.js' import exitHook from 'exit-hook' import { save, saveSync, Writelog } from './save.js' -import { DbSchema, deSerialize } from '@based/schema' -import { BLOCK_CAPACITY_DEFAULT } from '@based/schema/def' +import { deSerialize } from '@based/schema' import { bufToHex, equals, hexToBuf, wait } from '@based/utils' -import { SCHEMA_FILE, WRITELOG_FILE, SCHEMA_FILE_DEPRECATED } from '../types.js' +import { + SCHEMA_FILE, + WRITELOG_FILE, + SCHEMA_FILE_DEPRECATED, + BLOCK_CAPACITY_DEFAULT, +} from '../types.js' import { setSchemaOnServer } from './schema.js' export type StartOpts = { @@ -67,7 +71,7 @@ export async function start(db: DbServer, opts: StartOpts) { // Load schema const schema = await readFile(join(path, SCHEMA_FILE)).catch(noop) if (schema) { - const s = deSerialize(schema) as DbSchema + const s = deSerialize(schema) setSchemaOnServer(db, s) } else { const schemaJson = await readFile(join(path, SCHEMA_FILE_DEPRECATED)) @@ -79,7 +83,7 @@ export async function start(db: DbServer, opts: StartOpts) { // Load block dumps for (const typeId in writelog.rangeDumps) { const dumps = writelog.rangeDumps[typeId] - const def = db.schemaTypesParsedById[typeId] + const def = db.defs.byId[typeId] if (!def.partial && !opts?.noLoadDumps) { for (const dump of dumps) { @@ -107,10 +111,10 @@ export async function start(db: DbServer, opts: StartOpts) { } } - db.verifTree = new VerifTree(db.schemaTypesParsed) + db.verifTree = new VerifTree(db.defs.byName) for (const { typeId } of db.verifTree.types()) { - const def = db.schemaTypesParsedById[typeId] + const def = db.defs.byId[typeId] def.blockCapacity = writelog?.types[def.id]?.blockCapacity || def.blockCapacity || diff --git a/packages/db/src/server/tree.ts b/packages/db/src/server/tree.ts index 23d6239221..8452b644a8 100644 --- a/packages/db/src/server/tree.ts +++ b/packages/db/src/server/tree.ts @@ -1,5 +1,5 @@ +import type { TypeDef } from '@based/schema' import createDbHash from './dbHash.js' -import { SchemaTypeDef } from '@based/schema/def' export const destructureTreeKey = (key: number) => [ (key / 4294967296) | 0, // typeId @@ -25,7 +25,6 @@ export const makeTreeKeyFromNodeId = ( } type Hash = Uint8Array -const HASH_SIZE = 16 /** * Block state. @@ -69,25 +68,22 @@ export class VerifTree { #types: { [key: number]: VerifType } #h = createDbHash() - constructor(schemaTypesParsed: Record) { - this.#types = VerifTree.#makeTypes(schemaTypesParsed) + constructor(defs: Record) { + this.#types = VerifTree.#makeTypes(defs) } - static #makeTypes(schemaTypesParsed: Record): { + static #makeTypes(defs: Record): { [key: number]: VerifType } { return Object.preventExtensions( - Object.keys(schemaTypesParsed) - .sort( - (a: string, b: string) => - schemaTypesParsed[a].id - schemaTypesParsed[b].id, - ) + Object.keys(defs) + .sort((a: string, b: string) => defs[a].id - defs[b].id) .reduce( ( obj: { [key: number]: VerifType }, key: string, ): { [key: number]: VerifType } => { - const def = schemaTypesParsed[key] + const def = defs[key] const typeId = def.id obj[typeId] = { typeId, @@ -183,9 +179,9 @@ export class VerifTree { return VerifTree.blockSdbFile(typeId, start, end) } - updateTypes(schemaTypesParsed: Record) { + updateTypes(defs: Record) { const oldTypes = this.#types - const newTypes = VerifTree.#makeTypes(schemaTypesParsed) + const newTypes = VerifTree.#makeTypes(defs) for (const k of Object.keys(oldTypes)) { const oldType = oldTypes[k] diff --git a/packages/db/src/shared/DbBase.ts b/packages/db/src/shared/DbBase.ts index 079fd30e3b..32148730dc 100644 --- a/packages/db/src/shared/DbBase.ts +++ b/packages/db/src/shared/DbBase.ts @@ -1,9 +1,8 @@ -import { SchemaTypeDef } from '@based/schema/def' -import { DbSchema } from '@based/schema' +import { type SchemaOut, type TypeDefs } from '@based/schema' import { Emitter } from './Emitter.js' export type EventMap = { - schema: DbSchema + schema: SchemaOut } export type Event = keyof EventMap @@ -11,7 +10,9 @@ export type Event = keyof EventMap export type Listener = (data: T) => void export class DbShared extends Emitter { - schema?: DbSchema - schemaTypesParsed?: Record = {} - schemaTypesParsedById?: Record = {} + schema?: SchemaOut + defs: TypeDefs = { + byName: {}, + byId: {}, + } } diff --git a/packages/db/src/shared/Emitter.ts b/packages/db/src/shared/Emitter.ts index ff955385d1..cebe68e30a 100644 --- a/packages/db/src/shared/Emitter.ts +++ b/packages/db/src/shared/Emitter.ts @@ -1,7 +1,7 @@ -import { DbSchema } from '@based/schema' +import type { SchemaOut } from '@based/schema' export type EventMap = { - schema: DbSchema + schema: SchemaOut info: string error: string } diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index c02c1b2a35..e833f903bf 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -3,6 +3,10 @@ export const SCHEMA_FILE = 'schema.bin' export const WRITELOG_FILE = 'writelog.json' export const COMMON_SDB_FILE = 'common.sdb' +export const BLOCK_CAPACITY_MIN = 1025 +export const BLOCK_CAPACITY_MAX = 2147483647 +export const BLOCK_CAPACITY_DEFAULT = 100_000 + export type BasedDbOpts = { path: string | null maxModifySize?: number diff --git a/packages/db/test/capped.ts b/packages/db/test/capped.ts index 842ef58fc1..e035f9b8cd 100644 --- a/packages/db/test/capped.ts +++ b/packages/db/test/capped.ts @@ -142,12 +142,6 @@ await test('capped references', async (t) => { deepEqual(await db.query('user', user).include('**').get(), { id: 1, - latestArticles: [ - { id: 6 }, - { id: 7 }, - { id: 8 }, - { id: 9 }, - { id: 10 }, - ] + latestArticles: [{ id: 6 }, { id: 7 }, { id: 8 }, { id: 9 }, { id: 10 }], }) }) diff --git a/packages/db/test/cardinality.ts b/packages/db/test/cardinality.ts index 836603c7a6..750ff1d3ad 100644 --- a/packages/db/test/cardinality.ts +++ b/packages/db/test/cardinality.ts @@ -409,9 +409,9 @@ await test('defaultPrecision', async (t) => { t.after(() => db.stop()) await db.setSchema({ - props: { - myRootCount: 'cardinality', - }, + // props: { + // myRootCount: 'cardinality', + // }, types: { stores: { name: 'string', diff --git a/packages/db/test/colvec.ts b/packages/db/test/colvec.ts index 3c88b88cb3..088b30e4b2 100644 --- a/packages/db/test/colvec.ts +++ b/packages/db/test/colvec.ts @@ -26,7 +26,7 @@ await test.skip('colvec', async (t) => { }, }) - deepEqual(db.server.schemaTypesParsed.col.blockCapacity, 10_000) + deepEqual(db.server.defs.byName.col.blockCapacity, 10_000) let seed = 100 const next = () => (seed = (214013 * seed + 2531011) % 10e3) diff --git a/packages/db/test/edges/edgeString.ts b/packages/db/test/edges/edgeString.ts new file mode 100644 index 0000000000..b5e1afc83e --- /dev/null +++ b/packages/db/test/edges/edgeString.ts @@ -0,0 +1,76 @@ +import { BasedDb } from '../../src/index.js' +import test from '../shared/test.js' +import { deepEqual } from '../shared/assert.js' + +await test('edge string', async (t) => { + const db = new BasedDb({ + path: t.tmp, + }) + + await db.start({ clean: true }) + t.after(() => t.backup(db)) + await db.setSchema({ + types: { + other: { + name: 'string', + }, + user: { + others: { + items: { + ref: 'other', + prop: 'users', + $rating: 'uint8', + $userString: 'string', + }, + }, + }, + }, + }) + + const user1 = db.create('other') + const user2 = db.create('other') + const user3 = db.create('other') + const user4 = await db.create('user', { + others: [ + { + id: user1, + $rating: 5, + $userString: 'abc', + }, + { + id: user2, + $rating: 5, + $userString: 'abc', + }, + { + id: user3, + $rating: 5, + $userString: 'abc', + }, + ], + }) + + await db.setSchema({ + types: { + other: { + name: 'string', + }, + user: { + name: 'string', + others: { + items: { + ref: 'other', + prop: 'users', + $rating: 'uint8', + $userString: 'string', + }, + }, + }, + }, + }) + + deepEqual(await db.query('user').include('**').get().toObject(), [ + { id: 1, others: [{ id: 2, $string: 'abc', name: '' }] }, + { id: 2, others: [{ id: 1, $string: 'abc', name: '' }] }, + ]) +}) diff --git a/packages/db/test/migration.ts b/packages/db/test/migration.ts index 8fa23d11f1..528195e282 100644 --- a/packages/db/test/migration.ts +++ b/packages/db/test/migration.ts @@ -1,4 +1,4 @@ -import { NonStrictSchema } from '@based/schema' +import { SchemaIn } from '@based/schema' import { BasedDb } from '../src/index.js' import test from './shared/test.js' import { deepEqual, equal, notEqual, throws } from './shared/assert.js' @@ -104,7 +104,7 @@ await test('migration', async (t) => { }, } - const schemas: NonStrictSchema[] = [ + const schemas: SchemaIn[] = [ { version: '2.0.0', types: { diff --git a/packages/db/test/migrationDebug.ts b/packages/db/test/migrationDebug.ts new file mode 100644 index 0000000000..4c49eb3fc2 --- /dev/null +++ b/packages/db/test/migrationDebug.ts @@ -0,0 +1,444 @@ +import { SchemaIn } from '@based/schema' +import { BasedDb } from '../src/index.js' +import test from './shared/test.js' +import { deepEqual, equal, notEqual, throws } from './shared/assert.js' + +await test('migration', async (t) => { + const db = new BasedDb({ + path: t.tmp, + }) + + await db.start({ clean: true }) + + t.after(() => t.backup(db)) + + await db.setSchema({ + version: '1.0.0', + types: { + user: { + // firstName: 'string', + // lastName: 'string', + // age: 'uint8', + // email: 'string', + // uniqueViews: 'cardinality', + }, + person: { + // email: 'string', + // user: { + // ref: 'user', + // prop: 'persons1', + // $relation: ['buddy', 'bff'], + // $userString: 'string', + // }, + users: { + items: { + ref: 'user', + prop: 'persons2', + $rating: 'uint8', + $userString: 'string', + }, + }, + }, + }, + }) + + let i = 3 + const _users = [] + while (i--) { + _users.push( + db.create('user', { + // firstName: 'John' + i, + // lastName: 'Doe' + i, + // email: 'johndoe' + i + '@example.com', + // age: i + 20, + // uniqueViews: ['a', 'b', 'c'], + }), + ) + } + i = _users.length + while (i--) { + db.create('person', { + // email: 'person' + i + '@example.com', + // user: { + // id: _users[i], + // $relation: 'buddy', + // $userString: 'abc', + // }, + users: _users.map((user) => { + return { id: user, $rating: 5, $userString: 'abc' } + }), + }) + } + + await db.drain() + + const hooksThatShouldBeIgnoredByMigration = { + create() { + return { + name: 'disaster', + } + }, + read() { + return { + name: 'its a secret', + } + }, + } + + const schemas: SchemaIn[] = [ + { + // version: '2.0.0', + types: { + user: { + props: { + fullName: 'string', + age: 'uint8', + email: 'string', + uniqueViews: 'cardinality', + }, + // hooks: hooksThatShouldBeIgnoredByMigration, + }, + person: { + email: 'string', + user: { + ref: 'user', + prop: 'persons1', + // $relation: ['buddy', 'bff'], + $userString: 'string', + }, + users: { + items: { + ref: 'user', + prop: 'persons2', + $rating: 'uint8', + $userString: 'string', + }, + }, + }, + }, + // migrations: [ + // { + // version: '<2', + // migrate: { + // user({ firstName, lastName, ...rest }) { + // return { + // fullName: firstName + ' ' + lastName, + // ...rest, + // } + // }, + // }, + // }, + // ], + }, + // { + // version: '3.0.0', + // types: { + // user: { + // props: { + // name: 'string', + // age: 'uint8', + // email: 'string', + // uniqueViews: 'cardinality', + // }, + // hooks: hooksThatShouldBeIgnoredByMigration, + // }, + // person: { + // email: 'string', + // emailPrimary: 'string', + // user: { + // ref: 'user', + // prop: 'persons1', + // $relation: ['buddy', 'bff'], + // $userString: 'string', + // }, + // users: { + // items: { + // ref: 'user', + // prop: 'persons2', + // $rating: 'uint8', + // $userString: 'string', + // }, + // }, + // }, + // }, + // migrations: [ + // { + // version: '2', + // migrate: { + // user({ fullName, ...rest }) { + // return { + // name: fullName, + // ...rest, + // } + // }, + // }, + // }, + // { + // version: '<3', + // migrate: { + // person({ email, ...rest }) { + // return { + // emailPrimary: email, + // email, + // ...rest, + // } + // }, + // }, + // }, + // ], + // }, + ] + + for (const schema of schemas) { + await db.setSchema(schema) + } + + const users = await db.query('user').get().toObject() + const people = await db.query('person').include('*', '**').get().toObject() + + equal(users.length, 10) + equal(people.length, 10) + + deepEqual(users, [ + { + id: 1, + age: 29, + name: 'John9 Doe9', + email: 'johndoe9@example.com', + uniqueViews: 3, + }, + { + id: 2, + age: 28, + name: 'John8 Doe8', + email: 'johndoe8@example.com', + uniqueViews: 3, + }, + { + id: 3, + age: 27, + name: 'John7 Doe7', + email: 'johndoe7@example.com', + uniqueViews: 3, + }, + { + id: 4, + age: 26, + name: 'John6 Doe6', + email: 'johndoe6@example.com', + uniqueViews: 3, + }, + { + id: 5, + age: 25, + name: 'John5 Doe5', + email: 'johndoe5@example.com', + uniqueViews: 3, + }, + { + id: 6, + age: 24, + name: 'John4 Doe4', + email: 'johndoe4@example.com', + uniqueViews: 3, + }, + { + id: 7, + age: 23, + name: 'John3 Doe3', + email: 'johndoe3@example.com', + uniqueViews: 3, + }, + { + id: 8, + age: 22, + name: 'John2 Doe2', + email: 'johndoe2@example.com', + uniqueViews: 3, + }, + { + id: 9, + age: 21, + name: 'John1 Doe1', + email: 'johndoe1@example.com', + uniqueViews: 3, + }, + { + id: 10, + age: 20, + name: 'John0 Doe0', + email: 'johndoe0@example.com', + uniqueViews: 3, + }, + ]) + + deepEqual(people, [ + { + id: 1, + email: 'person9@example.com', + emailPrimary: 'person9@example.com', + user: { + ...users.at(-1), + $relation: 'buddy', + $userString: 'abc', + }, + users: users.map((user) => { + return { + ...user, + $rating: 5, + } + }), + }, + { + id: 2, + email: 'person8@example.com', + emailPrimary: 'person8@example.com', + user: { + ...users.at(-2), + $relation: 'buddy', + $userString: 'abc', + }, + users: users.map((user) => { + return { + ...user, + $rating: 5, + } + }), + }, + { + id: 3, + email: 'person7@example.com', + emailPrimary: 'person7@example.com', + user: { + ...users.at(-3), + $relation: 'buddy', + $userString: 'abc', + }, + users: users.map((user) => { + return { + ...user, + $rating: 5, + } + }), + }, + { + id: 4, + email: 'person6@example.com', + emailPrimary: 'person6@example.com', + user: { + ...users.at(-4), + $relation: 'buddy', + $userString: 'abc', + }, + users: users.map((user) => { + return { + ...user, + $rating: 5, + } + }), + }, + { + id: 5, + email: 'person5@example.com', + emailPrimary: 'person5@example.com', + user: { + ...users.at(-5), + $relation: 'buddy', + $userString: 'abc', + }, + users: users.map((user) => { + return { + ...user, + $rating: 5, + } + }), + }, + { + id: 6, + email: 'person4@example.com', + emailPrimary: 'person4@example.com', + user: { + ...users.at(-6), + $relation: 'buddy', + $userString: 'abc', + }, + users: users.map((user) => { + return { + ...user, + $rating: 5, + } + }), + }, + { + id: 7, + email: 'person3@example.com', + emailPrimary: 'person3@example.com', + user: { + ...users.at(-7), + $relation: 'buddy', + $userString: 'abc', + }, + users: users.map((user) => { + return { + ...user, + $rating: 5, + } + }), + }, + { + id: 8, + email: 'person2@example.com', + emailPrimary: 'person2@example.com', + user: { + ...users.at(-8), + $relation: 'buddy', + $userString: 'abc', + }, + users: users.map((user) => { + return { + ...user, + $rating: 5, + } + }), + }, + { + id: 9, + email: 'person1@example.com', + emailPrimary: 'person1@example.com', + user: { + ...users.at(-9), + $relation: 'buddy', + $userString: 'abc', + }, + users: users.map((user) => { + return { + ...user, + $rating: 5, + } + }), + }, + { + id: 10, + email: 'person0@example.com', + emailPrimary: 'person0@example.com', + user: { + ...users.at(-10), + $relation: 'buddy', + $userString: 'abc', + }, + users: users.map((user) => { + return { + ...user, + $rating: 5, + } + }), + }, + ]) + + const lastSchema = schemas.at(-1) + lastSchema.types.user.props.age = 'string' + const checksum1 = db.client.schema.hash + await db.setSchema(lastSchema) + const checksum2 = db.client.schema.hash + notEqual(checksum1, checksum2) +}) diff --git a/packages/db/test/partial.ts b/packages/db/test/partial.ts index d080e23292..f97616a2fb 100644 --- a/packages/db/test/partial.ts +++ b/packages/db/test/partial.ts @@ -3,7 +3,7 @@ import test from './shared/test.js' import { deepEqual } from './shared/assert.js' import { throws } from './shared/assert.js' import { wait } from '@based/utils' -import {makeTreeKey} from '../src/server/tree.js' +import { makeTreeKey } from '../src/server/tree.js' await test('partial', async (t) => { const db = new BasedDb({ @@ -132,7 +132,7 @@ await test('simple load/unload', async (t) => { props: { sku: 'number', flap: 'number', - } + }, }, }, }) @@ -170,7 +170,12 @@ await test('simple load/unload', async (t) => { db2.server.verifTree.foreachBlock((block) => deepEqual(block.inmem, false)) await db2.server.loadBlock('product', 100_001) - deepEqual(db.server.verifTree.getBlock(makeTreeKey(db.server.schemaTypesParsed['product'].id, 100_001)).inmem, true) + deepEqual( + db.server.verifTree.getBlock( + makeTreeKey(db.server.defs.byName['product'].id, 100_001), + ).inmem, + true, + ) const d2 = await db2 .query('product') diff --git a/packages/db/test/rootProps.ts b/packages/db/test/rootProps.ts deleted file mode 100644 index 1301bf77dd..0000000000 --- a/packages/db/test/rootProps.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { BLOCK_CAPACITY_MAX } from '@based/schema/def' -import { BasedDb } from '../src/index.js' -import { deepEqual } from './shared/assert.js' -import test from './shared/test.js' - -await test('rootProps', async (t) => { - const db = new BasedDb({ - path: t.tmp, - }) - - await db.start({ clean: true }) - t.after(async () => t.backup(db)) - - await db.setSchema({ - props: { - myString: 'string', - myBoolean: 'boolean', - bestArticles: { - items: { - ref: 'article', - }, - }, - }, - types: { - article: { - props: { - name: 'string', - body: 'string', - }, - }, - }, - }) - - deepEqual( - db.server.schemaTypesParsed['_root'].blockCapacity, - BLOCK_CAPACITY_MAX, - ) - - const rootData = { - myString: 'im the root', - myBoolean: true, - } - - const article = db.create('article', { - name: 'best article', - body: 'success', - }) - - await db.update(rootData) - - let rootRes = await db.query().get().toObject() - - deepEqual(rootRes, rootData) - - await db.update({ - bestArticles: [article], - }) - - rootRes = await db.query().include('bestArticles').get().toObject() - - deepEqual(rootRes, { - bestArticles: [{ id: 1, name: 'best article', body: 'success' }], - }) - - const rootRes2 = await db - .query('_root', 1) - .include('bestArticles') - .get() - .toObject() - - deepEqual(rootRes, rootRes2) - - await db.update('_root', 1, { myBoolean: false }) - deepEqual(await db.query('_root', 1).include('myBoolean').get().toObject(), { myBoolean: false }) -}) diff --git a/packages/db/test/save/noload.ts b/packages/db/test/save/noload.ts index f8ddc33f8c..b76cf8e609 100644 --- a/packages/db/test/save/noload.ts +++ b/packages/db/test/save/noload.ts @@ -63,7 +63,13 @@ await test('noLoadDumps', async (t) => { .subordinates.length, 750, ) - deepEqual(await db.query('employee', 2).include((s) => s('subordinates').count()).get(), { id: 2, subordinates: { count: 750 } }) + deepEqual( + await db + .query('employee', 2) + .include((s) => s('subordinates').count()) + .get(), + { id: 2, subordinates: { count: 750 } }, + ) await db.drain() await db.save() @@ -75,11 +81,11 @@ await test('noLoadDumps', async (t) => { const getBlock1 = () => db2.server.verifTree.getBlock( - makeTreeKey(db2.client.schema.types['employee'].id, 1), + makeTreeKey(db2.client.defs.byName['employee'].id, 1), ) const getBlock2 = () => db2.server.verifTree.getBlock( - makeTreeKey(db2.client.schema.types['employee'].id, 1200), + makeTreeKey(db2.client.defs.byName['employee'].id, 1200), ) deepEqual(await db2.query('employee').include('*').get().toObject(), []) @@ -96,7 +102,13 @@ await test('noLoadDumps', async (t) => { .subordinates.length, 511, ) - deepEqual(await db2.query('employee', 2).include((s) => s('subordinates').count()).get(), { id: 2, subordinates: { count: 750 } }) + deepEqual( + await db2 + .query('employee', 2) + .include((s) => s('subordinates').count()) + .get(), + { id: 2, subordinates: { count: 750 } }, + ) deepEqual(getBlock1().inmem, true) deepEqual(getBlock2().inmem, false) @@ -108,12 +120,28 @@ await test('noLoadDumps', async (t) => { .subordinates.length, 750, ) - deepEqual(await db2.query('employee', 2).include((s) => s('subordinates').count()).get(), { id: 2, subordinates: { count: 750 } }) + deepEqual( + await db2 + .query('employee', 2) + .include((s) => s('subordinates').count()) + .get(), + { id: 2, subordinates: { count: 750 } }, + ) await db2.server.unloadBlock('employee', 1100) deepEqual(getBlock2().inmem, false) - deepEqual((await db2.query('employee', 2).include('subordinates').get().toObject()).subordinates.length, 511) - deepEqual(await db2.query('employee', 2).include((s) => s('subordinates').count()).get(), { id: 2, subordinates: { count: 750 } }) + deepEqual( + (await db2.query('employee', 2).include('subordinates').get().toObject()) + .subordinates.length, + 511, + ) + deepEqual( + await db2 + .query('employee', 2) + .include((s) => s('subordinates').count()) + .get(), + { id: 2, subordinates: { count: 750 } }, + ) // for (const type of db2.server.verifTree.types()) { // for (const block of db2.server.verifTree.blocks(type)) { diff --git a/packages/db/test/save/save.ts b/packages/db/test/save/save.ts index f98483f4fc..b17e278257 100644 --- a/packages/db/test/save/save.ts +++ b/packages/db/test/save/save.ts @@ -27,14 +27,14 @@ await test('simple', async (t) => { cs: { required: true }, et: { required: true }, }, - props: { - coolname: 'string', - users: { - items: { - ref: 'user', - }, - }, - }, + // props: { + // coolname: 'string', + // users: { + // items: { + // ref: 'user', + // }, + // }, + // }, types: { user: { props: { @@ -134,9 +134,9 @@ await test('empty root', async (t) => { t.after(() => db.destroy()) await db.setSchema({ - props: { - rando: { type: 'string' }, - }, + // props: { + // rando: { type: 'string' }, + // }, types: { user: { props: { diff --git a/packages/db/test/schema/schema.ts b/packages/db/test/schema/schema.ts index 9aa2b9bc36..02de3f6a3e 100644 --- a/packages/db/test/schema/schema.ts +++ b/packages/db/test/schema/schema.ts @@ -2,22 +2,22 @@ import test from '../shared/test.js' import { BasedDb } from '../../src/index.js' import { setTimeout } from 'node:timers/promises' import { deepEqual, throws } from '../shared/assert.js' -await test('support many fields on root', async (t) => { - const db = new BasedDb({ - path: t.tmp, - }) - await db.start({ clean: true }) - t.after(() => db.destroy()) - - const props = {} - for (let i = 0; i < 248; i++) { - props['myProp' + i] = 'string' - } - - await db.setSchema({ - props, - }) -}) +// await test('support many fields on root', async (t) => { +// const db = new BasedDb({ +// path: t.tmp, +// }) +// await db.start({ clean: true }) +// t.after(() => db.destroy()) + +// const props = {} +// for (let i = 0; i < 248; i++) { +// props['myProp' + i] = 'string' +// } + +// await db.setSchema({ +// props, +// }) +// }) await test('support many fields on type', async (t) => { const db = new BasedDb({ diff --git a/packages/db/test/schema/schemaChange.ts b/packages/db/test/schema/schemaChange.ts deleted file mode 100644 index 3fed85721b..0000000000 --- a/packages/db/test/schema/schemaChange.ts +++ /dev/null @@ -1,126 +0,0 @@ -import test from '../shared/test.js' -import { BasedDb } from '../../src/index.js' -import { deepCopy } from '@based/utils' -import { Schema } from '@based/schema' -import { deepEqual } from '../shared/assert.js' - -await test('set schema dont migrate', async (t) => { - const db = new BasedDb({ - path: t.tmp, - }) - - await db.start({ clean: true }) - - t.after(() => db.destroy()) - - let schema = { - props: { - haha: 'boolean', - }, - } - - let updates = 0 - db.client.on('schema', () => { - updates++ - }) - - await db.setSchema(deepCopy(schema) as Schema) - await db.setSchema(deepCopy(schema) as Schema) - await db.setSchema(deepCopy(schema) as Schema) - - deepEqual(updates, 1, '1 update') - // deepEqual(migrates, 0, '0 migrates') - - await db.setSchema({ - props: { - coolguy: 'string', - badguy: 'boolean', - }, - types: { - yes: { - name: 'string', - }, - nope: { - name: 'string', - }, - }, - }) - - // TODO: when https://linear.app/1ce/issue/FDN-1304 changes ignore this as no change - // await db.setSchema({ - // props: { - // badguy: 'boolean', - // coolguy: 'string', - // }, - // types: { - // nope: { - // name: 'string', - // }, - // yes: { - // name: 'string', - // }, - // }, - // }) - - deepEqual(updates, 2, '2 update') - // deepEqual(migrates, 1, '1 migrates') - - await db.setSchema({ - props: { - badguy: 'boolean', - coolguy: 'string', - okguy: 'boolean', - }, - types: { - nope: { - name: 'string', - }, - yes: { - name: 'string', - success: 'boolean', - }, - }, - }) - - deepEqual(updates, 3, '3 update') - // deepEqual(migrates, 2, '2 migrates') - await db.update({ - badguy: true, - coolguy: 'arnold', - okguy: true, - }) - - await db.create('nope', { - name: 'abe', - }) - - await db.create('yes', { - name: 'bill', - success: true, - }) - - await db.setSchema({ - props: { - badguy: 'boolean', - coolguy: 'string', - okguy: 'boolean', - excludesNopes: { - items: { - ref: 'nope', - }, - }, - }, - types: { - nope: { - name: 'string', - }, - yes: { - name: 'string', - success: 'boolean', - }, - }, - }) - - deepEqual(updates, 4, '4 update') - // deepEqual(migrates, 3, '3 migrates') -}) diff --git a/packages/db/test/schema/schemaDebug.ts b/packages/db/test/schema/schemaDebug.ts index f0309f0576..d26f2928b2 100644 --- a/packages/db/test/schema/schemaDebug.ts +++ b/packages/db/test/schema/schemaDebug.ts @@ -3,7 +3,6 @@ import { DbClient, DbClientHooks, DbServer } from '../../src/index.js' import test from '../shared/test.js' import { deepCopy, deepMerge, wait } from '@based/utils' import { copy, emptyDir } from 'fs-extra/esm' -import { deepEqual, equal } from '../shared/assert.js' const cleanProps = (props) => { for (const i in props) { @@ -36,17 +35,10 @@ const removeInverseProps = (props) => { const cleanSchema = (schema: DbServer['schema']) => { const schemaCopy = deepCopy(schema) - delete schemaCopy.lastId delete schemaCopy.hash for (const type in schemaCopy.types) { cleanProps(schemaCopy.types[type].props) - if (type === '_root') { - // @ts-ignore - schemaCopy.props = schemaCopy.types[type].props - removeInverseProps(schemaCopy.props) - delete schemaCopy.types[type] - } } return schemaCopy } diff --git a/packages/db/test/schema/schemaProblems.ts b/packages/db/test/schema/schemaProblems.ts index 7d4c2a1084..1ea0c9969b 100644 --- a/packages/db/test/schema/schemaProblems.ts +++ b/packages/db/test/schema/schemaProblems.ts @@ -236,10 +236,10 @@ await test.skip('schema problems', async (t) => { // validates the schema // setServerLocalSchema(SERVER) // add lastSchemaId and makes checksum - // schemaTypesParsed() + // defs() // setLocalSchema (client) // adds client.schema and emits 'schema' - // schemaTypesParsed() + // defs() // subscribeSchema (client hook) // listen on incoming schema (over network) and calls setLocalSchema // migrateSchema diff --git a/packages/db/test/shared/assert.ts b/packages/db/test/shared/assert.ts index a0a752aa9b..bdcd96d40b 100644 --- a/packages/db/test/shared/assert.ts +++ b/packages/db/test/shared/assert.ts @@ -2,7 +2,6 @@ import { deepEqual as uDeepEqual } from '@based/utils' import { styleText } from 'node:util' import util from 'node:util' import { BasedQueryResponse } from '../../src/client/query/BasedQueryResponse.js' -import { REVERSE_TYPE_INDEX_MAP } from '@based/schema/def' export { perf } from './perf.js' // add fn @@ -55,7 +54,7 @@ export const isSorted = ( const propDef = a.def?.schema?.props?.[field] if (propDef) { - fieldType = ' ' + REVERSE_TYPE_INDEX_MAP[propDef.typeIndex] + fieldType = ' ' + propDef.type } for (const result of a) { diff --git a/packages/db/test/shared/test.ts b/packages/db/test/shared/test.ts index edf0b7fa0c..ab54271b2e 100644 --- a/packages/db/test/shared/test.ts +++ b/packages/db/test/shared/test.ts @@ -36,7 +36,7 @@ export type T = { const test = async ( name: string, - fn: (t?: T) => Promise, + fn: (t: T) => Promise, ): Promise => { if ( process.env.TEST_TO_RUN && @@ -50,7 +50,7 @@ const test = async ( let hasErrored = false console.log(styleText('gray', `\nstart ${name}`)) const d = performance.now() - const afters = [] + const afters: any[] = [] const t: T = { after: (fn: () => Promise | void, push?: boolean) => { if (push) { @@ -78,8 +78,11 @@ const test = async ( for (const type in db.server.schema?.types) { let x = await db.query(type).include(fields).get() + // @ts-ignore checksums.push(x.checksum) + // @ts-ignore data.push(x.toObject()) + // @ts-ignore counts.push(await db.query(type).count().get().toObject().count) } @@ -155,6 +158,7 @@ const test = async ( deepEqual( b[di], a[di], + // @ts-ignore `Mismatch after backup (len:${b.length}) ${Object.keys(db.server.schema.types)[di]}`, ) } @@ -163,6 +167,7 @@ const test = async ( deepEqual( c[ci], counts[ci], + // @ts-ignore `Mismatching count after backup (len:${b.length}) ${Object.keys(db.server.schema.types)[ci]}`, ) } @@ -229,6 +234,7 @@ const test = async ( try { await afters[i]() } catch (err) { + // @ts-ignore errs.push(err) } } diff --git a/packages/db/test/sort/sortHll.ts b/packages/db/test/sort/sortHll.ts index a5a9163273..f40875a2c4 100644 --- a/packages/db/test/sort/sortHll.ts +++ b/packages/db/test/sort/sortHll.ts @@ -1,4 +1,3 @@ -import { numberTypes } from '@based/schema/def' import { BasedDb, xxHash64 } from '../../src/index.js' import test from '../shared/test.js' import { deepEqual, equal } from 'node:assert' diff --git a/packages/db/test/subscription/subscriptionSchemaChanges.ts b/packages/db/test/subscription/subscriptionSchemaChanges.ts index da041cfea7..789e126b1b 100644 --- a/packages/db/test/subscription/subscriptionSchemaChanges.ts +++ b/packages/db/test/subscription/subscriptionSchemaChanges.ts @@ -74,7 +74,7 @@ await test('subscription schema changes', async (t) => { await wait(20) q.reset() deepEqual(result1, q.get(), 'first schema change results are correct') - const subResults = [] + const subResults: any = [] const close = q.subscribe((q) => { subResults.push(q.toObject()) cnt++ @@ -132,7 +132,7 @@ await test('better subscription schema changes', async (t) => { }, }) - const results = [] + const results: any = [] db.query('user').subscribe((res) => { const obj = res.toObject() results.push(obj) diff --git a/packages/db/test/text/textFilter.ts b/packages/db/test/text/textFilter.ts index e6344f9b6f..bea98b96fb 100644 --- a/packages/db/test/text/textFilter.ts +++ b/packages/db/test/text/textFilter.ts @@ -104,7 +104,7 @@ await test('textFilter', async (t) => { searchTerms.push('F') } - const q = [] + const q: any[] = [] for (const term of searchTerms) { q.push( (async () => { diff --git a/packages/db/test/update.ts b/packages/db/test/update.ts index 0085f823c0..f4804ca7ad 100644 --- a/packages/db/test/update.ts +++ b/packages/db/test/update.ts @@ -1,6 +1,6 @@ import { BasedDb } from '../src/index.js' import test from './shared/test.js' -import { deepEqual, equal, throws, perf } from './shared/assert.js' +import { deepEqual, equal, throws } from './shared/assert.js' await test('update with payload.id', async (t) => { const db = new BasedDb({ @@ -197,7 +197,7 @@ await test('update', async (t) => { }, ]) - const ids = [] + const ids: any[] = [] let snurpId = 1 for (; snurpId <= 1e6; snurpId++) { ids.push(snurpId) @@ -250,7 +250,7 @@ await test('update', async (t) => { ], ) - const promises = [] + const promises: any[] = [] for (var j = 0; j < 1; j++) { for (var i = 0; i < 1e5; i++) { promises.push(db.query('snurp', i).include('a').get()) diff --git a/packages/db/tsconfig.json b/packages/db/tsconfig.json index 80da4f4faf..9614b1cb8b 100644 --- a/packages/db/tsconfig.json +++ b/packages/db/tsconfig.json @@ -6,7 +6,8 @@ "allowJs": true, "noPropertyAccessFromIndexSignature": false, "rootDir": "./", - "composite": true + "composite": true, + "strictNullChecks": false }, "include": ["./src", "./test"], "exclude": ["./dist"] diff --git a/packages/exporter/src/exporter.ts b/packages/exporter/src/exporter.ts index 6f298b2e5f..41fc43e790 100644 --- a/packages/exporter/src/exporter.ts +++ b/packages/exporter/src/exporter.ts @@ -41,7 +41,7 @@ const getCsvFileName = ( const processBlockAndExportToCsv = async (db: BasedDb, blockKey: number) => { const [typeId, startNodeId] = destructureTreeKey(blockKey) - const def = db.client.schemaTypesParsedById[typeId] + const def = db.client.defsById[typeId] log( `Processing block: type "${def.type}" (id: ${typeId}), starting from node: ${startNodeId}`, ) diff --git a/packages/protocol/src/db-read/aggregate.ts b/packages/protocol/src/db-read/aggregate.ts index c4fee242fa..c504ef9a7d 100644 --- a/packages/protocol/src/db-read/aggregate.ts +++ b/packages/protocol/src/db-read/aggregate.ts @@ -1,17 +1,4 @@ -import { - ENUM, - isNumberType, - TIMESTAMP, - REFERENCE, - INT16, - INT32, - INT8, - NUMBER, - UINT16, - UINT32, - UINT8, - TypeIndex, -} from '@based/schema/prop-types' +import { numberTypes, typeIndexMap, type TypeIndex } from '@based/schema' import { ReaderSchema, AggregateType } from './types.js' import { readInt64, @@ -29,23 +16,27 @@ const readNumber = ( type: TypeIndex, ): any => { switch (type) { - case NUMBER: + case typeIndexMap.number: return readDoubleLE(value, offset) - case UINT16: + case typeIndexMap.uint16: return readUint16(value, offset) - case UINT32: + case typeIndexMap.uint32: return readUint32(value, offset) - case INT16: + case typeIndexMap.int16: return readInt16(value, offset) - case INT32: + case typeIndexMap.int32: return readInt32(value, offset) - case UINT8: + case typeIndexMap.uint8: return value[offset] - case INT8: + case typeIndexMap.int8: return value[offset] } } +const numberTypeIndexSet = new Set( + numberTypes.map((type) => typeIndexMap[type]), +) + export const readAggregate = ( // only need agg q: ReaderSchema, @@ -68,25 +59,25 @@ export const readAggregate = ( // } i += 2 } else { - if (q.aggregate.groupBy.typeIndex == ENUM) { + if (q.aggregate.groupBy.typeIndex == typeIndexMap.enum) { i += 2 key = q.aggregate.groupBy.enum[result[i] - 1] i++ - } else if (isNumberType(q.aggregate.groupBy.typeIndex)) { + } else if (numberTypeIndexSet.has(q.aggregate.groupBy.typeIndex)) { keyLen = readUint16(result, i) i += 2 key = readNumber(result, i, q.aggregate.groupBy.typeIndex) i += keyLen } else if ( - q.aggregate.groupBy.typeIndex == TIMESTAMP && + q.aggregate.groupBy.typeIndex == typeIndexMap.timestamp && q.aggregate.groupBy.stepType ) { keyLen = readUint16(result, i) i += 2 - key = readNumber(result, i, INT32) + key = readNumber(result, i, typeIndexMap.int32) i += keyLen } else if ( - q.aggregate.groupBy.typeIndex == TIMESTAMP && + q.aggregate.groupBy.typeIndex == typeIndexMap.timestamp && q.aggregate.groupBy.stepRange !== 0 ) { keyLen = readUint16(result, i) @@ -105,10 +96,10 @@ export const readAggregate = ( key = dtFormat.format(readInt64(result, i)) } i += keyLen - } else if (q.aggregate.groupBy.typeIndex == REFERENCE) { + } else if (q.aggregate.groupBy.typeIndex == typeIndexMap.reference) { keyLen = readUint16(result, i) i += 2 - key = readNumber(result, i, INT32) + key = readNumber(result, i, typeIndexMap.int32) i += keyLen } else { keyLen = readUint16(result, i) diff --git a/packages/protocol/src/db-read/main.ts b/packages/protocol/src/db-read/main.ts index 786e50bf12..ab25429ee6 100644 --- a/packages/protocol/src/db-read/main.ts +++ b/packages/protocol/src/db-read/main.ts @@ -9,23 +9,9 @@ import { readInt32, readUtf8, } from '@based/utils' -import { - INT8, - INT16, - INT32, - UINT8, - UINT16, - UINT32, - NUMBER, - TIMESTAMP, - BOOLEAN, - ENUM, - STRING, - JSON, - BINARY, -} from '@based/schema/prop-types' import { Item } from './types.js' import { readMetaMainString } from './meta.js' +import { typeIndexMap } from '@based/schema' const readMainValue = ( prop: ReaderPropDef, @@ -34,21 +20,21 @@ const readMainValue = ( item: Item, ) => { const typeIndex = prop.typeIndex - if (typeIndex === TIMESTAMP) { + if (typeIndex === typeIndexMap.timestamp) { addProp(prop, readInt64(result, i), item) - } else if (typeIndex === NUMBER) { + } else if (typeIndex === typeIndexMap.number) { addProp(prop, readDoubleLE(result, i), item) - } else if (typeIndex === UINT32) { + } else if (typeIndex === typeIndexMap.uint32) { addProp(prop, readUint32(result, i), item) - } else if (typeIndex === BOOLEAN) { + } else if (typeIndex === typeIndexMap.boolean) { addProp(prop, Boolean(result[i]), item) - } else if (typeIndex === ENUM) { + } else if (typeIndex === typeIndexMap.enum) { if (result[i] === 0) { addProp(prop, undefined, item) } else { addProp(prop, prop.enum[result[i] - 1], item) } - } else if (typeIndex === STRING) { + } else if (typeIndex === typeIndexMap.string) { const len = result[i] i++ const value = len === 0 ? '' : readUtf8(result, i, len) @@ -62,26 +48,26 @@ const readMainValue = ( } else { addProp(prop, value, item) } - } else if (typeIndex === JSON) { + } else if (typeIndex === typeIndexMap.json) { const len = result[i] i++ const value = len === 0 ? null : global.JSON.parse(readUtf8(result, i, len)) addProp(prop, value, item) - } else if (typeIndex === BINARY) { + } else if (typeIndex === typeIndexMap.binary) { const len = result[i] i++ const value = len === 0 ? new Uint8Array(0) : result.subarray(i, i + len) addProp(prop, value, item) - } else if (typeIndex === INT8) { + } else if (typeIndex === typeIndexMap.int8) { const signedVal = (result[i] << 24) >> 24 addProp(prop, signedVal, item) - } else if (typeIndex === UINT8) { + } else if (typeIndex === typeIndexMap.uint8) { addProp(prop, result[i], item) - } else if (typeIndex === INT16) { + } else if (typeIndex === typeIndexMap.int16) { addProp(prop, readInt16(result, i), item) - } else if (typeIndex === UINT16) { + } else if (typeIndex === typeIndexMap.uint16) { addProp(prop, readUint16(result, i), item) - } else if (typeIndex === INT32) { + } else if (typeIndex === typeIndexMap.int32) { addProp(prop, readInt32(result, i), item) } } diff --git a/packages/protocol/src/db-read/prop.ts b/packages/protocol/src/db-read/prop.ts index 414b444630..8a6982f47a 100644 --- a/packages/protocol/src/db-read/prop.ts +++ b/packages/protocol/src/db-read/prop.ts @@ -1,18 +1,9 @@ import { readUint32, readUtf8 } from '@based/utils/dist/src/uint8.js' import { Item, ReaderPropDef, ReaderSchema } from './types.js' import { addProp, addLangProp } from './addProps.js' -import { - ALIAS, - BINARY, - CARDINALITY, - COLVEC, - JSON, - STRING, - TEXT, - VECTOR, -} from '@based/schema/def' import { readString } from './string.js' import { readVector } from './vector.js' +import { typeIndexMap } from '@based/schema' const readStringProp = ( prop: ReaderPropDef, @@ -21,16 +12,16 @@ const readStringProp = ( size: number, ) => { if ( - prop.typeIndex === TEXT || - prop.typeIndex === STRING || - prop.typeIndex === ALIAS + prop.typeIndex === typeIndexMap.text || + prop.typeIndex === typeIndexMap.string || + prop.typeIndex === typeIndexMap.alias ) { return readString(buf, offset, size, true) } - if (prop.typeIndex === JSON) { + if (prop.typeIndex === typeIndexMap.json) { return global.JSON.parse(readString(buf, offset, size, true)) } - if (prop.typeIndex === BINARY) { + if (prop.typeIndex === typeIndexMap.binary) { return buf.subarray(offset + 2, size + offset) } } @@ -45,23 +36,23 @@ export const readProp = ( const prop = q.props[instruction] prop.readBy = q.readId - if (prop.typeIndex === CARDINALITY) { + if (prop.typeIndex === typeIndexMap.cardinality) { const size = readUint32(result, i) addProp(prop, readUint32(result, i + 4), item) i += size + 4 - } else if (prop.typeIndex === JSON) { + } else if (prop.typeIndex === typeIndexMap.json) { const size = readUint32(result, i) addProp(prop, readStringProp(prop, result, i + 4, size), item) i += size + 4 - } else if (prop.typeIndex === BINARY) { + } else if (prop.typeIndex === typeIndexMap.binary) { const size = readUint32(result, i) addProp(prop, readStringProp(prop, result, i + 4, size), item) i += size + 4 - } else if (prop.typeIndex === STRING) { + } else if (prop.typeIndex === typeIndexMap.string) { const size = readUint32(result, i) addProp(prop, readStringProp(prop, result, i + 4, size), item) i += size + 4 - } else if (prop.typeIndex == TEXT) { + } else if (prop.typeIndex == typeIndexMap.text) { const size = readUint32(result, i) if (size === 0) { // do nothing @@ -78,7 +69,7 @@ export const readProp = ( } } i += size + 4 - } else if (prop.typeIndex === ALIAS) { + } else if (prop.typeIndex === typeIndexMap.alias) { const size = readUint32(result, i) i += 4 if (size === 0) { @@ -88,7 +79,10 @@ export const readProp = ( i += size addProp(prop, string, item) } - } else if (prop.typeIndex === VECTOR || prop.typeIndex === COLVEC) { + } else if ( + prop.typeIndex === typeIndexMap.vector || + prop.typeIndex === typeIndexMap.colvec + ) { const tmp = result.slice(i, i + prop.len) // maybe align? addProp(prop, readVector(prop, tmp), item) i += prop.len diff --git a/packages/protocol/src/db-read/read.ts b/packages/protocol/src/db-read/read.ts index 6f12f236d6..1c8dcfdc58 100644 --- a/packages/protocol/src/db-read/read.ts +++ b/packages/protocol/src/db-read/read.ts @@ -20,7 +20,8 @@ import { addLangMetaProp, addMetaProp, addProp } from './addProps.js' import { readProp } from './prop.js' import { readMain } from './main.js' import { undefinedProps } from './undefined.js' -import { TEXT } from '@based/schema/prop-types' +import { typeIndexMap } from '@based/schema' + export * from './types.js' export * from './string.js' export * from './schema/deserialize.js' @@ -32,7 +33,7 @@ const meta: ReadInstruction = (q, result, i, item) => { const lang = result[i] i++ prop.readBy = q.readId - if (prop.typeIndex === TEXT && prop.locales) { + if (prop.typeIndex === typeIndexMap.text && prop.locales) { addLangMetaProp(prop, readMetaSeperate(result, i), item, lang) } else { addMetaProp(prop, readMetaSeperate(result, i), item) diff --git a/packages/protocol/src/db-read/schema/deserialize.ts b/packages/protocol/src/db-read/schema/deserialize.ts index 09d0a8e140..a42cead432 100644 --- a/packages/protocol/src/db-read/schema/deserialize.ts +++ b/packages/protocol/src/db-read/schema/deserialize.ts @@ -1,4 +1,4 @@ -import { TypeIndex } from '@based/schema/prop-types' +import type { TypeIndex } from '@based/schema' import { ReaderPropDef, ReaderSchema, diff --git a/packages/protocol/src/db-read/types.ts b/packages/protocol/src/db-read/types.ts index eef2a014ac..83d2fcd746 100644 --- a/packages/protocol/src/db-read/types.ts +++ b/packages/protocol/src/db-read/types.ts @@ -1,5 +1,4 @@ -import { TypeIndex, VectorBaseType } from '@based/schema/prop-types' -import type { HLLRegisterRepresentation, SchemaHooks } from '@based/schema' +import type { SchemaHooks, TypeIndex } from '@based/schema' export type Item = { id: number @@ -46,6 +45,17 @@ export type ReadInstruction = ( export type ReaderLocales = { [langCode: string]: string } +export enum VectorBaseType { + Int8 = 1, + Uint8 = 2, + Int16 = 3, + Uint16 = 4, + Int32 = 5, + Uint32 = 6, + Float32 = 7, + Float64 = 8, +} + export type ReaderPropDef = { path: string[] typeIndex: TypeIndex diff --git a/packages/protocol/src/db-read/undefined.ts b/packages/protocol/src/db-read/undefined.ts index f6e6c5ba4b..64238bc546 100644 --- a/packages/protocol/src/db-read/undefined.ts +++ b/packages/protocol/src/db-read/undefined.ts @@ -1,43 +1,33 @@ -import { - STRING, - JSON, - BINARY, - CARDINALITY, - REFERENCES, - REFERENCE, - VECTOR, - TEXT, - ALIAS, -} from '@based/schema/prop-types' import { Item, ReaderMeta, ReaderPropDef, ReaderSchema } from './types.js' import { addLangMetaProp, addMetaProp, addProp } from './addProps.js' import { readVector } from './vector.js' import { emptyMeta } from './meta.js' +import { typeIndexMap } from '@based/schema' const undefinedValue = (prop: ReaderPropDef) => { const typeIndex = prop.typeIndex - if (typeIndex === STRING || typeIndex === ALIAS) { + if (typeIndex === typeIndexMap.string || typeIndex === typeIndexMap.alias) { return '' } - if (typeIndex === JSON) { + if (typeIndex === typeIndexMap.json) { return null } - if (typeIndex === BINARY) { + if (typeIndex === typeIndexMap.binary) { return new Uint8Array() } - if (typeIndex === CARDINALITY) { + if (typeIndex === typeIndexMap.cardinality) { return 0 } - if (typeIndex === REFERENCES) { + if (typeIndex === typeIndexMap.references) { return [] } - if (typeIndex === REFERENCE) { + if (typeIndex === typeIndexMap.reference) { return null } - if (typeIndex === VECTOR) { + if (typeIndex === typeIndexMap.vector) { return readVector(prop, new Uint8Array()) } - if (typeIndex === TEXT) { + if (typeIndex === typeIndexMap.text) { if (prop.locales) { const codes = {} for (const code in prop.locales) { @@ -57,7 +47,7 @@ export const undefinedProps = (q: ReaderSchema, item: Item) => { if (p.readBy !== q.readId) { p.readBy = q.readId if (p.meta) { - if (p.typeIndex === TEXT && p.locales) { + if (p.typeIndex === typeIndexMap.text && p.locales) { for (const code in p.locales) { const meta = emptyMeta() if (p.meta === ReaderMeta.combined) { diff --git a/packages/protocol/src/db-read/vector.ts b/packages/protocol/src/db-read/vector.ts index 442fd54073..5338293ca6 100644 --- a/packages/protocol/src/db-read/vector.ts +++ b/packages/protocol/src/db-read/vector.ts @@ -1,5 +1,4 @@ -import { VectorBaseType } from '@based/schema/prop-types' -import { ReaderPropDef } from './types.js' +import { ReaderPropDef, VectorBaseType } from './types.js' export const readVector = (prop: ReaderPropDef, tmp: Uint8Array) => { switch (prop.vectorBaseType) { diff --git a/packages/schema-diagram/src/utils.ts b/packages/schema-diagram/src/utils.ts index c4df6f0bd3..e5e661add0 100644 --- a/packages/schema-diagram/src/utils.ts +++ b/packages/schema-diagram/src/utils.ts @@ -1,10 +1,11 @@ -import { SchemaType, SchemaProp, Schema, StrictSchema } from '@based/schema' +import { SchemaType, SchemaProp, type SchemaOut } from '@based/schema' import { SchemaDiagram } from './SchemaDiagram.js' import { FilterOps } from './types.js' import { getByPath, setByPath } from '@based/utils' +import type { SchemaObject } from '@based/schema/dist/schema/object.js' export const walkProps = ( - type: SchemaType, + type: SchemaType | SchemaObject, collect: { [key: string]: SchemaProp }, path = [], ) => { @@ -13,7 +14,7 @@ export const walkProps = ( const schemaProp = target[key] const propPath = [...path, key] const propType = schemaProp.type - if (propType === 'object' || 'props' in schemaProp) { + if (propType === 'object') { walkProps(schemaProp, collect, propPath) } else { if (propType || schemaProp.items || schemaProp.enum || schemaProp.ref) { @@ -30,7 +31,7 @@ export const filterSchema = (ctx: SchemaDiagram, ops: FilterOps) => { ctx.schema = originalSchema ctx.filterInternal = undefined } - const filteredSchema: StrictSchema = { types: {} } + const filteredSchema: SchemaOut = { types: {} } const walk = (type: SchemaType, path: string[], all?: boolean) => { const target = type.props path = [...path, 'props'] diff --git a/packages/schema/package.json b/packages/schema/package.json index 0b308674db..04e7c660d4 100644 --- a/packages/schema/package.json +++ b/packages/schema/package.json @@ -8,12 +8,6 @@ "!dist/**/*.map" ], "main": "./dist/index.js", - "exports": { - "./def": "./dist/def/index.js", - "./lang": "./dist/lang.js", - "./prop-types": "./dist/def/typeIndexes.js", - ".": "./dist/index.js" - }, "scripts": { "clean": "rm -rf .node_modules ./dist ./tsconfig.tsbuildinfo", "build": "tsc", @@ -27,12 +21,16 @@ "@saulx/prettier-config": "2.0.0", "@saulx/tsconfig": "^1.1.0", "@types/node": "^22.5.3", + "@types/validator": "^13.15.10", "tsx": "^4.19.0", "typescript": "^5.6.3" }, "dependencies": { - "@based/utils": "1.2.0", + "@based/hash": "^1.1.0", + "@based/utils": "^1.2.0", + "flat": "^6.0.1", "picocolors": "^1.1.0", + "valibot": "^1.1.0", "validator": "^13.15.20" } } diff --git a/packages/schema/src/dbSchema.ts b/packages/schema/src/dbSchema.ts deleted file mode 100644 index 78972a98ef..0000000000 --- a/packages/schema/src/dbSchema.ts +++ /dev/null @@ -1,172 +0,0 @@ -import { hash } from '@based/hash' -import { getPropType } from './parse/utils.js' -import { - SchemaPropOneWay, - SchemaProps, - SchemaTypes, - StrictSchema, -} from './types.js' -import { deepCopy } from '@based/utils' - -export type DbSchema = StrictSchema & { lastId: number; hash: number } -export type SchemaChecksum = number - -function _makeEdgeTypes( - newTypes: SchemaTypes, - typeName: string, - props: SchemaProps, - propPrefix: string, -): void { - type EdgeProps = Record<`$${string}`, SchemaPropOneWay> - const putEdgeProps = ( - from: `${string}_${string}`, - to: `${string}_${string}`, - edgeProps: EdgeProps, - ) => (newTypes[`_${[from, to].sort().join(':')}`] = { props: edgeProps }) - - for (const propName in props) { - const prop = props[propName] - const propType = getPropType(prop) - const nextPropPrefix = propPrefix ? `${propPrefix}_${propName}` : propName - if (propType === 'object') { - _makeEdgeTypes(newTypes, typeName, prop.props, nextPropPrefix) - } else if (propType === 'reference') { - const edgeProps: Record<`$${string}`, SchemaPropOneWay> = {} - Object.keys(prop) - .filter((k) => k[0] === '$') - .forEach((k) => (edgeProps[k] = prop[k])) - - if (Object.keys(edgeProps).length > 0) { - putEdgeProps( - `${typeName}_${nextPropPrefix}`, - `${prop.ref}_${prop.prop}`, - edgeProps, - ) - } - } else if (propType === 'references') { - const edgeProps: Record<`$${string}`, SchemaPropOneWay> = {} - - Object.keys(prop.items) - .filter((k) => k[0] === '$') - .forEach((k) => (edgeProps[k] = prop.items[k])) - - if (Object.keys(edgeProps).length > 0) { - putEdgeProps( - `${typeName}_${nextPropPrefix}`, - `${prop.items.ref}_${prop.items.prop}`, - edgeProps, - ) - } - } - } -} - -function makeEdgeTypes(types: SchemaTypes): SchemaTypes { - const newTypes = {} - - for (const typeName in types) { - const type = types[typeName] - _makeEdgeTypes(newTypes, typeName, type.props, '') - } - - return newTypes -} - -export const strictSchemaToDbSchema = (schema: StrictSchema): DbSchema => { - // @ts-ignore - let dbSchema: DbSchema = deepCopy(schema) - - // reserve 1 for root (even if you dont have it) - dbSchema.lastId = 1 - - // Make the _root type - if (dbSchema.props) { - for (const key in dbSchema.props) { - const prop = dbSchema.props[key] - const propType = getPropType(prop) - let refProp: any - - if (propType === 'reference') { - refProp = prop - } else if (propType === 'references') { - refProp = prop.items - prop.items = refProp - } - - if (refProp) { - const type = dbSchema.types[refProp.ref] - const inverseKey = '_' + key - dbSchema.types[refProp.ref] = { - ...type, - props: { - ...type.props, - [inverseKey]: { - items: { - ref: '_root', - prop: key, - }, - }, - }, - } - refProp.prop = inverseKey - } - } - - dbSchema.types ??= {} - // @ts-ignore This creates an internal type to use for root props - dbSchema.types._root = { - id: 1, - props: dbSchema.props, - } - delete dbSchema.props - } - - const edgeTypes = makeEdgeTypes(dbSchema.types) - // Create inverse props for reference(s) - for (const et in edgeTypes) { - dbSchema.types[et] = edgeTypes[et] - - for (const key in edgeTypes[et].props) { - const prop = edgeTypes[et].props[key] - const propType = getPropType(prop) - let refProp: any - - if (propType === 'reference') { - refProp = prop - } else if (propType === 'references') { - refProp = prop.items - } else { - continue // not a ref - } - - const type = dbSchema.types[refProp.ref] - const inverseKey = `_${et}_${key}` - dbSchema.types[refProp.ref] = { - ...type, - props: { - ...type.props, - [inverseKey]: { - items: { - ref: et, - prop: key, - }, - }, - }, - } - refProp.prop = inverseKey - } - } - - // Assign typeIds - for (const typeName in dbSchema.types) { - if (!('id' in dbSchema.types[typeName])) { - dbSchema.lastId++ - dbSchema.types[typeName].id = dbSchema.lastId - } - } - - delete dbSchema.hash - dbSchema.hash = hash(dbSchema) - - return dbSchema -} diff --git a/packages/schema/src/def/addEdges.ts b/packages/schema/src/def/addEdges.ts deleted file mode 100644 index 42624c4461..0000000000 --- a/packages/schema/src/def/addEdges.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { getPropType, getValidator, SchemaReference } from '../index.js' -import { DEFAULT_MAP } from './defaultMap.js' -import { fillEmptyMain } from './fillEmptyMain.js' -import { - PropDef, - TYPE_INDEX_MAP, - PropDefEdge, - REFERENCES, - REFERENCE, - ENUM, - NUMBER, -} from './types.js' -import { - getPropLen, - isSeparate, - parseMinMaxStep, - sortMainProps, -} from './utils.js' -import { defaultValidation, VALIDATION_MAP } from './validation.js' - -export const addEdges = (prop: PropDef, refProp: SchemaReference) => { - const mainEdges: PropDefEdge[] = [] - for (const key in refProp) { - if (key[0] === '$') { - if (!prop.edges) { - prop.edgeMainLen = 0 - prop.edges = {} - prop.reverseSeperateEdges = {} - prop.reverseMainEdges = {} - prop.edgesSeperateCnt = 0 - } - const edgeProp = refProp[key] - const edgeType = getPropType(edgeProp) - const len = getPropLen(edgeProp) - const separate = isSeparate(edgeProp, len) - if (separate) { - prop.edgesSeperateCnt++ - } - const typeIndex = TYPE_INDEX_MAP[edgeType] - - if (edgeProp.default !== undefined) { - prop.hasDefaultEdges = true - } - - // add default - const edge: PropDefEdge = { - schema: edgeProp, - __isPropDef: true, - __isEdge: true, - prop: separate ? prop.edgesSeperateCnt : 0, - validation: getValidator(edgeProp), - name: key, - typeIndex, - len, - separate, - path: [...prop.path, key], - default: edgeProp.default ?? DEFAULT_MAP[typeIndex], - // start: prop.edgeMainLen, - } - - if (!separate) { - mainEdges.push(edge) - } - - if (edgeProp.max !== undefined) { - edgeProp.max = edge.max = parseMinMaxStep(edgeProp.max) - } - - if (edgeProp.min !== undefined) { - edgeProp.min = edge.min = parseMinMaxStep(edgeProp.min) - } - - if (edgeProp.step !== undefined) { - edgeProp.step = edge.step = parseMinMaxStep(edgeProp.step) - } - - if (edge.typeIndex !== NUMBER && edge.step === undefined) { - edge.step = 1 - } - - if (edge.typeIndex === ENUM) { - edge.enum = Array.isArray(refProp[key]) - ? refProp[key] - : refProp[key].enum - edge.reverseEnum = {} - for (let i = 0; i < edge.enum.length; i++) { - edge.reverseEnum[edge.enum[i]] = i - } - } else if (edge.typeIndex === REFERENCES) { - edge.inverseTypeName = refProp[key].items.ref - } else if (edge.typeIndex === REFERENCE) { - edge.inverseTypeName = refProp[key].ref - } - prop.edges[key] = edge - if (separate) { - prop.reverseSeperateEdges[edge.prop] = edge - } - } - } - - mainEdges.sort(sortMainProps) - for (const edge of mainEdges) { - edge.start = prop.edgeMainLen - prop.edgeMainLen += edge.len - prop.reverseMainEdges[edge.start] = edge - } - - prop.edgeMainEmpty = fillEmptyMain(mainEdges, prop.edgeMainLen) -} diff --git a/packages/schema/src/def/createEmptyDef.ts b/packages/schema/src/def/createEmptyDef.ts deleted file mode 100644 index 6b6385a0c8..0000000000 --- a/packages/schema/src/def/createEmptyDef.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { SchemaLocales, SchemaObject, StrictSchemaType } from '../types.js' -import { hashObjectIgnoreKeyOrder } from '@based/hash' - -export const createEmptyDef = ( - typeName: string, - type: StrictSchemaType | SchemaObject, - locales: Partial, -) => { - return { - cnt: 0, - blockCapacity: 0, - capped: 0, - insertOnly: false, - partial: false, - checksum: hashObjectIgnoreKeyOrder(type), - type: typeName, - props: {}, - reverseProps: {}, - idUint8: new Uint8Array([0, 0]), - // empty main buffer - id: 0, - mainEmpty: new Uint8Array(0), - mainLen: 0, - separate: [], - tree: {}, - total: 0, - lastId: 0, - locales: {}, - main: {}, - separateSortProps: 0, - separateSortText: 0, - localeSize: 0, - hasSeperateSort: false, - separateSort: { - size: 0, - props: [], - buffer: new Uint8Array([]), - bufferTmp: new Uint8Array([]), - }, - hasSeperateTextSort: false, - separateTextSort: { - size: 0, // prop len - props: [], - buffer: new Uint8Array([]), - noUndefined: new Uint8Array( - new Array(Object.keys(locales).length).fill(0), - ), - bufferTmp: new Uint8Array([]), - localeStringToIndex: new Map(), - localeToIndex: new Map(), - }, - } -} diff --git a/packages/schema/src/def/defaultMap.ts b/packages/schema/src/def/defaultMap.ts deleted file mode 100644 index 16d09e760a..0000000000 --- a/packages/schema/src/def/defaultMap.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { - TypeIndex, - ALIAS, - BINARY, - JSON, - BOOLEAN, - CARDINALITY, - TIMESTAMP, - INT16, - INT32, - INT8, - UINT8, - UINT16, - UINT32, - NUMBER, - ENUM, - ID, - MICRO_BUFFER, - REFERENCE, - REFERENCES, - STRING, - TEXT, - ALIASES, - VECTOR, - COLVEC, - NULL, - OBJECT, -} from './types.js' - -export const DEFAULT_MAP: Record = { - [NULL]: 0, - [OBJECT]: 0, - [ALIAS]: '', - [BINARY]: new Uint8Array(), - [BOOLEAN]: false, - [CARDINALITY]: [], - [NUMBER]: 0, - [TIMESTAMP]: 0, - [ENUM]: 0, - [ID]: 0, - [INT16]: 0, - [INT32]: 0, - [INT8]: 0, - [UINT8]: 0, - [UINT16]: 0, - [UINT32]: 0, - [JSON]: null, - [MICRO_BUFFER]: undefined, - [REFERENCE]: undefined, - [REFERENCES]: [], - [STRING]: '', - [ALIASES]: [], - [TEXT]: {}, - [VECTOR]: undefined, // maybe not can set a vec with 0 - [COLVEC]: undefined, // maybe not can set a vec with 0 -} diff --git a/packages/schema/src/def/enums.ts b/packages/schema/src/def/enums.ts new file mode 100644 index 0000000000..ba526c4742 --- /dev/null +++ b/packages/schema/src/def/enums.ts @@ -0,0 +1,45 @@ +type ReverseMap> = { + [P in keyof T as T[P]]: P +} + +const reverseMap =

>( + obj: P, +): ReverseMap

=> { + const reverse: any = {} + for (const k in obj) { + reverse[k] = obj[k] + } + return reverse +} + +export const typeIndexMap = { + null: 0, + timestamp: 1, + number: 4, + cardinality: 5, + int8: 20, + uint8: 6, + int16: 21, + uint16: 22, + int32: 23, + uint32: 7, + boolean: 9, + enum: 10, + string: 11, + text: 12, + reference: 13, + references: 14, + microbuffer: 17, + alias: 18, + // aliases: 19, + binary: 25, + // id: 26, + vector: 27, + json: 28, + // object: 29, + colvec: 30, +} as const + +export const reverseTypeMap = reverseMap(typeIndexMap) +export type TypeName = keyof typeof typeIndexMap +export type TypeIndex = keyof typeof reverseTypeMap diff --git a/packages/schema/src/def/fillEmptyMain.ts b/packages/schema/src/def/fillEmptyMain.ts deleted file mode 100644 index 8cc69948ec..0000000000 --- a/packages/schema/src/def/fillEmptyMain.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { convertToTimestamp, writeInt64 } from '@based/utils' -import { - BINARY, - BOOLEAN, - ENUM, - INT16, - INT32, - INT8, - NUMBER, - PropDef, - PropDefEdge, - STRING, - TIMESTAMP, - UINT16, - UINT32, - UINT8, -} from './types.js' - -export const ENCODER = new TextEncoder() - -export const fillEmptyMain = ( - vals: (PropDef | PropDefEdge)[], - mainLen: number, -) => { - const mainEmpty = new Uint8Array(mainLen) - for (const f of vals) { - if (f.separate) { - continue - } - const t = f.typeIndex - const s = f.start - let val = f.default - - if (t === ENUM) { - mainEmpty[s] = f.default ?? 0 - } else if (t === INT8 || t === UINT8) { - mainEmpty[s] = val - } else if (t === BOOLEAN) { - mainEmpty[s] = val ? 1 : 0 - } else if (t === UINT32 || t === INT32) { - mainEmpty[s] = val - mainEmpty[s + 1] = val >>>= 8 - mainEmpty[s + 2] = val >>>= 8 - mainEmpty[s + 3] = val >>>= 8 - } else if (t === UINT16 || t === INT16) { - mainEmpty[s] = val - mainEmpty[s + 1] = val >>>= 8 - } else if (t === TIMESTAMP) { - writeInt64(mainEmpty, convertToTimestamp(val), s) - } else if (t === NUMBER) { - const view = new DataView(mainEmpty.buffer, s, 8) - view.setFloat64(0, val, true) - } else if (t === STRING) { - val = ENCODER.encode(val) - mainEmpty[s] = val.byteLength - mainEmpty.set(val, s + 1) - } else if (t === BINARY) { - if (val !== undefined) { - mainEmpty.set(val, s) - } - } - } - - return mainEmpty -} - -export const isZeroes = (buf: Uint8Array) => { - let i = 0 - while (i < buf.byteLength) { - if (buf[i] !== 0) { - return false - } - i++ - } - return true -} diff --git a/packages/schema/src/def/index.ts b/packages/schema/src/def/index.ts index a0ebdd9399..75d3f0ed76 100644 --- a/packages/schema/src/def/index.ts +++ b/packages/schema/src/def/index.ts @@ -1,7 +1,341 @@ -// flap -export * from './types.js' -export * from './typeDef.js' -export * from './utils.js' -export * from './createEmptyDef.js' -export * from './defaultMap.js' -export * from './validation.js' +import type { SchemaEnum } from '../schema/enum.js' +import type { SchemaObject } from '../schema/object.js' +import type { SchemaProp } from '../schema/prop.js' +import type { SchemaReference } from '../schema/reference.js' +import type { SchemaReferences } from '../schema/references.js' +import type { SchemaLocale, SchemaOut } from '../schema/schema.js' +import type { SchemaString } from '../schema/string.js' +import type { SchemaProps, SchemaType } from '../schema/type.js' +import type { SchemaVector } from '../schema/vector.js' +import { getValidator, type Validation } from './validation.js' +import type { SchemaPropHooks } from '../schema/hooks.js' +import type { LangName } from '../schema/lang.js' +import { typeIndexMap, type TypeIndex, type TypeName } from './enums.js' +import type { Base } from '../schema/base.js' + +type BaseProp = Base & { + typeDef: TypeDef + path: string[] + validation: Validation +} + +type DbProp = BaseProp & { + id: number + typeIndex: TypeIndex +} + +type IdPropDef = DbProp & { + type: 'id' + id: 255 + typeIndex: typeof typeIndexMap.null +} + +type ErrorPropDef = BaseProp & { + type: 'error' + id: -1 + typeIndex: -1 +} + +type EnumPropDef = DbProp & + SchemaEnum & { + enumMap: Record + } + +type RefLike = SchemaReference | SchemaReferences + +export type ObjPropDef = SchemaObject & + BaseProp & { props: Record } + +type VectorPropDef = SchemaVector & DbProp & { baseSize: number } +type PropDefRest = Exclude< + SchemaProp, + RefLike | SchemaObject | SchemaEnum | SchemaVector +> & + DbProp + +export type RefPropDef = RefLike & + DbProp & { target: RefPropDef; edgesDef?: TypeDef } + +export type SeparateDef = RefPropDef | PropDefRest | EnumPropDef | VectorPropDef +export type MainDef = SeparateDef & { + main: { + start: number + size: number + } +} +export type LeafDef = SeparateDef | MainDef +export type PropDef = ObjPropDef | LeafDef +export type QueryPropDef = LeafDef | IdPropDef // | ErrorPropDef + +export type TypeDef = Omit, 'props'> & { + id: number + name: string + size: number + props: Record + edge?: true + separate: SeparateDef[] + propHooks: Partial> + locales: Partial> + queryProps: { id: IdPropDef; [key: string]: QueryPropDef } + dbProps: { [key: string]: LeafDef } +} +export type BranchDef = TypeDef | ObjPropDef +export type TypeDefs = { + byName: Record + byId: Record +} + +const stringLen = ({ maxBytes = Infinity, max = Infinity }: SchemaString) => + maxBytes < 61 ? maxBytes + 1 : max < 31 ? max * 2 + 1 : 0 + +const numberLen = (type?: string) => + (type && Number(type.replace(/[^0-9]/g, '')) / 4) || 8 + +type SizeMap = Partial< + Record) => number)> +> + +export const mainSizeMap = { + boolean: 1, + enum: 1, + int8: 1, + uint8: 1, + int16: 2, + uint16: 2, + int32: 4, + uint32: 4, + number: 8, + timestamp: 8, + binary: stringLen, + string: stringLen, +} as const satisfies SizeMap + +export const sizeMap = { + // cardinality: stringLen, + vector: ({ size }: SchemaVector) => size * 4, + colvec: ({ size, baseType }: SchemaVector) => size * numberLen(baseType), +} as const satisfies SizeMap + +export const createIdProp = (typeDef: TypeDef): IdPropDef => ({ + id: 255, + type: 'id', + typeIndex: 0, + path: ['id'], + typeDef, + validation: () => true, +}) + +export const createErrorProp = (typeDef: TypeDef): ErrorPropDef => ({ + id: -1, + type: 'error', + typeIndex: -1, + path: [], + typeDef, + validation: () => true, +}) + +const sortMainProps = (a: LeafDef, b: LeafDef) => { + const sizeA = ('main' in a && a.main.size) || 0 + const sizeB = ('main' in b && b.main.size) || 0 + if (sizeA === 8) { + return -1 + } + if (sizeA === 4 && sizeB !== 8) { + return -1 + } + if (sizeA === sizeB) { + return 0 + } + return 1 +} + +const sortDbPropOrder = ({ type }) => { + if (type === 'reference' || type === 'references') { + return -1 + } + if (type === 'alias' || type === 'colvec') { + return 300 + } + return 299 +} + +export const schemaToTypeDefs = (schema: SchemaOut): TypeDefs => { + const defs: TypeDefs = { + byName: {}, + byId: {}, + } + + let idCnt = 0 + + const locales = {} + for (const lang in schema.locales) { + const v = schema.locales[lang] + locales[lang] = typeof v === 'object' ? v : {} + } + + for (const type in schema.types) { + const typeDef = parseType(type, schema.types[type]) + defs.byName[type] = typeDef + defs.byId[typeDef.id] = typeDef + } + + for (const type in defs.byName) { + for (const prop in defs.byName[type].props) { + const propDef = defs.byName[type].props[prop] + if ('target' in propDef) { + parseRefLike(type, prop, propDef) + } + } + } + + return defs + + function parseType(name: string, typeSchema: SchemaType): TypeDef { + const typeProps = {} + const typeDef = { + id: idCnt++, + name, + size: 0, + propHooks: {}, + blockCapacity: 100_000, + ...typeSchema, + locales, + props: {}, + dbProps: {}, + } as TypeDef + + typeDef.separate = [] + typeDef.queryProps = {} as typeof typeDef.queryProps + typeDef.props = parseProps(typeSchema.props, typeProps, []) + typeDef.queryProps.id = createIdProp(typeDef) + + const sorted = Object.values(typeDef.dbProps).sort(sortDbPropOrder) + + let propIdCnt = 1 + for (const prop of sorted) { + if (!('main' in prop)) { + prop.id = propIdCnt++ + } + } + + sorted.sort(sortMainProps) + let start = 0 + for (const prop of sorted) { + if ('main' in prop) { + prop.main.start = start + start += prop.main.size + } + } + + return typeDef + + function parseProps( + schemaProps: Record>, + defProps: Record, + path: string[], + ): Record { + for (const prop in schemaProps) { + const propSchema = schemaProps[prop] + const { type: propType, ...rest } = propSchema + const propPath = [...path, prop] + const validation = getValidator(propSchema) + + if (propType === 'object') { + defProps[prop] = { + type: propType, + path: propPath, + typeDef, + ...rest, + props: parseProps(propSchema.props, {}, propPath), + validation, + } + continue + } + + const getMainSize = mainSizeMap[propType] + const mainSize = + typeof getMainSize === 'function' + ? getMainSize(propSchema) + : getMainSize || 0 + + const propDef = { + id: 0, // temporary id because we have to sort before giving id later + type: propType, + typeIndex: typeIndexMap[propType], + typeDef, + path: propPath, + ...rest, + validation, + } as LeafDef + + if ('enum' in propDef) { + propDef.enumMap = Object.fromEntries( + propDef.enum.map((val, index) => [val, index + 1]), + ) + } else if (propDef.type === 'vector') { + propDef.baseSize = 4 + } else if (propDef.type === 'colvec') { + propDef.baseSize = numberLen(propDef.baseType) + } + + if (mainSize) { + typeDef.size += mainSize + ;(propDef as MainDef).main = { + start: 0, // temporary start because we have to sort before giving start later + size: mainSize, + } + } else { + typeDef.separate.push(propDef) + } + + if (propDef.hooks) { + for (const key in propDef.hooks) { + typeDef.propHooks[key] ??= [] + typeDef.propHooks[key].push(propDef) + } + } + + const dotPath = propDef.path.join('.') + typeDef.queryProps[dotPath] = propDef + typeDef.dbProps[dotPath] = propDef + defProps[prop] = propDef + } + return defProps + } + } + + function parseRefLike(type: string, prop: string, refDef: RefPropDef) { + const refSchema = 'items' in refDef ? refDef.items : refDef + const refType = refSchema.ref + const target = defs.byName[refType].props[refSchema.prop] as RefPropDef + + refDef.target = target + + if (type > refType) return + + let edges: SchemaProps | undefined + for (const key in refSchema) { + if (key[0] === '$') { + edges ??= {} + edges[key] = refSchema[key] + } + } + + if (!edges) return + + const targetSchema = 'items' in target ? target.items : target + const edgeTypeName = `$${type}_${prop}` + const edgeType = parseType(edgeTypeName, { props: edges }) + + for (const key in edges) { + const edgeProp = edgeType.props[key] + refSchema[key] = edgeProp + targetSchema[key] = edgeProp + refDef.edgesDef = edgeType + target.edgesDef = edgeType + } + + defs.byName[edgeTypeName] = edgeType + edgeType.edge = true + } +} diff --git a/packages/schema/src/def/makeSeparateSort.ts b/packages/schema/src/def/makeSeparateSort.ts deleted file mode 100644 index a8dcefb2fe..0000000000 --- a/packages/schema/src/def/makeSeparateSort.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { SchemaTypeDef, STRING, ALIAS, CARDINALITY } from './types.js' - -export function makeSeparateSort(result: Partial) { - result.hasSeperateSort = true - let max = 0 - for (const f of result.separate) { - if ( - f.typeIndex === STRING || - f.typeIndex === ALIAS || - f.typeIndex === CARDINALITY - ) { - if (f.prop > max) { - max = f.prop - } - } - } - - result.separateSort.buffer = new Uint8Array(max + 1) - for (const f of result.separate) { - if ( - f.typeIndex === STRING || - f.typeIndex === ALIAS || - f.typeIndex === CARDINALITY - ) { - result.separateSort.buffer[f.prop] = 1 - result.separateSort.props.push(f) - result.separateSort.size++ - } - } - result.separateSort.bufferTmp = new Uint8Array(max + 1) - result.separateSort.buffer.set(result.separateSort.bufferTmp) -} diff --git a/packages/schema/src/def/makeSeparateTextSort.ts b/packages/schema/src/def/makeSeparateTextSort.ts deleted file mode 100644 index 2538eae8b1..0000000000 --- a/packages/schema/src/def/makeSeparateTextSort.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { langCodesMap } from '../lang.js' -import { SchemaTypeDef, TEXT } from './types.js' - -export function makeSeparateTextSort(result: Partial) { - result.hasSeperateTextSort = true - let max = 0 - for (const f of result.separate) { - if (f.typeIndex === TEXT) { - if (f.prop > max) { - max = f.prop - } - } - } - - const bufLen = (max + 1) * (result.localeSize + 1) - result.separateTextSort.buffer = new Uint8Array(bufLen) - let index = 0 - for (const code in result.locales) { - const codeLang = langCodesMap.get(code) - result.separateTextSort.localeStringToIndex.set( - code, - new Uint8Array([index + 1, codeLang]), - ) - result.separateTextSort.localeToIndex.set(codeLang, index + 1) - index++ - } - for (const f of result.separate) { - if (f.typeIndex === TEXT) { - const index = f.prop * (result.localeSize + 1) - result.separateTextSort.buffer[index] = result.localeSize - for (const [, locales] of result.separateTextSort.localeStringToIndex) { - result.separateTextSort.buffer[locales[0] + index] = locales[1] - } - result.separateTextSort.props.push(f) - result.separateTextSort.size += result.localeSize - } - } - result.separateTextSort.props.sort((a, b) => (a.prop > b.prop ? 1 : -1)) - result.separateTextSort.bufferTmp = new Uint8Array(bufLen) - result.separateTextSort.bufferTmp.fill(0) - result.separateTextSort.bufferTmp.set(result.separateTextSort.buffer) -} diff --git a/packages/schema/src/def/typeDef.ts b/packages/schema/src/def/typeDef.ts deleted file mode 100644 index 9c056c061a..0000000000 --- a/packages/schema/src/def/typeDef.ts +++ /dev/null @@ -1,370 +0,0 @@ -import { - isPropType, - SchemaObject, - StrictSchemaType, - getPropType, - SchemaLocales, - SchemaHooks, - getValidator, -} from '../index.js' -import { setByPath } from '@based/utils' -import { - PropDef, - SchemaTypeDef, - TYPE_INDEX_MAP, - REFERENCES, - REFERENCE, - NUMBER, - BLOCK_CAPACITY_MAX, - BLOCK_CAPACITY_DEFAULT, - BLOCK_CAPACITY_MIN, - VECTOR, - COLVEC, - CARDINALITY, -} from './types.js' -import { DEFAULT_MAP } from './defaultMap.js' -import { StrictSchema } from '../types.js' -import { makeSeparateTextSort } from './makeSeparateTextSort.js' -import { makeSeparateSort } from './makeSeparateSort.js' -import { - getPropLen, - isSeparate, - parseMinMaxStep, - reorderProps, - schemaVectorBaseTypeToEnum, - sortMainProps, - cardinalityModeToEnum, -} from './utils.js' -import { addEdges } from './addEdges.js' -import { createEmptyDef } from './createEmptyDef.js' -import { fillEmptyMain, isZeroes } from './fillEmptyMain.js' - -export const updateTypeDefs = (schema: StrictSchema) => { - const schemaTypesParsed: { [key: string]: SchemaTypeDef } = {} - const schemaTypesParsedById: { [id: number]: SchemaTypeDef } = {} - - for (const typeName in schema.types) { - const type = schema.types[typeName] - if (!type.id) { - throw new Error('NEED ID ON TYPE') - } - const def = createSchemaTypeDef( - typeName, - type, - schema.locales ?? { - en: {}, - }, - ) - schemaTypesParsed[typeName] = def - schemaTypesParsedById[type.id] = def - } - - for (const schema of Object.values(schemaTypesParsed)) { - for (const prop of Object.values(schema.props)) { - if (prop.typeIndex === REFERENCE || prop.typeIndex === REFERENCES) { - // FIXME Now references in edgeType are missing __isEdge - // However, we can soon just delete weak refs - if (!prop.__isEdge && !prop.inversePropName) { - prop.__isEdge = true - } - - if (!prop.__isEdge) { - // Update inverseProps in references - const dstType: SchemaTypeDef = schemaTypesParsed[prop.inverseTypeName] - prop.inverseTypeId = dstType.id - prop.inversePropNumber = dstType.props[prop.inversePropName].prop - - if (prop.edges) { - if (dstType.props[prop.inversePropName].edges) { - // this currently is not allowed, but might be - const mergedEdges = { - ...dstType.props[prop.inversePropName].edges, - ...prop.edges, - } - dstType.props[prop.inversePropName].edges = mergedEdges - prop.edges = mergedEdges - } else { - dstType.props[prop.inversePropName].edges = prop.edges - } - } - - // Update edgeNodeTypeId - if (!prop.edgeNodeTypeId) { - if (prop.edges) { - const edgeTypeName = `_${[`${schema.type}_${prop.path.join('_')}`, `${dstType.type}_${dstType.props[prop.inversePropName].path.join('_')}`].sort().join(':')}` - const edgeType = schemaTypesParsed[edgeTypeName] - prop.edgeNodeTypeId = edgeType.id - dstType.props[prop.inversePropName].edgeNodeTypeId = edgeType.id - } else { - prop.edgeNodeTypeId = 0 - } - } - } - } - } - } - - return { schemaTypesParsed, schemaTypesParsedById } -} - -const createSchemaTypeDef = ( - typeName: string, - type: StrictSchemaType | SchemaObject, - locales: Partial, - result: Partial = createEmptyDef(typeName, type, locales), - path: string[] = [], - top: boolean = true, -): SchemaTypeDef => { - if (top) { - if (result.id == 0) { - if ('id' in type) { - result.id = type.id - } else { - throw new Error(`Invalid schema type id ${result.type}`) - } - } - if (result.blockCapacity == 0) { - if ('blockCapacity' in type) { - if ( - typeof type.blockCapacity !== 'number' || - type.blockCapacity < BLOCK_CAPACITY_MIN || - type.blockCapacity > BLOCK_CAPACITY_MAX - ) { - throw new Error('Invalid blockCapacity') - } - result.blockCapacity = type.blockCapacity - } else { - result.blockCapacity = - typeName === '_root' ? BLOCK_CAPACITY_MAX : BLOCK_CAPACITY_DEFAULT - } - } - if (result.capped == 0) { - if ('capped' in type) { - if (typeof type.capped !== 'number' || type.capped < 0) { - throw new Error('Invalid capped') - } - result.capped = type.capped - } - } - if (result.insertOnly == false && 'insertOnly' in type) { - result.insertOnly = !!type.insertOnly - } - if (result.partial == false && 'partial' in type) { - result.partial = !!type.partial - } - if ('hooks' in type) { - result.hooks = type.hooks as SchemaHooks - } - } - result.locales = locales - result.localeSize = Object.keys(locales).length - result.idUint8[0] = result.id & 255 - result.idUint8[1] = result.id >> 8 - const target = type.props - - for (const key in target) { - // Create prop def - const schemaProp = target[key] - const propPath = [...path, key] - const propType = getPropType(schemaProp) - if (propType === 'object') { - createSchemaTypeDef( - typeName, - schemaProp as SchemaObject, - locales, - result, - propPath, - false, - ) - continue - } - - const len = getPropLen(schemaProp) - if ( - isPropType('string', schemaProp) || - isPropType('alias', schemaProp) || - isPropType('cardinality', schemaProp) - ) { - if (typeof schemaProp === 'object') { - if ( - !(schemaProp.maxBytes < 61) || - !('max' in schemaProp && schemaProp.max < 31) - ) { - result.separateSortProps++ - } - } else { - result.separateSortProps++ - } - } else if (isPropType('text', schemaProp)) { - result.separateSortText++ - } else if (isPropType('colvec', schemaProp)) { - if (!result.insertOnly) { - throw new Error('colvec requires insertOnly') - } - } - const isseparate = isSeparate(schemaProp, len) - const typeIndex = TYPE_INDEX_MAP[propType] - const prop: PropDef = { - schema: schemaProp, - typeIndex, - __isPropDef: true, - separate: isseparate, - path: propPath, - start: 0, - validation: getValidator(schemaProp), - len, - default: schemaProp.default ?? DEFAULT_MAP[typeIndex], - prop: isseparate ? ++result.cnt : 0, - } - - if (schemaProp.hooks) { - result.propHooks ??= {} - for (const key in schemaProp.hooks) { - prop.hooks = schemaProp.hooks - result.propHooks[key] ??= new Set() - result.propHooks[key].add(prop) - } - } - - if (schemaProp.max !== undefined) { - schemaProp.max = prop.max = parseMinMaxStep(schemaProp.max) - } - - if (schemaProp.min !== undefined) { - schemaProp.min = prop.min = parseMinMaxStep(schemaProp.min) - } - - if (schemaProp.step !== undefined) { - schemaProp.step = prop.step = parseMinMaxStep(schemaProp.step) - } - - if (prop.typeIndex !== NUMBER && prop.step === undefined) { - prop.step = 1 - } - if (prop.typeIndex === VECTOR || prop.typeIndex === COLVEC) { - prop.vectorBaseType = schemaVectorBaseTypeToEnum( - schemaProp.baseType ?? 'number', - ) - } - - if (prop.typeIndex === CARDINALITY) { - prop.cardinalityMode ??= cardinalityModeToEnum( - (schemaProp.mode ??= 'sparse'), - ) - const prec = typeName == '_root' ? 14 : 8 - prop.cardinalityPrecision ??= schemaProp.precision ??= prec - } - - if (isPropType('enum', schemaProp)) { - prop.enum = Array.isArray(schemaProp) ? schemaProp : schemaProp.enum - prop.reverseEnum = {} - for (let i = 0; i < prop.enum.length; i++) { - prop.reverseEnum[prop.enum[i]] = i - } - } else if (isPropType('references', schemaProp)) { - if (result.partial) { - throw new Error('references is not supported with partial') - } - - prop.inversePropName = schemaProp.items.prop - prop.inverseTypeName = schemaProp.items.ref - prop.dependent = schemaProp.items.dependent - prop.referencesCapped = schemaProp.capped ?? 0 - addEdges(prop, schemaProp.items) - } else if (isPropType('reference', schemaProp)) { - if (result.partial) { - throw new Error('reference is not supported with partial') - } - - prop.inversePropName = schemaProp.prop - prop.inverseTypeName = schemaProp.ref - prop.dependent = schemaProp.dependent - addEdges(prop, schemaProp) - } else if (typeof schemaProp === 'object') { - if (isPropType('string', schemaProp) || isPropType('text', schemaProp)) { - prop.compression = - 'compression' in schemaProp && schemaProp.compression === 'none' - ? 0 - : 1 - } else if (isPropType('timestamp', schemaProp) && 'on' in schemaProp) { - if (schemaProp.on[0] === 'c') { - result.createTs ??= [] - result.createTs.push(prop) - } else if (schemaProp.on[0] === 'u') { - result.createTs ??= [] - result.createTs.push(prop) - result.updateTs ??= [] - result.updateTs.push(prop) - } - } - } - result.props[propPath.join('.')] = prop - if (isseparate) { - result.separate.push(prop) - } - } - - if (top) { - // Put top level together - const vals = Object.values(result.props) - reorderProps(vals) - let len = 2 - let biggestSeperatePropDefault = 0 - - for (const f of vals) { - if (f.separate) { - len += 2 - setByPath(result.tree, f.path, f) - if (f.default !== undefined) { - result.hasSeperateDefaults = true - if (!result.separateDefaults) { - result.separateDefaults = { - props: new Map(), - bufferTmp: new Uint8Array(), - } - } - result.separateDefaults.props.set(f.prop, f) - if (f.prop > biggestSeperatePropDefault) { - biggestSeperatePropDefault = f.prop - } - } - } - } - - const mainProps = vals.filter((v) => !v.separate).sort(sortMainProps) - for (const f of mainProps) { - if (!result.mainLen) { - len += 2 - } - len += 1 - f.start = result.mainLen - result.mainLen += f.len - setByPath(result.tree, f.path, f) - } - - if (result.hasSeperateDefaults) { - result.separateDefaults.bufferTmp = new Uint8Array( - biggestSeperatePropDefault + 1, - ) - } - - result.mainEmpty = fillEmptyMain(vals, result.mainLen) - result.mainEmptyAllZeroes = isZeroes(result.mainEmpty) - - if (result.separateSortText > 0) { - makeSeparateTextSort(result) - } - if (result.separateSortProps > 0) { - makeSeparateSort(result) - } - for (const p in result.props) { - const x = result.props[p] - if (!x.separate) { - result.main[x.start] = x - } else { - result.reverseProps[x.prop] = x - } - } - } - return result as SchemaTypeDef -} diff --git a/packages/schema/src/def/typeIndexes.ts b/packages/schema/src/def/typeIndexes.ts deleted file mode 100644 index cc2b5cd3ce..0000000000 --- a/packages/schema/src/def/typeIndexes.ts +++ /dev/null @@ -1,77 +0,0 @@ -// WARN: The following type codes are used in js and zig but selva has its own typing. -export const NULL = 0 -export const TIMESTAMP = 1 -export const NUMBER = 4 -export const CARDINALITY = 5 -export const INT8 = 20 -export const UINT8 = 6 -export const INT16 = 21 -export const UINT16 = 22 -export const INT32 = 23 -export const UINT32 = 7 -export const BOOLEAN = 9 -export const ENUM = 10 -export const STRING = 11 -export const TEXT = 12 -export const REFERENCE = 13 -export const REFERENCES = 14 -export const MICRO_BUFFER = 17 -export const ALIAS = 18 -export const ALIASES = 19 -export const BINARY = 25 -export const ID = 26 -export const VECTOR = 27 -export const JSON = 28 -export const OBJECT = 29 -export const COLVEC = 30 - -export type TypeIndex = - | typeof NULL - | typeof TIMESTAMP - | typeof NUMBER - | typeof CARDINALITY - | typeof INT8 - | typeof UINT8 - | typeof INT16 - | typeof UINT16 - | typeof INT32 - | typeof UINT32 - | typeof BOOLEAN - | typeof ENUM - | typeof STRING - | typeof TEXT - | typeof REFERENCE - | typeof REFERENCES - | typeof MICRO_BUFFER - | typeof ALIAS - | typeof ALIASES - | typeof BINARY - | typeof ID - | typeof VECTOR - | typeof JSON - | typeof OBJECT - | typeof COLVEC - -export enum VectorBaseType { - Int8 = 1, - Uint8 = 2, - Int16 = 3, - Uint16 = 4, - Int32 = 5, - Uint32 = 6, - Float32 = 7, - Float64 = 8, -} - -export const isNumberType = (type: TypeIndex): boolean => { - return ( - type === NUMBER || - type === UINT16 || - type === UINT32 || - type === INT16 || - type === INT32 || - type == UINT8 || - type === INT8 || - type === CARDINALITY - ) -} diff --git a/packages/schema/src/def/types.ts b/packages/schema/src/def/types.ts deleted file mode 100644 index a0b172a7e4..0000000000 --- a/packages/schema/src/def/types.ts +++ /dev/null @@ -1,304 +0,0 @@ -import type { - LangCode, - SchemaHooks, - SchemaLocales, - SchemaProp, - SchemaPropHooks, -} from '../index.js' -import { Validation } from './validation.js' -import { - ALIAS, - ALIASES, - BINARY, - BOOLEAN, - CARDINALITY, - COLVEC, - ENUM, - INT16, - INT32, - INT8, - JSON, - MICRO_BUFFER, - NULL, - NUMBER, - OBJECT, - REFERENCE, - REFERENCES, - STRING, - TEXT, - TIMESTAMP, - UINT16, - UINT32, - UINT8, - VECTOR, - TypeIndex, - VectorBaseType, - ID, -} from './typeIndexes.js' - -export * from './typeIndexes.js' - -export const TYPE_INDEX_MAP: Record = { - alias: ALIAS, - aliases: ALIASES, - microbuffer: MICRO_BUFFER, - references: REFERENCES, - reference: REFERENCE, - timestamp: TIMESTAMP, - boolean: BOOLEAN, - number: NUMBER, - string: STRING, - text: TEXT, - uint16: UINT16, - uint32: UINT32, - int16: INT16, - int32: INT32, - uint8: UINT8, - enum: ENUM, - int8: INT8, - id: NULL, - binary: BINARY, - vector: VECTOR, - cardinality: CARDINALITY, - json: JSON, - object: OBJECT, - colvec: COLVEC, -} - -export const enum numberTypes { - number = NUMBER, - uint16 = UINT16, - uint32 = UINT32, - int16 = INT16, - int32 = INT32, - uint8 = UINT8, - int8 = INT8, - cardinality = CARDINALITY, -} - -export type InternalSchemaProp = keyof typeof TYPE_INDEX_MAP - -export type PropDef = { - __isPropDef: true - schema: SchemaProp - prop: number // (0-250) - typeIndex: TypeIndex - separate: boolean - path: string[] - start: number - len: number // bytes or count - compression?: 0 | 1 // 0 == none , 1 == standard deflate - enum?: any[] - dependent?: boolean - // default here? - validation: Validation - default: any - // references - inverseTypeName?: string - inversePropName?: string - inverseTypeId?: number - inversePropNumber?: number - referencesCapped?: number - // vectors - vectorBaseType?: VectorBaseType - vectorSize?: number - // cardinality - cardinalityMode?: number - cardinalityPrecision?: number - // edge stuff - edgeNodeTypeId?: number - edgeMainLen?: 0 - hasDefaultEdges?: boolean - reverseEnum?: { [key: string]: number } - edgesSeperateCnt?: number - edges?: { - [key: string]: PropDefEdge - } - reverseSeperateEdges?: { - [prop: string]: PropDefEdge - } - reverseMainEdges?: { - [start: string]: PropDefEdge - } - edgeMainEmpty?: Uint8Array - __isEdge?: boolean - // Schema stuff - max?: any - min?: any - step?: any - hooks?: SchemaPropHooks -} - -export type PropDefEdge = Partial & { - __isPropDef: true - typeIndex: TypeIndex - schema: SchemaProp - len: number - prop: number // (0-250) - name: string - edgesTotalLen?: number - __isEdge: true -} - -export type PropDefAggregate = Partial & { - __isPropDef: true - typeIndex: TypeIndex - len: number - prop: number // (0-250) - name: string -} - -export type SchemaPropTree = { [key: string]: SchemaPropTree | PropDef } - -export type SchemaSortUndefinedHandler = { - size: number // number of text fields - buffer: Uint8Array - bufferTmp: Uint8Array // Gets reused in modify to avoid extra Alloc - props: PropDef[] -} - -export const BLOCK_CAPACITY_MIN = 1025 -export const BLOCK_CAPACITY_MAX = 2147483647 -export const BLOCK_CAPACITY_DEFAULT = 100_000 - -export type SchemaTypeDef = { - cnt: number - checksum: number - type: string - blockCapacity: number - capped: number // Maximum number of nodes in the type. This creates a circularly collected type. - insertOnly: boolean // delete not allowed - partial: boolean // only active block(s) should be loaded in-mem - mainLen: number - buf: Uint8Array - propNames: Uint8Array - props: { - [path: string]: PropDef - } - reverseProps: { - [field: string]: PropDef - } - id: number // u16 number - idUint8: Uint8Array - separate: PropDef[] - main: { - [start: string]: PropDef - } - mainEmpty: Uint8Array - mainEmptyAllZeroes: boolean - tree: SchemaPropTree - separateSortProps: number - separateSortText: number - hasSeperateSort: boolean - separateSort: SchemaSortUndefinedHandler - hasSeperateTextSort: boolean - separateTextSort: SchemaSortUndefinedHandler & { - noUndefined: Uint8Array - localeStringToIndex: Map // [langCode][index] - localeToIndex: Map - } - hasSeperateDefaults: boolean - separateDefaults?: { props: Map; bufferTmp: Uint8Array } - createTs?: PropDef[] - updateTs?: PropDef[] - locales: Partial - localeSize: number - hooks?: SchemaHooks - propHooks?: { - [K in keyof SchemaPropHooks]: Set - } -} - -export const VECTOR_BASE_TYPE_SIZE_MAP: Record = { - [VectorBaseType.Int8]: 1, - [VectorBaseType.Uint8]: 1, - [VectorBaseType.Int16]: 2, - [VectorBaseType.Uint16]: 2, - [VectorBaseType.Int32]: 4, - [VectorBaseType.Uint32]: 4, - [VectorBaseType.Float32]: 4, - [VectorBaseType.Float64]: 8, -} - -export const SIZE_MAP: Record = { - timestamp: 8, // 64bit - // double-precision 64-bit binary format IEEE 754 value - number: 8, // 64bit - int8: 1, - uint8: 1, - int16: 2, - uint16: 2, - int32: 4, - uint32: 4, - boolean: 1, - reference: 0, // separate - enum: 1, // enum - string: 0, // separate - text: 0, // separate - cardinality: 0, // separate - references: 0, // separate - microbuffer: 0, // separate - alias: 0, - aliases: 0, - id: 4, - binary: 0, - vector: 0, // separate - json: 0, - object: 0, - colvec: 0, // separate -} - -const reverseMap: any = {} -for (const k in TYPE_INDEX_MAP) { - reverseMap[TYPE_INDEX_MAP[k]] = k -} - -// @ts-ignore -export const REVERSE_SIZE_MAP: Record = {} - -for (const k in SIZE_MAP) { - REVERSE_SIZE_MAP[TYPE_INDEX_MAP[k]] = SIZE_MAP[k] -} - -export const REVERSE_TYPE_INDEX_MAP: Record = - reverseMap - -export const ID_FIELD_DEF: PropDef = { - schema: null, - typeIndex: NULL, - separate: true, - path: ['id'], - start: 0, - prop: 255, - default: 0, - len: 4, - validation: () => true, - __isPropDef: true, -} - -export const EMPTY_MICRO_BUFFER: PropDef = { - schema: null, - typeIndex: MICRO_BUFFER, - separate: true, - path: [''], - start: 0, - default: undefined, - prop: 0, - len: 1, - validation: () => true, - __isPropDef: true, -} - -export const getPropTypeName = (propType: TypeIndex): InternalSchemaProp => { - return REVERSE_TYPE_INDEX_MAP[propType] -} - -export const isPropDef = (prop: any): prop is PropDef => { - if ('__isPropDef' in prop && prop.__isPropDef === true) { - return true - } - return false -} - -export type SchemaTypesParsed = { [key: string]: SchemaTypeDef } -export type SchemaTypesParsedById = Record diff --git a/packages/schema/src/def/utils.ts b/packages/schema/src/def/utils.ts deleted file mode 100644 index 879e07b27c..0000000000 --- a/packages/schema/src/def/utils.ts +++ /dev/null @@ -1,183 +0,0 @@ -import { - INT16, - INT32, - INT8, - UINT16, - UINT32, - UINT8, - NUMBER, - TIMESTAMP, - PropDef, - PropDefEdge, - SIZE_MAP, - VECTOR_BASE_TYPE_SIZE_MAP, - VectorBaseType, - REVERSE_SIZE_MAP, - REFERENCES, - REFERENCE, - ALIAS, - ALIASES, - COLVEC, -} from './types.js' - -import { - SchemaProp, - SchemaVectorBaseType, - isPropType, - HLLRegisterRepresentation, -} from '../types.js' -import { getPropType } from '../parse/utils.js' -import { convertToTimestamp } from '@based/utils' - -export function isSeparate(schemaProp: SchemaProp, len: number) { - return ( - len === 0 || - isPropType('vector', schemaProp) || - isPropType('colvec', schemaProp) - ) -} - -export const propIsSigned = (prop: PropDef | PropDefEdge): boolean => { - const t = prop.typeIndex - if (t === INT16 || t === INT32 || t === INT8) { - return true - } - return false -} - -export const propIsNumerical = (prop: PropDef | PropDefEdge) => { - const t = prop.typeIndex - if ( - t === INT16 || - t === INT32 || - t === INT8 || - t === UINT8 || - t === UINT16 || - t === UINT32 || - t === NUMBER || - t === TIMESTAMP - ) { - return true - } - return false -} - -export const schemaVectorBaseTypeToEnum = ( - vector: SchemaVectorBaseType, -): VectorBaseType => { - switch (vector) { - case 'int8': - return VectorBaseType.Int8 - case 'uint8': - return VectorBaseType.Uint8 - case 'int16': - return VectorBaseType.Int16 - case 'uint16': - return VectorBaseType.Uint16 - case 'int32': - return VectorBaseType.Int32 - case 'uint32': - return VectorBaseType.Uint32 - case 'float32': - return VectorBaseType.Float32 - case 'float64': - return VectorBaseType.Float64 - case 'number': - return VectorBaseType.Float64 - } -} - -export const cardinalityModeToEnum = (mode: HLLRegisterRepresentation) => { - if (mode === 'dense') return 1 - else 0 -} - -export function getPropLen(schemaProp: SchemaProp) { - let len = SIZE_MAP[getPropType(schemaProp)] - if ( - isPropType('string', schemaProp) || - isPropType('alias', schemaProp) || - isPropType('cardinality', schemaProp) - ) { - if (typeof schemaProp === 'object') { - if (schemaProp.maxBytes < 61) { - len = schemaProp.maxBytes + 1 - } else if ('max' in schemaProp && schemaProp.max < 31) { - len = schemaProp.max * 2 + 1 - } - } - } else if (isPropType('vector', schemaProp)) { - len = 4 * schemaProp.size - } else if (isPropType('colvec', schemaProp)) { - len = - schemaProp.size * - VECTOR_BASE_TYPE_SIZE_MAP[ - schemaVectorBaseTypeToEnum(schemaProp.baseType) ?? - VectorBaseType.Float64 - ] - } - - return len -} - -export const parseMinMaxStep = (val: any) => { - if (typeof val === 'number') { - return val - } - if (typeof val === 'string') { - if (!val.includes('now')) { - return convertToTimestamp(val) - } - return val - } -} - -export const sortMainProps = ( - a: PropDef | PropDefEdge, - b: PropDef | PropDefEdge, -) => { - const sizeA = REVERSE_SIZE_MAP[a.typeIndex] - const sizeB = REVERSE_SIZE_MAP[b.typeIndex] - if (sizeA === 8) { - return -1 - } - if (sizeA === 4 && sizeB !== 8) { - return -1 - } - if (sizeA === sizeB) { - return 0 - } - return 1 -} - -export const propIndexOffset = (prop: PropDef) => { - if (!prop.separate) { - return 0 - } - - switch (prop.typeIndex) { - case REFERENCES: - case REFERENCE: - return -300 - case ALIAS: - case ALIASES: - case COLVEC: - return 300 - default: - return 0 - } -} - -export const reorderProps = (props: PropDef[]) => { - props.sort( - (a, b) => a.prop + propIndexOffset(a) - (b.prop + propIndexOffset(b)), - ) - - // Reassign prop indices - let lastProp = 0 - for (const p of props) { - if (p.separate) { - p.prop = ++lastProp - } - } -} diff --git a/packages/schema/src/def/validation.ts b/packages/schema/src/def/validation.ts index 04998d872f..231b680811 100644 --- a/packages/schema/src/def/validation.ts +++ b/packages/schema/src/def/validation.ts @@ -1,428 +1,269 @@ import { convertToTimestamp } from '@based/utils' -import { - TypeIndex, - ALIAS, - BINARY, - JSON, - BOOLEAN, - CARDINALITY, - TIMESTAMP, - INT16, - INT32, - INT8, - UINT8, - UINT16, - UINT32, - NUMBER, - ENUM, - ID, - MICRO_BUFFER, - REFERENCE, - REFERENCES, - STRING, - TEXT, - ALIASES, - VECTOR, - COLVEC, - NULL, - OBJECT, - TYPE_INDEX_MAP, - PropDef, - PropDefEdge, -} from './types.js' -import { - MAX_ID, - MIN_ID, - SchemaEnum, - SchemaNumber, - SchemaObject, - SchemaProp, - SchemaProps, - SchemaString, - SchemaTimestamp, - SchemaType, - StrictSchema, -} from '../types.js' -import v from 'validator' -import { getPropType } from '../parse/index.js' +import type { SchemaProp } from '../schema/prop.js' +import isEmail from 'validator/lib/isEmail.js' +import isURL from 'validator/lib/isURL.js' +import isMACAddress from 'validator/lib/isMACAddress.js' +import isIP from 'validator/lib/isIP.js' +import isIPRange from 'validator/lib/isIPRange.js' +import isFQDN from 'validator/lib/isFQDN.js' +import isIBAN from 'validator/lib/isIBAN.js' +import isBIC from 'validator/lib/isBIC.js' +import isAlpha from 'validator/lib/isAlpha.js' +import isAlphanumeric from 'validator/lib/isAlphanumeric.js' +import isPassportNumber from 'validator/lib/isPassportNumber.js' +import isPort from 'validator/lib/isPort.js' +import isLowercase from 'validator/lib/isLowercase.js' +import isUppercase from 'validator/lib/isUppercase.js' +import isAscii from 'validator/lib/isAscii.js' +import isSemVer from 'validator/lib/isSemVer.js' +import isSurrogatePair from 'validator/lib/isSurrogatePair.js' +import isIMEI from 'validator/lib/isIMEI.js' +import isHexadecimal from 'validator/lib/isHexadecimal.js' +import isOctal from 'validator/lib/isOctal.js' +import isHexColor from 'validator/lib/isHexColor.js' +import isRgbColor from 'validator/lib/isRgbColor.js' +import isHSL from 'validator/lib/isHSL.js' +import isISRC from 'validator/lib/isISRC.js' +import isMD5 from 'validator/lib/isMD5.js' +import isJWT from 'validator/lib/isJWT.js' +import isUUID from 'validator/lib/isUUID.js' +import isLuhnNumber from 'validator/lib/isLuhnNumber.js' +import isCreditCard from 'validator/lib/isCreditCard.js' +import isEAN from 'validator/lib/isEAN.js' +import isISIN from 'validator/lib/isISIN.js' +import isISBN from 'validator/lib/isISBN.js' +import isISSN from 'validator/lib/isISSN.js' +import isMobilePhone from 'validator/lib/isMobilePhone.js' +import isPostalCode from 'validator/lib/isPostalCode.js' +import isEthereumAddress from 'validator/lib/isEthereumAddress.js' +import isCurrency from 'validator/lib/isCurrency.js' +import isBtcAddress from 'validator/lib/isBtcAddress.js' +import isISO6391 from 'validator/lib/isISO6391.js' +import isISO8601 from 'validator/lib/isISO8601.js' +import isRFC3339 from 'validator/lib/isRFC3339.js' +import isISO31661Alpha2 from 'validator/lib/isISO31661Alpha2.js' +import isISO31661Alpha3 from 'validator/lib/isISO31661Alpha3.js' +import isISO4217 from 'validator/lib/isISO4217.js' +import isBase32 from 'validator/lib/isBase32.js' +import isBase58 from 'validator/lib/isBase58.js' +import isBase64 from 'validator/lib/isBase64.js' +import isDataURI from 'validator/lib/isDataURI.js' +import isMagnetURI from 'validator/lib/isMagnetURI.js' +import isMimeType from 'validator/lib/isMimeType.js' +import isLatLong from 'validator/lib/isLatLong.js' +import isSlug from 'validator/lib/isSlug.js' +import isStrongPassword from 'validator/lib/isStrongPassword.js' +import isTaxID from 'validator/lib/isTaxID.js' +import isLicensePlate from 'validator/lib/isLicensePlate.js' +import isVAT from 'validator/lib/isVAT.js' +import { isBoolean, isRecord, isString } from '../schema/shared.js' +import type { SchemaTimestamp } from '../schema/timestamp.js' +import type { SchemaNumber } from '../schema/number.js' +import type { SchemaEnum } from '../schema/enum.js' +import type { SchemaString } from '../schema/string.js' +import type { SchemaObject } from '../schema/object.js' +import type { SchemaOut } from '../schema/schema.js' export type Validation = ( payload: any, schema: SchemaProp, ) => boolean | string -const EPSILON = 1e-9 // Small tolerance for floating point comparisons -const validators: Record boolean> = { - email: v.isEmail, - URL: v.isURL, - MACAddress: v.isMACAddress, - IP: v.isIP, - IPRange: v.isIPRange, - FQDN: v.isFQDN, - IBAN: v.isIBAN, - BIC: v.isBIC, - alpha: v.isAlpha, - alphaLocales: v.isAlphaLocales, - alphanumeric: v.isAlphanumeric, - alphanumericLocales: v.isAlphanumericLocales, - passportNumber: v.isPassportNumber, - port: v.isPort, - lowercase: v.isLowercase, - uppercase: v.isUppercase, - ascii: v.isAscii, - semVer: v.isSemVer, - surrogatePair: v.isSurrogatePair, - IMEI: v.isIMEI, - hexadecimal: v.isHexadecimal, - octal: v.isOctal, - hexColor: v.isHexColor, - rgbColor: v.isRgbColor, - HSL: v.isHSL, - ISRC: v.isISRC, - MD5: v.isMD5, - JWT: v.isJWT, - UUID: v.isUUID, - luhnNumber: v.isLuhnNumber, - creditCard: v.isCreditCard, - identityCard: v.isIdentityCard, - EAN: v.isEAN, - ISIN: v.isISIN, - ISBN: v.isISBN, - ISSN: v.isISSN, - mobilePhone: v.isMobilePhone, - mobilePhoneLocales: v.isMobilePhoneLocales, - postalCode: v.isPostalCode, - postalCodeLocales: v.isPostalCodeLocales, - ethereumAddress: v.isEthereumAddress, - currency: v.isCurrency, - btcAddress: v.isBtcAddress, - ISO6391: v.isISO6391, - ISO8601: v.isISO8601, - RFC3339: v.isRFC3339, - ISO31661Alpha2: v.isISO31661Alpha2, - ISO31661Alpha3: v.isISO31661Alpha3, - ISO4217: v.isISO4217, - base32: v.isBase32, - base58: v.isBase58, - base64: v.isBase64, - dataURI: v.isDataURI, - magnetURI: v.isMagnetURI, - mimeType: v.isMimeType, - latLong: v.isLatLong, - slug: v.isSlug, - password: v.isStrongPassword, - taxID: v.isTaxID, - licensePlate: v.isLicensePlate, - VAT: v.isVAT, - code: () => true, - javascript: () => true, - typescript: () => true, - python: () => true, - rust: () => true, - css: () => true, - html: () => true, - json: () => true, - markdown: () => true, - clike: () => true, -} -export const VALIDATION_MAP: Record = { - [NULL]: () => true, - [OBJECT]: () => true, - [COLVEC]: () => true, - [ALIAS]: (value) => { - if (typeof value !== 'string') { + +const allGood: Validation = () => true +const wrap = (validator: any) => (v: unknown) => validator(v) +export const validators = { + email: wrap(isEmail), + URL: wrap(isURL), + MACAddress: wrap(isMACAddress), + IP: wrap(isIP), + IPRange: wrap(isIPRange), + FQDN: wrap(isFQDN), + IBAN: wrap(isIBAN), + BIC: wrap(isBIC), + alpha: wrap(isAlpha), + alphanumeric: wrap(isAlphanumeric), + passportNumber: wrap(isPassportNumber), + port: wrap(isPort), + lowercase: wrap(isLowercase), + uppercase: wrap(isUppercase), + ascii: wrap(isAscii), + semVer: wrap(isSemVer), + surrogatePair: wrap(isSurrogatePair), + IMEI: wrap(isIMEI), + hexadecimal: wrap(isHexadecimal), + octal: wrap(isOctal), + hexColor: wrap(isHexColor), + rgbColor: wrap(isRgbColor), + HSL: wrap(isHSL), + ISRC: wrap(isISRC), + MD5: wrap(isMD5), + JWT: wrap(isJWT), + UUID: wrap(isUUID), + luhnNumber: wrap(isLuhnNumber), + creditCard: wrap(isCreditCard), + EAN: wrap(isEAN), + ISIN: wrap(isISIN), + ISBN: wrap(isISBN), + ISSN: wrap(isISSN), + mobilePhone: wrap(isMobilePhone), + postalCode: wrap(isPostalCode), + ethereumAddress: wrap(isEthereumAddress), + currency: wrap(isCurrency), + btcAddress: wrap(isBtcAddress), + ISO6391: wrap(isISO6391), + ISO8601: wrap(isISO8601), + RFC3339: wrap(isRFC3339), + ISO31661Alpha2: wrap(isISO31661Alpha2), + ISO31661Alpha3: wrap(isISO31661Alpha3), + ISO4217: wrap(isISO4217), + base32: wrap(isBase32), + base58: wrap(isBase58), + base64: wrap(isBase64), + dataURI: wrap(isDataURI), + magnetURI: wrap(isMagnetURI), + mimeType: wrap(isMimeType), + latLong: wrap(isLatLong), + slug: wrap(isSlug), + password: wrap(isStrongPassword), + taxID: wrap(isTaxID), + licensePlate: wrap(isLicensePlate), + VAT: wrap(isVAT), + code: allGood, + javascript: allGood, + typescript: allGood, + python: allGood, + rust: allGood, + css: allGood, + html: allGood, + json: allGood, + markdown: allGood, + clike: allGood, +} as const + +export const MAX_ID = 4_294_967_295 +const epsilon = 1e-9 // Small tolerance for floating point comparisons +export const isValidId = (v) => v > 0 && v <= MAX_ID +const getIntValidaton = + (min: number, max: number): Validation => + (value, t: SchemaNumber) => { + if ( + typeof value !== 'number' || + (t.step && value % t.step !== 0) || + isNaN(value) + ) { return false } - return true - }, - [BINARY]: (value) => { - if (value instanceof Uint8Array) { - return true + if (value > max || value < min) { + return false } - return false - }, - [BOOLEAN]: (value) => { - if (typeof value !== 'boolean') { + if (t.min !== undefined && value < t.min) { + return false + } + if (t.max !== undefined && value > t.max) { return false } return true - }, - [CARDINALITY]: (val) => { - return ( - typeof val === 'string' || - (val instanceof Uint8Array && val.byteLength === 8) - ) - }, - [TIMESTAMP]: (value, t: SchemaTimestamp) => { - if ( - typeof value !== 'number' || - (t.step && value % t.step !== 0) || - isNaN(value) - ) { + } +const isStringLike: Validation = (v, t: SchemaString) => { + if (v instanceof Uint8Array) { + return !!v[1] + } + if (typeof v !== 'string') { + return false + } + if (t.max !== undefined && v.length > t.max) { + return false + } + if (t.min !== undefined && v.length < t.min) { + return false + } + if (t.format !== undefined && t.format in validators) { + return validators[t.format](v, t) + } + return true +} + +export const validationMap: Record['type'], Validation> = { + object: isRecord, + colvec: allGood, + alias: isString, + binary: (v) => v instanceof Uint8Array, + boolean: isBoolean, + cardinality: (v) => + isString(v) || (v instanceof Uint8Array && v.byteLength === 8), + timestamp: (v, t: SchemaTimestamp) => { + if (typeof v !== 'number' || (t.step && v % t.step !== 0) || isNaN(v)) { return false } if (t.min !== undefined) { if (typeof t.min === 'number') { - if (value < t.min) { + if (v < t.min) { return false } - } else if (value < convertToTimestamp(t.min)) { + } else if (v < convertToTimestamp(t.min)) { return false } } if (t.max !== undefined) { if (typeof t.max === 'number') { - if (value > t.max) { + if (v > t.max) { return false } - } else if (value > convertToTimestamp(t.max)) { + } else if (v > convertToTimestamp(t.max)) { return false } } return true }, - [INT16]: (value, t: SchemaNumber) => { - if ( - typeof value !== 'number' || - (t.step && value % t.step !== 0) || - isNaN(value) - ) { - return false - } - if (value > 32767 || value < -32768) { - return false - } - if (t.min !== undefined && value < t.min) { - return false - } - if (t.max !== undefined && value > t.max) { - return false - } - return true - }, - [INT32]: (value, t: SchemaNumber) => { - if ( - typeof value !== 'number' || - (t.step && value % t.step !== 0) || - isNaN(value) - ) { - return false - } - if (value > 2147483647 || value < -2147483648) { - return false - } - if (t.min !== undefined && value < t.min) { - return false - } - if (t.max !== undefined && value > t.max) { - return false - } - return true - }, - [INT8]: (value, t: SchemaNumber) => { - // use % for steps size - if ( - typeof value !== 'number' || - (t.step && value % t.step !== 0) || - isNaN(value) - ) { - return false - } - if (value > 127 || value < -128) { - return false - } - if (t.min !== undefined && value < t.min) { - return false - } - if (t.max !== undefined && value > t.max) { - return false - } - return true - }, - [UINT8]: (value, t: SchemaNumber) => { - if ( - typeof value !== 'number' || - (t.step && value % t.step !== 0) || - isNaN(value) - ) { - return false - } - if (value > 255 || value < 0) { - return false - } - if (t.min !== undefined && value < t.min) { - return false - } - if (t.max !== undefined && value > t.max) { - return false - } - return true - }, - [UINT16]: (value, t: SchemaNumber) => { - if ( - typeof value !== 'number' || - (t.step && value % t.step !== 0) || - isNaN(value) - ) { - return false - } - if (value > 65535 || value < 0) { - return false - } - if (t.min !== undefined && value < t.min) { - return false - } - if (t.max !== undefined && value > t.max) { - return false - } - return true - }, - [UINT32]: (value, t: SchemaNumber) => { - if ( - typeof value !== 'number' || - (t.step && value % t.step !== 0) || - isNaN(value) - ) { - return false - } - if (value > 4294967295 || value < 0) { - return false - } - if (t.min !== undefined && value < t.min) { - return false - } - if (t.max !== undefined && value > t.max) { - return false - } - return true - }, - [NUMBER]: (value, t: SchemaNumber) => { + int8: getIntValidaton(-128, 127), + int16: getIntValidaton(-32_768, 32_767), + int32: getIntValidaton(-2_147_483_648, 2_147_483_647), + uint8: getIntValidaton(0, 255), + uint16: getIntValidaton(0, 65_535), + uint32: getIntValidaton(0, MAX_ID), + number: (v, t: SchemaNumber) => { if (t.step) { - const div = value / t.step - if (Math.abs(div - Math.round(div)) > EPSILON) { + const div = v / t.step + if (Math.abs(div - Math.round(div)) > epsilon) { return false } } - if (typeof value !== 'number') { + if (typeof v !== 'number') { return false } - if (t.min !== undefined && value < t.min) { + if (t.min !== undefined && v < t.min) { return false } - if (t.max !== undefined && value > t.max) { + if (t.max !== undefined && v > t.max) { return false } return true }, - [ENUM]: (value, t: SchemaEnum) => { - if (value === null) { + enum: (v, t: SchemaEnum) => { + if (v === null) { return true } const arr = t.enum for (let i = 0; i < arr.length; i++) { - if (value === arr[i]) { + if (v === arr[i]) { return true } } return false }, - [ID]: (value) => { - if (typeof value !== 'number' || value % 1 !== 0) { - return false - } - return true - }, - [JSON]: (value) => { - // mep - return true - }, - [MICRO_BUFFER]: (value) => { - if (!(value instanceof Uint8Array)) { - return false - } - return true - }, - [REFERENCE]: (v) => { - if (typeof v !== 'number') { - return false - } - if (v === 0 || v > MAX_ID) { - return false - } - return true - }, - [REFERENCES]: (v) => { - if (typeof v !== 'number') { - return false - } - if (v === 0 || v > MAX_ID) { - return false - } - return true - }, - [STRING]: (v, t: SchemaString) => { - if (v instanceof Uint8Array) { - return !!v[1] - } - if (typeof v !== 'string') { - return false - } - if (t.max !== undefined && v.length > t.max) { - return false - } - if (t.min !== undefined && v.length < t.min) { - return false - } - if (t.format !== undefined && t.format in validators) { - return validators[t.format](v) - } - return true - }, - [ALIASES]: (value) => { - if (!Array.isArray(value)) { - return false - } - const len = value.length - for (let i = 0; i < len; i++) { - if (typeof value[i] !== 'string') { - return false - } - } - return true - }, - [VECTOR]: (value) => { - // Array should be supported - if (!(value instanceof Float32Array)) { - return false - } - return true - }, - [TEXT]: null, -} - -VALIDATION_MAP[TEXT] = VALIDATION_MAP[STRING] - -export const defaultValidation = () => true - -export const isValidId = (id: number) => { - if (typeof id != 'number' || id < MIN_ID || id > MAX_ID) { - return false - } - return true -} - -export const isValidString = (v: any) => { - const isVal = - typeof v === 'string' || - (v as any) instanceof Uint8Array || - ArrayBuffer.isView(v) - return isVal + json: allGood, + reference: isValidId, + references: isValidId, + string: isStringLike, + text: isStringLike, + vector: (v) => v instanceof Float32Array, } type ValidationErrors = { path: string[]; value: unknown; error: string }[] const validateObj = ( value: unknown, - props: SchemaProps | SchemaObject, + props: Record> | SchemaObject, errors: ValidationErrors, path: string[], required: boolean, @@ -501,7 +342,7 @@ const validateObj = ( } } else { const msg = - getPropType(prop) === 'reference' && typeof val === 'object' + prop.type === 'reference' && typeof val === 'object' ? test(val?.id, prop) : test(val, prop) if (msg !== true) { @@ -517,8 +358,7 @@ const validateObj = ( } export const getValidator = (prop: SchemaProp): Validation => { - const validator = - VALIDATION_MAP[TYPE_INDEX_MAP[getPropType(prop)]] ?? defaultValidation + const validator = validationMap[prop.type] const custom = prop.validation if (custom) { return (a, b) => { @@ -529,7 +369,7 @@ export const getValidator = (prop: SchemaProp): Validation => { return validator } -export function validate( +export function validate( schema: S, type: keyof S['types'], payload: unknown, diff --git a/packages/schema/src/index.ts b/packages/schema/src/index.ts index bf6471c3ae..7727467106 100644 --- a/packages/schema/src/index.ts +++ b/packages/schema/src/index.ts @@ -1,8 +1,82 @@ -export * from './types.js' -export * from './dbSchema.js' -export * from './parse/index.js' -export * from './lang.js' +import type { LeafDef, ObjPropDef, PropDef, TypeDef } from './def/index.js' +import type { SchemaHooks } from './schema/hooks.js' +import type { numberTypes } from './schema/number.js' +import type { SchemaProp } from './schema/prop.js' +import { + parseSchema, + type Schema, + type SchemaIn, + type SchemaMigrateFns, + type SchemaOut, +} from './schema/schema.js' +import type { SchemaProps, SchemaType } from './schema/type.js' + +export * from './def/enums.js' +export * from './schema/lang.js' +export * from './def/index.js' export * from './def/validation.js' +export * as semver from './semver/mod.js' +export { type SchemaVector } from './schema/vector.js' +export { type SchemaTimestamp } from './schema/timestamp.js' +export { type SchemaCardinality } from './schema/cardinality.js' +export { type SchemaBinary } from './schema/binary.js' +export { type SchemaBoolean } from './schema/boolean.js' +export { type SchemaString } from './schema/string.js' +export { type SchemaNumber, numberTypes } from './schema/number.js' +export { type SchemaAlias } from './schema/alias.js' +export { type SchemaEnum } from './schema/enum.js' export * from './serialize.js' -export * from './infer.js' -export * as semver from './parse/semver/mod.js' + +export type { + Schema, + SchemaIn, + SchemaOut, + SchemaMigrateFns, + SchemaProps, + SchemaProp, + SchemaType, + SchemaHooks, +} + +export const parse = (input: SchemaIn): { schema: SchemaOut } => { + const schema = parseSchema(input) + return { schema } +} + +export const getPropChain = ( + typeDef: TypeDef, + path: string[], +): (PropDef | void)[] => { + let next: TypeDef | PropDef | void = typeDef + return path.map((key) => { + if (next) { + if (key[0] === '$') { + next = 'items' in next ? next.items[key] : next[key] + } else if ('props' in next) { + next = next.props[key] + } else if ('target' in next) { + next = next.target.typeDef.props[key] + } else { + next = undefined + } + } + return next as any + }) +} + +export function* getAllProps( + def: ObjPropDef | TypeDef, +): IterableIterator { + for (const key in def.props) { + const propDef = def.props[key] + + if ('props' in propDef) { + yield* getAllProps(propDef) + } else if (!('target' in propDef)) { + yield propDef + } + } +} + +export const getProp = (typeDef: TypeDef, path: string[]): PropDef | void => + getPropChain(typeDef, path).at(-1) diff --git a/packages/schema/src/infer.ts b/packages/schema/src/infer.ts index 23ede3e6a6..c49f1b2aa1 100644 --- a/packages/schema/src/infer.ts +++ b/packages/schema/src/infer.ts @@ -1,6 +1,3 @@ -import { VectorBaseType } from './def/typeIndexes.js' -import { Schema, SchemaVector } from './types.js' - // type BasedTypeMap = { // uint8: Uint8Array // float32: Float32Array @@ -8,6 +5,9 @@ import { Schema, SchemaVector } from './types.js' // float64: Float64Array // } +import type { SchemaObject } from './schema/object.js' +import type { Schema } from './schema/schema.js' + type TypedArray = | Uint8Array | Float32Array @@ -54,14 +54,12 @@ type InferReferences< ? Array<{ id: number } & InferSchemaType> : Array<{ id: number }> -type InferObject, Types> = { - [K in keyof T]: InferProp -} - type InferSet = InferProp[] -type InferProp = T extends { props: infer P } - ? InferObject +type InferProp = T extends SchemaObject + ? { + [K in keyof T['props']]: InferProp + } : T extends { items: { ref: infer R extends string } } ? InferReferences : T extends { items: infer I } diff --git a/packages/schema/src/parse/assert.ts b/packages/schema/src/parse/assert.ts deleted file mode 100644 index c97de1e5ef..0000000000 --- a/packages/schema/src/parse/assert.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { - EXPECTED_ARR, - EXPECTED_BOOL, - EXPECTED_FN, - EXPECTED_NUM, - EXPECTED_OBJ, - EXPECTED_STR, -} from './errors.js' -import { parse } from './semver/parse.js' -import { SemVer } from './semver/types.js' - -export const expectVersion = (obj: any): SemVer => { - try { - return parse(obj) - } catch (e) { - throw Error('invalid semver version') - } -} - -export const expectObject = (obj: any, msg?: string) => { - if (typeof obj !== 'object' || obj === null) { - throw Error(msg || EXPECTED_OBJ) - } -} - -export const expectArray = (obj: any) => { - if (!Array.isArray(obj)) { - throw Error(EXPECTED_ARR) - } -} - -export const expectFloat32Array = (arr: any) => { - if (!(arr instanceof Float32Array)) { - throw Error(EXPECTED_ARR) - } -} - -export const expectString = (obj: any) => { - if (typeof obj !== 'string') { - throw Error(EXPECTED_STR) - } -} - -export const expectBoolean = (v: any) => { - if (v !== true && v !== false) { - throw Error(EXPECTED_BOOL) - } -} - -export const expectFunction = (v: any) => { - if (typeof v !== 'function') { - throw Error(EXPECTED_FN) - } -} - -export const expectNumber = (v: any) => { - if (typeof v !== 'number') { - throw Error(EXPECTED_NUM) - } -} - -export const expectPositiveNumber = (v: any) => { - expectNumber(v) - if (v < 0) { - throw Error('Expected positive number') - } -} - -export const expectTimezoneName = (v: any) => { - expectString(v) - Intl.DateTimeFormat(undefined, { timeZone: v }) -} diff --git a/packages/schema/src/parse/errors.ts b/packages/schema/src/parse/errors.ts deleted file mode 100644 index f33d303bfb..0000000000 --- a/packages/schema/src/parse/errors.ts +++ /dev/null @@ -1,20 +0,0 @@ -export const OUT_OF_RANGE = 'Value is out of range' -export const EXPECTED_VALUE_IN_ENUM = 'Expected value in "enum" field' -export const EXPECTED_DATE = 'Expected number or Date' -export const EXPECTED_BOOL = 'Expected boolean' -export const EXPECTED_OBJ = 'Expected object' -export const EXPECTED_ARR = 'Expected array' -export const EXPECTED_NUM = 'Expected number' -export const EXPECTED_STR = 'Expected string' -export const EXPECTED_FN = 'Expected function' -export const EXPECTED_PRIMITIVE = 'Expected primitive value' -export const INVALID_VALUE = 'Invalid value' -export const INVALID_KEY = 'Invalid key' -export const INVALID_SCHEMA = 'Invalid schema' -export const MIN_MAX = 'Max value must be larger than min value' -export const UNKNOWN_PROP = 'Unknown property' -export const MISSING_TYPE = 'Missing type' -export const INVALID_TYPE = 'Invalid type' -export const TEXT_REQUIRES_LOCALES = 'Type text requires locales' -export const TYPE_MISMATCH = 'Types do not match' -export const NOT_ALLOWED_IN_ITEMS = 'Property not allowed in items definition' diff --git a/packages/schema/src/parse/index.ts b/packages/schema/src/parse/index.ts deleted file mode 100644 index 50ce98ef7a..0000000000 --- a/packages/schema/src/parse/index.ts +++ /dev/null @@ -1,206 +0,0 @@ -import { Schema, SchemaProps, SchemaType, StrictSchema } from '../types.js' -import { INVALID_KEY, INVALID_VALUE, UNKNOWN_PROP } from './errors.js' -import { getPropType } from './utils.js' -import propParsers from './props.js' -import pc from 'picocolors' -import { - expectArray, - expectBoolean, - expectFunction, - expectObject, - expectTimezoneName, - expectVersion, -} from './assert.js' -import { deepCopy } from '@based/utils' -import { parse as parseVersion } from './semver/parse.js' -import { parseRange } from './semver/parse_range.js' -import { satisfies } from './semver/satisfies.js' -import { rangeIntersects } from './semver/range_intersects.js' -import type { Range } from './semver/types.js' -export { getPropType } - -export class SchemaParser { - constructor(schema: Schema) { - // uint8Array is not working - this.schema = deepCopy(schema) - } - - isItems: boolean - inQuery: boolean - schema: Schema - type: SchemaType - path: string[] = [] - lvl = 0 - - parseTypes() { - this.path[this.lvl++] = 'types' - const { types } = this.schema - expectObject(types) - for (const type in types) { - this.lvl++ - if (type === '_root') { - throw new Error(INVALID_KEY) - } - this.path[this.lvl] = type - expectObject(types[type]) - if (!('props' in types[type])) { - types[type] = { props: types[type] } as SchemaProps - } - this.lvl-- - } - for (const type in types) { - this.path[this.lvl++] = type - this.parseProps(types[type].props, types[type]) - this.lvl-- - } - this.lvl-- - } - - parseProps(props: any, schemaType: SchemaType = null) { - this.path[this.lvl++] = 'props' - expectObject(props) - this.type = schemaType - for (const key in props) { - if (key[0] === '_') { - throw new Error(INVALID_KEY) - } - const prop = props[key] - const type = getPropType(prop, props, key) - this.path[this.lvl++] = key - if (type in propParsers) { - propParsers[type](prop, this) - } else { - throw Error(INVALID_VALUE) - } - this.lvl-- - } - this.lvl-- - } - - parseLocales() { - const { locales } = this.schema - expectObject(locales) - for (const locale in locales) { - const opts = locales[locale] - if (opts === true) { - continue - } - expectObject(opts) - for (const key in opts) { - const val = opts[key] - if (key === 'required') { - expectBoolean(val) - } else if (key === 'fallback') { - if (Array.isArray(val) || typeof val !== 'string') { - throw Error(INVALID_VALUE) - } - } else { - throw Error(UNKNOWN_PROP) - } - } - } - } - - parseDefaultTimezone() { - expectTimezoneName(this.schema.defaultTimezone) - } - - parseMigrations() { - const { migrations, version } = this.schema - const ranges = new Map() - - expectArray(migrations) - for (const item of migrations) { - expectObject(item) - expectObject(item.migrate) - const targetRange = parseRange(item.version) - const currentVersion = parseVersion(version) - if (satisfies(currentVersion, targetRange)) { - throw Error('migration version can not match current version') - } - if (Object.keys(item).length > 2) { - throw new Error( - 'migrations can only have "version" and "migrate" properties', - ) - } - for (const type in item.migrate) { - expectFunction(item.migrate[type]) - if (ranges.has(type)) { - const otherRanges = ranges.get(type) - for (const otherRange of otherRanges) { - if (rangeIntersects(targetRange, otherRange)) { - throw Error( - 'invalid overlapping version for migration for ' + - type + - ` ${JSON.stringify(otherRange)} ${JSON.stringify(targetRange)} ${otherRange === targetRange}`, - ) - } - } - ranges.get(type).push(targetRange) - } else { - ranges.set(type, [targetRange]) - } - } - } - } - - parse(): StrictSchema { - expectObject(this.schema) - // always do types first because it removes props shorthand - if ('types' in this.schema) { - this.parseTypes() - } - for (const key in this.schema) { - if (key === 'version') { - expectVersion(this.schema.version) - } else if (key === 'props') { - this.parseProps(this.schema.props) - } else if (key === 'locales') { - this.parseLocales() - } else if (key === 'defaultTimezone') { - this.parseDefaultTimezone() - } else if (key === 'migrations') { - this.parseMigrations() - } else if (key !== 'types') { - throw Error(UNKNOWN_PROP) - } - } - return this.schema as StrictSchema - } -} - -export const print = (schema: Schema, path: string[]) => { - let obj = schema - const depth = path.length - 1 - const lines: string[] = path.map((key, lvl) => { - if (typeof obj !== 'object') { - return '' - } - const v = obj[key] - const padding = ' '.repeat(lvl) - const prefix = key === Object.keys(obj)[0] ? '' : `${padding}...\n` - if (lvl === depth && lvl !== 0) { - const err = - key in obj - ? `${key}: ${typeof v === 'object' && v !== null && !Array.isArray(v) ? `{..}` : JSON.stringify(v)}` - : key - return `${prefix}${'--'.repeat(lvl - 1)}> ${pc.red(err)}` - } - obj = v - return `${prefix}${padding}${key}: {` - }) - return lines.join('\n') -} - -export const parse = (schema: Schema): { schema: StrictSchema } => { - const parser = new SchemaParser(schema) - - try { - return { schema: parser.parse() } - } catch (e) { - const cause = parser.path.slice(0, Math.min(4, parser.lvl) + 1) - e.message += '\n\n' + print(schema, cause) + '\n' - e.cause = cause - throw e - } -} diff --git a/packages/schema/src/parse/props.ts b/packages/schema/src/parse/props.ts deleted file mode 100644 index bf4fa40339..0000000000 --- a/packages/schema/src/parse/props.ts +++ /dev/null @@ -1,619 +0,0 @@ -import { convertToTimestamp } from '@based/utils' -import { NUMBER, PropDef, TYPE_INDEX_MAP, TypeIndex } from '../def/types.js' -import { getValidator, VALIDATION_MAP } from '../def/validation.js' -import { - SchemaAnyProp, - SchemaBoolean, - SchemaCardinality, - SchemaEnum, - SchemaNumber, - SchemaReferenceOneWay, - SchemaReference, - SchemaString, - SchemaTimestamp, - SchemaText, - SchemaObject, - SchemaObjectOneWay, - SchemaReferences, - SchemaAlias, - stringFormats, - SchemaVector, - SchemaJson, - SchemaColvec, -} from '../types.js' -import { - expectBoolean, - expectFunction, - expectNumber, - expectObject, - expectString, -} from './assert.js' -import { - EXPECTED_ARR, - EXPECTED_OBJ, - EXPECTED_PRIMITIVE, - INVALID_VALUE, - MIN_MAX, - MISSING_TYPE, - TEXT_REQUIRES_LOCALES, - TYPE_MISMATCH, - UNKNOWN_PROP, - NOT_ALLOWED_IN_ITEMS, -} from './errors.js' -import type { SchemaParser } from './index.js' -import { getPropType } from './utils.js' -import { DEFAULT_MAP } from '../def/defaultMap.js' -import { parseMinMaxStep } from '../def/utils.js' - -let stringFormatsSet: Set -type PropsFns = Record< - string, - (val: any, prop: PropType, ctx: SchemaParser, key?: string) => void -> -const STUB = {} -const shared: PropsFns = { - type() {}, - role(val) { - expectString(val) - }, - required(val, _prop, ctx) { - if (ctx.isItems) { - throw new Error(NOT_ALLOWED_IN_ITEMS) - } - expectBoolean(val) - }, - query(val) { - expectFunction(val) - }, - path(val, prop, ctx) { - expectString(val) - const path = val.split('.') - let t: any = ctx.type - for (const key of path) { - if ('items' in t) { - t = t.items - } - if ('ref' in t) { - t = ctx.schema.types[t.ref] - } - t = t.props[key] - expectObject(t) - } - if (t.type !== getPropType(prop)) { - throw Error(TYPE_MISMATCH) - } - }, - title(val) { - expectString(val) - }, - description(val) { - expectString(val) - }, - examples(val) { - expectString(val) - }, - validation(val) { - expectFunction(val) - }, - hooks(val, prop, ctx) { - expectObject(val) - for (const key in val) { - expectFunction(val[key]) - } - }, -} - -function propParser( - required: PropsFns, - optional: PropsFns, - allowShorthand?: number, -) { - return (prop, ctx: SchemaParser) => { - if (typeof prop === 'string') { - // allow string - if (allowShorthand === 0) { - // @ts-ignore - required?.type?.(prop, { type: prop }, ctx) - return - } - throw Error(EXPECTED_OBJ) - } - - if (Array.isArray(prop)) { - // allow array - if (allowShorthand === 1) { - return - } - throw Error(EXPECTED_OBJ) - } - - for (const key in required) { - ctx.path[ctx.lvl] = key - const changed = required[key](prop[key], prop, ctx) - if (changed !== undefined) { - prop[key] = changed - } - } - - for (const key in prop) { - ctx.path[ctx.lvl] = key - const val = prop[key] - let changed: any - if (key in optional) { - changed = optional[key](val, prop, ctx) - } else if (key in shared) { - changed = shared[key](val, prop, ctx) - } else if (!(key in required)) { - if (key[0] === '$' && 'ref' in prop) { - optional.edge(val, prop, ctx, key) - } else { - throw Error(UNKNOWN_PROP) - } - } - if (changed !== undefined) { - prop[key] = changed - } - } - } -} - -const p: Record> = {} - -export const isDefault = (val, prop, ctx) => { - let typeIndex: TypeIndex - typeIndex = TYPE_INDEX_MAP[prop.type] - if ('enum' in prop) { - typeIndex = TYPE_INDEX_MAP['enum'] - } - if (prop.type === 'timestamp') { - val = convertToTimestamp(val) - } - - const validation = getValidator(prop) - const tmpProp = Object.assign({}, prop, { - step: parseMinMaxStep((prop.step ?? typeIndex === NUMBER) ? 0 : 1), - max: parseMinMaxStep(prop.max), - min: parseMinMaxStep(prop.min), - }) - - if (prop.type === 'text') { - if (typeof val === 'object') { - for (const key in val) { - if (!ctx.schema.locales || !(key in ctx.schema.locales)) { - throw new Error(`Incorrect default for type "text" lang "${key}"`) - } - - if (!validation(val[key], tmpProp)) { - throw new Error(`Incorrect default for type "text" lang "${key}"`) - } - } - } else { - if (!validation(val, tmpProp)) { - throw new Error(`Incorrect default for type "text"`) - } - val = {} - for (const key in ctx.schema.locales) { - val[key] = ctx.schema.locales[key] - } - } - - return val - } - - if (!validation(val, tmpProp)) { - throw new Error(`Incorrect default for type "${prop.type ?? 'enum'}"`) - } - if ('enum' in prop) { - if (val === undefined) { - return 0 - } - return prop.enum.findIndex((v) => v === val) + 1 - } - return val -} - -p.boolean = propParser( - STUB, - { - default(val, prop, ctx) { - return isDefault(val, prop, ctx) - }, - }, - 0, -) - -p.vector = propParser( - { - size(val) { - expectNumber(val) - }, - }, - { - default(val, prop, ctx) { - return isDefault(val, prop, ctx) - }, - baseType(val, prop, ctx) { - if ( - ![ - 'number', - 'int8', - 'uint8', - 'int16', - 'uint16', - 'int32', - 'uint32', - 'float32', - 'float64', - ].includes(val) - ) { - throw Error(INVALID_VALUE) - } - }, - }, - 0, -) - -p.colvec = propParser( - { - size(val) { - expectNumber(val) - }, - }, - { - default(val, prop, ctx) { - return isDefault(val, prop, ctx) - }, - baseType(val, prop, ctx) { - if ( - ![ - 'number', - 'int8', - , - 'uint8', - 'int16', - 'uint16', - 'int32', - 'uint32', - 'float32', - 'float64', - ].includes(val) - ) { - throw Error(INVALID_VALUE) - } - }, - }, - 0, -) - -p.enum = propParser( - { - enum(items) { - if (!Array.isArray(items)) { - throw Error(EXPECTED_ARR) - } - if (items.length > 255) { - throw Error('Max enum length (255) exceeded') - } - for (const item of items) { - if (typeof item === 'object') { - throw Error(EXPECTED_PRIMITIVE) - } - } - }, - }, - { - default(val, prop, ctx) { - return isDefault(val, prop, ctx) - }, - }, - 1, -) - -const numberOpts = { - min(val: any, prop, ctx, key) { - expectNumber(val) - }, - max(val: any, prop, ctx, key) { - expectNumber(val) - if (prop.min > val) { - throw Error(MIN_MAX) - } - }, - step(val: any, prop, ctx, key) { - if (typeof val !== 'number' && val !== 'any') { - throw Error(INVALID_VALUE) - } - }, - default(val: any, prop: any, ctx: any) { - return isDefault(val, prop, ctx) - }, -} - -p.number = propParser(STUB, numberOpts, 0) -p.int8 = propParser(STUB, numberOpts, 0) -p.uint8 = propParser(STUB, numberOpts, 0) -p.int16 = propParser(STUB, numberOpts, 0) -p.uint16 = propParser(STUB, numberOpts, 0) -p.int32 = propParser(STUB, numberOpts, 0) -p.uint32 = propParser(STUB, numberOpts, 0) - -p.object = propParser( - { - props(val, _prop, ctx) { - ctx.parseProps(val, ctx.type) - }, - }, - {}, -) - -p.references = propParser( - { - items(items, prop, ctx) { - expectObject(items) - const itemsType = getPropType(items) - if (itemsType === 'reference') { - ctx.inQuery = 'query' in prop - ctx.isItems = true - p[itemsType](items, ctx) - ctx.inQuery = false - ctx.isItems = false - } else { - throw new Error(INVALID_VALUE) - } - }, - capped(val) { - if (val !== undefined && (typeof val !== 'number' || val < 0 || !Number.isInteger(val))) { - throw Error(INVALID_VALUE) - } - }, - }, - { - default(val, prop, ctx) { - return isDefault(val, prop, ctx) - }, - }, -) - -const binaryOpts = { - default(val, prop, ctx) { - return isDefault(val, prop, ctx) - }, - format(val) { - expectString(val) - stringFormatsSet ??= new Set(stringFormats) - stringFormatsSet.has(val) - }, - mime(val) { - if (Array.isArray(val)) { - val.forEach(expectString) - } else { - expectString(val) - } - }, - maxBytes(val) { - expectNumber(val) - }, - compression(val) { - // return the actualy string! - return val - }, -} - -p.binary = propParser(STUB, binaryOpts, 0) - -p.string = propParser( - STUB, - { - ...binaryOpts, - min(val) { - expectNumber(val) - }, - max(val) { - expectNumber(val) - }, - }, - 0, -) - -p.text = propParser( - { - type(_val, _prop, { schema }) { - if (schema.locales) { - for (const _ in schema.locales) { - return - } - } - throw Error(TEXT_REQUIRES_LOCALES) - }, - }, - { - compression(val) { - return val - }, - format: binaryOpts.format, - default(val, prop, ctx) { - return isDefault(val, prop, ctx) - }, - }, - 0, -) - -p.timestamp = propParser( - STUB, - { - min(val) { - if (typeof val !== 'string' && typeof val !== 'number') { - throw Error(INVALID_VALUE) - } - }, - max(val) { - if (typeof val !== 'string' && typeof val !== 'number') { - throw Error(INVALID_VALUE) - } - }, - step(val) { - if (typeof val !== 'string' && typeof val !== 'number') { - throw Error(INVALID_VALUE) - } - if (typeof val === 'string' && val.includes('now')) { - throw Error(INVALID_VALUE) - } - }, - default(val, prop, ctx) { - return isDefault(val, prop, ctx) - }, - on(val) { - if (val !== 'create' && val !== 'update') { - throw Error(INVALID_VALUE) - } - }, - }, - 0, -) - -p.reference = propParser( - { - ref(ref, _prop, { schema }) { - if (!schema.types[ref]?.props) { - throw Error(MISSING_TYPE) - } - }, - prop(propKey, prop, { schema, type, inQuery, path, lvl }) { - const propAllowed = type && !inQuery - - if (propAllowed) { - expectString(propKey) - - const propPath = propKey.split('.') - let targetProp: any = schema.types[prop.ref] - - expectObject(targetProp, 'expected type') - - let create - for (const key of propPath) { - if (!targetProp.props[key]) { - create = true - targetProp.props[key] = {} - } - targetProp = targetProp.props[key] - } - - if (create) { - const ref = path[1] - let prop = '' - for (let i = 3; i < lvl; i += 2) { - prop += prop ? `.${path[i]}` : path[i] - } - targetProp.items = { - ref, - prop, - } - } - - if ('items' in targetProp) { - targetProp = targetProp.items - } - - if ('ref' in targetProp && 'prop' in targetProp) { - const inversePath = targetProp.prop.split('.') - let inverseProp: any = schema.types[targetProp.ref] - for (const key of inversePath) { - inverseProp = inverseProp.props[key] - } - if (inverseProp && 'items' in inverseProp) { - inverseProp = inverseProp.items - } - - if (prop === inverseProp) { - return - } - } - - throw Error('expected inverse property') - } - - if (propKey !== undefined) { - throw Error('ref prop not supported on root or edge p') - } - }, - }, - { - mime: binaryOpts.mime, - default(val, prop, ctx) { - return isDefault(val, prop, ctx) - }, - edge(val, prop, ctx, key) { - const edgeAllowed = ctx.type && !ctx.inQuery - if (edgeAllowed) { - let t: any = ctx.schema.types[prop.ref].props[prop.prop] - t = t.items || t - - // if (t[key] && t !== prop) { - // throw Error('Edge can not be defined on both props') - // } - // @ts-ignore - const edgePropType = getPropType(val, prop, key) - const inType = ctx.type - ctx.type = null - p[edgePropType](val, ctx) - t[key] = prop[key] - ctx.type = inType - return - } - - throw Error('ref edge not supported on root or edge property') - }, - dependent(val, prop, ctx, key) { - expectBoolean(val) - const dependentAllowed = ctx.type && !ctx.inQuery - if (!dependentAllowed) { - throw Error('ref dependency not supported on root or edge property') - } - }, - }, -) - -p.alias = propParser( - STUB, - { - default(val, prop, ctx) { - return isDefault(val, prop, ctx) - }, - format: binaryOpts.format, - }, - 0, -) - -p.cardinality = propParser( - STUB, - { - default(val, prop, ctx) { - return isDefault(val, prop, ctx) - }, - mode(val, prop, ctx) { - if (!['dense', 'sparse'].includes(val)) { - throw Error(INVALID_VALUE) - } - p.mode = val - }, - precision(val, prop, ctx) { - if (val < 2 || val > 16) { - throw Error(INVALID_VALUE) - } - p.precision = val - }, - }, - 0, -) - -p.json = propParser( - STUB, - { - default(val, prop, ctx) { - return isDefault(val, prop, ctx) - }, - }, - 0, -) - -export default p diff --git a/packages/schema/src/parse/utils.ts b/packages/schema/src/parse/utils.ts deleted file mode 100644 index d7148cab9b..0000000000 --- a/packages/schema/src/parse/utils.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { SchemaAnyProp, SchemaProps, SchemaPropTypes } from '../types.js' -import { INVALID_TYPE, MISSING_TYPE } from './errors.js' - -export const getPropType = ( - prop: SchemaAnyProp, - props?: SchemaProps, - key?: string, -): SchemaPropTypes => { - if (typeof prop === 'string') { - if (props) { - props[key] = { type: prop } - } - return prop - } - - if ('type' in prop) { - if (typeof prop.type !== 'string') { - throw Error(INVALID_TYPE) - } - return prop.type - } - - if ('ref' in prop) { - return 'reference' - } - - if ('items' in prop && getPropType(prop.items) === 'reference') { - Object.keys(prop.items) - .filter((v) => v[0] === '$') - .forEach((v) => { - if (typeof prop.items[v] === 'string') { - prop.items[v] = { type: prop.items[v] } - } - }) - - return 'references' - } - - if ('props' in prop) { - return 'object' - } - - if ('enum' in prop) { - return 'enum' - } - - if (Array.isArray(prop)) { - if (props) { - props[key] = { enum: prop } - } - return 'enum' - } - - throw Error(MISSING_TYPE) -} diff --git a/packages/schema/src/schema/alias.ts b/packages/schema/src/schema/alias.ts new file mode 100644 index 0000000000..d3dc1f9ae6 --- /dev/null +++ b/packages/schema/src/schema/alias.ts @@ -0,0 +1,11 @@ +import { parseString, type SchemaString } from './string.js' + +export type SchemaAlias = Omit & { + type: 'alias' +} + +export const parseAlias = (def: Record): SchemaAlias => { + def.type = 'string' + const { type, ...rest } = parseString(def) + return { type: 'alias', ...rest } +} diff --git a/packages/schema/src/schema/base.ts b/packages/schema/src/schema/base.ts new file mode 100644 index 0000000000..dfe31f2f7c --- /dev/null +++ b/packages/schema/src/schema/base.ts @@ -0,0 +1,71 @@ +import { + assert, + deleteUndefined, + isBoolean, + isFunction, + isRecord, + isString, +} from './shared.js' +import type { SchemaProp } from './prop.js' +import { isHooks, type SchemaPropHooks } from './hooks.js' +import { getValidator } from '../def/validation.js' + +type Validation = (payload: any, schema: SchemaProp) => boolean | string + +export type Base = { + required?: boolean + title?: string + description?: string + validation?: Validation + hooks?: SchemaPropHooks +} + +const isValidation = (v: unknown): v is Validation => isFunction(v) + +export const parseBase = >( + def: Record, + result: T, +): T => { + assert( + def.required === undefined || isBoolean(def.required), + 'Required should be boolean', + ) + assert( + def.title === undefined || isString(def.title), + 'Title should be string', + ) + assert( + def.description === undefined || isString(def.description), + 'Description should be string', + ) + assert( + def.validation === undefined || isValidation(def.validation), + 'Invalid validation', + ) + assert(def.hooks === undefined || isHooks(def.hooks), 'Invalid hooks') + + result.required = def.required + result.title = def.title + result.description = def.description + result.validation = def.validation + result.hooks = def.hooks + + const unexpectedKey = Object.keys(def).find((key) => !(key in result)) + assert(unexpectedKey === undefined, `Unexpected property: ${unexpectedKey}`) + + if ('default' in result && result.default !== undefined) { + const validation = getValidator(result) + const msg = `Default should be valid ${('format' in result && result.format) || result.type}` + if (isRecord(def.default)) { + for (const k in def.default) { + assert(validation(def.default[k], result), msg) + } + } else { + assert(validation(def.default, result), msg) + } + + result.default = def.default + } + + return deleteUndefined(result) +} diff --git a/packages/schema/src/schema/binary.ts b/packages/schema/src/schema/binary.ts new file mode 100644 index 0000000000..7d5f378f92 --- /dev/null +++ b/packages/schema/src/schema/binary.ts @@ -0,0 +1,32 @@ +import { isFormat, isMime, type Mime, type StringFormat } from './string.js' +import { assert, isNatural, isRecord } from './shared.js' +import { parseBase, type Base } from './base.js' + +export type SchemaBinary = Base & { + type: 'binary' + default?: Uint8Array + maxBytes?: number + mime?: Mime + format?: StringFormat +} + +export const parseBinary = (def: Record): SchemaBinary => { + assert( + def.default === undefined || def.default instanceof Uint8Array, + 'Default should be Uint8Array', + ) + assert( + def.maxBytes === undefined || isNatural(def.maxBytes), + 'Max Bytes should be a natural number', + ) + assert(def.mime === undefined || isMime(def.mime), 'Invalid mime') + assert(def.format === undefined || isFormat(def.format), 'Invalid format') + + return parseBase(def, { + type: 'binary', + default: def.default, + maxBytes: def.maxBytes, + mime: def.mime, + format: def.format, + }) +} diff --git a/packages/schema/src/schema/boolean.ts b/packages/schema/src/schema/boolean.ts new file mode 100644 index 0000000000..1ec93e1714 --- /dev/null +++ b/packages/schema/src/schema/boolean.ts @@ -0,0 +1,19 @@ +import { assert, isBoolean, isRecord } from './shared.js' +import { parseBase, type Base } from './base.js' + +export type SchemaBoolean = Base & { + type: 'boolean' + default?: boolean +} + +export const parseBoolean = (def: Record): SchemaBoolean => { + assert( + def.default === undefined || isBoolean(def.default), + 'Default should be boolean', + ) + + return parseBase(def, { + type: 'boolean', + default: def.default, + }) +} diff --git a/packages/schema/src/schema/cardinality.ts b/packages/schema/src/schema/cardinality.ts new file mode 100644 index 0000000000..6de8c60fbb --- /dev/null +++ b/packages/schema/src/schema/cardinality.ts @@ -0,0 +1,33 @@ +import { assert, isNatural, isRecord } from './shared.js' +import { parseBase, type Base } from './base.js' + +export type SchemaCardinality = Base & { + type: 'cardinality' + maxBytes?: number + precision?: number + mode?: 'sparse' | 'dense' +} + +export const parseCardinality = ( + def: Record, +): SchemaCardinality => { + assert( + def.maxBytes === undefined || isNatural(def.maxBytes), + 'Max Bytes should be natural number', + ) + assert( + def.precision === undefined || isNatural(def.precision), + 'Precision should be natural number', + ) + assert( + def.mode === undefined || def.mode === 'sparse' || def.mode === 'dense', + "Mode should be 'sparse' or 'dense'", + ) + + return parseBase(def, { + type: 'cardinality', + maxBytes: def.maxBytes, + precision: def.precision, + mode: def.mode, + }) +} diff --git a/packages/schema/src/schema/enum.ts b/packages/schema/src/schema/enum.ts new file mode 100644 index 0000000000..760ed91c97 --- /dev/null +++ b/packages/schema/src/schema/enum.ts @@ -0,0 +1,32 @@ +import { + assert, + isBoolean, + isNumber, + isString, + type RequiredIfStrict, +} from './shared.js' +import { parseBase, type Base } from './base.js' + +export type EnumItem = string | number +export type SchemaEnum = Base & + RequiredIfStrict<{ type: 'enum' }, strict> & { + default?: EnumItem + enum: EnumItem[] + } + +const isEnumItem = (v: unknown): v is EnumItem => + isString(v) || isNumber(v) || isBoolean(v) + +export const parseEnum = (def: Record): SchemaEnum => { + assert( + def.default === undefined || isEnumItem(def.default), + 'Default should be valid enum', + ) + assert(Array.isArray(def.enum) && def.enum.every(isEnumItem), 'Invalid enum') + + return parseBase>(def, { + type: 'enum', + default: def.default, + enum: def.enum, + }) +} diff --git a/packages/schema/src/schema/hooks.ts b/packages/schema/src/schema/hooks.ts new file mode 100644 index 0000000000..6c06a5d6c0 --- /dev/null +++ b/packages/schema/src/schema/hooks.ts @@ -0,0 +1,59 @@ +// import type { BasedDbQuery, Operator } from '@based/db' +import { isFunction, isRecord } from './shared.js' + +type BasedDbQuery = any +type Operator = any + +export type SchemaHooks = { + create?: (payload: Record) => void | Record + update?: (payload: Record) => void | Record + read?: (result: Record) => void | null | Record + search?: (query: BasedDbQuery, fields: Set) => void + include?: ( + query: BasedDbQuery, + fields: Map< + string, + { + field: string + opts?: any // temp this type + } + >, + ) => void + filter?: ( + query: BasedDbQuery, + field: string, + operator: Operator, + value: any, + ) => void + groupBy?: (query: BasedDbQuery, field: string) => void + aggregate?: (query: BasedDbQuery, fields: Set) => void +} + +export type SchemaPropHooks = { + create?: (value: any, payload: Record) => any + update?: (value: any, payload: Record) => any + read?: (value: any, result: Record) => any + aggregate?: (query: BasedDbQuery, fields: Set) => void + search?: (query: BasedDbQuery, fields: Set) => void + groupBy?: (query: BasedDbQuery, field: string) => void + filter?: ( + query: BasedDbQuery, + field: string, + operator: Operator, + value: any, + ) => void + include?: ( + query: BasedDbQuery, + fields: Map< + string, + { + field: string + opts?: any // temp this type + } + >, + ) => void +} + +export const isHooks = ( + v: unknown, +): v is Hooks => isRecord(v) && Object.values(v).every(isFunction) diff --git a/packages/schema/src/schema/json.ts b/packages/schema/src/schema/json.ts new file mode 100644 index 0000000000..d7efb6963f --- /dev/null +++ b/packages/schema/src/schema/json.ts @@ -0,0 +1,13 @@ +import { assert, isRecord } from './shared.js' +import { parseBase, type Base } from './base.js' + +export type SchemaJson = Base & { + type: 'json' + default?: any +} + +export const parseJson = (def: Record): SchemaJson => + parseBase(def, { + type: 'json', + default: def.default, + }) diff --git a/packages/schema/src/lang.ts b/packages/schema/src/schema/lang.ts similarity index 82% rename from packages/schema/src/lang.ts rename to packages/schema/src/schema/lang.ts index 14568ec0b1..50cd46c63c 100644 --- a/packages/schema/src/lang.ts +++ b/packages/schema/src/schema/lang.ts @@ -155,8 +155,3 @@ langCodesMap.forEach((v, k) => { }) export type LangName = keyof typeof langCodes export type LangCode = (typeof langCodes)[LangName] - -// generate this -// __aaabafaksqamarangyasaevaeyazeubebnbisbsbrbgmycakmcezhcvkwcohrxsdadvnldzenetfofifrffgdgldegswelklguhtahahehihuisiiganiaiuikgaitjaknkskkrwkokukylolalvlblilnltmkmgmsmlmtgvmironmnesenoenbnocoroospsfaplptqurmrusmsascrsdsisklslsostnresswssvtlttattethbotitotstntrktkugukuruzveviwacyfywoxhyiyozukacnr -// for the reader index = code * 2 [0] [1] -// add a file to the build step diff --git a/packages/schema/src/schema/number.ts b/packages/schema/src/schema/number.ts new file mode 100644 index 0000000000..86bf605886 --- /dev/null +++ b/packages/schema/src/schema/number.ts @@ -0,0 +1,44 @@ +import { assert, isNumber, isRecord, isString } from './shared.js' +import { parseBase, type Base } from './base.js' + +export const numberTypes = [ + 'number', + 'int8', + 'uint8', + 'int16', + 'uint16', + 'int32', + 'uint32', +] as const + +export type NumberType = (typeof numberTypes)[number] + +export type SchemaNumber = Base & { + type: NumberType + default?: number + min?: number + max?: number + step?: number +} + +const isNumberType = (v: unknown): v is NumberType => + isString(v) && numberTypes.includes(v as NumberType) + +export const parseNumber = (def: Record): SchemaNumber => { + assert(isNumberType(def.type), `Type should be one of ${numberTypes}`) + assert( + def.default === undefined || isNumber(def.default), + 'Default should be number', + ) + assert(def.min === undefined || isNumber(def.min), 'Min should be number') + assert(def.max === undefined || isNumber(def.max), 'Max should be number') + assert(def.step === undefined || isNumber(def.step), 'Step should be number') + + return parseBase(def, { + type: def.type, + default: def.default, + min: def.min, + max: def.max, + step: def.step, + }) +} diff --git a/packages/schema/src/schema/object.ts b/packages/schema/src/schema/object.ts new file mode 100644 index 0000000000..14fc6d06ff --- /dev/null +++ b/packages/schema/src/schema/object.ts @@ -0,0 +1,23 @@ +import { assert, isRecord, type RequiredIfStrict } from './shared.js' +import { parseBase, type Base } from './base.js' +import { parseProp, type SchemaProp } from './prop.js' + +export type SchemaObject = Base & { + props: Record> +} & RequiredIfStrict<{ type: 'object' }, strict> + +export const parseObject = ( + def: Record, +): SchemaObject => { + assert(isRecord(def.props), 'Props should be record') + + const props = {} + for (const prop in def.props) { + props[prop] = parseProp(def.props, prop) + } + + return parseBase>(def, { + type: 'object', + props, + }) +} diff --git a/packages/schema/src/schema/prop.ts b/packages/schema/src/schema/prop.ts new file mode 100644 index 0000000000..25f270126d --- /dev/null +++ b/packages/schema/src/schema/prop.ts @@ -0,0 +1,111 @@ +import { assert, isRecord, isString } from './shared.js' +import { parseAlias, type SchemaAlias } from './alias.js' +import { parseBinary, type SchemaBinary } from './binary.js' +import { parseBoolean, type SchemaBoolean } from './boolean.js' +import { parseCardinality, type SchemaCardinality } from './cardinality.js' +import { parseEnum, type EnumItem, type SchemaEnum } from './enum.js' +import { parseJson, type SchemaJson } from './json.js' +import { + numberTypes, + parseNumber, + type NumberType, + type SchemaNumber, +} from './number.js' +import { parseReferences, type SchemaReferences } from './references.js' +import { parseReference, type SchemaReference } from './reference.js' +import { parseString, type SchemaString } from './string.js' +import { parseText, type SchemaText } from './text.js' +import { parseTimestamp, type SchemaTimestamp } from './timestamp.js' +import { parseVector, type SchemaVector } from './vector.js' +import { parseObject, type SchemaObject } from './object.js' + +type SchemaPropObj = + | SchemaAlias + | SchemaBinary + | SchemaBoolean + | SchemaCardinality + | SchemaEnum + | SchemaJson + | SchemaNumber + | SchemaReferences + | SchemaReference + | SchemaString + | SchemaText + | SchemaTimestamp + | SchemaVector + | SchemaObject + +type SchemaPropShorthand = + | 'timestamp' + | 'binary' + | 'boolean' + | 'string' + | 'alias' + | 'text' + | 'json' + | 'cardinality' + | NumberType + | EnumItem[] + +export type SchemaProp = + | SchemaPropObj + | (strict extends true ? never : SchemaPropShorthand) + +export const parseProp = ( + parent: Record, + key: string, +): SchemaProp => { + let def = parent[key] + if (isString(def)) { + def = { type: def } + } else if (Array.isArray(def)) { + def = { enum: def } + } + + assert(isRecord(def), 'Invalid property') + + let type = def.type + + if (type === undefined) { + if ('enum' in def) { + type = 'enum' + } else if ('props' in def) { + type = 'object' + } else if ('ref' in def) { + type = 'reference' + } else if ('items' in def) { + type = 'references' + } + } + if (type === 'object') { + return parseObject(def) + } else if (type === 'alias') { + return parseAlias(def) + } else if (type === 'binary') { + return parseBinary(def) + } else if (type === 'boolean') { + return parseBoolean(def) + } else if (type === 'cardinality') { + return parseCardinality(def) + } else if (type === 'enum') { + return parseEnum(def) + } else if (type === 'json') { + return parseJson(def) + } else if (type === 'reference') { + return parseReference(def) + } else if (type === 'references') { + return parseReferences(def) + } else if (type === 'string') { + return parseString(def) + } else if (type === 'text') { + return parseText(def) + } else if (type === 'timestamp') { + return parseTimestamp(def) + } else if (type === 'vector' || type === 'colvec') { + return parseVector(def) + } else if (numberTypes.includes(type as any)) { + return parseNumber(def) + } + + throw 'Unknown property' +} diff --git a/packages/schema/src/schema/reference.ts b/packages/schema/src/schema/reference.ts new file mode 100644 index 0000000000..30710d9560 --- /dev/null +++ b/packages/schema/src/schema/reference.ts @@ -0,0 +1,55 @@ +import { parseBase, type Base } from './base.js' +import { assert, isBoolean, isString, type RequiredIfStrict } from './shared.js' +import { parseProp, type SchemaProp } from './prop.js' +import type { SchemaReferences } from './references.js' + +type EdgeExcludedProps = 'prop' | `$${string}` + +export type SchemaReference = Base & + RequiredIfStrict<{ type: 'reference' }, strict> & { + ref: string + } & { + prop: string + dependent?: boolean + [edge: `$${string}`]: + | Exclude, SchemaReferences> + | (Omit, 'items'> & { + items: Omit, EdgeExcludedProps> + }) + } + +let parsingEdges: boolean +export const parseReference = ( + def: Record, +): SchemaReference => { + assert(isString(def.ref), 'Ref should be string') + + if (parsingEdges) { + return parseBase>(def, { + type: 'reference', + ref: def.ref, + } as SchemaReference) + } + + assert(isString(def.prop), 'Prop should be string') + assert( + def.dependent === undefined || isBoolean(def.dependent), + 'Dependent should be boolean', + ) + + const result: SchemaReference = { + type: 'reference', + ref: def.ref, + prop: def.prop, + } + + parsingEdges = true + for (const key in def) { + if (key.startsWith('$')) { + result[key] = parseProp(def, key) + } + } + parsingEdges = false + + return parseBase>(def, result) +} diff --git a/packages/schema/src/schema/references.ts b/packages/schema/src/schema/references.ts new file mode 100644 index 0000000000..c1a78221b6 --- /dev/null +++ b/packages/schema/src/schema/references.ts @@ -0,0 +1,19 @@ +import { parseBase, type Base } from './base.js' +import { parseReference, type SchemaReference } from './reference.js' +import { assert, isRecord, type RequiredIfStrict } from './shared.js' + +export type SchemaReferences = Base & + RequiredIfStrict<{ type: 'references' }, strict> & { + capped?: number + items: SchemaReference + } + +export const parseReferences = ( + def: Record, +): SchemaReferences => { + assert(isRecord(def.items), 'Items should be record') + return parseBase(def, { + type: 'references', + items: parseReference(def.items), + }) +} diff --git a/packages/schema/src/schema/schema.ts b/packages/schema/src/schema/schema.ts new file mode 100644 index 0000000000..c835081d82 --- /dev/null +++ b/packages/schema/src/schema/schema.ts @@ -0,0 +1,198 @@ +import { + assert, + deleteUndefined, + isBoolean, + isFunction, + isRecord, + isString, + type RequiredIfStrict, +} from './shared.js' +import { parseType, type SchemaType } from './type.js' +import { langCodesMap, type LangName } from './lang.js' +import type { SchemaProp } from './prop.js' +import { hash } from '@based/hash' +import { inspect } from 'node:util' + +type SchemaTypes = Record> +export type SchemaLocale = { + required?: boolean + fallback?: LangName // not multiple - 1 is enough else it becomes too complex +} +type SchemaLocales = Partial> + +type MigrateFn = ( + node: Record, +) => Record | [string, Record] +export type SchemaMigrateFns = Record +export type SchemaMigrations = { + version: string + migrate: SchemaMigrateFns +}[] +export type Schema = { + version?: string + types: SchemaTypes + defaultTimezone?: string + migrations?: SchemaMigrations +} & RequiredIfStrict<{ locales: SchemaLocales; hash: number }, strict> + +export type SchemaIn = Schema +export type SchemaOut = Schema + +const isMigrations = (v: unknown): v is SchemaMigrations => + isRecord(v) && + Object.values(v).every( + (m) => + isRecord(m) && + isString(m.version) && + isRecord(m.migrate) && + Object.values(m.migrate).every(isFunction), + ) + +const isLocales = (v: unknown): v is SchemaLocales => + isRecord(v) && + Object.entries(v).every(([k, v]) => { + if (langCodesMap.has(k)) { + // TODO make more strict! + return isBoolean(v) || isRecord(v) + } + }) + +const parseRefs = ( + types: SchemaTypes, + type: keyof SchemaTypes, + prop: SchemaProp, + path: string[], +) => { + if (prop.type === 'reference') { + assert(prop.ref in types, `Ref type ${prop.ref} should be defined`) + let inverse: any = types[prop.ref] + for (const key of prop.prop.split('.')) { + let next = 'props' in inverse ? inverse.props?.[key] : inverse[key] + if (!next) { + inverse.props ??= {} + next = inverse.props[key] = {} + } + inverse = next + } + const dotPath = path.join('.') + if (!inverse.type) { + inverse.type = 'references' + inverse.items = { + type: 'reference', + ref: type, + prop: dotPath, + } + } + if (inverse.items) { + inverse = inverse.items + } + + for (const key in inverse) { + if (key[0] === '$') { + prop[key] = inverse[key] + } + } + + for (const key in prop) { + if (key[0] === '$') { + inverse[key] = prop[key] + } + } + + assert(inverse.ref === type, `Ref should be ${type}`) + assert(inverse.prop === dotPath, `Prop should be ${dotPath}`) + } else if ('items' in prop) { + parseRefs(types, type, prop.items, path) + } else if ('props' in prop) { + for (const k in prop.props) { + parseRefs(types, type, prop.props[k], [...path, k]) + } + } +} + +const getPath = ( + obj: Record, + def: Record, + path: string[], +): string[] | undefined => { + for (const k in obj) { + const v = obj[k] + if (v === def) { + return path + } + if (isRecord(v)) { + const res = getPath(v, def, [...path, k]) + if (res) return res + } + } +} + +let tracking +let path: string[] = [] +let value: any + +const _track =

>( + input: P, + depth: number, +): P => + new Proxy(input, { + get(obj, key: string) { + let val = obj[key] + value = val + path[depth] = key + if (path.length > depth + 1) path = path.slice(0, depth + 1) + return isRecord(val) ? _track(val, depth + 1) : val + }, + }) +const track =

>(input: P): P => { + tracking = input + return _track(input, 0) +} + +export const parseSchema = (input: SchemaIn): SchemaOut => { + const v: unknown = track(input) + assert(isRecord(v), 'Schema should be record') + try { + assert(isRecord(v.types), 'Types should be record') + assert( + v.version === undefined || isString(v.version), + 'Version should be string', + ) + assert(v.locales === undefined || isLocales(v.locales), 'Invalid locales') + assert( + v.migrations === undefined || isMigrations(v.migrations), + 'Invalid migrations', + ) + assert( + v.defaultTimezone === undefined || isString(v.defaultTimezone), + 'Invalid Default Timezone', + ) + + let types: SchemaTypes = {} + for (const key in v.types) { + const type = v.types[key] + assert(isRecord(type), 'Type should be object') + types[key] = parseType(type) + } + + const result = deleteUndefined({ + version: v.version, + locales: v.locales || {}, + defaultTimezone: v.defaultTimezone, + migrations: v.migrations, + types, + }) as SchemaOut + + const tracked = track(result) + for (const type in tracked.types) { + for (const k in tracked.types[type].props) { + parseRefs(tracked.types, type, tracked.types[type].props[k], [k]) + } + } + + result.hash = hash(result) + return result + } catch (e) { + throw Error(`${path.join('.')}: ${inspect(value)} - ${e}`, { cause: e }) + } +} diff --git a/packages/schema/src/schema/shared.ts b/packages/schema/src/schema/shared.ts new file mode 100644 index 0000000000..12957e2e1b --- /dev/null +++ b/packages/schema/src/schema/shared.ts @@ -0,0 +1,28 @@ +export const isRecord = (v: unknown): v is Record => + typeof v === 'object' && v !== null + +export const isEmpty = (v: object) => Object.keys(v).length === 0 +export const isString = (v: unknown): v is string => typeof v === 'string' +export const isBoolean = (v: unknown): v is boolean => typeof v === 'boolean' +export const isFunction = (v: unknown): v is Function => typeof v === 'function' +export const isNumber = (v: unknown): v is number => typeof v === 'number' +export const isInteger = (v: unknown): v is number => + isNumber(v) && Number.isSafeInteger(v) +export const isNatural = (v: unknown): v is number => isInteger(v) && v > 0 + +export function assert(condition: unknown, msg: string): asserts condition { + if (!condition) throw Error(msg) +} + +export type RequiredIfStrict = strict extends true ? T : Partial + +export const deleteUndefined =

>( + obj: P, +): P => { + for (const key in obj) { + if (obj[key] === undefined) { + delete obj[key] + } + } + return obj +} diff --git a/packages/schema/src/schema/string.ts b/packages/schema/src/schema/string.ts new file mode 100644 index 0000000000..648ebedf65 --- /dev/null +++ b/packages/schema/src/schema/string.ts @@ -0,0 +1,86 @@ +import { isString, isNatural, isRecord, assert } from './shared.js' +import { parseBase, type Base } from './base.js' +import { validators } from '../def/validation.js' + +export type StringFormat = keyof typeof validators + +export const stringFormats = Object.keys(validators) as StringFormat[] +export const stringCompressions = ['none', 'deflate'] as const + +export type StringCompression = (typeof stringCompressions)[number] +type KnownMimeTypes = + | 'text/html' + | 'text/plain' + | 'text/markdown' + | 'image/png' + | 'image/jpeg' + | 'video/mp4' + | 'video/quicktime' + | 'image/*' + | 'video/*' + | 'audio/*' + | '*/*' + +type GeneralMimeType = `${string}/${string}` +type MimeString = KnownMimeTypes | (GeneralMimeType & {}) + +export type Mime = MimeString | MimeString[] + +const isMimeString = (v: unknown): v is MimeString => + isString(v) && v.includes('/') + +export const isMime = (v: unknown): v is Mime => + Array.isArray(v) ? v.every(isMimeString) : isMimeString(v) + +export const isFormat = (v: unknown): v is StringFormat => + stringFormats.includes(v as any) + +export const isCompression = (v: unknown): v is StringCompression => + stringCompressions.includes(v as any) + +export type SchemaString = Base & { + type: 'string' + default?: string + maxBytes?: number + min?: number + max?: number + mime?: Mime + format?: StringFormat + compression?: StringCompression +} + +export const parseString = (def: Record): SchemaString => { + assert( + def.default === undefined || isString(def.default), + 'Default should be string', + ) + assert( + def.maxBytes === undefined || isNatural(def.maxBytes), + 'Max bytes should be natural number', + ) + assert( + def.min === undefined || isNatural(def.min), + 'Min should be natural number', + ) + assert( + def.max === undefined || isNatural(def.max), + 'Max should be natural number', + ) + assert(def.mime === undefined || isMime(def.mime), 'Invalid mime') + assert(def.format === undefined || isFormat(def.format), 'Invalid format') + assert( + def.compression === undefined || isCompression(def.compression), + 'Invalid compression', + ) + + return parseBase(def, { + type: 'string', + default: def.default, + maxBytes: def.maxBytes, + min: def.min, + max: def.max, + mime: def.mime, + format: def.format, + compression: def.compression, + }) +} diff --git a/packages/schema/src/schema/text.ts b/packages/schema/src/schema/text.ts new file mode 100644 index 0000000000..f93ebf40ce --- /dev/null +++ b/packages/schema/src/schema/text.ts @@ -0,0 +1,25 @@ +import { parseBase, type Base } from './base.js' +import { assert, isRecord, isString } from './shared.js' +import type { StringCompression, StringFormat } from './string.js' + +export type SchemaText = Base & { + type: 'text' + default?: Record + format?: StringFormat + compression?: StringCompression +} + +export const parseText = (def: Record): SchemaText => { + if (def.default) { + assert(isRecord(def.default), 'Default should be record of strings') + assert( + Object.values(def.default).every(isString), + 'Default should be record of strings', + ) + } + + return parseBase(def, { + type: 'text', + default: def.default as SchemaText['default'], + }) +} diff --git a/packages/schema/src/schema/timestamp.ts b/packages/schema/src/schema/timestamp.ts new file mode 100644 index 0000000000..f738101412 --- /dev/null +++ b/packages/schema/src/schema/timestamp.ts @@ -0,0 +1,48 @@ +import { assert, isNumber, isRecord, isString } from './shared.js' +import { parseBase, type Base } from './base.js' +import { convertToTimestamp } from '@based/utils' + +type Timestamp = number | Date | string + +export type SchemaTimestamp = Base & { + type: 'timestamp' + on?: 'create' | 'update' + min?: strict extends true ? number : Timestamp + max?: strict extends true ? number : Timestamp + default?: strict extends true ? number : Timestamp + step?: strict extends true ? number : number | string +} + +const isTimestamp = (v: unknown): v is Timestamp => + v instanceof Date || isNumber(v) || isString(v) + +const convertToTsIfDefined = (v: Timestamp | undefined): number | undefined => + v === undefined ? v : convertToTimestamp(v) + +export const parseTimestamp = ( + def: Record, +): SchemaTimestamp => { + assert( + def.on === undefined || def.on === 'create' || def.on === 'update', + "On should be one of 'create' or 'update", + ) + assert(def.min === undefined || isTimestamp(def.min), 'Invalid timestamp') + assert(def.max === undefined || isTimestamp(def.max), 'Invalid max timestamp') + assert( + def.step === undefined || isNumber(def.step) || isString(def.step), + 'Invalid step', + ) + assert( + def.default === undefined || isTimestamp(def.default), + 'Invalid default timestamp', + ) + + return parseBase>(def, { + type: 'timestamp', + on: def.on, + min: convertToTsIfDefined(def.min), + max: convertToTsIfDefined(def.max), + step: convertToTsIfDefined(def.step), + default: convertToTsIfDefined(def.default), + }) +} diff --git a/packages/schema/src/schema/type.ts b/packages/schema/src/schema/type.ts new file mode 100644 index 0000000000..f6156a601d --- /dev/null +++ b/packages/schema/src/schema/type.ts @@ -0,0 +1,92 @@ +import type { Schema } from './schema.js' +import { + assert, + deleteUndefined, + isBoolean, + isNatural, + isRecord, +} from './shared.js' +import { parseProp, type SchemaProp } from './prop.js' +import { isHooks } from './hooks.js' + +type BasedDbQuery = any +type Operator = any + +type SchemaHooks = { + create?: (payload: Record) => void | Record + update?: (payload: Record) => void | Record + read?: (result: Record) => void | null | Record + search?: (query: BasedDbQuery, fields: Set) => void + include?: ( + query: BasedDbQuery, + fields: Map< + string, + { + field: string + opts?: any // temp this type + } + >, + ) => void + filter?: ( + query: BasedDbQuery, + field: string, + operator: Operator, + value: any, + ) => void + groupBy?: (query: BasedDbQuery, field: string) => void + aggregate?: (query: BasedDbQuery, fields: Set) => void +} + +export type SchemaProps = Record> +type SchemaTypeObj = { + hooks?: SchemaHooks + blockCapacity?: number + insertOnly?: boolean + capped?: number + partial?: boolean + props: SchemaProps +} + +export type SchemaType = strict extends true + ? SchemaTypeObj + : SchemaTypeObj | ({ props?: never } & SchemaProps) + +const parseProps = (typeProps: Record) => { + const props: SchemaProps = {} + for (const key in typeProps) { + props[key] = parseProp(typeProps, key) + } + return props +} + +export const parseType = (type: Record): SchemaType => { + if (type.props === undefined) { + return { props: parseProps(type) } + } + assert( + type.hooks === undefined || isHooks(type.hooks), + 'Invalid hooks', + ) + assert( + type.blockCapacity === undefined || isNatural(type.blockCapacity), + 'Should be natural number', + ) + assert( + type.capped === undefined || isNatural(type.capped), + 'Should be natural number', + ) + assert( + type.partial === undefined || isBoolean(type.partial), + 'Should be boolean', + ) + assert(isRecord(type.props), 'Should be record') + const result = { + hooks: type.hooks, + blockCapacity: type.blockCapacity, + capped: type.capped, + partial: type.partial, + props: parseProps(type.props), + } + + return deleteUndefined(result) +} diff --git a/packages/schema/src/schema/vector.ts b/packages/schema/src/schema/vector.ts new file mode 100644 index 0000000000..6e86937d9a --- /dev/null +++ b/packages/schema/src/schema/vector.ts @@ -0,0 +1,36 @@ +import { assert, isNatural, isRecord, isString } from './shared.js' +import { parseBase, type Base } from './base.js' +import { numberTypes } from './number.js' + +const vectorBaseTypes = [...numberTypes, 'float32', 'float64'] as const + +export type SchemaVector = Base & { + type: 'vector' | 'colvec' + /** + * Number of elements in the vector. + */ + size: number + /** + * Base type of the vector. + * float64 == number + */ + baseType?: (typeof vectorBaseTypes)[number] +} + +export const parseVector = (def: Record): SchemaVector => { + assert( + def.type === 'vector' || def.type === 'colvec', + "Type should be one of 'vector' or 'colvec'", + ) + assert(isNatural(def.size), 'Size should be natural number') + assert( + isString(def.baseType) && vectorBaseTypes.includes(def.baseType as any), + 'Invalid baseType', + ) + + return parseBase(def, { + type: def.type, + size: def.size, + baseType: def.baseType as SchemaVector['baseType'], + }) +} diff --git a/packages/schema/src/parse/semver/_constants.ts b/packages/schema/src/semver/_constants.ts similarity index 100% rename from packages/schema/src/parse/semver/_constants.ts rename to packages/schema/src/semver/_constants.ts diff --git a/packages/schema/src/parse/semver/_shared.ts b/packages/schema/src/semver/_shared.ts similarity index 100% rename from packages/schema/src/parse/semver/_shared.ts rename to packages/schema/src/semver/_shared.ts diff --git a/packages/schema/src/parse/semver/_test_comparator_set.ts b/packages/schema/src/semver/_test_comparator_set.ts similarity index 100% rename from packages/schema/src/parse/semver/_test_comparator_set.ts rename to packages/schema/src/semver/_test_comparator_set.ts diff --git a/packages/schema/src/parse/semver/can_parse.ts b/packages/schema/src/semver/can_parse.ts similarity index 100% rename from packages/schema/src/parse/semver/can_parse.ts rename to packages/schema/src/semver/can_parse.ts diff --git a/packages/schema/src/parse/semver/compare.ts b/packages/schema/src/semver/compare.ts similarity index 100% rename from packages/schema/src/parse/semver/compare.ts rename to packages/schema/src/semver/compare.ts diff --git a/packages/schema/src/parse/semver/difference.ts b/packages/schema/src/semver/difference.ts similarity index 100% rename from packages/schema/src/parse/semver/difference.ts rename to packages/schema/src/semver/difference.ts diff --git a/packages/schema/src/parse/semver/equals.ts b/packages/schema/src/semver/equals.ts similarity index 100% rename from packages/schema/src/parse/semver/equals.ts rename to packages/schema/src/semver/equals.ts diff --git a/packages/schema/src/parse/semver/format.ts b/packages/schema/src/semver/format.ts similarity index 100% rename from packages/schema/src/parse/semver/format.ts rename to packages/schema/src/semver/format.ts diff --git a/packages/schema/src/parse/semver/format_range.ts b/packages/schema/src/semver/format_range.ts similarity index 100% rename from packages/schema/src/parse/semver/format_range.ts rename to packages/schema/src/semver/format_range.ts diff --git a/packages/schema/src/parse/semver/greater_or_equal.ts b/packages/schema/src/semver/greater_or_equal.ts similarity index 100% rename from packages/schema/src/parse/semver/greater_or_equal.ts rename to packages/schema/src/semver/greater_or_equal.ts diff --git a/packages/schema/src/parse/semver/greater_than.ts b/packages/schema/src/semver/greater_than.ts similarity index 100% rename from packages/schema/src/parse/semver/greater_than.ts rename to packages/schema/src/semver/greater_than.ts diff --git a/packages/schema/src/parse/semver/greater_than_range.ts b/packages/schema/src/semver/greater_than_range.ts similarity index 100% rename from packages/schema/src/parse/semver/greater_than_range.ts rename to packages/schema/src/semver/greater_than_range.ts diff --git a/packages/schema/src/parse/semver/increment.ts b/packages/schema/src/semver/increment.ts similarity index 100% rename from packages/schema/src/parse/semver/increment.ts rename to packages/schema/src/semver/increment.ts diff --git a/packages/schema/src/parse/semver/is_range.ts b/packages/schema/src/semver/is_range.ts similarity index 100% rename from packages/schema/src/parse/semver/is_range.ts rename to packages/schema/src/semver/is_range.ts diff --git a/packages/schema/src/parse/semver/is_semver.ts b/packages/schema/src/semver/is_semver.ts similarity index 100% rename from packages/schema/src/parse/semver/is_semver.ts rename to packages/schema/src/semver/is_semver.ts diff --git a/packages/schema/src/parse/semver/less_or_equal.ts b/packages/schema/src/semver/less_or_equal.ts similarity index 100% rename from packages/schema/src/parse/semver/less_or_equal.ts rename to packages/schema/src/semver/less_or_equal.ts diff --git a/packages/schema/src/parse/semver/less_than.ts b/packages/schema/src/semver/less_than.ts similarity index 100% rename from packages/schema/src/parse/semver/less_than.ts rename to packages/schema/src/semver/less_than.ts diff --git a/packages/schema/src/parse/semver/less_than_range.ts b/packages/schema/src/semver/less_than_range.ts similarity index 100% rename from packages/schema/src/parse/semver/less_than_range.ts rename to packages/schema/src/semver/less_than_range.ts diff --git a/packages/schema/src/parse/semver/max_satisfying.ts b/packages/schema/src/semver/max_satisfying.ts similarity index 100% rename from packages/schema/src/parse/semver/max_satisfying.ts rename to packages/schema/src/semver/max_satisfying.ts diff --git a/packages/schema/src/parse/semver/min_satisfying.ts b/packages/schema/src/semver/min_satisfying.ts similarity index 100% rename from packages/schema/src/parse/semver/min_satisfying.ts rename to packages/schema/src/semver/min_satisfying.ts diff --git a/packages/schema/src/parse/semver/mod.ts b/packages/schema/src/semver/mod.ts similarity index 100% rename from packages/schema/src/parse/semver/mod.ts rename to packages/schema/src/semver/mod.ts diff --git a/packages/schema/src/parse/semver/not_equals.ts b/packages/schema/src/semver/not_equals.ts similarity index 100% rename from packages/schema/src/parse/semver/not_equals.ts rename to packages/schema/src/semver/not_equals.ts diff --git a/packages/schema/src/parse/semver/parse.ts b/packages/schema/src/semver/parse.ts similarity index 100% rename from packages/schema/src/parse/semver/parse.ts rename to packages/schema/src/semver/parse.ts diff --git a/packages/schema/src/parse/semver/parse_range.ts b/packages/schema/src/semver/parse_range.ts similarity index 100% rename from packages/schema/src/parse/semver/parse_range.ts rename to packages/schema/src/semver/parse_range.ts diff --git a/packages/schema/src/parse/semver/range_intersects.ts b/packages/schema/src/semver/range_intersects.ts similarity index 100% rename from packages/schema/src/parse/semver/range_intersects.ts rename to packages/schema/src/semver/range_intersects.ts diff --git a/packages/schema/src/parse/semver/satisfies.ts b/packages/schema/src/semver/satisfies.ts similarity index 100% rename from packages/schema/src/parse/semver/satisfies.ts rename to packages/schema/src/semver/satisfies.ts diff --git a/packages/schema/src/parse/semver/try_parse.ts b/packages/schema/src/semver/try_parse.ts similarity index 100% rename from packages/schema/src/parse/semver/try_parse.ts rename to packages/schema/src/semver/try_parse.ts diff --git a/packages/schema/src/parse/semver/try_parse_range.ts b/packages/schema/src/semver/try_parse_range.ts similarity index 100% rename from packages/schema/src/parse/semver/try_parse_range.ts rename to packages/schema/src/semver/try_parse_range.ts diff --git a/packages/schema/src/parse/semver/types.ts b/packages/schema/src/semver/types.ts similarity index 100% rename from packages/schema/src/parse/semver/types.ts rename to packages/schema/src/semver/types.ts diff --git a/packages/schema/src/serialize.ts b/packages/schema/src/serialize.ts index 5844997807..643a4f357d 100644 --- a/packages/schema/src/serialize.ts +++ b/packages/schema/src/serialize.ts @@ -1,6 +1,3 @@ -// import * as deflate from 'fflate' -import { StrictSchema, stringFormats } from './types.js' -import { ENUM, REVERSE_TYPE_INDEX_MAP, TYPE_INDEX_MAP } from './def/types.js' import { readDoubleLE, readUint16, @@ -13,6 +10,9 @@ import { ENCODER, DECODER, } from '@based/utils' +import { stringFormats } from './schema/string.js' +import { reverseTypeMap, typeIndexMap } from './def/enums.js' +import { Schema } from './schema/schema.js' const UINT8 = 245 const FALSE = 246 @@ -41,6 +41,7 @@ const REF = 8 const PROP = 9 const KEY_OPTS = PROP +const ENUM = typeIndexMap.enum type DictMapUsed = { key: DictMapKey; address: number } @@ -237,12 +238,12 @@ const walk = ( const isArray = Array.isArray(obj) const isFromObj = prev2?.type === 'object' || fromObject === false const isSchemaProp = - ('enum' in obj || ('type' in obj && TYPE_INDEX_MAP[obj.type])) && isFromObj + ('enum' in obj || ('type' in obj && typeIndexMap[obj.type])) && isFromObj ensureCapacity(1 + 5) // Type byte + size if (isSchemaProp) { schemaBuffer.buf[schemaBuffer.len++] = SCHEMA_PROP - const typeIndex = TYPE_INDEX_MAP['enum' in obj ? 'enum' : obj.type] + const typeIndex = typeIndexMap['enum' in obj ? 'enum' : obj.type] schemaBuffer.buf[schemaBuffer.len++] = typeIndex } else { schemaBuffer.buf[schemaBuffer.len++] = isArray ? ARRAY : OBJECT @@ -468,7 +469,7 @@ export const deSerializeInner = ( if (isSchemaProp) { const type = buf[i] - const parsedType = REVERSE_TYPE_INDEX_MAP[type] + const parsedType = reverseTypeMap[type] if (type !== ENUM) { obj.type = parsedType } @@ -594,9 +595,9 @@ export const deSerializeInner = ( return i - start } -export const deSerialize = (buf: Uint8Array): StrictSchema => { +export const deSerialize = (buf: Uint8Array): Schema => { // if first byte is deflate - const schema: any = {} + const schema = {} as Schema deSerializeInner(buf, schema, 0, false) - return schema as StrictSchema + return schema } diff --git a/packages/schema/src/types.ts b/packages/schema/src/types.ts deleted file mode 100644 index f7a84aed58..0000000000 --- a/packages/schema/src/types.ts +++ /dev/null @@ -1,528 +0,0 @@ -import { getPropType } from './parse/utils.js' -import type { LangName } from './lang.js' -import type { Validation } from './def/validation.js' - -export const stringFormats = [ - 'alpha', - 'alphaLocales', - 'alphanumeric', - 'alphanumericLocales', - 'ascii', - 'base32', - 'base58', - 'base64', - 'BIC', - 'btcAddress', - 'clike', - 'code', - 'creditCard', - 'css', - 'currency', - 'dataURI', - 'EAN', - 'email', - 'ethereumAddress', - 'FQDN', - 'hexadecimal', - 'hexColor', - 'HSL', - 'html', - 'IBAN', - 'identityCard', - 'IMEI', - 'IP', - 'IPRange', - 'ISBN', - 'ISIN', - 'ISO31661Alpha2', - 'ISO31661Alpha3', - 'ISO4217', - 'ISO6391', - 'ISO8601', - 'ISRC', - 'ISSN', - 'javascript', - 'json', - 'JWT', - 'latLong', - 'licensePlate', - 'lowercase', - 'luhnNumber', - 'MACAddress', - 'magnetURI', - 'markdown', - 'MD5', - 'mimeType', - 'mobilePhone', - 'mobilePhoneLocales', - 'octal', - 'password', - 'passportNumber', - 'port', - 'postalCode', - 'postalCodeLocales', - 'python', - 'RFC3339', - 'rgbColor', - 'rust', - 'semVer', - 'slug', - 'surrogatePair', - 'taxID', - 'typescript', - 'uppercase', - 'URL', - 'UUID', - 'VAT', -] as const - -type StringFormat = (typeof stringFormats)[number] - -type MimeString = - | 'text/html' - | 'text/plain' - | 'text/markdown' - | 'image/png' - | 'image/jpeg' - | 'video/mp4' - | 'video/quicktime' - | 'image/*' - | 'video/*' - | 'audio/*' - | '*/*' - | `${string}/${string}` // this is overriding the previous - -type Mime = MimeString | MimeString[] - -type Letter = - | 'A' - | 'B' - | 'C' - | 'D' - | 'E' - | 'F' - | 'G' - | 'H' - | 'I' - | 'J' - | 'K' - | 'L' - | 'M' - | 'N' - | 'O' - | 'P' - | 'Q' - | 'R' - | 'S' - | 'T' - | 'U' - | 'V' - | 'W' - | 'X' - | 'Y' - | 'Z' - | 'a' - | 'b' - | 'c' - | 'd' - | 'e' - | 'f' - | 'g' - | 'h' - | 'i' - | 'j' - | 'k' - | 'l' - | 'm' - | 'n' - | 'o' - | 'p' - | 'q' - | 'r' - | 's' - | 't' - | 'u' - | 'v' - | 'w' - | 'x' - | 'y' - | 'z' - -type AllowedKey = `${Letter}${string}` -type QueryFn = Function -type PropValues = { - type?: string - default?: any - validation?: Validation -} -type Prop = { - required?: boolean - title?: string | Record - description?: string | Record - // path?: string - // query?: QueryFn - // role?: Role - // readOnly?: boolean - examples?: string[] - validation?: Validation - hooks?: SchemaPropHooks -} & V - -type EnumItem = string | number | boolean -type NeverInItems = { required?: never } - -export type SchemaReferences = Prop<{ - type?: 'references' - default?: number[] - capped?: number - items: SchemaReference & NeverInItems -}> - -export type SchemaReferencesOneWay = Prop<{ - type?: 'references' - default?: number[] - items: SchemaReferenceOneWay & NeverInItems -}> - -export type SchemaText = Prop<{ - type: 'text' - default?: Record - format?: StringFormat - compression?: 'none' | 'deflate' -}> - -type NumberType = - | 'number' - | 'int8' - | 'uint8' - | 'int16' - | 'uint16' - | 'int32' - | 'uint32' - -export type SchemaNumber = Prop<{ - type: NumberType - default?: number - min?: number - max?: number - step?: number -}> - -export type SchemaString = Prop<{ - type: 'string' - default?: string - maxBytes?: number - max?: number - min?: number - mime?: Mime - format?: StringFormat - compression?: 'none' | 'deflate' -}> - -export type SchemaBinary = Prop<{ - type: 'binary' - default?: Uint8Array - maxBytes?: number - mime?: Mime - format?: StringFormat -}> - -export type SchemaJson = Prop<{ - type: 'json' - default?: Record | null -}> - -export type SchemaBoolean = Prop<{ - type: 'boolean' - default?: boolean -}> - -export type HLLRegisterRepresentation = 'sparse' | 'dense' - -export type SchemaCardinality = Prop<{ - type: 'cardinality' - maxBytes?: number - precision?: number - mode?: HLLRegisterRepresentation -}> - -type VectorDefaultType = - | Int8Array - | Uint8Array - | Int16Array - | Uint16Array - | Int32Array - | Uint32Array - | Float32Array - | Float64Array -export type SchemaVectorBaseType = NumberType | 'float32' | 'float64' - -export type SchemaVector = Prop<{ - type: 'vector' - default?: VectorDefaultType - /** - * Number of elements in the vector. - */ - size: number - /** - * Base type of the vector. - * float64 == number - */ - baseType?: SchemaVectorBaseType -}> - -export type SchemaColvec = Prop<{ - type: 'colvec' - default?: VectorDefaultType - /** - * Number of elements in the vector. - */ - size: number - /** - * Base type of the vector. - * float64 == number - */ - baseType?: SchemaVectorBaseType -}> - -export type SchemaTimestamp = Prop<{ - type: 'timestamp' - default?: number | Date | string - on?: 'create' | 'update' - min?: number | string - max?: number | string - step?: isStrict extends true ? number : number | string -}> - -export type SchemaReferenceOneWay = Prop<{ - type?: 'reference' - default?: number - ref: string - mime?: Mime -}> - -export type SchemaReference = Prop<{ - type?: 'reference' - default?: number - ref: string - prop: string - dependent?: boolean - mime?: Mime -}> & - Record<`$${string}`, SchemaPropOneWay> - -export type SchemaObject = Prop<{ - type?: 'object' - props: SchemaProps -}> - -export type SchemaObjectOneWay = Prop<{ - type?: 'object' - props: SchemaPropsOneWay -}> - -export type SchemaReferenceWithQuery = SchemaReferenceOneWay & { - query: QueryFn -} -export type SchemaReferencesWithQuery = SchemaReferencesOneWay & { - query: QueryFn -} - -export type SchemaEnum = Prop<{ - type?: 'enum' - default?: EnumItem | undefined - enum: EnumItem[] -}> - -export type SchemaAlias = Omit & { type: 'alias' } - -export type SchemaPropShorthand = - | 'timestamp' - | 'binary' - | 'boolean' - | 'string' - | 'alias' - | 'text' - | 'json' - | 'cardinality' - | NumberType - | EnumItem[] - -type NonRefSchemaProps = - | SchemaTimestamp - | SchemaBoolean - | SchemaNumber - | SchemaString - | SchemaAlias - | SchemaText - | SchemaEnum - | SchemaJson - | SchemaBinary - | SchemaCardinality - | SchemaVector - | SchemaColvec - | (isStrict extends true ? never : SchemaPropShorthand) - -export type SchemaProp = - | SchemaReferencesWithQuery - | SchemaReferenceWithQuery - | NonRefSchemaProps - | SchemaReferences - | SchemaReference - | SchemaObject - | SchemaBinary - -export type SchemaPropOneWay = - | SchemaReferencesOneWay - | SchemaReferenceOneWay - | SchemaObjectOneWay - | NonRefSchemaProps - -export type SchemaAnyProp = SchemaPropOneWay | SchemaProp -export type SchemaProps = Record< - AllowedKey, - SchemaProp -> & { id?: never } - -// TODO: export these types in a pkg (not db => circular!) -type BasedDbQuery = any -type Operator = string - -export type SchemaHooks = { - create?: (payload: Record) => void | Record - update?: (payload: Record) => void | Record - read?: (result: Record) => void | null | Record - search?: (query: BasedDbQuery, fields: Set) => void - include?: ( - query: BasedDbQuery, - fields: Map< - string, - { - field: string - opts?: any // temp this type - } - >, - ) => void - filter?: ( - query: BasedDbQuery, - field: string, - operator: Operator, - value: any, - ) => void - groupBy?: (query: BasedDbQuery, field: string) => void - aggregate?: (query: BasedDbQuery, fields: Set) => void -} - -export type SchemaPropHooks = { - create?: (value: any, payload: Record) => any - update?: (value: any, payload: Record) => any - read?: (value: any, result: Record) => any - aggregate?: (query: BasedDbQuery, fields: Set) => void - search?: (query: BasedDbQuery, fields: Set) => void - groupBy?: (query: BasedDbQuery, field: string) => void - filter?: ( - query: BasedDbQuery, - field: string, - operator: Operator, - value: any, - ) => void - include?: ( - query: BasedDbQuery, - fields: Map< - string, - { - field: string - opts?: any // temp this type - } - >, - ) => void -} - -type GenericSchemaType = { - hooks?: SchemaHooks - id?: number - blockCapacity?: number - capped?: number - insertOnly?: boolean - partial?: boolean - props: SchemaProps -} - -export type StrictSchemaType = GenericSchemaType -export type SchemaType = isStrict extends true - ? StrictSchemaType - : - | StrictSchemaType - | GenericSchemaType - | (SchemaProps & { props?: never }) - -export type SchemaTypeName = Exclude -export type SchemaTypes = Record< - SchemaTypeName, - SchemaType -> & { _root?: never } - -export type SchemaPropsOneWay = Record< - AllowedKey, - SchemaPropOneWay -> & { id?: never } - -type MigrateFn = ( - node: Record, -) => Record | [string, Record] - -export type MigrateFns = Record - -type GenericSchema = { - version?: string - types?: SchemaTypes - props?: SchemaPropsOneWay - locales?: Partial - defaultTimezone?: string - migrations?: { - version: string - migrate: MigrateFns - }[] -} - -export type StrictSchema = GenericSchema -export type NonStrictSchema = GenericSchema -export type Schema = NonStrictSchema | StrictSchema - -export type SchemaLocales = Record< - LangName, - | true - | { - required?: boolean - fallback?: LangName // not multiple - 1 is enough else it becomes too complex - } -> - -export type SchemaPropTypeMap = { - references: SchemaReferences - timestamp: SchemaTimestamp - reference: SchemaReference - boolean: SchemaBoolean - string: SchemaString - object: SchemaObject - alias: SchemaAlias - enum: SchemaEnum - text: SchemaText - json: SchemaJson - binary: SchemaBinary - cardinality: SchemaCardinality - vector: SchemaVector - colvec: SchemaColvec -} & Record - -export type SchemaPropTypes = keyof SchemaPropTypeMap - -export const isPropType = ( - type: T, - prop: SchemaProp, -): prop is SchemaPropTypeMap[T] => { - return getPropType(prop) === type -} - -export const MAX_ID = 4294967295 -export const MIN_ID = 1 diff --git a/packages/schema/test/alias.ts b/packages/schema/test/alias.ts deleted file mode 100644 index b4c74c82fd..0000000000 --- a/packages/schema/test/alias.ts +++ /dev/null @@ -1,22 +0,0 @@ -import test from 'node:test' -import { parse } from '@based/schema' - -await test('alias', () => { - parse({ - types: { - article: { - props: { - externalId: { - type: 'alias', - }, - friendlyUrl: { - type: 'alias', - }, - body: { - type: 'string', - }, - }, - }, - }, - }) -}) diff --git a/packages/schema/test/basic.ts b/packages/schema/test/basic.ts deleted file mode 100644 index 061ec5b858..0000000000 --- a/packages/schema/test/basic.ts +++ /dev/null @@ -1,41 +0,0 @@ -import test from 'node:test' -import { parse } from '@based/schema' - -await test('basic', () => { - parse({ - locales: { - en: {}, - }, - types: { - user: { - props: { - string: { type: 'string' }, - number: { type: 'number' }, - binary: { type: 'binary' }, - boolean: { type: 'boolean' }, - timestamp: { type: 'timestamp' }, - enum: { type: 'enum', enum: ['foo', 'bar', 'baz'] }, - text: { type: 'text' }, - object: { - type: 'object', - props: { - foo: { type: 'string' }, - }, - }, - reference: { type: 'reference', ref: 'file', prop: 'referenceFor' }, - references: { - type: 'references', - items: { ref: 'file', prop: 'referencesFor' }, - }, - }, - }, - file: { - props: { - src: { type: 'string' }, - referenceFor: { type: 'reference', ref: 'user', prop: 'reference' }, - referencesFor: { type: 'reference', ref: 'user', prop: 'references' }, - }, - }, - }, - }) -}) diff --git a/packages/schema/test/boolean.ts b/packages/schema/test/boolean.ts deleted file mode 100644 index 732978e2d9..0000000000 --- a/packages/schema/test/boolean.ts +++ /dev/null @@ -1,26 +0,0 @@ -import test from 'node:test' -import { throws } from 'node:assert' -import { parse } from '@based/schema' - -await test('boolean', () => { - parse({ - props: { - myBoolean: { - type: 'boolean', - default: true, - }, - }, - }) - - throws(() => { - parse({ - props: { - myBoolean: { - type: 'boolean', - // @ts-ignore - default: 'hello', - }, - }, - }) - }, 'only allow booleans') -}) diff --git a/packages/schema/test/defaultTimezone.ts b/packages/schema/test/defaultTimezone.ts deleted file mode 100644 index 14c33e5dd0..0000000000 --- a/packages/schema/test/defaultTimezone.ts +++ /dev/null @@ -1,51 +0,0 @@ -import test from 'node:test' -import { parse } from '@based/schema' -import assert, { throws } from 'assert' - -await test('defaultTimezone', () => { - let parsed = parse({ - types: { - user: { - props: { - timestamp: { type: 'timestamp' }, - }, - }, - }, - }) - assert(parsed.schema.defaultTimezone === undefined) - - parsed = parse({ - defaultTimezone: 'Europe/Helsinki', - types: { - user: { - props: { - timestamp: { type: 'timestamp' }, - }, - }, - }, - }) - assert(parsed.schema.defaultTimezone === 'Europe/Helsinki') - - throws(() => parse({ - defaultTimezone: 'Europe/Turku', - types: { - user: { - props: { - timestamp: { type: 'timestamp' }, - }, - }, - }, - })) - - throws(() => parse({ - // @ts-ignore - defaultTimezone: 2, - types: { - user: { - props: { - timestamp: { type: 'timestamp' }, - }, - }, - }, - })) -}) diff --git a/packages/schema/test/dependency.ts b/packages/schema/test/dependency.ts deleted file mode 100644 index a7a6e77825..0000000000 --- a/packages/schema/test/dependency.ts +++ /dev/null @@ -1,19 +0,0 @@ -import test from 'node:test' -import { parse } from '@based/schema' - -await test('dependency', () => { - parse({ - types: { - user: { - name: 'string', - }, - file: { - owner: { - ref: 'user', - prop: 'uploadedFiles', - dependent: true, // <==== if there is no reference assigned, this item will be deleted - }, - }, - }, - }) -}) diff --git a/packages/schema/test/edges.ts b/packages/schema/test/edges.ts deleted file mode 100644 index b1c01652b7..0000000000 --- a/packages/schema/test/edges.ts +++ /dev/null @@ -1,71 +0,0 @@ -import test from 'node:test' -import { parse } from '@based/schema' -import { deepEqual } from 'assert' - -await test('edges', () => { - try { - const { schema } = parse({ - types: { - dude: {}, - user: { - friend: { - ref: 'dude', - prop: 'friend', - $nerds: 'boolean', - $option: ['cool', 'dazzling'], - }, - friends: { - items: { - ref: 'dude', - prop: 'friends', - $nerds: 'boolean', - }, - }, - }, - }, - }) - - deepEqual(schema, { - types: { - dude: { - props: { - friend: { - items: { - ref: 'user', - prop: 'friend', - $nerds: { type: 'boolean' }, - $option: { enum: ['cool', 'dazzling'] }, - }, - }, - friends: { - items: { - ref: 'user', - prop: 'friends', - $nerds: { type: 'boolean' }, - }, - }, - }, - }, - user: { - props: { - friend: { - ref: 'dude', - prop: 'friend', - $nerds: { type: 'boolean' }, - $option: { enum: ['cool', 'dazzling'] }, - }, - friends: { - items: { - ref: 'dude', - prop: 'friends', - $nerds: { type: 'boolean' }, - }, - }, - }, - }, - }, - }) - } catch (e) { - console.error(e) - } -}) diff --git a/packages/schema/test/enum.ts b/packages/schema/test/enum.ts deleted file mode 100644 index 105a69731d..0000000000 --- a/packages/schema/test/enum.ts +++ /dev/null @@ -1,31 +0,0 @@ -import test from 'node:test' -import { throws } from 'node:assert' -import { parse } from '@based/schema' - -await test('enum', () => { - parse({ - props: { - myEnum: { - enum: ['published', 'draft'], - default: 'published', - }, - }, - }) - - parse({ - props: { - myEnum: ['published', 'draft'], - }, - }) - - throws(() => { - parse({ - props: { - myEnum: { - enum: ['published', 'draft'], - default: 'blurdo', - }, - }, - }) - }, 'disallow non defined default') -}) diff --git a/packages/schema/test/errors.ts b/packages/schema/test/errors.ts new file mode 100644 index 0000000000..32d959008f --- /dev/null +++ b/packages/schema/test/errors.ts @@ -0,0 +1,40 @@ +import test from 'node:test' +import { parseSchema } from '../src/schema/schema.js' +import assert from 'node:assert' + +await test('errors', () => { + assert.throws( + () => + parseSchema({ + types: { + role: { + // @ts-expect-error + name: { + type: 'string', + default: 1, + }, + }, + }, + }), + { + message: /^types.role.name.default: 1/, + }, + ) + + assert.throws( + () => + parseSchema({ + types: { + article: { + author: { + ref: 'author', + prop: 'bla', + }, + }, + }, + }), + { + message: /^types.article.props.author.ref: 'author'/, + }, + ) +}) diff --git a/packages/schema/test/hll.ts b/packages/schema/test/hll.ts deleted file mode 100644 index c4142405e2..0000000000 --- a/packages/schema/test/hll.ts +++ /dev/null @@ -1,48 +0,0 @@ -import test from 'node:test' -import { parse } from '@based/schema' - -await test('cardinality', () => { - parse({ - props: { - myUniqueValuesCount: { - type: 'cardinality', - }, - }, - }) - - parse({ - props: { - myUniqueValuesCount: 'cardinality', - }, - }) -}) - -await test('cardinality props', () => { - parse({ - props: { - myUniqueValuesCountDense: { - type: 'cardinality', - mode: 'dense', - precision: 12, - }, - }, - }) - - parse({ - props: { - myUniqueValuesCountSparse: { - type: 'cardinality', - mode: 'sparse', - }, - }, - }) - - parse({ - props: { - myUniqueValuesCountSparse: { - type: 'cardinality', - precision: 2, - }, - }, - }) -}) diff --git a/packages/schema/test/hooks.ts b/packages/schema/test/hooks.ts deleted file mode 100644 index a41e549705..0000000000 --- a/packages/schema/test/hooks.ts +++ /dev/null @@ -1,34 +0,0 @@ -import test from 'node:test' -import { deSerialize, parse, serialize } from '@based/schema' -import { deepEqual } from 'assert' - -await test('hooks', () => { - const { schema } = parse({ - types: { - user: { - stringified: { - type: 'string', - hooks: { - create(value, payload) { - return JSON.stringify({ value, payload }) - }, - }, - }, - }, - }, - }) - - const serialized = serialize(schema) - const deserialized = deSerialize(serialized) - const value = 1 - const payload = { bla: true } - const result = JSON.stringify({ value, payload }) - const res1 = schema.types.user.props.stringified.hooks.create(value, payload) - const res2 = deserialized.types.user.props.stringified.hooks.create( - value, - payload, - ) - - deepEqual(result, res1) - deepEqual(result, res2) -}) diff --git a/packages/schema/test/index.ts b/packages/schema/test/index.ts new file mode 100644 index 0000000000..a367baf61f --- /dev/null +++ b/packages/schema/test/index.ts @@ -0,0 +1,80 @@ +import test from 'node:test' +import { parseSchema, type Schema } from '../src/schema/schema.js' +import { schemaToDefs } from '../src/def/index.js' +import { getAllProps, getPropChain } from '../src/index.js' + +await test('testings', () => { + const input: Schema = { + types: { + role: { + name: 'string', + }, + author: { + name: 'string', + age: 'uint8', + articles: { + ref: 'article', + prop: 'author', + }, + }, + article: { + address: { + props: { + street: 'string', + }, + }, + author: { + ref: 'author', + prop: 'articles', + $durt: { + ref: 'author', + }, + }, + collaborators: { + items: { + ref: 'author', + prop: 'collaborations', + $role: ['owner', 'reader'], + $roles: { + items: { + ref: 'role', + // prop: 'ballz', + // $test: 'string', + }, + }, + }, + }, + }, + }, + } as const satisfies Schema + + // console.log('-- input') + // console.dir(input, { depth: null }) + // const schema = parseSchema(input as Schema) + // console.log('-- strict schema', schema) + // console.dir(schema, { depth: null }) + // const defs = schemaToDefs(schema) + // console.log('-- schema defs') + // console.dir(defs, { depth: 6 }) + // console.log('-- schema path') + // console.log('collaborators', getPropChain(defs.article, ['collaborators'])) + // console.log( + // 'collaborators.$role', + // getPropChain(defs.article, ['collaborators', '$role']), + // ) + // console.log( + // 'collaborators.age', + // getPropChain(defs.article, ['collaborators', 'age']), + // ) + // console.log( + // 'collaborators.articles.address', + // getPropChain(defs.article, ['collaborators', 'articles', 'address']), + // ) + + // console.log('--------------------------------') + // for (const prop of getAllProps(defs.author)) { + // console.log('-', prop) + // } + + // types.user = { name: 'string' } +}) diff --git a/packages/schema/test/infer.ts b/packages/schema/test/infer.ts deleted file mode 100644 index e7db99cad4..0000000000 --- a/packages/schema/test/infer.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { Infer } from '../src/infer.js' -import { Schema } from '../src/index.js' - -// Test schema -const testSchema = { - types: { - user: { - props: { - name: { type: 'string' }, - age: { type: 'number' }, - status: { enum: ['active', 'inactive'] }, - friends: { items: { ref: 'user', prop: 'friends' } }, - profile: { props: { bio: { type: 'text' } } }, - }, - }, - post: { - title: { type: 'string' }, - content: { type: 'text' }, - author: { ref: 'user', prop: 'author' }, - metadata: { type: 'json' }, - published: { type: 'boolean' }, - derp: { type: 'alias' }, - createdAt: { type: 'timestamp' }, - flap: { type: 'vector', baseType: 'uint8', size: 32 }, - }, - }, -} as const satisfies Schema - -// Inferred type -type TestSchemaParsed = Infer - -// Type assertions to verify the inference works correctly -type User = TestSchemaParsed['user'] -type Post = TestSchemaParsed['post'] - -function createUser(data: User): User { - return data -} - -function createPost(data: Post): Post { - return data -} - -// Test the type inference -const user: User = { - id: 1, - name: 'John Doe', - age: 30, - status: 'active', // TypeScript will enforce this must be 'active' | 'inactive' - friends: [], // Array of numbers (user IDs) - profile: { - bio: 'Hello world!', - }, -} - -const post: Post = { - id: 1, - title: 'My First Post', - content: 'This is the content of my post', - author: { - id: 1, - name: 'John Doe', - age: 30, - status: 'active', - friends: [], - profile: { bio: 'Hello world!' }, - }, - metadata: { views: 100, likes: 5 }, - published: true, - createdAt: new Date(), // Date.now(), - flap: new Uint8Array(32), - derp: 'alias!', -} - -console.log('Type inference working correctly!') -console.log('User:', user) -console.log('Post:', post) - -export { testSchema, createUser, createPost } - -const x = createUser(user) - -console.log(x, user.status === 'inactive') diff --git a/packages/schema/test/json.ts b/packages/schema/test/json.ts deleted file mode 100644 index 1b04845dd9..0000000000 --- a/packages/schema/test/json.ts +++ /dev/null @@ -1,17 +0,0 @@ -import test from 'node:test' -import { parse } from '@based/schema' - -await test('json', () => { - parse({ - types: { - article: { - props: { - myCoolJson: 'json', - myCoolJson2: { - type: 'json', - }, - }, - }, - }, - }) -}) diff --git a/packages/schema/test/number.ts b/packages/schema/test/number.ts deleted file mode 100644 index e977012c41..0000000000 --- a/packages/schema/test/number.ts +++ /dev/null @@ -1,45 +0,0 @@ -import test from 'node:test' -import { throws } from 'node:assert' -import { parse } from '@based/schema' - -await test('number', () => { - parse({ - props: { - myNumber: { - type: 'number', - default: 11, - min: 10, - max: 100, - step: 0.5, - }, - }, - }) - - throws(() => { - parse({ - props: { - myNumber: { - type: 'number', - default: 200, - min: 10, - max: 100, - step: 0.5, - }, - }, - }) - }, 'should throw with out of range default') - - throws(() => { - parse({ - props: { - myNumber: { - type: 'number', - default: 110, - min: 10, - max: 100, - step: 0.7, - }, - }, - }) - }, 'should throw with unreachable default') -}) diff --git a/packages/schema/test/object.ts b/packages/schema/test/object.ts deleted file mode 100644 index 20865f47bf..0000000000 --- a/packages/schema/test/object.ts +++ /dev/null @@ -1,16 +0,0 @@ -import test from 'node:test' -import { parse } from '@based/schema' - -await test('object', () => { - parse({ - props: { - myObject: { - props: { - myField: { - type: 'boolean', - }, - }, - }, - }, - }) -}) diff --git a/packages/schema/test/parse.ts b/packages/schema/test/parse.ts deleted file mode 100644 index 3fb93017b0..0000000000 --- a/packages/schema/test/parse.ts +++ /dev/null @@ -1,27 +0,0 @@ -import test from 'node:test' -import { throws } from 'node:assert' -import { parse } from '@based/schema' - -await test('parse', () => { - parse({ - types: { - coolname: { - props: { - myTimestamp: { - type: 'timestamp', - }, - }, - }, - jurbo: { - props: { - flurk: { - type: 'string', - }, - }, - }, - }, - }) - - // types | - // [0, 'coolname', ] -}) diff --git a/packages/schema/test/path.ts b/packages/schema/test/path.ts deleted file mode 100644 index 9bb0567ba1..0000000000 --- a/packages/schema/test/path.ts +++ /dev/null @@ -1,67 +0,0 @@ -import test from 'node:test' -import { throws } from 'node:assert' -import { parse } from '@based/schema' - -await test('path', () => { - parse({ - types: { - club: { - props: { - logo: { - type: 'string', - }, - teams: { - items: { - ref: 'team', - prop: 'club', - }, - }, - }, - }, - team: { - props: { - logo: { - type: 'string', - path: 'club.logo', - }, - club: { - ref: 'club', - prop: 'teams', - }, - }, - }, - }, - }) - - throws(() => { - parse({ - types: { - club: { - props: { - logoWithDifferentType: { - type: 'boolean', - }, - teams: { - items: { - ref: 'team', - prop: 'club', - }, - }, - }, - }, - team: { - props: { - logo: { - type: 'string', - path: 'club.logoWithDifferentType', - }, - club: { - ref: 'club', - prop: 'teams', - }, - }, - }, - }, - }) - }, 'Mismatching types not allowed') -}) diff --git a/packages/schema/test/query.ts b/packages/schema/test/query.ts deleted file mode 100644 index 68f15edfdf..0000000000 --- a/packages/schema/test/query.ts +++ /dev/null @@ -1,48 +0,0 @@ -import test from 'node:test' -import { parse } from '@based/schema' - -await test('query', () => { - parse({ - types: { - author: { - props: { - top10Articles: { - items: { - ref: 'article', - }, - query: (query: any) => - query('articles').sort('views', 'desc').range(0, 10), - }, - articles: { - items: { - ref: 'article', - prop: 'author', - }, - }, - }, - }, - article: { - props: { - author: { - ref: 'author', - prop: 'articles', - }, - published: { - type: 'boolean', - query: (query: any) => - query().filter('status', 'published').boolean(), - }, - status: { - enum: ['published', 'draft', 'archived'], - }, - views: { - type: 'number', - min: 0, - max: Infinity, - step: 1, - }, - }, - }, - }, - }) -}) diff --git a/packages/schema/test/references.ts b/packages/schema/test/references.ts deleted file mode 100644 index 0eb3116d9b..0000000000 --- a/packages/schema/test/references.ts +++ /dev/null @@ -1,303 +0,0 @@ -import test from 'node:test' -import { deepEqual, throws } from 'node:assert' -import { parse } from '@based/schema' - -await test('references', (t) => { - parse({ - types: { - article: { - props: { - writer: { - ref: 'author', - prop: 'articles', - }, - }, - }, - author: { - props: { - articles: { - items: { - ref: 'article', - prop: 'writer', - }, - }, - }, - }, - }, - }) - - parse({ - props: { - articles: { - items: { - ref: 'article', - }, - }, - }, - types: { - article: { - name: 'string', - }, - }, - }) - - deepEqual( - parse({ - types: { - article: { - props: { - writer: { - ref: 'author', - prop: 'articles', - }, - }, - }, - author: { - props: {}, - }, - }, - }).schema, - { - types: { - article: { - props: { - writer: { - ref: 'author', - prop: 'articles', - }, - }, - }, - author: { - props: { - articles: { - items: { - ref: 'article', - prop: 'writer', - }, - }, - }, - }, - }, - }, - ) - - const { schema } = parse({ - types: { - article: { - props: { - writers: { - items: { - ref: 'author', - prop: 'articles', - }, - }, - }, - }, - author: { - props: {}, - }, - }, - }) - - deepEqual(schema, { - types: { - article: { - props: { - writers: { - items: { - ref: 'author', - prop: 'articles', - }, - }, - }, - }, - author: { - props: { - articles: { - items: { - ref: 'article', - prop: 'writers', - }, - }, - }, - }, - }, - }) - - throws(() => { - parse({ - types: { - author: { - props: { - articles: { - items: { - ref: 'article', - prop: 'author', - }, - }, - }, - }, - }, - }) - }, 'Disallow missing type for ref') - - throws(() => { - parse({ - props: { - myRefs: { - items: { - ref: 'user', - }, - }, - }, - }) - }, 'Disallow missing type for refs') - - throws(() => { - parse({ - types: { - article: { - props: { - author: { - ref: 'author', - prop: 'articles', - }, - }, - }, - author: { - props: { - articles: { - items: { - ref: 'article', - prop: 'author', - }, - }, - }, - }, - author2: { - props: { - articles: { - items: { - ref: 'article', - prop: 'author', - }, - }, - }, - }, - }, - }) - }, 'Disallow mixed ref types') - - throws(() => { - parse({ - types: { - article: { - props: { - author: { - ref: 'author', - prop: 'articles', - }, - }, - }, - author: { - props: { - // @ts-ignore - articles: { - // @ts-ignore - items: { - // @ts-ignore - required: true, - ref: 'article', - prop: 'author', - }, - }, - }, - }, - }, - }) - }, 'Disallow incorrect location of required prop') -}) - -await test('edges', () => { - parse({ - types: { - event: { - props: { - createdAt: { - type: 'timestamp', - on: 'create', - }, - }, - }, - article: { - props: { - author: { - ref: 'author', - prop: 'articles', - $role: { - enum: ['admin', 'collaborator'], - }, - $relatedEvent: { - ref: 'event', - }, - $enum: ['zzz'], - }, - }, - }, - author: { - props: { - articles: { - items: { - ref: 'article', - prop: 'author', - }, - }, - }, - }, - }, - }) -}) - -await test('references no auto on same prop', (t) => { - parse({ - types: { - contributor: { - name: 'string', - }, - vote: { - props: { - choice: { - ref: 'contributor', - prop: 'votes', - }, - }, - }, - }, - }) - - throws(() => { - parse({ - types: { - contributor: { - name: 'string', - }, - vote: { - props: { - choice: { - ref: 'contributor', - prop: 'votes', - }, - }, - }, - test: { - props: { - test: { - ref: 'contributor', - prop: 'votes', - }, - }, - }, - }, - }) - }, 'Disallow auto on same prop') -}) diff --git a/packages/schema/test/required.ts b/packages/schema/test/required.ts index 0990d4d1da..dcd594fceb 100644 --- a/packages/schema/test/required.ts +++ b/packages/schema/test/required.ts @@ -1,69 +1,19 @@ import test from 'node:test' -import { parse } from '@based/schema' +import { parseSchema } from '../src/schema/schema.js' -await test('dependency', () => { - parse({ - locales: { - en: {}, - }, +await test('required', () => { + parseSchema({ types: { - user: { - props: { - string: { - required: true, - type: 'string', - }, - number: { - required: true, - type: 'number', - }, - binary: { - required: true, - type: 'binary', - }, - boolean: { - required: true, - type: 'boolean', - }, - timestamp: { - required: true, - type: 'timestamp', - }, - enum: { - required: true, - type: 'enum', - enum: ['foo', 'bar', 'baz'], - }, - text: { - required: true, - type: 'text', - }, - - object: { - required: true, - type: 'object', - props: { - foo: { type: 'string' }, - }, - }, - reference: { - required: true, - type: 'reference', - ref: 'file', - prop: 'referenceFor', - }, - references: { - required: true, - type: 'references', - items: { ref: 'file', prop: 'referencesFor' }, - }, + role: { + name: { + type: 'string', + default: 'John Doe', + required: true, }, - }, - file: { - props: { - src: { type: 'string' }, - referenceFor: { type: 'reference', ref: 'user', prop: 'reference' }, - referencesFor: { type: 'reference', ref: 'user', prop: 'references' }, + age: { + type: 'uint8', + default: 21, + required: false, }, }, }, diff --git a/packages/schema/test/schema.ts b/packages/schema/test/schema.ts deleted file mode 100644 index 5b82da48db..0000000000 --- a/packages/schema/test/schema.ts +++ /dev/null @@ -1,19 +0,0 @@ -import test from 'node:test' -import { throws } from 'node:assert' -import { parse } from '@based/schema' - -await test('schema', () => { - parse({ - props: {}, - types: {}, - }) - - throws(() => { - parse({ - props: {}, - types: {}, - // @ts-ignore - unknownTsIgnoredField: true, - }) - }, 'Should throw with unknown fields') -}) diff --git a/packages/schema/test/schema/based.schema.ts b/packages/schema/test/schema/based.schema.ts deleted file mode 100644 index 563c8247e0..0000000000 --- a/packages/schema/test/schema/based.schema.ts +++ /dev/null @@ -1,361 +0,0 @@ -export default { - locales: { - en: { required: true }, - fr: { required: true }, - nl: { required: true }, - el: { required: true }, - he: { required: true }, - it: { required: true }, - lv: { required: true }, - lb: { required: true }, - ro: { required: true }, - sl: { required: true }, - es: { required: true }, - de: { required: true }, - cs: { required: true }, - et: { required: true }, - }, - props: { - info: { type: 'text', format: 'html' }, - terms: { - type: 'object', - props: { - title: { type: 'text' }, - introductionTitle: { type: 'text' }, - introductionText: { type: 'text', format: 'html' }, - rulesTitle: { type: 'text' }, - rulesText: { type: 'text', format: 'html' }, - procedureTitle: { type: 'text' }, - procedureText: { type: 'text', format: 'html' }, - noRefundsTitle: { type: 'text' }, - noRefundsText: { type: 'text', format: 'html' }, - dataTitle: { type: 'text' }, - dataText: { type: 'text', format: 'html' }, - voterRespTitle: { type: 'text' }, - voterRespText: { type: 'text', format: 'html' }, - votingWindowTitle: { type: 'text' }, - votingWindowText: { type: 'text', format: 'html' }, - generalCondTitle: { type: 'text' }, - generalCondText: { type: 'text', format: 'html' }, - jurisdictionTitle: { type: 'text' }, - jurisdictionText: { type: 'text', format: 'html' }, - changesTitle: { type: 'text' }, - changesText: { type: 'text', format: 'html' }, - lastUpdatedText: { type: 'text', format: 'html' }, - lastUpdated: { type: 'timestamp' }, - }, - }, - privacy: { type: 'text', format: 'html' }, - excludedCountries: { items: { ref: 'country' } }, - activeSequence: { ref: 'sequence' }, - coreDataLock: { type: 'boolean' }, - winner: { type: 'string' }, - dictionary: { - type: 'object', - props: { - addVotes: { type: 'text' }, - appleMusic: { type: 'text' }, - appleMusicDisclaimer: { type: 'text' }, - buttonCall: { type: 'text' }, - buttonSMS: { type: 'text' }, - callNow: { type: 'text' }, - changeDescription: { type: 'text' }, - changeLocation: { type: 'text' }, - continue: { type: 'text' }, - cookieDisclaimer: { type: 'text' }, - imprint: { type: 'text' }, - overviewButton: { type: 'text' }, - phoneDescription: { type: 'text' }, - phoneInfo: { type: 'text' }, - phoneLocation: { type: 'text' }, - phoneSelection: { type: 'text' }, - privacy: { type: 'text' }, - selectionButton: { type: 'text' }, - selectionDescription: { type: 'text' }, - selectionSecure: { type: 'text' }, - selectionTitle: { type: 'text' }, - sendSMS: { type: 'text' }, - summaryButton: { type: 'text' }, - summaryCheckbox: { type: 'text' }, - summaryDisclaimer: { type: 'text' }, - summaryEmail: { type: 'text' }, - summaryProcessing: { type: 'text' }, - summaryProcessingMsg: { type: 'text' }, - summaryRefund: { type: 'text' }, - summarySecure: { type: 'text' }, - summarySelect: { type: 'text' }, - summaryTitle: { type: 'text' }, - summaryTotal: { type: 'text' }, - terms: { type: 'text' }, - thankYou: { type: 'text' }, - thankYouMax: { type: 'text' }, - thankYouMaxButton: { type: 'text' }, - thankYouOnline: { type: 'text' }, - thankYouOnlineButton: { type: 'text' }, - thankYouPhone: { type: 'text' }, - thankYouPhoneButton: { type: 'text' }, - votingInfo: { type: 'text' }, - whyImportant: { type: 'text' }, - days: { type: 'text' }, - hours: { type: 'text' }, - minutes: { type: 'text' }, - seconds: { type: 'text' }, - }, - }, - onboarding: { - type: 'object', - props: { - restOfTheWorld: { - type: 'object', - props: { title: { type: 'text' }, description: { type: 'text' } }, - }, - sanMarino: { - type: 'object', - props: { title: { type: 'text' }, description: { type: 'text' } }, - }, - notAllowedSemi1: { - type: 'object', - props: { title: { type: 'text' }, description: { type: 'text' } }, - }, - notAllowedSemi2: { - type: 'object', - props: { title: { type: 'text' }, description: { type: 'text' } }, - }, - online: { - type: 'object', - props: { title: { type: 'text' }, description: { type: 'text' } }, - }, - parentCode: { - type: 'object', - props: { title: { type: 'text' }, description: { type: 'text' } }, - }, - phone: { - type: 'object', - props: { title: { type: 'text' }, description: { type: 'text' } }, - }, - overseasCode: { - type: 'object', - props: { title: { type: 'text' }, description: { type: 'text' } }, - }, - noCode: { - type: 'object', - props: { title: { type: 'text' }, description: { type: 'text' } }, - }, - }, - }, - }, - types: { - country: { - props: { - name: { type: 'alias' }, - currency: { - enum: [ - 'all', - 'amd', - 'aud', - 'azn', - 'chf', - 'czk', - 'dkk', - 'eur', - 'gbp', - 'gel', - 'ils', - 'isk', - 'mdl', - 'nok', - 'pln', - 'rsd', - 'sek', - 'uah', - ], - }, - isOverSeasTerritory: { type: 'boolean' }, - prequalifiedFor: { ref: 'round', prop: 'prequalifiedContries' }, - voteType: { enum: ['sms_text', 'sms_suffix', 'online', 'call_suffix'] }, - maxVotes: { type: 'uint8' }, - isTerritoryOf: { ref: 'country', prop: 'overseasTerritories' }, - price: { type: 'uint16' }, - destination: { type: 'string' }, - votingText: { type: 'text' }, - votingLegal: { type: 'text' }, - overseasTerritories: { - items: { ref: 'country', prop: 'isTerritoryOf' }, - }, - votingRounds: { - items: { ref: 'round', prop: 'canVote' }, - }, - contestants: { - items: { ref: 'contestant', prop: 'country' }, - }, - }, - }, - sequence: { - props: { - name: { type: 'alias' }, - displayName: { type: 'string' }, - goLiveTime: { type: 'timestamp' }, - recapVideo: { ref: 'file', prop: 'sequenceRecapVideo' }, - recapTitle: { type: 'text' }, - title: { type: 'text' }, - description: { type: 'text' }, - countdown: { type: 'timestamp' }, - round: { ref: 'round', prop: 'sequences' }, - row: { - props: { - title: { type: 'text' }, - description: { type: 'text' }, - countdown: { type: 'timestamp' }, - }, - }, - }, - }, - round: { - props: { - name: { type: 'alias' }, - displayName: { type: 'string' }, - contestants: { items: { ref: 'contestant', prop: 'rounds' } }, - canVote: { items: { ref: 'country', prop: 'votingRounds' } }, - prequalifiedContries: { - items: { ref: 'country', prop: 'prequalifiedFor' }, - }, - votes: { items: { ref: 'vote', prop: 'round' } }, - createdBy: { ref: 'user', prop: 'createdRounds' }, - sequences: { - items: { ref: 'sequence', prop: 'round' }, - }, - }, - }, - contestant: { - props: { - name: { type: 'string' }, - ddi: { type: 'alias' }, - song: { type: 'string' }, - lyrics: { type: 'string' }, - country: { ref: 'country', prop: 'contestants' }, - image: { ref: 'file', prop: 'contestant_image' }, - video: { ref: 'file', prop: 'contestant_video' }, - thankYouVideos: { items: { ref: 'file', prop: 'contestant_ty_video' } }, - rounds: { - items: { ref: 'round', prop: 'contestants' }, - }, - }, - }, - user: { - props: { - name: { type: 'string' }, - email: { type: 'alias', format: 'email' }, - role: { enum: ['admin', 'translator', 'viewer'] }, - translatorOf: { items: { ref: 'language', prop: 'translators' } }, - avatar: { ref: 'file', prop: 'avatarOf' }, - currentToken: { type: 'alias' }, - status: { enum: ['login', 'clear', 'invited'] }, - createdAt: { type: 'timestamp', on: 'create' }, - updatedAt: { type: 'timestamp', on: 'update' }, - invited: { items: { ref: 'user', prop: 'invitedBy' } }, - invitedBy: { ref: 'user', prop: 'invited' }, - inactive: { type: 'boolean' }, - createdRounds: { - items: { ref: 'round', prop: 'createdBy' }, - }, - }, - }, - file: { - props: { - name: { type: 'string' }, - src: { type: 'string', format: 'URL' }, - mimeType: { type: 'string' }, - description: { type: 'string' }, - progress: { type: 'number', display: 'ratio' }, - status: { enum: ['uploading', 'transcoding', 'ready', 'error'] }, - statusText: { type: 'string' }, - size: { type: 'int32' }, - createdAt: { type: 'timestamp', on: 'create' }, - updatedAt: { type: 'timestamp', on: 'update' }, - thumbnail: { type: 'string', format: 'URL' }, - hls: { type: 'string', format: 'URL' }, - dash: { type: 'string', format: 'URL' }, - videoPreview: { type: 'string', format: 'URL' }, - selvaId: { type: 'alias' }, - sequenceRecapVideo: { - items: { ref: 'sequence', prop: 'recapVideo' }, - }, - contestant_image: { - items: { ref: 'contestant', prop: 'image' }, - }, - contestant_video: { - items: { ref: 'contestant', prop: 'video' }, - }, - contestant_ty_video: { - items: { ref: 'contestant', prop: 'thankYouVideos' }, - }, - items: { ref: 'user', prop: 'avatar' }, - }, - }, - }, - language: { - props: { - iso: { type: 'alias', required: true }, - name: { type: 'string' }, - completionStatus: { type: 'number', min: 0, max: 1 }, - translators: { items: { ref: 'user', prop: 'translatorOf' } }, - }, - }, - payment: { - props: { - confirmationTokenId: { type: 'string' }, - intentId: { type: 'alias' }, - fingerprint: { type: 'alias' }, - amount: { type: 'number' }, - currency: { type: 'string' }, - status: { - enum: [ - 'requested', - 'ready_for_payment', - 'intent_success', - 'intent_failed', - 'webhook_success', - 'webhook_failed', - 'capture_captured', - 'capture_canceled', - ], - }, - bankCountry: { type: 'string' }, - failReason: { type: 'string' }, - vote: { ref: 'vote', prop: 'payment' }, - createdAt: { type: 'timestamp', on: 'create' }, - updatedAt: { type: 'timestamp', on: 'update' }, - }, - }, - vote: { - props: { - fingerprintAndRound: { type: 'alias', required: true }, - payment: { ref: 'payment', prop: 'vote', required: true }, - round: { ref: 'round', prop: 'votes', required: true }, - ddi: { - props: { - ddi1: { type: 'uint8' }, - ddi2: { type: 'uint8' }, - ddi3: { type: 'uint8' }, - ddi4: { type: 'uint8' }, - ddi5: { type: 'uint8' }, - ddi6: { type: 'uint8' }, - ddi7: { type: 'uint8' }, - ddi8: { type: 'uint8' }, - ddi9: { type: 'uint8' }, - ddi10: { type: 'uint8' }, - ddi11: { type: 'uint8' }, - ddi12: { type: 'uint8' }, - ddi13: { type: 'uint8' }, - ddi14: { type: 'uint8' }, - ddi15: { type: 'uint8' }, - ddi16: { type: 'uint8' }, - ddi17: { type: 'uint8' }, - ddi18: { type: 'uint8' }, - ddi19: { type: 'uint8' }, - ddi20: { type: 'uint8' }, - }, - }, - }, - }, -} diff --git a/packages/schema/test/serialize.ts b/packages/schema/test/serialize.ts deleted file mode 100644 index dfc73c232e..0000000000 --- a/packages/schema/test/serialize.ts +++ /dev/null @@ -1,225 +0,0 @@ -import test from 'node:test' -import { ok } from 'node:assert' -import { serialize, deSerialize, StrictSchema, parse } from '../src/index.js' -import eurovisionSchema from './schema/based.schema.js' -import { deflateSync } from 'node:zlib' - -function deepEqual(a: any, b: any): boolean { - if (a === b) return true - if (a === null || a === undefined || b === null || b === undefined) { - return a === b - } - if (a.constructor !== b.constructor) return false - if (a instanceof Uint8Array && b instanceof Uint8Array) { - if (a.length !== b.length) return false - for (let i = 0; i < a.length; i++) { - if (a[i] !== b[i]) return false - } - return true - } - if (Array.isArray(a)) { - if (a.length !== b.length) return false - for (let i = 0; i < a.length; i++) { - if (!deepEqual(a[i], b[i])) return false - } - return true - } - if (typeof a === 'object') { - const keysA = Object.keys(a).filter((key) => typeof a[key] !== 'function') - const keysB = new Set( - Object.keys(b).filter((key) => typeof b[key] !== 'function'), - ) - if (keysA.length !== keysB.size) return false - for (const key of keysA) { - if (!keysB.has(key) || !deepEqual(a[key], b[key])) { - return false - } - } - return true - } - return false -} - -test('serialize and deserialize basic schema', () => { - const basicSchema: StrictSchema = { - locales: { - en: { required: true }, - nl: {}, - }, - types: { - thing: { - props: { - name: { type: 'string', default: 'thingy' }, - nested: { - type: 'object', - props: { - field: { type: 'number' }, - }, - }, - ref: { - ref: 'other', - prop: 'backref', - }, - validationFn: { - type: 'string', - validation: (v) => v.startsWith('valid'), - }, - }, - }, - other: { - props: { - backref: { - items: { - ref: 'thing', - prop: 'ref', - }, - }, - }, - }, - }, - } - - const serialized = serialize(basicSchema) - const deserialized = deSerialize(serialized) - - ok(deepEqual(basicSchema, deserialized), 'Basic schema did not match') -}) - -test('serialize and deserialize complex (Eurovision) schema', () => { - const serialized = serialize(eurovisionSchema) - const deserialized = deSerialize(serialized) - - ok( - deepEqual(eurovisionSchema, deserialized), - 'Eurovision schema did not match after roundtrip', - ) - - // console.log( - // serialized.byteLength, - // deflateSync(JSON.stringify(eurovisionSchema)).byteLength, - // ) -}) - -test('serialize with readOnly option strips validation and defaults', () => { - const schema: StrictSchema = { - types: { - thing: { - props: { - name: { type: 'string', default: 'thingy' }, - age: { type: 'number', validation: (v) => v > 18 }, - }, - }, - }, - } - - const serialized = serialize(schema, { readOnly: true }) - const deserialized = deSerialize(serialized) - - const expected: StrictSchema = { - types: { - thing: { - props: { - name: { type: 'string' }, // default removed - age: { type: 'number' }, // validation removed - }, - }, - }, - } - ok( - deepEqual(deserialized, expected), - 'readOnly option did not strip fields correctly', - ) -}) - -// optimize this with an extra map -// keep serialized schema in MEM -test('big schema', () => { - const makeALot = (n: number) => { - const props: any = {} - for (let i = 0; i < n; i++) { - props[`f${i}`] = { type: 'int32' } - } - return props - } - - const basicSchema: StrictSchema = { - locales: { - en: { required: true }, - nl: {}, - }, - types: { - thing: { - props: { - ...makeALot(16000), - }, - }, - }, - } - - const serialized = serialize(basicSchema) - const deserialized = deSerialize(serialized) - - ok(deepEqual(basicSchema, deserialized), 'Big schema did not match') -}) - -test('Simple shared prop', () => { - const basicSchema: StrictSchema = parse({ - types: { - article: { - props: { - body: { type: 'string' }, - }, - }, - italy: { - props: { - body: { type: 'string' }, - }, - }, - }, - }).schema - - const serialized = serialize(basicSchema) - const deserialized = deSerialize(serialized) - ok(deepEqual(basicSchema, deserialized), 'Mismatch') -}) - -test('empty schema', () => { - const serialized = serialize({}) - const deserialized = deSerialize(serialized) - ok(deepEqual({}, deserialized), 'Mismatch') -}) - -test('schema with 1 unit8array', () => { - const x = { x: new Uint8Array([1, 2, 3]) } - const serialized = serialize(x) - const deserialized = deSerialize(serialized) - ok(deepEqual(x, deserialized), 'Mismatch') -}) - -test('schema with 1 unit8array array', () => { - const x = { - x: [{ x: new Uint8Array([1, 2, 3]) }, { x: new Uint8Array([1, 2, 3]) }], - } - const serialized = serialize(x) - const deserialized = deSerialize(serialized) - ok(deepEqual(x, deserialized), 'Mismatch') -}) - -test('empty uint8Array', () => { - const deserialized = deSerialize(new Uint8Array()) - ok(deepEqual({}, deserialized), 'Mismatch') -}) - -test('schema with hash', () => { - const serialized = serialize({ hash: 14986952164472 }) - const deserialized = deSerialize(serialized) - ok(deepEqual({ hash: 14986952164472 }, deserialized), 'Mismatch') -}) - -// make something like serialize for payloads in the server -test('serialize random object', () => { - const obj = { bla: [1, 23, 2, 12, { x: 1 }] } - const serialized = serialize(obj) - const deserialized = deSerialize(serialized) - ok(deepEqual(obj, deserialized), 'Mismatch') -}) diff --git a/packages/schema/test/shorthands.ts b/packages/schema/test/shorthands.ts deleted file mode 100644 index b199346244..0000000000 --- a/packages/schema/test/shorthands.ts +++ /dev/null @@ -1,102 +0,0 @@ -import test from 'node:test' -import { deepEqual } from 'node:assert' -import { parse } from '@based/schema' - -await test('shorthands', () => { - deepEqual( - parse({ - locales: { en: {} }, - types: { - hello: { - props: { - myText: 'text', - myString: 'string', - myNumber: 'number', - myEnum: [1, 2, 3], - }, - }, - }, - props: { - myText: 'text', - }, - }).schema, - { - locales: { en: {} }, - types: { - hello: { - props: { - myText: { type: 'text' }, - myString: { type: 'string' }, - myNumber: { type: 'number' }, - myEnum: { enum: [1, 2, 3] }, - }, - }, - }, - props: { - myText: { type: 'text' }, - }, - }, - ) - - deepEqual( - parse({ - locales: { en: {} }, - types: { - article: { - header: 'text', - body: 'string', - views: 'number', - }, - }, - }).schema, - { - locales: { en: {} }, - types: { - article: { - props: { - header: { - type: 'text', - }, - body: { - type: 'string', - }, - views: { - type: 'number', - }, - }, - }, - }, - }, - ) - - deepEqual( - parse({ - locales: { en: {} }, - types: { - article: { - header: 'text', - body: 'string', - views: 'number', - }, - }, - }).schema, - { - locales: { en: {} }, - types: { - article: { - props: { - header: { - type: 'text', - }, - body: { - type: 'string', - }, - views: { - type: 'number', - }, - }, - }, - }, - }, - ) -}) diff --git a/packages/schema/test/string.ts b/packages/schema/test/string.ts deleted file mode 100644 index 28eb9b5224..0000000000 --- a/packages/schema/test/string.ts +++ /dev/null @@ -1,24 +0,0 @@ -import test from 'node:test' -import { throws } from 'node:assert' -import { parse } from '@based/schema' - -await test('string', () => { - parse({ - props: { - myString: { - type: 'string', - }, - }, - }) - - throws(() => { - parse({ - props: { - // @ts-expect-error - myEnum: { - enum: [{ invalidObj: true }], - }, - }, - }) - }, 'should throw with non primitive enum') -}) diff --git a/packages/schema/test/text.ts b/packages/schema/test/text.ts deleted file mode 100644 index ff5357b0ee..0000000000 --- a/packages/schema/test/text.ts +++ /dev/null @@ -1,42 +0,0 @@ -import test from 'node:test' -import { throws } from 'node:assert' -import { parse } from '@based/schema' - -await test('text', () => { - parse({ - locales: { - en: { - required: true, - }, - de: {}, - nl: { - fallback: 'en', - }, - }, - props: { - myText: { - type: 'text', - }, - }, - }) - - throws(() => { - parse({ - props: { - myText: { - type: 'text', - }, - }, - }) - }, 'type text requires locales to be defined') - - throws(() => { - parse({ - types: { - product: { - description: 'text', - }, - }, - }) - }, 'type text requires locales to be defined 2') -}) diff --git a/packages/schema/test/timestamp.ts b/packages/schema/test/timestamp.ts deleted file mode 100644 index 834aff715e..0000000000 --- a/packages/schema/test/timestamp.ts +++ /dev/null @@ -1,22 +0,0 @@ -import test from 'node:test' -import { parse } from '@based/schema' - -await test('timestamp', () => { - parse({ - props: { - myTimestamp: { - type: 'timestamp', - }, - created: { - type: 'timestamp', - on: 'create', - }, - lastModified: { - type: 'timestamp', - on: 'update', - }, - }, - }) -}) - -// ADD STRING OPTION HERE diff --git a/packages/schema/test/types.ts b/packages/schema/test/types.ts deleted file mode 100644 index 179f09aae4..0000000000 --- a/packages/schema/test/types.ts +++ /dev/null @@ -1,32 +0,0 @@ -import test from 'node:test' -import { SchemaProps, SchemaType } from '@based/schema' - -await test('types', () => { - // const notificationSubjects: string[] = [ - // 'flight', - // 'incident', - // 'workspace', - // 'team', - // 'training', - // 'drone', - // 'groundStation', - // 'equipment', - // 'battery', - // ] - // const notification: SchemaProps = { - // subjectType: notificationSubjects, - // ...subjectRelations(), - // } - // function subjectRelations(): SchemaProps { - // return notificationSubjects.reduce>( - // (schema: SchemaType, subjectType) => ({ - // ...schema, - // [subjectType]: { - // ref: subjectType, - // prop: 'notifications', - // }, - // }), - // {} as SchemaProps, - // ) as SchemaProps - // } -}) diff --git a/packages/schema/test/validate.ts b/packages/schema/test/validate.ts deleted file mode 100644 index 4b1a03188b..0000000000 --- a/packages/schema/test/validate.ts +++ /dev/null @@ -1,259 +0,0 @@ -import test from 'node:test' -import { deepEqual, equal } from 'node:assert' -import { parse } from '@based/schema' -import { validate } from '../src/index.js' - -await test('validate - basic', () => { - const { schema } = parse({ - types: { - user: { - name: 'string', - email: { type: 'string', format: 'email' }, - age: { - type: 'uint8', - min: 18, - max: 120, - }, - address: { - props: { - street: { - type: 'string', - required: true, - }, - }, - }, - badValidator: { - type: 'number', - validation: () => true, - }, - }, - }, - }) - - equal( - validate(schema, 'user', { - name: 'youri', - address: { - street: 'downtown', - }, - }).valid, - true, - ) - - equal( - validate(schema, 'user', { - name: 1, - }).valid, - false, - ) - - equal( - validate(schema, 'user', { - badValidator: 'snurk', - }).valid, - false, - 'Also checks default validation', - ) -}) - -await test('validate - messages', () => { - const { schema } = parse({ - types: { - user: { - name: 'string', - email: { - type: 'string', - format: 'email', - }, - age: { - type: 'uint8', - min: 18, - max: 120, - validation: (val) => - (18 <= val && val <= 120) || 'Expected age between 18 and 120', - }, - }, - }, - }) - - deepEqual( - validate(schema, 'user', { - name: 'youri', - age: 17, - }), - { - valid: false, - errors: [ - { - path: ['age'], - value: 17, - error: 'Expected age between 18 and 120', - }, - ], - }, - ) - - deepEqual( - validate(schema, 'user', { - name: 'youri', - age: 17, - email: 'not-an-email', - }), - { - errors: [ - { - error: 'Invalid value', - path: ['email'], - value: 'not-an-email', - }, - { - error: 'Expected age between 18 and 120', - path: ['age'], - value: 17, - }, - ], - valid: false, - }, - ) -}) - -await test('validate - reference', () => { - const { schema } = parse({ - types: { - user: { - name: 'string', - bff: { - ref: 'user', - prop: 'bff', - }, - }, - }, - }) - - deepEqual( - validate(schema, 'user', { - name: 'youri', - bff: 1, - }), - { - valid: true, - errors: [], - }, - ) - - deepEqual( - validate(schema, 'user', { - name: 'youri', - bff: { id: 1 }, - }), - { - valid: true, - errors: [], - }, - ) -}) - -await test('validate - references', () => { - const { schema } = parse({ - types: { - user: { - name: 'string', - friends: { - items: { - ref: 'user', - prop: 'friends', - }, - }, - }, - }, - }) - - deepEqual( - validate(schema, 'user', { - name: 'youri', - friends: [1], - }), - { - valid: true, - errors: [], - }, - ) - - deepEqual( - validate(schema, 'user', { - name: 'youri', - friends: [{ id: 1 }], - }), - { - valid: true, - errors: [], - }, - ) - - deepEqual( - validate(schema, 'user', { - name: 'youri', - friends: { - add: [{ id: 1 }], - }, - }), - { - valid: true, - errors: [], - }, - ) - - deepEqual( - validate(schema, 'user', { - name: 'youri', - friends: { - update: [{ id: 1 }], - }, - }), - { - valid: true, - errors: [], - }, - ) - - deepEqual( - validate(schema, 'user', { - name: 'youri', - friends: { - delete: [{ id: 1 }], - }, - }), - { - valid: true, - errors: [], - }, - ) - - deepEqual( - validate(schema, 'user', { - name: 'youri', - friends: { - add: [{ id: 1 }], - update: [{ id: 1 }], - delete: [{ id: 1 }], - }, - }), - { - valid: true, - errors: [], - }, - ) - - deepEqual( - validate(schema, 'user', { - name: 'youri', - friends: { - add: [{ id: 1 }], - update: [{ id: 1 }], - delete: [{ id: 1 }], - onknown: [], - }, - }).valid, - false, - ) -}) diff --git a/packages/schema/tsconfig.json b/packages/schema/tsconfig.json index 385cecc794..3e832e8368 100644 --- a/packages/schema/tsconfig.json +++ b/packages/schema/tsconfig.json @@ -5,8 +5,8 @@ "outDir": "dist", "rootDir": "src", "declarationMap": true, - "esModuleInterop": true + "esModuleInterop": true, + "strictNullChecks": true }, - "include": ["src"], - "exclude": ["dist"] + "include": ["src/**/*.ts"] } diff --git a/packages/server/src/incoming/http/handleRequest.ts b/packages/server/src/incoming/http/handleRequest.ts index c952c02f6c..b9e1808282 100644 --- a/packages/server/src/incoming/http/handleRequest.ts +++ b/packages/server/src/incoming/http/handleRequest.ts @@ -2,7 +2,6 @@ import { BasedServer } from '../../server.js' import { HttpSession, Context, BasedRouteComplete } from '@based/functions' import { readBody } from './readBody.js' import payloadParser from './payloadParser.js' -import { SchemaType } from '../../../../schema/dist/types.js' export const handleRequest = ( server: BasedServer, diff --git a/packages/types/README.md b/packages/types/README.md new file mode 100644 index 0000000000..cebefa777b --- /dev/null +++ b/packages/types/README.md @@ -0,0 +1 @@ +# @based/types diff --git a/packages/types/index.d.ts b/packages/types/index.d.ts new file mode 100644 index 0000000000..da4faed198 --- /dev/null +++ b/packages/types/index.d.ts @@ -0,0 +1 @@ +export { BasedDbQuery } from '@based/db' diff --git a/packages/types/package.json b/packages/types/package.json new file mode 100644 index 0000000000..8b7f44bf6a --- /dev/null +++ b/packages/types/package.json @@ -0,0 +1,9 @@ +{ + "name": "@based/types", + "version": "1.0.0", + "type": "module", + "main": "index.d.ts", + "files": [ + "index.d.ts" + ] +} diff --git a/packages/types/tsconfig.json b/packages/types/tsconfig.json new file mode 100644 index 0000000000..716ceeac12 --- /dev/null +++ b/packages/types/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "@saulx/tsconfig/default.json", + "compilerOptions": { + "composite": true, + "outDir": "dist", + "rootDir": "src", + "esModuleInterop": true, + "strictNullChecks": true + } +} diff --git a/packages/utils/src/timestamp.ts b/packages/utils/src/timestamp.ts index 2273355232..d3021a20fe 100644 --- a/packages/utils/src/timestamp.ts +++ b/packages/utils/src/timestamp.ts @@ -17,7 +17,7 @@ const timeToNumber = (ex: string): number => { return 1 } -export const convertToTimestamp = (value: string | Date | number) => { +export const convertToTimestamp = (value: string | Date | number): number => { if (value instanceof Date) { return value.valueOf() } diff --git a/tsconfig.json b/tsconfig.json index d08d6fb027..871f6b0308 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -5,5 +5,9 @@ "strict": true, "noEmit": true }, - "include": ["packages/*/src", "packages/*/test"] + "include": [ + "packages/*/src", + "packages/*/test", + "packages/schema/test/index.ts" + ] }