From a8c20d36759f494506c3614fe1d35c7b13a40927 Mon Sep 17 00:00:00 2001 From: dswbx Date: Fri, 10 Jan 2025 14:43:39 +0100 Subject: [PATCH] Refactor module schema handling and add sync mechanism Redesigned entity and index management with methods to streamline schema updates and added a sync flag to signal required DB syncs post-build. Enhanced test coverage and functionality for schema modifications, including support for additional fields. --- app/__test__/Module.spec.ts | 38 ---- app/__test__/modules/AppAuth.spec.ts | 30 +++ app/__test__/modules/AppMedia.spec.ts | 48 ++++- app/__test__/modules/Module.spec.ts | 200 ++++++++++++++++++ .../{ => modules}/ModuleManager.spec.ts | 10 +- app/src/auth/AppAuth.ts | 43 ++-- app/src/data/prototype/index.ts | 16 +- app/src/media/AppMedia.ts | 41 ++-- app/src/modules/Module.ts | 47 +++- app/src/modules/ModuleManager.ts | 15 +- app/vite.dev.ts | 34 ++- 11 files changed, 413 insertions(+), 109 deletions(-) delete mode 100644 app/__test__/Module.spec.ts create mode 100644 app/__test__/modules/Module.spec.ts rename app/__test__/{ => modules}/ModuleManager.spec.ts (96%) diff --git a/app/__test__/Module.spec.ts b/app/__test__/Module.spec.ts deleted file mode 100644 index 4089ab1..0000000 --- a/app/__test__/Module.spec.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { type TSchema, Type, stripMark } from "../src/core/utils"; -import { Module } from "../src/modules/Module"; - -function createModule(schema: Schema) { - class TestModule extends Module { - getSchema() { - return schema; - } - toJSON() { - return this.config; - } - useForceParse() { - return true; - } - } - - return TestModule; -} - -describe("Module", async () => { - test("basic", async () => {}); - - test("listener", async () => { - let result: any; - - const module = createModule(Type.Object({ a: Type.String() })); - const m = new module({ a: "test" }); - - await m.schema().set({ a: "test2" }); - m.setListener(async (c) => { - await new Promise((r) => setTimeout(r, 10)); - result = stripMark(c); - }); - await m.schema().set({ a: "test3" }); - expect(result).toEqual({ a: "test3" }); - }); -}); diff --git a/app/__test__/modules/AppAuth.spec.ts b/app/__test__/modules/AppAuth.spec.ts index d8866c7..849d8dd 100644 --- a/app/__test__/modules/AppAuth.spec.ts +++ b/app/__test__/modules/AppAuth.spec.ts @@ -1,6 +1,7 @@ import { afterAll, beforeAll, beforeEach, describe, expect, spyOn, test } from "bun:test"; import { createApp } from "../../src"; import { AuthController } from "../../src/auth/api/AuthController"; +import { em, entity, text } from "../../src/data"; import { AppAuth, type ModuleBuildContext } from "../../src/modules"; import { disableConsoleLog, enableConsoleLog } from "../helper"; import { makeCtx, moduleTestSuite } from "./module-test-suite"; @@ -102,4 +103,33 @@ describe("AppAuth", () => { expect(spy.mock.calls.length).toBe(2); }); + + test("should allow additional user fields", async () => { + const app = createApp({ + initialConfig: { + auth: { + entity_name: "users", + enabled: true + }, + data: em({ + users: entity("users", { + additional: text() + }) + }).toJSON() + } + }); + + await app.build(); + + const userfields = app.modules.em.entity("users").fields.map((f) => f.name); + expect(userfields).toContain("additional"); + expect(userfields).toEqual([ + "id", + "additional", + "email", + "strategy", + "strategy_value", + "role" + ]); + }); }); diff --git a/app/__test__/modules/AppMedia.spec.ts b/app/__test__/modules/AppMedia.spec.ts index 6f1b0f5..c1fc2f9 100644 --- a/app/__test__/modules/AppMedia.spec.ts +++ b/app/__test__/modules/AppMedia.spec.ts @@ -1,7 +1,53 @@ -import { describe } from "bun:test"; +import { describe, expect, test } from "bun:test"; +import { createApp, registries } from "../../src"; +import { em, entity, text } from "../../src/data"; +import { StorageLocalAdapter } from "../../src/media/storage/adapters/StorageLocalAdapter"; import { AppMedia } from "../../src/modules"; import { moduleTestSuite } from "./module-test-suite"; describe("AppMedia", () => { moduleTestSuite(AppMedia); + + test("should allow additional fields", async () => { + registries.media.register("local", StorageLocalAdapter); + + const app = createApp({ + initialConfig: { + media: { + entity_name: "media", + enabled: true, + adapter: { + type: "local", + config: { + path: "./" + } + } + }, + data: em({ + media: entity("media", { + additional: text() + }) + }).toJSON() + } + }); + + await app.build(); + + const fields = app.modules.em.entity("media").fields.map((f) => f.name); + expect(fields).toContain("additional"); + expect(fields).toEqual([ + "id", + "additional", + "path", + "folder", + "mime_type", + "size", + "scope", + "etag", + "modified_at", + "reference", + "entity_id", + "metadata" + ]); + }); }); diff --git a/app/__test__/modules/Module.spec.ts b/app/__test__/modules/Module.spec.ts new file mode 100644 index 0000000..b53dcfa --- /dev/null +++ b/app/__test__/modules/Module.spec.ts @@ -0,0 +1,200 @@ +import { describe, expect, test } from "bun:test"; +import { type TSchema, Type, stripMark } from "../../src/core/utils"; +import { EntityManager, em, entity, index, text } from "../../src/data"; +import { DummyConnection } from "../../src/data/connection/DummyConnection"; +import { Module } from "../../src/modules/Module"; + +function createModule(schema: Schema) { + class TestModule extends Module { + getSchema() { + return schema; + } + toJSON() { + return this.config; + } + useForceParse() { + return true; + } + } + + return TestModule; +} + +describe("Module", async () => { + describe("basic", () => { + test("listener", async () => { + let result: any; + + const module = createModule(Type.Object({ a: Type.String() })); + const m = new module({ a: "test" }); + + await m.schema().set({ a: "test2" }); + m.setListener(async (c) => { + await new Promise((r) => setTimeout(r, 10)); + result = stripMark(c); + }); + await m.schema().set({ a: "test3" }); + expect(result).toEqual({ a: "test3" }); + }); + }); + + describe("db schema", () => { + class M extends Module { + override getSchema() { + return Type.Object({}); + } + + prt = { + ensureEntity: this.ensureEntity.bind(this), + ensureIndex: this.ensureIndex.bind(this), + ensureSchema: this.ensureSchema.bind(this) + }; + + get em() { + return this.ctx.em; + } + } + + function make(_em: ReturnType) { + const em = new EntityManager( + Object.values(_em.entities), + new DummyConnection(), + _em.relations, + _em.indices + ); + return new M({} as any, { em, flags: Module.ctx_flags } as any); + } + function flat(_em: EntityManager) { + return { + entities: _em.entities.map((e) => ({ + name: e.name, + fields: e.fields.map((f) => f.name) + })), + indices: _em.indices.map((i) => ({ + name: i.name, + entity: i.entity.name, + fields: i.fields.map((f) => f.name), + unique: i.unique + })) + }; + } + + test("no change", () => { + const initial = em({}); + + const m = make(initial); + expect(m.ctx.flags.sync_required).toBe(false); + + expect(flat(make(initial).em)).toEqual({ + entities: [], + indices: [] + }); + }); + + test("init", () => { + const initial = em({ + users: entity("u", { + name: text() + }) + }); + + const m = make(initial); + expect(m.ctx.flags.sync_required).toBe(false); + + expect(flat(m.em)).toEqual({ + entities: [ + { + name: "u", + fields: ["id", "name"] + } + ], + indices: [] + }); + }); + + test("ensure entity", () => { + const initial = em({ + users: entity("u", { + name: text() + }) + }); + + const m = make(initial); + expect(flat(m.em)).toEqual({ + entities: [ + { + name: "u", + fields: ["id", "name"] + } + ], + indices: [] + }); + + // this should add a new entity + m.prt.ensureEntity( + entity("p", { + title: text() + }) + ); + + // this should only add the field "important" + m.prt.ensureEntity( + entity("u", { + important: text() + }) + ); + + expect(m.ctx.flags.sync_required).toBe(true); + expect(flat(m.em)).toEqual({ + entities: [ + { + name: "u", + fields: ["id", "name", "important"] + }, + { + name: "p", + fields: ["id", "title"] + } + ], + indices: [] + }); + }); + + test("ensure index", () => { + const users = entity("u", { + name: text(), + title: text() + }); + const initial = em({ users }, ({ index }, { users }) => { + index(users).on(["title"]); + }); + + const m = make(initial); + m.prt.ensureIndex(index(users).on(["name"])); + + expect(m.ctx.flags.sync_required).toBe(true); + expect(flat(m.em)).toEqual({ + entities: [ + { + name: "u", + fields: ["id", "name", "title"] + } + ], + indices: [ + { + name: "idx_u_title", + entity: "u", + fields: ["title"], + unique: false + }, + { + name: "idx_u_name", + entity: "u", + fields: ["name"], + unique: false + } + ] + }); + }); + }); +}); diff --git a/app/__test__/ModuleManager.spec.ts b/app/__test__/modules/ModuleManager.spec.ts similarity index 96% rename from app/__test__/ModuleManager.spec.ts rename to app/__test__/modules/ModuleManager.spec.ts index 2e928d6..e22afff 100644 --- a/app/__test__/ModuleManager.spec.ts +++ b/app/__test__/modules/ModuleManager.spec.ts @@ -1,9 +1,9 @@ import { describe, expect, test } from "bun:test"; -import { mark, stripMark } from "../src/core/utils"; -import { entity, text } from "../src/data"; -import { ModuleManager, getDefaultConfig } from "../src/modules/ModuleManager"; -import { CURRENT_VERSION, TABLE_NAME } from "../src/modules/migrations"; -import { getDummyConnection } from "./helper"; +import { stripMark } from "../../src/core/utils"; +import { entity, text } from "../../src/data"; +import { ModuleManager, getDefaultConfig } from "../../src/modules/ModuleManager"; +import { CURRENT_VERSION, TABLE_NAME } from "../../src/modules/migrations"; +import { getDummyConnection } from "../helper"; describe("ModuleManager", async () => { test("s1: no config, no build", async () => { diff --git a/app/src/auth/AppAuth.ts b/app/src/auth/AppAuth.ts index a65b251..8899495 100644 --- a/app/src/auth/AppAuth.ts +++ b/app/src/auth/AppAuth.ts @@ -4,7 +4,7 @@ import { auth } from "auth/middlewares"; import { type DB, Exception, type PrimaryFieldType } from "core"; import { type Static, secureRandomString, transformObject } from "core/utils"; import { type Entity, EntityIndex, type EntityManager } from "data"; -import { type FieldSchema, entity, enumm, make, text } from "data/prototype"; +import { type FieldSchema, em, entity, enumm, make, text } from "data/prototype"; import type { Hono } from "hono"; import { pick } from "lodash-es"; import { Module } from "modules/Module"; @@ -250,43 +250,30 @@ export class AppAuth extends Module { }; registerEntities() { - const users = this.getUsersEntity(); - - if (!this.em.hasEntity(users.name)) { - this.em.addEntity(users); - } else { - // if exists, check all fields required are there - // @todo: add to context: "needs sync" flag - const _entity = this.getUsersEntity(true); - for (const field of _entity.fields) { - const _field = users.field(field.name); - if (!_field) { - users.addField(field); + const name = this.config.entity_name as "users"; + const { + entities: { users } + } = this.ensureSchema( + em( + { + [name]: entity(name, AppAuth.usersFields) + }, + ({ index }, { users }) => { + index(users).on(["email"], true).on(["strategy"]).on(["strategy_value"]); } - } - } - - const indices = [ - new EntityIndex(users, [users.field("email")!], true), - new EntityIndex(users, [users.field("strategy")!]), - new EntityIndex(users, [users.field("strategy_value")!]) - ]; - indices.forEach((index) => { - if (!this.em.hasIndex(index)) { - this.em.addIndex(index); - } - }); + ) + ); try { const roles = Object.keys(this.config.roles ?? {}); const field = make("role", enumm({ enum: roles })); - this.em.entity(users.name).__experimental_replaceField("role", field); + users.__experimental_replaceField("role", field); } catch (e) {} try { const strategies = Object.keys(this.config.strategies ?? {}); const field = make("strategy", enumm({ enum: strategies })); - this.em.entity(users.name).__experimental_replaceField("strategy", field); + users.__experimental_replaceField("strategy", field); } catch (e) {} } diff --git a/app/src/data/prototype/index.ts b/app/src/data/prototype/index.ts index e9e868f..6a05f72 100644 --- a/app/src/data/prototype/index.ts +++ b/app/src/data/prototype/index.ts @@ -272,18 +272,22 @@ class EntityManagerPrototype> extends En } } -type Chained any, Rt = ReturnType> = ( - e: E -) => { - [K in keyof Rt]: Rt[K] extends (...args: any[]) => any - ? (...args: Parameters) => Rt +type Chained any>> = { + [K in keyof R]: R[K] extends (...args: any[]) => any + ? (...args: Parameters) => Chained : never; }; +type ChainedFn< + Fn extends (...args: any[]) => Record any>, + Return extends ReturnType = ReturnType +> = (e: Entity) => { + [K in keyof Return]: (...args: Parameters) => Chained; +}; export function em>( entities: Entities, schema?: ( - fns: { relation: Chained; index: Chained }, + fns: { relation: ChainedFn; index: ChainedFn }, entities: Entities ) => void ) { diff --git a/app/src/media/AppMedia.ts b/app/src/media/AppMedia.ts index 789dae9..97f0a9b 100644 --- a/app/src/media/AppMedia.ts +++ b/app/src/media/AppMedia.ts @@ -1,8 +1,17 @@ import type { PrimaryFieldType } from "core"; -import { EntityIndex, type EntityManager } from "data"; +import { type Entity, EntityIndex, type EntityManager } from "data"; import { type FileUploadedEventData, Storage, type StorageAdapter } from "media"; import { Module } from "modules/Module"; -import { type FieldSchema, boolean, datetime, entity, json, number, text } from "../data/prototype"; +import { + type FieldSchema, + boolean, + datetime, + em, + entity, + json, + number, + text +} from "../data/prototype"; import { MediaController } from "./api/MediaController"; import { ADAPTERS, buildMediaSchema, type mediaConfigSchema, registry } from "./media-schema"; @@ -17,6 +26,7 @@ export class AppMedia extends Module { private _storage?: Storage; override async build() { + console.log("building"); if (!this.config.enabled) { this.setBuilt(); return; @@ -38,18 +48,13 @@ export class AppMedia extends Module { this.setupListeners(); this.ctx.server.route(this.basepath, new MediaController(this).getController()); - // @todo: add check for media entity - const mediaEntity = this.getMediaEntity(); - if (!this.ctx.em.hasEntity(mediaEntity)) { - this.ctx.em.addEntity(mediaEntity); - } - - const pathIndex = new EntityIndex(mediaEntity, [mediaEntity.field("path")!], true); - if (!this.ctx.em.hasIndex(pathIndex)) { - this.ctx.em.addIndex(pathIndex); - } - - // @todo: check indices + const mediaEntity = this.getMediaEntity(true); + const name = mediaEntity.name as "media"; + this.ensureSchema( + em({ [name]: mediaEntity }, ({ index }, { media }) => { + index(media).on(["path"], true).on(["reference"]); + }) + ); } catch (e) { console.error(e); throw new Error( @@ -94,13 +99,13 @@ export class AppMedia extends Module { metadata: json() }; - getMediaEntity() { + getMediaEntity(forceCreate?: boolean): Entity<"media", typeof AppMedia.mediaFields> { const entity_name = this.config.entity_name; - if (!this.em.hasEntity(entity_name)) { - return entity(entity_name, AppMedia.mediaFields, undefined, "system"); + if (forceCreate || !this.em.hasEntity(entity_name)) { + return entity(entity_name as "media", AppMedia.mediaFields, undefined, "system"); } - return this.em.entity(entity_name); + return this.em.entity(entity_name) as any; } get em(): EntityManager { diff --git a/app/src/modules/Module.ts b/app/src/modules/Module.ts index e9e2933..e304c4a 100644 --- a/app/src/modules/Module.ts +++ b/app/src/modules/Module.ts @@ -3,7 +3,7 @@ import type { Guard } from "auth"; import { SchemaObject } from "core"; import type { EventManager } from "core/events"; import type { Static, TSchema } from "core/utils"; -import type { Connection, EntityManager } from "data"; +import type { Connection, Entity, EntityIndex, EntityManager, em as prototypeEm } from "data"; import type { Hono } from "hono"; export type ServerEnv = { @@ -21,6 +21,7 @@ export type ModuleBuildContext = { em: EntityManager; emgr: EventManager; guard: Guard; + flags: (typeof Module)["ctx_flags"]; }; export abstract class Module> { @@ -43,6 +44,13 @@ export abstract class Module { return to; } @@ -129,4 +137,41 @@ export abstract class Module> { return this.config; } + + // @todo: add a method to signal the requirement of database sync!!! + + protected ensureEntity(entity: Entity) { + // check fields + if (!this.ctx.em.hasEntity(entity.name)) { + this.ctx.em.addEntity(entity); + this.ctx.flags.sync_required = true; + return; + } + + const instance = this.ctx.em.entity(entity.name); + + // if exists, check all fields required are there + // @todo: check if the field also equal + for (const field of entity.fields) { + const _field = instance.field(field.name); + if (!_field) { + instance.addField(field); + this.ctx.flags.sync_required = true; + } + } + } + + protected ensureIndex(index: EntityIndex) { + if (!this.ctx.em.hasIndex(index)) { + this.ctx.em.addIndex(index); + this.ctx.flags.sync_required = true; + } + } + + protected ensureSchema>(schema: Schema): Schema { + Object.values(schema.entities ?? {}).forEach(this.ensureEntity.bind(this)); + schema.indices?.forEach(this.ensureIndex.bind(this)); + + return schema; + } } diff --git a/app/src/modules/ModuleManager.ts b/app/src/modules/ModuleManager.ts index 558b3d8..af6da0d 100644 --- a/app/src/modules/ModuleManager.ts +++ b/app/src/modules/ModuleManager.ts @@ -1,4 +1,3 @@ -import type { App } from "App"; import { Guard } from "auth"; import { BkndError, DebugLogger } from "core"; import { EventManager } from "core/events"; @@ -34,7 +33,7 @@ import { AppAuth } from "../auth/AppAuth"; import { AppData } from "../data/AppData"; import { AppFlows } from "../flows/AppFlows"; import { AppMedia } from "../media/AppMedia"; -import type { Module, ModuleBuildContext, ServerEnv } from "./Module"; +import { Module, type ModuleBuildContext, type ServerEnv } from "./Module"; export type { ModuleBuildContext }; @@ -230,7 +229,8 @@ export class ModuleManager { server: this.server, em: this.em, emgr: this.emgr, - guard: this.guard + guard: this.guard, + flags: Module.ctx_flags }; } @@ -415,7 +415,14 @@ export class ModuleManager { } this._built = true; - this.logger.log("modules built"); + this.logger.log("modules built", ctx.flags); + + if (ctx.flags.sync_required) { + this.logger.log("db sync requested"); + await ctx.em.schema().sync({ force: true }); + await this.save(); + ctx.flags.sync_required = false; // reset + } } async build() { diff --git a/app/vite.dev.ts b/app/vite.dev.ts index b48e450..6050997 100644 --- a/app/vite.dev.ts +++ b/app/vite.dev.ts @@ -1,4 +1,10 @@ -import { serve } from "./src/adapter/vite"; +import { serveStatic } from "@hono/node-server/serve-static"; +import { createClient } from "@libsql/client/node"; +import { App, registries } from "./src"; +import { LibsqlConnection } from "./src/data"; +import { StorageLocalAdapter } from "./src/media/storage/adapters/StorageLocalAdapter"; + +registries.media.register("local", StorageLocalAdapter); const credentials = { url: import.meta.env.VITE_DB_URL!, @@ -8,10 +14,22 @@ if (!credentials.url) { throw new Error("Missing VITE_DB_URL env variable. Add it to .env file"); } -export default serve({ - connection: { - type: "libsql", - config: credentials - }, - forceDev: true -}); +const connection = new LibsqlConnection(createClient(credentials)); + +export default { + async fetch(request: Request) { + const app = App.create({ connection }); + + app.emgr.onEvent( + App.Events.AppBuiltEvent, + async () => { + app.registerAdminController({ forceDev: true }); + app.module.server.client.get("/assets/*", serveStatic({ root: "./" })); + }, + "sync" + ); + await app.build(); + + return app.fetch(request); + } +};