Refactor module schema handling and add sync mechanism

Redesigned entity and index management with methods to streamline schema updates and added a sync flag to signal required DB syncs post-build. Enhanced test coverage and functionality for schema modifications, including support for additional fields.

app/__test__/Module.spec.ts

@@ -1,38 +0,0 @@
-import { describe, expect, test } from "bun:test";
-import { type TSchema, Type, stripMark } from "../src/core/utils";
-import { Module } from "../src/modules/Module";
-
-function createModule<Schema extends TSchema>(schema: Schema) {
-   class TestModule extends Module<typeof schema> {
-      getSchema() {
-         return schema;
-      }
-      toJSON() {
-         return this.config;
-      }
-      useForceParse() {
-         return true;
-      }
-   }
-
-   return TestModule;
-}
-
-describe("Module", async () => {
-   test("basic", async () => {});
-
-   test("listener", async () => {
-      let result: any;
-
-      const module = createModule(Type.Object({ a: Type.String() }));
-      const m = new module({ a: "test" });
-
-      await m.schema().set({ a: "test2" });
-      m.setListener(async (c) => {
-         await new Promise((r) => setTimeout(r, 10));
-         result = stripMark(c);
-      });
-      await m.schema().set({ a: "test3" });
-      expect(result).toEqual({ a: "test3" });
-   });
-});

app/__test__/modules/AppAuth.spec.ts

@@ -1,6 +1,7 @@
 import { afterAll, beforeAll, beforeEach, describe, expect, spyOn, test } from "bun:test";
 import { createApp } from "../../src";
 import { AuthController } from "../../src/auth/api/AuthController";
+import { em, entity, text } from "../../src/data";
 import { AppAuth, type ModuleBuildContext } from "../../src/modules";
 import { disableConsoleLog, enableConsoleLog } from "../helper";
 import { makeCtx, moduleTestSuite } from "./module-test-suite";
@@ -102,4 +103,33 @@ describe("AppAuth", () => {
 
       expect(spy.mock.calls.length).toBe(2);
    });
+
+   test("should allow additional user fields", async () => {
+      const app = createApp({
+         initialConfig: {
+            auth: {
+               entity_name: "users",
+               enabled: true
+            },
+            data: em({
+               users: entity("users", {
+                  additional: text()
+               })
+            }).toJSON()
+         }
+      });
+
+      await app.build();
+
+      const userfields = app.modules.em.entity("users").fields.map((f) => f.name);
+      expect(userfields).toContain("additional");
+      expect(userfields).toEqual([
+         "id",
+         "additional",
+         "email",
+         "strategy",
+         "strategy_value",
+         "role"
+      ]);
+   });
 });

app/__test__/modules/AppMedia.spec.ts

@@ -1,7 +1,53 @@
-import { describe } from "bun:test";
+import { describe, expect, test } from "bun:test";
+import { createApp, registries } from "../../src";
+import { em, entity, text } from "../../src/data";
+import { StorageLocalAdapter } from "../../src/media/storage/adapters/StorageLocalAdapter";
 import { AppMedia } from "../../src/modules";
 import { moduleTestSuite } from "./module-test-suite";
 
 describe("AppMedia", () => {
    moduleTestSuite(AppMedia);
+
+   test("should allow additional fields", async () => {
+      registries.media.register("local", StorageLocalAdapter);
+
+      const app = createApp({
+         initialConfig: {
+            media: {
+               entity_name: "media",
+               enabled: true,
+               adapter: {
+                  type: "local",
+                  config: {
+                     path: "./"
+                  }
+               }
+            },
+            data: em({
+               media: entity("media", {
+                  additional: text()
+               })
+            }).toJSON()
+         }
+      });
+
+      await app.build();
+
+      const fields = app.modules.em.entity("media").fields.map((f) => f.name);
+      expect(fields).toContain("additional");
+      expect(fields).toEqual([
+         "id",
+         "additional",
+         "path",
+         "folder",
+         "mime_type",
+         "size",
+         "scope",
+         "etag",
+         "modified_at",
+         "reference",
+         "entity_id",
+         "metadata"
+      ]);
+   });
 });

app/__test__/modules/Module.spec.ts

@@ -0,0 +1,200 @@
+import { describe, expect, test } from "bun:test";
+import { type TSchema, Type, stripMark } from "../../src/core/utils";
+import { EntityManager, em, entity, index, text } from "../../src/data";
+import { DummyConnection } from "../../src/data/connection/DummyConnection";
+import { Module } from "../../src/modules/Module";
+
+function createModule<Schema extends TSchema>(schema: Schema) {
+   class TestModule extends Module<typeof schema> {
+      getSchema() {
+         return schema;
+      }
+      toJSON() {
+         return this.config;
+      }
+      useForceParse() {
+         return true;
+      }
+   }
+
+   return TestModule;
+}
+
+describe("Module", async () => {
+   describe("basic", () => {
+      test("listener", async () => {
+         let result: any;
+
+         const module = createModule(Type.Object({ a: Type.String() }));
+         const m = new module({ a: "test" });
+
+         await m.schema().set({ a: "test2" });
+         m.setListener(async (c) => {
+            await new Promise((r) => setTimeout(r, 10));
+            result = stripMark(c);
+         });
+         await m.schema().set({ a: "test3" });
+         expect(result).toEqual({ a: "test3" });
+      });
+   });
+
+   describe("db schema", () => {
+      class M extends Module {
+         override getSchema() {
+            return Type.Object({});
+         }
+
+         prt = {
+            ensureEntity: this.ensureEntity.bind(this),
+            ensureIndex: this.ensureIndex.bind(this),
+            ensureSchema: this.ensureSchema.bind(this)
+         };
+
+         get em() {
+            return this.ctx.em;
+         }
+      }
+
+      function make(_em: ReturnType<typeof em>) {
+         const em = new EntityManager(
+            Object.values(_em.entities),
+            new DummyConnection(),
+            _em.relations,
+            _em.indices
+         );
+         return new M({} as any, { em, flags: Module.ctx_flags } as any);
+      }
+      function flat(_em: EntityManager) {
+         return {
+            entities: _em.entities.map((e) => ({
+               name: e.name,
+               fields: e.fields.map((f) => f.name)
+            })),
+            indices: _em.indices.map((i) => ({
+               name: i.name,
+               entity: i.entity.name,
+               fields: i.fields.map((f) => f.name),
+               unique: i.unique
+            }))
+         };
+      }
+
+      test("no change", () => {
+         const initial = em({});
+
+         const m = make(initial);
+         expect(m.ctx.flags.sync_required).toBe(false);
+
+         expect(flat(make(initial).em)).toEqual({
+            entities: [],
+            indices: []
+         });
+      });
+
+      test("init", () => {
+         const initial = em({
+            users: entity("u", {
+               name: text()
+            })
+         });
+
+         const m = make(initial);
+         expect(m.ctx.flags.sync_required).toBe(false);
+
+         expect(flat(m.em)).toEqual({
+            entities: [
+               {
+                  name: "u",
+                  fields: ["id", "name"]
+               }
+            ],
+            indices: []
+         });
+      });
+
+      test("ensure entity", () => {
+         const initial = em({
+            users: entity("u", {
+               name: text()
+            })
+         });
+
+         const m = make(initial);
+         expect(flat(m.em)).toEqual({
+            entities: [
+               {
+                  name: "u",
+                  fields: ["id", "name"]
+               }
+            ],
+            indices: []
+         });
+
+         // this should add a new entity
+         m.prt.ensureEntity(
+            entity("p", {
+               title: text()
+            })
+         );
+
+         // this should only add the field "important"
+         m.prt.ensureEntity(
+            entity("u", {
+               important: text()
+            })
+         );
+
+         expect(m.ctx.flags.sync_required).toBe(true);
+         expect(flat(m.em)).toEqual({
+            entities: [
+               {
+                  name: "u",
+                  fields: ["id", "name", "important"]
+               },
+               {
+                  name: "p",
+                  fields: ["id", "title"]
+               }
+            ],
+            indices: []
+         });
+      });
+
+      test("ensure index", () => {
+         const users = entity("u", {
+            name: text(),
+            title: text()
+         });
+         const initial = em({ users }, ({ index }, { users }) => {
+            index(users).on(["title"]);
+         });
+
+         const m = make(initial);
+         m.prt.ensureIndex(index(users).on(["name"]));
+
+         expect(m.ctx.flags.sync_required).toBe(true);
+         expect(flat(m.em)).toEqual({
+            entities: [
+               {
+                  name: "u",
+                  fields: ["id", "name", "title"]
+               }
+            ],
+            indices: [
+               {
+                  name: "idx_u_title",
+                  entity: "u",
+                  fields: ["title"],
+                  unique: false
+               },
+               {
+                  name: "idx_u_name",
+                  entity: "u",
+                  fields: ["name"],
+                  unique: false
+               }
+            ]
+         });
+      });
+   });
+});

app/__test__/modules/ModuleManager.spec.ts

@@ -1,9 +1,9 @@
 import { describe, expect, test } from "bun:test";
-import { mark, stripMark } from "../src/core/utils";
+import { stripMark } from "../../src/core/utils";
-import { entity, text } from "../src/data";
+import { entity, text } from "../../src/data";
-import { ModuleManager, getDefaultConfig } from "../src/modules/ModuleManager";
+import { ModuleManager, getDefaultConfig } from "../../src/modules/ModuleManager";
-import { CURRENT_VERSION, TABLE_NAME } from "../src/modules/migrations";
+import { CURRENT_VERSION, TABLE_NAME } from "../../src/modules/migrations";
-import { getDummyConnection } from "./helper";
+import { getDummyConnection } from "../helper";
 
 describe("ModuleManager", async () => {
    test("s1: no config, no build", async () => {

app/src/auth/AppAuth.ts

@@ -4,7 +4,7 @@ import { auth } from "auth/middlewares";
 import { type DB, Exception, type PrimaryFieldType } from "core";
 import { type Static, secureRandomString, transformObject } from "core/utils";
 import { type Entity, EntityIndex, type EntityManager } from "data";
-import { type FieldSchema, entity, enumm, make, text } from "data/prototype";
+import { type FieldSchema, em, entity, enumm, make, text } from "data/prototype";
 import type { Hono } from "hono";
 import { pick } from "lodash-es";
 import { Module } from "modules/Module";
@@ -250,43 +250,30 @@ export class AppAuth extends Module<typeof authConfigSchema> {
    };
 
    registerEntities() {
-      const users = this.getUsersEntity();
+      const name = this.config.entity_name as "users";
-
+      const {
-      if (!this.em.hasEntity(users.name)) {
+         entities: { users }
-         this.em.addEntity(users);
+      } = this.ensureSchema(
-      } else {
+         em(
-         // if exists, check all fields required are there
+            {
-         // @todo: add to context: "needs sync" flag
+               [name]: entity(name, AppAuth.usersFields)
-         const _entity = this.getUsersEntity(true);
+            },
-         for (const field of _entity.fields) {
+            ({ index }, { users }) => {
-            const _field = users.field(field.name);
+               index(users).on(["email"], true).on(["strategy"]).on(["strategy_value"]);
-            if (!_field) {
-               users.addField(field);
             }
-         }
+         )
-      }
+      );
-
-      const indices = [
-         new EntityIndex(users, [users.field("email")!], true),
-         new EntityIndex(users, [users.field("strategy")!]),
-         new EntityIndex(users, [users.field("strategy_value")!])
-      ];
-      indices.forEach((index) => {
-         if (!this.em.hasIndex(index)) {
-            this.em.addIndex(index);
-         }
-      });
 
       try {
          const roles = Object.keys(this.config.roles ?? {});
          const field = make("role", enumm({ enum: roles }));
-         this.em.entity(users.name).__experimental_replaceField("role", field);
+         users.__experimental_replaceField("role", field);
       } catch (e) {}
 
       try {
          const strategies = Object.keys(this.config.strategies ?? {});
          const field = make("strategy", enumm({ enum: strategies }));
-         this.em.entity(users.name).__experimental_replaceField("strategy", field);
+         users.__experimental_replaceField("strategy", field);
       } catch (e) {}
    }
 

app/src/data/prototype/index.ts

@@ -272,18 +272,22 @@ class EntityManagerPrototype<Entities extends Record<string, Entity>> extends En
    }
 }
 
-type Chained<Fn extends (...args: any[]) => any, Rt = ReturnType<Fn>> = <E extends Entity>(
+type Chained<R extends Record<string, (...args: any[]) => any>> = {
-   e: E
+   [K in keyof R]: R[K] extends (...args: any[]) => any
-) => {
+      ? (...args: Parameters<R[K]>) => Chained<R>
-   [K in keyof Rt]: Rt[K] extends (...args: any[]) => any
-      ? (...args: Parameters<Rt[K]>) => Rt
       : never;
 };
+type ChainedFn<
+   Fn extends (...args: any[]) => Record<string, (...args: any[]) => any>,
+   Return extends ReturnType<Fn> = ReturnType<Fn>
+> = (e: Entity) => {
+   [K in keyof Return]: (...args: Parameters<Return[K]>) => Chained<Return>;
+};
 
 export function em<Entities extends Record<string, Entity>>(
    entities: Entities,
    schema?: (
-      fns: { relation: Chained<typeof relation>; index: Chained<typeof index> },
+      fns: { relation: ChainedFn<typeof relation>; index: ChainedFn<typeof index> },
       entities: Entities
    ) => void
 ) {

app/src/media/AppMedia.ts

@@ -1,8 +1,17 @@
 import type { PrimaryFieldType } from "core";
-import { EntityIndex, type EntityManager } from "data";
+import { type Entity, EntityIndex, type EntityManager } from "data";
 import { type FileUploadedEventData, Storage, type StorageAdapter } from "media";
 import { Module } from "modules/Module";
-import { type FieldSchema, boolean, datetime, entity, json, number, text } from "../data/prototype";
+import {
+   type FieldSchema,
+   boolean,
+   datetime,
+   em,
+   entity,
+   json,
+   number,
+   text
+} from "../data/prototype";
 import { MediaController } from "./api/MediaController";
 import { ADAPTERS, buildMediaSchema, type mediaConfigSchema, registry } from "./media-schema";
 
@@ -17,6 +26,7 @@ export class AppMedia extends Module<typeof mediaConfigSchema> {
    private _storage?: Storage;
 
    override async build() {
+      console.log("building");
       if (!this.config.enabled) {
          this.setBuilt();
          return;
@@ -38,18 +48,13 @@ export class AppMedia extends Module<typeof mediaConfigSchema> {
          this.setupListeners();
          this.ctx.server.route(this.basepath, new MediaController(this).getController());
 
-         // @todo: add check for media entity
+         const mediaEntity = this.getMediaEntity(true);
-         const mediaEntity = this.getMediaEntity();
+         const name = mediaEntity.name as "media";
-         if (!this.ctx.em.hasEntity(mediaEntity)) {
+         this.ensureSchema(
-            this.ctx.em.addEntity(mediaEntity);
+            em({ [name]: mediaEntity }, ({ index }, { media }) => {
-         }
+               index(media).on(["path"], true).on(["reference"]);
-
+            })
-         const pathIndex = new EntityIndex(mediaEntity, [mediaEntity.field("path")!], true);
+         );
-         if (!this.ctx.em.hasIndex(pathIndex)) {
-            this.ctx.em.addIndex(pathIndex);
-         }
-
-         // @todo: check indices
       } catch (e) {
          console.error(e);
          throw new Error(
@@ -94,13 +99,13 @@ export class AppMedia extends Module<typeof mediaConfigSchema> {
       metadata: json()
    };
 
-   getMediaEntity() {
+   getMediaEntity(forceCreate?: boolean): Entity<"media", typeof AppMedia.mediaFields> {
       const entity_name = this.config.entity_name;
-      if (!this.em.hasEntity(entity_name)) {
+      if (forceCreate || !this.em.hasEntity(entity_name)) {
-         return entity(entity_name, AppMedia.mediaFields, undefined, "system");
+         return entity(entity_name as "media", AppMedia.mediaFields, undefined, "system");
       }
 
-      return this.em.entity(entity_name);
+      return this.em.entity(entity_name) as any;
    }
 
    get em(): EntityManager {

app/src/modules/Module.ts

@@ -3,7 +3,7 @@ import type { Guard } from "auth";
 import { SchemaObject } from "core";
 import type { EventManager } from "core/events";
 import type { Static, TSchema } from "core/utils";
-import type { Connection, EntityManager } from "data";
+import type { Connection, Entity, EntityIndex, EntityManager, em as prototypeEm } from "data";
 import type { Hono } from "hono";
 
 export type ServerEnv = {
@@ -21,6 +21,7 @@ export type ModuleBuildContext = {
    em: EntityManager;
    emgr: EventManager<any>;
    guard: Guard;
+   flags: (typeof Module)["ctx_flags"];
 };
 
 export abstract class Module<Schema extends TSchema = TSchema, ConfigSchema = Static<Schema>> {
@@ -43,6 +44,13 @@ export abstract class Module<Schema extends TSchema = TSchema, ConfigSchema = St
       });
    }
 
+   static ctx_flags = {
+      sync_required: false
+   } as {
+      // signal that a sync is required at the end of build
+      sync_required: boolean;
+   };
+
    onBeforeUpdate(from: ConfigSchema, to: ConfigSchema): ConfigSchema | Promise<ConfigSchema> {
       return to;
    }
@@ -129,4 +137,41 @@ export abstract class Module<Schema extends TSchema = TSchema, ConfigSchema = St
    toJSON(secrets?: boolean): Static<ReturnType<(typeof this)["getSchema"]>> {
       return this.config;
    }
+
+   // @todo: add a method to signal the requirement of database sync!!!
+
+   protected ensureEntity(entity: Entity) {
+      // check fields
+      if (!this.ctx.em.hasEntity(entity.name)) {
+         this.ctx.em.addEntity(entity);
+         this.ctx.flags.sync_required = true;
+         return;
+      }
+
+      const instance = this.ctx.em.entity(entity.name);
+
+      // if exists, check all fields required are there
+      // @todo: check if the field also equal
+      for (const field of entity.fields) {
+         const _field = instance.field(field.name);
+         if (!_field) {
+            instance.addField(field);
+            this.ctx.flags.sync_required = true;
+         }
+      }
+   }
+
+   protected ensureIndex(index: EntityIndex) {
+      if (!this.ctx.em.hasIndex(index)) {
+         this.ctx.em.addIndex(index);
+         this.ctx.flags.sync_required = true;
+      }
+   }
+
+   protected ensureSchema<Schema extends ReturnType<typeof prototypeEm>>(schema: Schema): Schema {
+      Object.values(schema.entities ?? {}).forEach(this.ensureEntity.bind(this));
+      schema.indices?.forEach(this.ensureIndex.bind(this));
+
+      return schema;
+   }
 }

app/src/modules/ModuleManager.ts

@@ -1,4 +1,3 @@
-import type { App } from "App";
 import { Guard } from "auth";
 import { BkndError, DebugLogger } from "core";
 import { EventManager } from "core/events";
@@ -34,7 +33,7 @@ import { AppAuth } from "../auth/AppAuth";
 import { AppData } from "../data/AppData";
 import { AppFlows } from "../flows/AppFlows";
 import { AppMedia } from "../media/AppMedia";
-import type { Module, ModuleBuildContext, ServerEnv } from "./Module";
+import { Module, type ModuleBuildContext, type ServerEnv } from "./Module";
 
 export type { ModuleBuildContext };
 
@@ -230,7 +229,8 @@ export class ModuleManager {
          server: this.server,
          em: this.em,
          emgr: this.emgr,
-         guard: this.guard
+         guard: this.guard,
+         flags: Module.ctx_flags
       };
    }
 
@@ -415,7 +415,14 @@ export class ModuleManager {
       }
 
       this._built = true;
-      this.logger.log("modules built");
+      this.logger.log("modules built", ctx.flags);
+
+      if (ctx.flags.sync_required) {
+         this.logger.log("db sync requested");
+         await ctx.em.schema().sync({ force: true });
+         await this.save();
+         ctx.flags.sync_required = false; // reset
+      }
    }
 
    async build() {

app/vite.dev.ts

@@ -1,4 +1,10 @@
-import { serve } from "./src/adapter/vite";
+import { serveStatic } from "@hono/node-server/serve-static";
+import { createClient } from "@libsql/client/node";
+import { App, registries } from "./src";
+import { LibsqlConnection } from "./src/data";
+import { StorageLocalAdapter } from "./src/media/storage/adapters/StorageLocalAdapter";
+
+registries.media.register("local", StorageLocalAdapter);
 
 const credentials = {
    url: import.meta.env.VITE_DB_URL!,
@@ -8,10 +14,22 @@ if (!credentials.url) {
    throw new Error("Missing VITE_DB_URL env variable. Add it to .env file");
 }
 
-export default serve({
+const connection = new LibsqlConnection(createClient(credentials));
-   connection: {
+
-      type: "libsql",
+export default {
-      config: credentials
+   async fetch(request: Request) {
+      const app = App.create({ connection });
+
+      app.emgr.onEvent(
+         App.Events.AppBuiltEvent,
+         async () => {
+            app.registerAdminController({ forceDev: true });
+            app.module.server.client.get("/assets/*", serveStatic({ root: "./" }));
          },
-   forceDev: true
+         "sync"
-});
+      );
+      await app.build();
+
+      return app.fetch(request);
+   }
+};