Refactor module schema handling and add sync mechanism

Redesigned entity and index management with methods to streamline schema updates and added a sync flag to signal required DB syncs post-build. Enhanced test coverage and functionality for schema modifications, including support for additional fields.

app/src/auth/AppAuth.ts

@@ -4,7 +4,7 @@ import { auth } from "auth/middlewares";
 import { type DB, Exception, type PrimaryFieldType } from "core";
 import { type Static, secureRandomString, transformObject } from "core/utils";
 import { type Entity, EntityIndex, type EntityManager } from "data";
-import { type FieldSchema, entity, enumm, make, text } from "data/prototype";
+import { type FieldSchema, em, entity, enumm, make, text } from "data/prototype";
 import type { Hono } from "hono";
 import { pick } from "lodash-es";
 import { Module } from "modules/Module";
@@ -250,43 +250,30 @@ export class AppAuth extends Module<typeof authConfigSchema> {
    };
 
    registerEntities() {
-      const users = this.getUsersEntity();
-
-      if (!this.em.hasEntity(users.name)) {
-         this.em.addEntity(users);
-      } else {
-         // if exists, check all fields required are there
-         // @todo: add to context: "needs sync" flag
-         const _entity = this.getUsersEntity(true);
-         for (const field of _entity.fields) {
-            const _field = users.field(field.name);
-            if (!_field) {
-               users.addField(field);
+      const name = this.config.entity_name as "users";
+      const {
+         entities: { users }
+      } = this.ensureSchema(
+         em(
+            {
+               [name]: entity(name, AppAuth.usersFields)
+            },
+            ({ index }, { users }) => {
+               index(users).on(["email"], true).on(["strategy"]).on(["strategy_value"]);
             }
-         }
-      }
-
-      const indices = [
-         new EntityIndex(users, [users.field("email")!], true),
-         new EntityIndex(users, [users.field("strategy")!]),
-         new EntityIndex(users, [users.field("strategy_value")!])
-      ];
-      indices.forEach((index) => {
-         if (!this.em.hasIndex(index)) {
-            this.em.addIndex(index);
-         }
-      });
+         )
+      );
 
       try {
          const roles = Object.keys(this.config.roles ?? {});
          const field = make("role", enumm({ enum: roles }));
-         this.em.entity(users.name).__experimental_replaceField("role", field);
+         users.__experimental_replaceField("role", field);
       } catch (e) {}
 
       try {
          const strategies = Object.keys(this.config.strategies ?? {});
          const field = make("strategy", enumm({ enum: strategies }));
-         this.em.entity(users.name).__experimental_replaceField("strategy", field);
+         users.__experimental_replaceField("strategy", field);
       } catch (e) {}
    }
 

app/src/data/prototype/index.ts

@@ -272,18 +272,22 @@ class EntityManagerPrototype<Entities extends Record<string, Entity>> extends En
    }
 }
 
-type Chained<Fn extends (...args: any[]) => any, Rt = ReturnType<Fn>> = <E extends Entity>(
-   e: E
-) => {
-   [K in keyof Rt]: Rt[K] extends (...args: any[]) => any
-      ? (...args: Parameters<Rt[K]>) => Rt
+type Chained<R extends Record<string, (...args: any[]) => any>> = {
+   [K in keyof R]: R[K] extends (...args: any[]) => any
+      ? (...args: Parameters<R[K]>) => Chained<R>
       : never;
 };
+type ChainedFn<
+   Fn extends (...args: any[]) => Record<string, (...args: any[]) => any>,
+   Return extends ReturnType<Fn> = ReturnType<Fn>
+> = (e: Entity) => {
+   [K in keyof Return]: (...args: Parameters<Return[K]>) => Chained<Return>;
+};
 
 export function em<Entities extends Record<string, Entity>>(
    entities: Entities,
    schema?: (
-      fns: { relation: Chained<typeof relation>; index: Chained<typeof index> },
+      fns: { relation: ChainedFn<typeof relation>; index: ChainedFn<typeof index> },
       entities: Entities
    ) => void
 ) {

app/src/media/AppMedia.ts

@@ -1,8 +1,17 @@
 import type { PrimaryFieldType } from "core";
-import { EntityIndex, type EntityManager } from "data";
+import { type Entity, EntityIndex, type EntityManager } from "data";
 import { type FileUploadedEventData, Storage, type StorageAdapter } from "media";
 import { Module } from "modules/Module";
-import { type FieldSchema, boolean, datetime, entity, json, number, text } from "../data/prototype";
+import {
+   type FieldSchema,
+   boolean,
+   datetime,
+   em,
+   entity,
+   json,
+   number,
+   text
+} from "../data/prototype";
 import { MediaController } from "./api/MediaController";
 import { ADAPTERS, buildMediaSchema, type mediaConfigSchema, registry } from "./media-schema";
 
@@ -17,6 +26,7 @@ export class AppMedia extends Module<typeof mediaConfigSchema> {
    private _storage?: Storage;
 
    override async build() {
+      console.log("building");
       if (!this.config.enabled) {
          this.setBuilt();
          return;
@@ -38,18 +48,13 @@ export class AppMedia extends Module<typeof mediaConfigSchema> {
          this.setupListeners();
          this.ctx.server.route(this.basepath, new MediaController(this).getController());
 
-         // @todo: add check for media entity
-         const mediaEntity = this.getMediaEntity();
-         if (!this.ctx.em.hasEntity(mediaEntity)) {
-            this.ctx.em.addEntity(mediaEntity);
-         }
-
-         const pathIndex = new EntityIndex(mediaEntity, [mediaEntity.field("path")!], true);
-         if (!this.ctx.em.hasIndex(pathIndex)) {
-            this.ctx.em.addIndex(pathIndex);
-         }
-
-         // @todo: check indices
+         const mediaEntity = this.getMediaEntity(true);
+         const name = mediaEntity.name as "media";
+         this.ensureSchema(
+            em({ [name]: mediaEntity }, ({ index }, { media }) => {
+               index(media).on(["path"], true).on(["reference"]);
+            })
+         );
       } catch (e) {
          console.error(e);
          throw new Error(
@@ -94,13 +99,13 @@ export class AppMedia extends Module<typeof mediaConfigSchema> {
       metadata: json()
    };
 
-   getMediaEntity() {
+   getMediaEntity(forceCreate?: boolean): Entity<"media", typeof AppMedia.mediaFields> {
       const entity_name = this.config.entity_name;
-      if (!this.em.hasEntity(entity_name)) {
-         return entity(entity_name, AppMedia.mediaFields, undefined, "system");
+      if (forceCreate || !this.em.hasEntity(entity_name)) {
+         return entity(entity_name as "media", AppMedia.mediaFields, undefined, "system");
       }
 
-      return this.em.entity(entity_name);
+      return this.em.entity(entity_name) as any;
    }
 
    get em(): EntityManager {

app/src/modules/Module.ts

@@ -3,7 +3,7 @@ import type { Guard } from "auth";
 import { SchemaObject } from "core";
 import type { EventManager } from "core/events";
 import type { Static, TSchema } from "core/utils";
-import type { Connection, EntityManager } from "data";
+import type { Connection, Entity, EntityIndex, EntityManager, em as prototypeEm } from "data";
 import type { Hono } from "hono";
 
 export type ServerEnv = {
@@ -21,6 +21,7 @@ export type ModuleBuildContext = {
    em: EntityManager;
    emgr: EventManager<any>;
    guard: Guard;
+   flags: (typeof Module)["ctx_flags"];
 };
 
 export abstract class Module<Schema extends TSchema = TSchema, ConfigSchema = Static<Schema>> {
@@ -43,6 +44,13 @@ export abstract class Module<Schema extends TSchema = TSchema, ConfigSchema = St
       });
    }
 
+   static ctx_flags = {
+      sync_required: false
+   } as {
+      // signal that a sync is required at the end of build
+      sync_required: boolean;
+   };
+
    onBeforeUpdate(from: ConfigSchema, to: ConfigSchema): ConfigSchema | Promise<ConfigSchema> {
       return to;
    }
@@ -129,4 +137,41 @@ export abstract class Module<Schema extends TSchema = TSchema, ConfigSchema = St
    toJSON(secrets?: boolean): Static<ReturnType<(typeof this)["getSchema"]>> {
       return this.config;
    }
+
+   // @todo: add a method to signal the requirement of database sync!!!
+
+   protected ensureEntity(entity: Entity) {
+      // check fields
+      if (!this.ctx.em.hasEntity(entity.name)) {
+         this.ctx.em.addEntity(entity);
+         this.ctx.flags.sync_required = true;
+         return;
+      }
+
+      const instance = this.ctx.em.entity(entity.name);
+
+      // if exists, check all fields required are there
+      // @todo: check if the field also equal
+      for (const field of entity.fields) {
+         const _field = instance.field(field.name);
+         if (!_field) {
+            instance.addField(field);
+            this.ctx.flags.sync_required = true;
+         }
+      }
+   }
+
+   protected ensureIndex(index: EntityIndex) {
+      if (!this.ctx.em.hasIndex(index)) {
+         this.ctx.em.addIndex(index);
+         this.ctx.flags.sync_required = true;
+      }
+   }
+
+   protected ensureSchema<Schema extends ReturnType<typeof prototypeEm>>(schema: Schema): Schema {
+      Object.values(schema.entities ?? {}).forEach(this.ensureEntity.bind(this));
+      schema.indices?.forEach(this.ensureIndex.bind(this));
+
+      return schema;
+   }
 }

app/src/modules/ModuleManager.ts

@@ -1,4 +1,3 @@
-import type { App } from "App";
 import { Guard } from "auth";
 import { BkndError, DebugLogger } from "core";
 import { EventManager } from "core/events";
@@ -34,7 +33,7 @@ import { AppAuth } from "../auth/AppAuth";
 import { AppData } from "../data/AppData";
 import { AppFlows } from "../flows/AppFlows";
 import { AppMedia } from "../media/AppMedia";
-import type { Module, ModuleBuildContext, ServerEnv } from "./Module";
+import { Module, type ModuleBuildContext, type ServerEnv } from "./Module";
 
 export type { ModuleBuildContext };
 
@@ -230,7 +229,8 @@ export class ModuleManager {
          server: this.server,
          em: this.em,
          emgr: this.emgr,
-         guard: this.guard
+         guard: this.guard,
+         flags: Module.ctx_flags
       };
    }
 
@@ -415,7 +415,14 @@ export class ModuleManager {
       }
 
       this._built = true;
-      this.logger.log("modules built");
+      this.logger.log("modules built", ctx.flags);
+
+      if (ctx.flags.sync_required) {
+         this.logger.log("db sync requested");
+         await ctx.em.schema().sync({ force: true });
+         await this.save();
+         ctx.flags.sync_required = false; // reset
+      }
    }
 
    async build() {