feat: improved abilities of plugins, moved schema fns to ctx

app/src/App.ts

@@ -1,7 +1,7 @@
 import type { CreateUserPayload } from "auth/AppAuth";
 import { $console } from "core";
 import { Event } from "core/events";
-import { Connection, type LibSqlCredentials, LibsqlConnection } from "data";
+import { Connection, type LibSqlCredentials, LibsqlConnection, type em as prototypeEm } from "data";
 import type { Hono } from "hono";
 import {
    ModuleManager,
@@ -14,12 +14,21 @@ import {
 import * as SystemPermissions from "modules/permissions";
 import { AdminController, type AdminControllerOptions } from "modules/server/AdminController";
 import { SystemController } from "modules/server/SystemController";
+import type { MaybePromise } from "core/types";
+import type { ServerEnv } from "modules/Controller";
 
 // biome-ignore format: must be here
 import { Api, type ApiOptions } from "Api";
-import type { ServerEnv } from "modules/Controller";
 
-export type AppPlugin = (app: App) => Promise<void> | void;
+export type AppPluginConfig = {
+   name: string;
+   schema?: () => MaybePromise<ReturnType<typeof prototypeEm> | void>;
+   beforeBuild?: () => MaybePromise<void>;
+   onBuilt?: () => MaybePromise<void>;
+   onServerInit?: (server: Hono<ServerEnv>) => MaybePromise<void>;
+   onFirstBoot?: () => MaybePromise<void>;
+};
+export type AppPlugin = (app: App) => AppPluginConfig;
 
 abstract class AppEvent<A = {}> extends Event<{ app: App } & A> {}
 export class AppConfigUpdatedEvent extends AppEvent {
@@ -73,9 +82,9 @@ export class App {
    modules: ModuleManager;
    adminController?: AdminController;
    _id: string = crypto.randomUUID();
+   plugins: AppPluginConfig[];
 
    private trigger_first_boot = false;
-   private plugins: AppPlugin[];
    private _building: boolean = false;
 
    constructor(
@@ -83,13 +92,14 @@ export class App {
       _initialConfig?: InitialModuleConfigs,
       private options?: AppOptions,
    ) {
-      this.plugins = options?.plugins ?? [];
+      this.plugins = (options?.plugins ?? []).map((plugin) => plugin(this));
       this.modules = new ModuleManager(connection, {
          ...(options?.manager ?? {}),
          initial: _initialConfig,
          onUpdated: this.onUpdated.bind(this),
          onFirstBoot: this.onFirstBoot.bind(this),
          onServerInit: this.onServerInit.bind(this),
+         onModulesBuilt: this.onModulesBuilt.bind(this),
       });
       this.modules.ctx().emgr.registerEvents(AppEvents);
    }
@@ -98,6 +108,32 @@ export class App {
       return this.modules.ctx().emgr;
    }
 
+   protected async runPlugins<Key extends keyof AppPluginConfig>(
+      key: Key,
+      ...args: any[]
+   ): Promise<{ name: string; result: any }[]> {
+      const results: { name: string; result: any }[] = [];
+      for (const plugin of this.plugins) {
+         try {
+            if (key in plugin && plugin[key]) {
+               const fn = plugin[key];
+               if (fn && typeof fn === "function") {
+                  $console.debug(`[Plugin:${plugin.name}] ${key}`);
+                  // @ts-expect-error
+                  const result = await fn(...args);
+                  results.push({
+                     name: plugin.name,
+                     result,
+                  });
+               }
+            }
+         } catch (e) {
+            $console.warn(`[Plugin:${plugin.name}] error running "${key}"`, String(e));
+         }
+      }
+      return results as any;
+   }
+
    async build(options?: { sync?: boolean; fetch?: boolean; forceBuild?: boolean }) {
       // prevent multiple concurrent builds
       if (this._building) {
@@ -106,6 +142,8 @@ export class App {
          }
          if (!options?.forceBuild) return;
       }
+
+      await this.runPlugins("beforeBuild");
       this._building = true;
 
       if (options?.sync) this.modules.ctx().flags.sync_required = true;
@@ -117,13 +155,10 @@ export class App {
       guard.registerPermissions(Object.values(SystemPermissions));
       server.route("/api/system", new SystemController(this).getController());
 
-      // load plugins
-      if (this.plugins.length > 0) {
-         await Promise.all(this.plugins.map((plugin) => plugin(this)));
-      }
-
+      // emit built event
       $console.log("App built");
       await this.emgr.emit(new AppBuiltEvent({ app: this }));
+      await this.runPlugins("onBuilt");
 
       // first boot is set from ModuleManager when there wasn't a config table
       if (this.trigger_first_boot) {
@@ -223,12 +258,13 @@ export class App {
       await this.emgr.emit(new AppConfigUpdatedEvent({ app: this }));
    }
 
-   async onFirstBoot() {
+   protected async onFirstBoot() {
       $console.log("App first boot");
       this.trigger_first_boot = true;
+      await this.runPlugins("onFirstBoot");
    }
 
-   async onServerInit(server: Hono<ServerEnv>) {
+   protected async onServerInit(server: Hono<ServerEnv>) {
       server.use(async (c, next) => {
          c.set("app", this);
          await this.emgr.emit(new AppRequest({ app: this, request: c.req.raw }));
@@ -258,6 +294,23 @@ export class App {
       if (this.options?.manager?.onServerInit) {
          this.options.manager.onServerInit(server);
       }
+
+      await this.runPlugins("onServerInit", server);
+   }
+
+   protected async onModulesBuilt(ctx: ModuleBuildContext) {
+      const results = (await this.runPlugins("schema")) as {
+         name: string;
+         result: ReturnType<typeof prototypeEm>;
+      }[];
+      if (results.length > 0) {
+         for (const { name, result } of results) {
+            if (result) {
+               $console.log(`[Plugin:${name}] schema`);
+               ctx.helper.ensureSchema(result);
+            }
+         }
+      }
    }
 }
 

app/src/auth/AppAuth.ts

@@ -140,7 +140,7 @@ export class AppAuth extends Module<typeof authConfigSchema> {
 
    registerEntities() {
       const users = this.getUsersEntity(true);
-      this.ensureSchema(
+      this.ctx.helper.ensureSchema(
          em(
             {
                [users.name as "users"]: users,
@@ -153,13 +153,13 @@ export class AppAuth extends Module<typeof authConfigSchema> {
 
       try {
          const roles = Object.keys(this.config.roles ?? {});
-         this.replaceEntityField(users, "role", enumm({ enum: roles }));
+         this.ctx.helper.replaceEntityField(users, "role", enumm({ enum: roles }));
       } catch (e) {}
 
       try {
          // also keep disabled strategies as a choice
          const strategies = Object.keys(this.config.strategies ?? {});
-         this.replaceEntityField(users, "strategy", enumm({ enum: strategies }));
+         this.ctx.helper.replaceEntityField(users, "strategy", enumm({ enum: strategies }));
       } catch (e) {}
    }
 

app/src/auth/api/AuthController.ts

@@ -184,6 +184,6 @@ export class AuthController extends Controller {
          this.registerStrategyActions(strategy, hono);
       }
 
-      return hono.all("*", (c) => c.notFound());
+      return hono;
    }
 }

app/src/core/index.ts

@@ -26,7 +26,16 @@ export {
 } from "./object/query/query";
 export { Registry, type Constructor } from "./registry/Registry";
 export { getFlashMessage } from "./server/flash";
-export { s, jsc, describeRoute } from "./object/schema";
+export {
+   s,
+   parse,
+   jsc,
+   describeRoute,
+   schemaToSpec,
+   openAPISpecs,
+   type ParseOptions,
+   InvalidSchemaError,
+} from "./object/schema";
 
 export * from "./console";
 export * from "./events";

app/src/core/types.ts

@@ -2,3 +2,5 @@ export interface Serializable<Class, Json extends object = object> {
    toJSON(): Json;
    fromJSON(json: Json): Class;
 }
+
+export type MaybePromise<T> = T | Promise<T>;

app/src/data/api/DataController.ts

@@ -225,7 +225,7 @@ export class DataController extends Controller {
          },
       );
 
-      return hono.all("*", (c) => c.notFound());
+      return hono;
    }
 
    private getEntityRoutes() {

app/src/data/relations/RelationAccessor.ts

@@ -12,6 +12,15 @@ export class RelationAccessor {
       return this._relations;
    }
 
+   exists(relation: EntityRelation): boolean {
+      return this._relations.some(
+         (r) =>
+            r.source.entity.name === relation.source.entity.name &&
+            r.target.entity.name === relation.target.entity.name &&
+            r.type === relation.type,
+      );
+   }
+
    /**
     * Searches for the relations of [entity_name]
     */

app/src/index.ts

@@ -18,6 +18,8 @@ export {
    type InitialModuleConfigs,
 } from "./modules/ModuleManager";
 
+export type { ServerEnv } from "modules/Controller";
+
 export * as middlewares from "modules/middlewares";
 export { registries } from "modules/registries";
 

app/src/media/AppMedia.ts

@@ -51,7 +51,7 @@ export class AppMedia extends Module<typeof mediaConfigSchema> {
          this.ctx.server.route(this.basepath, new MediaController(this).getController());
 
          const media = this.getMediaEntity(true);
-         this.ensureSchema(
+         this.ctx.helper.ensureSchema(
             em({ [media.name as "media"]: media }, ({ index }, { media }) => {
                index(media).on(["path"], true).on(["reference"]).on(["entity_id"]);
             }),

app/src/media/api/MediaController.ts

@@ -297,6 +297,6 @@ export class MediaController extends Controller {
          },
       );
 
-      return hono.all("*", (c) => c.notFound());
+      return hono;
    }
 }

app/src/modules/Module.ts

@@ -2,19 +2,10 @@ import type { Guard } from "auth";
 import { type DebugLogger, SchemaObject } from "core";
 import type { EventManager } from "core/events";
 import type { Static, TSchema } from "core/utils";
-import {
-   type Connection,
-   type EntityIndex,
-   type EntityManager,
-   type Field,
-   FieldPrototype,
-   make,
-   type em as prototypeEm,
-} from "data";
-import { Entity } from "data";
+import type { Connection, EntityManager } from "data";
 import type { Hono } from "hono";
-import { isEqual } from "lodash-es";
 import type { ServerEnv } from "modules/Controller";
+import type { ModuleHelper } from "./ModuleHelper";
 
 export type ModuleBuildContext = {
    connection: Connection;
@@ -24,6 +15,7 @@ export type ModuleBuildContext = {
    guard: Guard;
    logger: DebugLogger;
    flags: (typeof Module)["ctx_flags"];
+   helper: ModuleHelper;
 };
 
 export abstract class Module<Schema extends TSchema = TSchema, ConfigSchema = Static<Schema>> {
@@ -141,80 +133,4 @@ export abstract class Module<Schema extends TSchema = TSchema, ConfigSchema = St
    toJSON(secrets?: boolean): Static<ReturnType<(typeof this)["getSchema"]>> {
       return this.config;
    }
-
-   protected ensureEntity(entity: Entity) {
-      const instance = this.ctx.em.entity(entity.name, true);
-
-      // check fields
-      if (!instance) {
-         this.ctx.em.addEntity(entity);
-         this.ctx.flags.sync_required = true;
-         return;
-      }
-
-      // if exists, check all fields required are there
-      // @todo: check if the field also equal
-      for (const field of entity.fields) {
-         const instanceField = instance.field(field.name);
-         if (!instanceField) {
-            instance.addField(field);
-            this.ctx.flags.sync_required = true;
-         } else {
-            const changes = this.setEntityFieldConfigs(field, instanceField);
-            if (changes > 0) {
-               this.ctx.flags.sync_required = true;
-            }
-         }
-      }
-
-      // replace entity (mainly to keep the ensured type)
-      this.ctx.em.__replaceEntity(
-         new Entity(instance.name, instance.fields, instance.config, entity.type),
-      );
-   }
-
-   protected ensureIndex(index: EntityIndex) {
-      if (!this.ctx.em.hasIndex(index)) {
-         this.ctx.em.addIndex(index);
-         this.ctx.flags.sync_required = true;
-      }
-   }
-
-   protected ensureSchema<Schema extends ReturnType<typeof prototypeEm>>(schema: Schema): Schema {
-      Object.values(schema.entities ?? {}).forEach(this.ensureEntity.bind(this));
-      schema.indices?.forEach(this.ensureIndex.bind(this));
-
-      return schema;
-   }
-
-   protected setEntityFieldConfigs(
-      parent: Field,
-      child: Field,
-      props: string[] = ["hidden", "fillable", "required"],
-   ) {
-      let changes = 0;
-      for (const prop of props) {
-         if (!isEqual(child.config[prop], parent.config[prop])) {
-            child.config[prop] = parent.config[prop];
-            changes++;
-         }
-      }
-      return changes;
-   }
-
-   protected replaceEntityField(
-      _entity: string | Entity,
-      field: Field | string,
-      _newField: Field | FieldPrototype,
-   ) {
-      const entity = this.ctx.em.entity(_entity);
-      const name = typeof field === "string" ? field : field.name;
-      const newField =
-         _newField instanceof FieldPrototype ? make(name, _newField as any) : _newField;
-
-      // ensure keeping vital config
-      this.setEntityFieldConfigs(entity.field(name)!, newField);
-
-      entity.__replaceField(name, newField);
-   }
 }

app/src/modules/ModuleHelper.ts

@@ -0,0 +1,113 @@
+import {
+   type EntityIndex,
+   type EntityRelation,
+   type Field,
+   type em as prototypeEm,
+   FieldPrototype,
+   make,
+   Entity,
+   entityTypes,
+} from "data";
+import { isEqual } from "lodash-es";
+import type { ModuleBuildContext } from "./Module";
+
+export class ModuleHelper {
+   constructor(protected ctx: Omit<ModuleBuildContext, "helper">) {}
+
+   get em() {
+      return this.ctx.em;
+   }
+
+   get flags() {
+      return this.ctx.flags;
+   }
+
+   ensureEntity(entity: Entity) {
+      const instance = this.em.entity(entity.name, true);
+
+      // check fields
+      if (!instance) {
+         this.em.addEntity(entity);
+         this.flags.sync_required = true;
+         return;
+      }
+
+      // if exists, check all fields required are there
+      // @todo: potentially identify system and generated entities and take that as instance
+      // @todo: check if the field also equal
+      for (const field of entity.fields) {
+         const instanceField = instance.field(field.name);
+         if (!instanceField) {
+            instance.addField(field);
+            this.flags.sync_required = true;
+         } else {
+            const changes = this.setEntityFieldConfigs(field, instanceField);
+            if (changes > 0) {
+               this.flags.sync_required = true;
+            }
+         }
+      }
+
+      // if type is different, keep the highest
+      if (instance.type !== entity.type) {
+         const instance_i = entityTypes.indexOf(instance.type);
+         const entity_i = entityTypes.indexOf(entity.type);
+         const type = entity_i > instance_i ? entity.type : instance.type;
+
+         this.em.__replaceEntity(new Entity(instance.name, instance.fields, instance.config, type));
+      }
+   }
+
+   ensureIndex(index: EntityIndex) {
+      if (!this.em.hasIndex(index)) {
+         this.em.addIndex(index);
+         this.flags.sync_required = true;
+      }
+   }
+
+   ensureRelation(relation: EntityRelation) {
+      if (!this.em.relations.exists(relation)) {
+         this.em.addRelation(relation);
+         this.flags.sync_required = true;
+      }
+   }
+
+   ensureSchema<Schema extends ReturnType<typeof prototypeEm>>(schema: Schema): Schema {
+      Object.values(schema.entities ?? {}).forEach(this.ensureEntity.bind(this));
+      schema.indices?.forEach(this.ensureIndex.bind(this));
+      schema.relations?.forEach(this.ensureRelation.bind(this));
+
+      return schema;
+   }
+
+   setEntityFieldConfigs(
+      parent: Field,
+      child: Field,
+      props: string[] = ["hidden", "fillable", "required"],
+   ) {
+      let changes = 0;
+      for (const prop of props) {
+         if (!isEqual(child.config[prop], parent.config[prop])) {
+            child.config[prop] = parent.config[prop];
+            changes++;
+         }
+      }
+      return changes;
+   }
+
+   replaceEntityField(
+      _entity: string | Entity,
+      field: Field | string,
+      _newField: Field | FieldPrototype,
+   ) {
+      const entity = this.em.entity(_entity);
+      const name = typeof field === "string" ? field : field.name;
+      const newField =
+         _newField instanceof FieldPrototype ? make(name, _newField as any) : _newField;
+
+      // ensure keeping vital config
+      this.setEntityFieldConfigs(entity.field(name)!, newField);
+
+      entity.__replaceField(name, newField);
+   }
+}

app/src/modules/ModuleManager.ts

@@ -34,6 +34,7 @@ import { AppMedia } from "../media/AppMedia";
 import type { ServerEnv } from "./Controller";
 import { Module, type ModuleBuildContext } from "./Module";
 import * as tbbox from "@sinclair/typebox";
+import { ModuleHelper } from "./ModuleHelper";
 const { Type } = tbbox;
 
 export type { ModuleBuildContext };
@@ -92,6 +93,8 @@ export type ModuleManagerOptions = {
    trustFetched?: boolean;
    // runs when initial config provided on a fresh database
    seed?: (ctx: ModuleBuildContext) => Promise<void>;
+   // called right after modules are built, before finish
+   onModulesBuilt?: (ctx: ModuleBuildContext) => Promise<void>;
    /** @deprecated */
    verbosity?: Verbosity;
 };
@@ -267,7 +270,7 @@ export class ModuleManager {
          this.guard = new Guard();
       }
 
-      return {
+      const ctx = {
          connection: this.connection,
          server: this.server,
          em: this.em,
@@ -276,6 +279,11 @@ export class ModuleManager {
          flags: Module.ctx_flags,
          logger: this.logger,
       };
+
+      return {
+         ...ctx,
+         helper: new ModuleHelper(ctx),
+      };
    }
 
    private async fetch(): Promise<ConfigTable | undefined> {
@@ -549,6 +557,10 @@ export class ModuleManager {
       this._built = state.built = true;
       this.logger.log("modules built", ctx.flags);
 
+      if (this.options?.onModulesBuilt) {
+         await this.options.onModulesBuilt(ctx);
+      }
+
       if (options?.ignoreFlags !== true) {
          if (ctx.flags.sync_required) {
             ctx.flags.sync_required = false;