public commit

app/src/modules/Module.ts

@@ -0,0 +1,112 @@
+import type { Guard } from "auth";
+import { SchemaObject } from "core";
+import type { EventManager } from "core/events";
+import type { Static, TSchema } from "core/utils";
+import type { Connection, EntityManager } from "data";
+import type { Hono } from "hono";
+
+export type ModuleBuildContext = {
+   connection: Connection;
+   server: Hono<any>;
+   em: EntityManager<any>;
+   emgr: EventManager<any>;
+   guard: Guard;
+};
+
+export abstract class Module<Schema extends TSchema = TSchema> {
+   private _built = false;
+   private _schema: SchemaObject<ReturnType<(typeof this)["getSchema"]>>;
+   private _listener: any = () => null;
+
+   constructor(
+      initial?: Partial<Static<Schema>>,
+      protected _ctx?: ModuleBuildContext
+   ) {
+      this._schema = new SchemaObject(this.getSchema(), initial, {
+         forceParse: this.useForceParse(),
+         onUpdate: async (c) => {
+            await this._listener(c);
+         },
+         restrictPaths: this.getRestrictedPaths(),
+         overwritePaths: this.getOverwritePaths()
+      });
+   }
+
+   setListener(listener: (c: ReturnType<(typeof this)["getSchema"]>) => void | Promise<void>) {
+      this._listener = listener;
+      return this;
+   }
+
+   // @todo: test all getSchema() for additional properties
+   abstract getSchema();
+
+   useForceParse() {
+      return false;
+   }
+
+   getRestrictedPaths(): string[] | undefined {
+      return undefined;
+   }
+
+   /**
+    * These paths will be overwritten, even when "patch" is called.
+    * This is helpful if there are keys that contains records, which always be sent in full.
+    */
+   getOverwritePaths(): (RegExp | string)[] | undefined {
+      return undefined;
+   }
+
+   get configDefault(): Static<ReturnType<(typeof this)["getSchema"]>> {
+      return this._schema.default();
+   }
+
+   get config(): Static<ReturnType<(typeof this)["getSchema"]>> {
+      return this._schema.get();
+   }
+
+   setContext(ctx: ModuleBuildContext) {
+      this._ctx = ctx;
+      return this;
+   }
+
+   schema() {
+      return this._schema;
+   }
+
+   get ctx() {
+      if (!this._ctx) {
+         throw new Error("Context not set");
+      }
+      return this._ctx;
+   }
+
+   async build() {
+      throw new Error("Not implemented");
+   }
+
+   setBuilt() {
+      this._built = true;
+      this._schema = new SchemaObject(this.getSchema(), this.toJSON(true), {
+         onUpdate: async (c) => {
+            await this._listener(c);
+         },
+         forceParse: this.useForceParse(),
+         restrictPaths: this.getRestrictedPaths(),
+         overwritePaths: this.getOverwritePaths()
+      });
+   }
+
+   isBuilt() {
+      return this._built;
+   }
+
+   throwIfNotBuilt() {
+      if (!this._built) {
+         throw new Error("Config not built: " + this.constructor.name);
+      }
+   }
+
+   toJSON(secrets?: boolean): Static<ReturnType<(typeof this)["getSchema"]>> {
+      return this.config;
+   }
+}

app/src/modules/ModuleApi.ts

@@ -0,0 +1,149 @@
+import type { PrimaryFieldType } from "core";
+import { encodeSearch } from "core/utils";
+
+export type { PrimaryFieldType };
+export type BaseModuleApiOptions = {
+   host: string;
+   basepath?: string;
+   token?: string;
+};
+
+export type ApiResponse<Data = any> = {
+   success: boolean;
+   status: number;
+   body: Data;
+   data?: Data extends { data: infer R } ? R : any;
+   res: Response;
+};
+
+export abstract class ModuleApi<Options extends BaseModuleApiOptions> {
+   constructor(protected readonly _options: Partial<Options> = {}) {}
+
+   protected getDefaultOptions(): Partial<Options> {
+      return {};
+   }
+
+   get options(): Options {
+      return {
+         host: "http://localhost",
+         token: undefined,
+         ...this.getDefaultOptions(),
+         ...this._options
+      } as Options;
+   }
+
+   protected getUrl(path: string) {
+      return this.options.host + (this.options.basepath + "/" + path).replace(/\/\//g, "/");
+   }
+
+   protected async request<Data = any>(
+      _input: string | (string | number | PrimaryFieldType)[],
+      _query?: Record<string, any> | URLSearchParams,
+      _init?: RequestInit
+   ): Promise<ApiResponse<Data>> {
+      const method = _init?.method ?? "GET";
+      const input = Array.isArray(_input) ? _input.join("/") : _input;
+      let url = this.getUrl(input);
+
+      if (_query instanceof URLSearchParams) {
+         url += "?" + _query.toString();
+      } else if (typeof _query === "object") {
+         if (Object.keys(_query).length > 0) {
+            url += "?" + encodeSearch(_query);
+         }
+      }
+
+      const headers = new Headers(_init?.headers ?? {});
+      headers.set("Accept", "application/json");
+
+      if (this.options.token) {
+         //console.log("setting token", this.options.token);
+         headers.set("Authorization", `Bearer ${this.options.token}`);
+      } else {
+         //console.log("no token");
+      }
+
+      let body: any = _init?.body;
+      if (_init && "body" in _init && ["POST", "PATCH"].includes(method)) {
+         const requestContentType = (headers.get("Content-Type") as string) ?? undefined;
+         if (!requestContentType || requestContentType.startsWith("application/json")) {
+            body = JSON.stringify(_init.body);
+            headers.set("Content-Type", "application/json");
+         }
+      }
+
+      //console.log("url", url);
+      const res = await fetch(url, {
+         ..._init,
+         method,
+         body,
+         headers
+      });
+
+      let resBody: any;
+      let resData: any;
+
+      const contentType = res.headers.get("Content-Type") ?? "";
+      if (contentType.startsWith("application/json")) {
+         resBody = await res.json();
+         if (typeof resBody === "object") {
+            resData = "data" in resBody ? resBody.data : resBody;
+         }
+      } else if (contentType.startsWith("text")) {
+         resBody = await res.text();
+      }
+
+      return {
+         success: res.ok,
+         status: res.status,
+         body: resBody,
+         data: resData,
+         res
+      };
+   }
+
+   protected async get<Data = any>(
+      _input: string | (string | number | PrimaryFieldType)[],
+      _query?: Record<string, any> | URLSearchParams,
+      _init?: RequestInit
+   ) {
+      return this.request<Data>(_input, _query, {
+         ..._init,
+         method: "GET"
+      });
+   }
+
+   protected async post<Data = any>(
+      _input: string | (string | number | PrimaryFieldType)[],
+      body?: any,
+      _init?: RequestInit
+   ) {
+      return this.request<Data>(_input, undefined, {
+         ..._init,
+         body,
+         method: "POST"
+      });
+   }
+
+   protected async patch<Data = any>(
+      _input: string | (string | number | PrimaryFieldType)[],
+      body?: any,
+      _init?: RequestInit
+   ) {
+      return this.request<Data>(_input, undefined, {
+         ..._init,
+         body,
+         method: "PATCH"
+      });
+   }
+
+   protected async delete<Data = any>(
+      _input: string | (string | number | PrimaryFieldType)[],
+      _init?: RequestInit
+   ) {
+      return this.request<Data>(_input, undefined, {
+         ..._init,
+         method: "DELETE"
+      });
+   }
+}

app/src/modules/ModuleManager.ts

@@ -0,0 +1,443 @@
+import { Diff } from "@sinclair/typebox/value";
+import { Guard } from "auth";
+import { DebugLogger, isDebug } from "core";
+import { EventManager } from "core/events";
+import { Default, type Static, objectEach, transformObject } from "core/utils";
+import { type Connection, EntityManager } from "data";
+import { Hono } from "hono";
+import { type Kysely, sql } from "kysely";
+import { CURRENT_VERSION, TABLE_NAME, migrate, migrateSchema } from "modules/migrations";
+import { AppServer } from "modules/server/AppServer";
+import { AppAuth } from "../auth/AppAuth";
+import { AppData } from "../data/AppData";
+import { AppFlows } from "../flows/AppFlows";
+import { AppMedia } from "../media/AppMedia";
+import type { Module, ModuleBuildContext } from "./Module";
+
+export const MODULES = {
+   server: AppServer,
+   data: AppData<any>,
+   auth: AppAuth,
+   media: AppMedia,
+   flows: AppFlows
+} as const;
+
+// get names of MODULES as an array
+export const MODULE_NAMES = Object.keys(MODULES) as ModuleKey[];
+
+export type ModuleKey = keyof typeof MODULES;
+export type Modules = {
+   [K in keyof typeof MODULES]: InstanceType<(typeof MODULES)[K]>;
+};
+
+export type ModuleSchemas = {
+   [K in keyof typeof MODULES]: ReturnType<(typeof MODULES)[K]["prototype"]["getSchema"]>;
+};
+
+export type ModuleConfigs = {
+   [K in keyof ModuleSchemas]: Static<ModuleSchemas[K]>;
+};
+
+export type InitialModuleConfigs = {
+   version: number;
+} & Partial<ModuleConfigs>;
+
+export type ModuleManagerOptions = {
+   initial?: InitialModuleConfigs;
+   eventManager?: EventManager<any>;
+   onUpdated?: <Module extends keyof Modules>(
+      module: Module,
+      config: ModuleConfigs[Module]
+   ) => Promise<void>;
+   // base path for the hono instance
+   basePath?: string;
+};
+
+type ConfigTable<Json = ModuleConfigs> = {
+   version: number;
+   type: "config" | "diff" | "backup";
+   json: Json;
+   created_at?: Date;
+   updated_at?: Date;
+};
+
+export class ModuleManager {
+   private modules: Modules;
+   em!: EntityManager<any>;
+   server!: Hono;
+   emgr!: EventManager;
+   guard!: Guard;
+
+   private _version: number = 0;
+   private _built = false;
+   private _fetched = false;
+   private readonly _provided;
+
+   private logger = new DebugLogger(isDebug() && false);
+
+   constructor(
+      private readonly connection: Connection,
+      private options?: Partial<ModuleManagerOptions>
+   ) {
+      this.modules = {} as Modules;
+      this.emgr = new EventManager();
+      const context = this.ctx(true);
+      let initial = {} as Partial<ModuleConfigs>;
+
+      if (options?.initial) {
+         const { version, ...initialConfig } = options.initial;
+         if (version && initialConfig) {
+            this._version = version;
+            initial = initialConfig;
+
+            this._provided = true;
+         } else {
+            throw new Error("Initial was provided, but it needs a version!");
+         }
+      }
+
+      for (const key in MODULES) {
+         const moduleConfig = key in initial ? initial[key] : {};
+         const module = new MODULES[key](moduleConfig, context) as Module;
+         module.setListener(async (c) => {
+            await this.onModuleConfigUpdated(key, c);
+         });
+
+         this.modules[key] = module;
+      }
+   }
+
+   /**
+    * This is set through module's setListener
+    * It's called everytime a module's config is updated in SchemaObject
+    * Needs to rebuild modules and save to database
+    */
+   private async onModuleConfigUpdated(key: string, config: any) {
+      if (this.options?.onUpdated) {
+         await this.options.onUpdated(key as any, config);
+      } else {
+         this.buildModules();
+      }
+   }
+
+   private rebuildServer() {
+      this.server = new Hono();
+      if (this.options?.basePath) {
+         this.server = this.server.basePath(this.options.basePath);
+      }
+
+      // @todo: this is a current workaround, controllers must be reworked
+      objectEach(this.modules, (module) => {
+         if ("getMiddleware" in module) {
+            const middleware = module.getMiddleware();
+            if (middleware) {
+               this.server.use(middleware);
+            }
+         }
+      });
+   }
+
+   ctx(rebuild?: boolean): ModuleBuildContext {
+      if (rebuild) {
+         this.rebuildServer();
+         this.em = new EntityManager([], this.connection, [], [], this.emgr);
+         this.guard = new Guard();
+      }
+
+      return {
+         connection: this.connection,
+         server: this.server,
+         em: this.em,
+         emgr: this.emgr,
+         guard: this.guard
+      };
+   }
+
+   private get db() {
+      return this.connection.kysely as Kysely<{ table: ConfigTable }>;
+   }
+
+   get table() {
+      return TABLE_NAME as "table";
+   }
+
+   private async fetch(): Promise<ConfigTable> {
+      this.logger.context("fetch").log("fetching");
+
+      const startTime = performance.now();
+      const result = await this.db
+         .selectFrom(this.table)
+         .selectAll()
+         .where("type", "=", "config")
+         .orderBy("version", "desc")
+         .executeTakeFirstOrThrow();
+
+      this.logger.log("took", performance.now() - startTime, "ms", result).clear();
+      return result;
+   }
+
+   async save() {
+      this.logger.context("save").log("saving version", this.version());
+      const configs = this.configs();
+      const version = this.version();
+
+      const json = JSON.stringify(configs) as any;
+      const state = await this.fetch();
+
+      if (state.version !== version) {
+         // @todo: mark all others as "backup"
+         this.logger.log("version conflict, storing new version", state.version, version);
+         await this.db
+            .insertInto(this.table)
+            .values({
+               version,
+               type: "config",
+               json
+            })
+            .execute();
+      } else {
+         this.logger.log("version matches");
+
+         const diff = Diff(state.json, JSON.parse(json));
+         this.logger.log("checking diff", diff);
+
+         if (diff.length > 0) {
+            // store diff
+            await this.db
+               .insertInto(this.table)
+               .values({
+                  version,
+                  type: "diff",
+                  json: JSON.stringify(diff) as any
+               })
+               .execute();
+
+            await this.db
+               .updateTable(this.table)
+               .set({ version, json, updated_at: sql`CURRENT_TIMESTAMP` })
+               .where((eb) => eb.and([eb("type", "=", "config"), eb("version", "=", version)]))
+               .execute();
+         } else {
+            this.logger.log("no diff, not saving");
+         }
+      }
+
+      // cleanup
+      /*this.logger.log("cleaning up");
+      const result = await this.db
+         .deleteFrom(this.table)
+         .where((eb) =>
+            eb.or([
+               // empty migrations
+               eb.and([
+                  eb("type", "=", "config"),
+                  eb("version", "<", version),
+                  eb("json", "is", null)
+               ]),
+               // past diffs
+               eb.and([eb("type", "=", "diff"), eb("version", "<", version)])
+            ])
+         )
+         .executeTakeFirst();
+      this.logger.log("cleaned up", result.numDeletedRows);*/
+
+      this.logger.clear();
+      return this;
+   }
+
+   private async migrate() {
+      this.logger.context("migrate").log("migrating?", this.version(), CURRENT_VERSION);
+
+      if (this.version() < CURRENT_VERSION) {
+         this.logger.log("there are migrations, verify version");
+
+         // modules must be built before migration
+         await this.buildModules({ graceful: true });
+
+         try {
+            const state = await this.fetch();
+            if (state.version !== this.version()) {
+               // @todo: potentially drop provided config and use database version
+               throw new Error(
+                  `Given version (${this.version()}) and fetched version (${state.version}) do not match.`
+               );
+            }
+         } catch (e: any) {
+            this.logger.clear(); // fetch couldn't clear
+
+            // if table doesn't exist, migrate schema to version
+            if (e.message.includes("no such table")) {
+               this.logger.log("table has to created, migrating schema up to", this.version());
+               await migrateSchema(this.version(), { db: this.db });
+            } else {
+               throw new Error(`Version is ${this.version()}, fetch failed: ${e.message}`);
+            }
+         }
+
+         this.logger.log("now migrating");
+         let version = this.version();
+         let configs: any = this.configs();
+         //console.log("migrating with", version, configs);
+         if (Object.keys(configs).length === 0) {
+            throw new Error("No config to migrate");
+         }
+
+         const [_version, _configs] = await migrate(version, configs, {
+            db: this.db
+         });
+         version = _version;
+         configs = _configs;
+
+         this.setConfigs(configs);
+         /* objectEach(configs, (config, key) => {
+            this.get(key as any).setConfig(config);
+         }); */
+
+         this._version = version;
+         this.logger.log("migrated to", version);
+
+         await this.save();
+      } else {
+         this.logger.log("no migrations needed");
+      }
+
+      this.logger.clear();
+   }
+
+   private setConfigs(configs: ModuleConfigs): void {
+      objectEach(configs, (config, key) => {
+         try {
+            // setting "noEmit" to true, to not force listeners to update
+            this.modules[key].schema().set(config as any, true);
+         } catch (e) {
+            console.error(e);
+            throw new Error(
+               `Failed to set config for module ${key}: ${JSON.stringify(config, null, 2)}`
+            );
+         }
+      });
+   }
+
+   private async buildModules(options?: { graceful?: boolean }) {
+      this.logger.log("buildModules() triggered", options?.graceful, this._built);
+      if (options?.graceful && this._built) {
+         this.logger.log("skipping build (graceful)");
+         return;
+      }
+
+      const ctx = this.ctx(true);
+      for (const key in this.modules) {
+         this.logger.log(`building "${key}"`);
+         await this.modules[key].setContext(ctx).build();
+      }
+
+      this._built = true;
+      this.logger.log("modules built");
+   }
+
+   async build() {
+      this.logger.context("build").log("version", this.version());
+
+      // if no config provided, try fetch from db
+      if (this.version() === 0) {
+         this.logger.context("build no config").log("version is 0");
+         try {
+            const result = await this.fetch();
+
+            // set version and config from fetched
+            this._version = result.version;
+            this.setConfigs(result.json);
+         } catch (e: any) {
+            this.logger.clear(); // fetch couldn't clear
+
+            this.logger.context("error handler").log("fetch failed", e.message);
+            // if table doesn't exist, migrate schema, set default config and latest version
+            if (e.message.includes("no such table")) {
+               this.logger.log("migrate schema to", CURRENT_VERSION);
+               await migrateSchema(CURRENT_VERSION, { db: this.db });
+               this._version = CURRENT_VERSION;
+
+               // we can safely build modules, since config version is up to date
+               // it's up to date because we use default configs (no fetch result)
+               await this.buildModules();
+               await this.save();
+
+               this.logger.clear();
+               return this;
+            } else {
+               throw e;
+               //throw new Error("Issues connecting to the database. Reason: " + e.message);
+            }
+         }
+         this.logger.clear();
+      }
+
+      // migrate to latest if needed
+      await this.migrate();
+
+      this.logger.log("building");
+      await this.buildModules();
+      return this;
+   }
+
+   get<K extends keyof Modules>(key: K): Modules[K] {
+      if (!(key in this.modules)) {
+         throw new Error(`Module "${key}" doesn't exist, cannot get`);
+      }
+      return this.modules[key];
+   }
+
+   version() {
+      return this._version;
+   }
+
+   built() {
+      return this._built;
+   }
+
+   configs(): ModuleConfigs {
+      return transformObject(this.modules, (module) => module.toJSON(true)) as any;
+   }
+
+   getSchema() {
+      const schemas = transformObject(this.modules, (module) => module.getSchema());
+
+      return {
+         version: this.version(),
+         ...schemas
+      };
+   }
+
+   toJSON(secrets?: boolean): { version: number } & ModuleConfigs {
+      const modules = transformObject(this.modules, (module) => {
+         if (this._built) {
+            return module.isBuilt() ? module.toJSON(secrets) : module.configDefault;
+         }
+
+         // returns no config if the all modules are not built
+         return undefined;
+      });
+
+      return {
+         version: this.version(),
+         ...modules
+      } as any;
+   }
+}
+
+export function getDefaultSchema(pretty = false) {
+   const schema = {
+      type: "object",
+      ...transformObject(MODULES, (module) => module.prototype.getSchema())
+   };
+
+   return JSON.stringify(schema, null, pretty ? 2 : undefined);
+}
+
+export function getDefaultConfig(pretty = false): ModuleConfigs {
+   const config = transformObject(MODULES, (module) => {
+      return Default(module.prototype.getSchema(), {});
+   });
+
+   return JSON.stringify(config, null, pretty ? 2 : undefined) as any;
+}

app/src/modules/SystemApi.ts

@@ -0,0 +1,24 @@
+import { ModuleApi } from "./ModuleApi";
+import type { ModuleConfigs, ModuleSchemas } from "./ModuleManager";
+
+export type ApiSchemaResponse = {
+   version: number;
+   schema: ModuleSchemas;
+   config: ModuleConfigs;
+   permissions: string[];
+};
+
+export class SystemApi extends ModuleApi<any> {
+   protected override getDefaultOptions(): Partial<any> {
+      return {
+         basepath: "/api/system"
+      };
+   }
+
+   async readSchema(options?: { config?: boolean; secrets?: boolean }) {
+      return await this.get<ApiSchemaResponse>("schema", {
+         config: options?.config ? 1 : 0,
+         secrets: options?.secrets ? 1 : 0
+      });
+   }
+}

app/src/modules/index.ts

@@ -0,0 +1,21 @@
+import * as prototype from "data/prototype";
+export { prototype };
+
+export { AppAuth } from "auth/AppAuth";
+export { AppData } from "data/AppData";
+export { AppMedia, type MediaFieldSchema } from "media/AppMedia";
+export { AppFlows, type AppFlowsSchema } from "flows/AppFlows";
+export {
+   type ModuleConfigs,
+   type ModuleSchemas,
+   MODULE_NAMES,
+   type ModuleKey
+} from "./ModuleManager";
+export { /*Module,*/ type ModuleBuildContext } from "./Module";
+
+export {
+   type PrimaryFieldType,
+   type BaseModuleApiOptions,
+   type ApiResponse,
+   ModuleApi
+} from "./ModuleApi";

app/src/modules/migrations.ts

@@ -0,0 +1,143 @@
+import { _jsonp } from "core/utils";
+import { type Kysely, sql } from "kysely";
+import { set } from "lodash-es";
+
+export type MigrationContext = {
+   db: Kysely<any>;
+};
+export type GenericConfigObject = Record<string, any>;
+
+export type Migration = {
+   version: number;
+   schema?: true;
+   up: (config: GenericConfigObject, ctx: MigrationContext) => Promise<GenericConfigObject>;
+};
+
+export const migrations: Migration[] = [
+   {
+      version: 1,
+      schema: true,
+      up: async (config, { db }) => {
+         //console.log("config given", config);
+         await db.schema
+            .createTable(TABLE_NAME)
+            .addColumn("id", "integer", (col) => col.primaryKey().notNull().autoIncrement())
+            .addColumn("version", "integer", (col) => col.notNull())
+            .addColumn("type", "text", (col) => col.notNull())
+            .addColumn("json", "text")
+            .addColumn("created_at", "datetime", (col) => col.defaultTo(sql`CURRENT_TIMESTAMP`))
+            .addColumn("updated_at", "datetime", (col) => col.defaultTo(sql`CURRENT_TIMESTAMP`))
+            .execute();
+
+         await db
+            .insertInto(TABLE_NAME)
+            .values({ version: 1, type: "config", json: null })
+            .execute();
+
+         return config;
+      }
+   },
+   {
+      version: 2,
+      up: async (config, { db }) => {
+         return config;
+      }
+   },
+   {
+      version: 3,
+      schema: true,
+      up: async (config, { db }) => {
+         await db.schema.alterTable(TABLE_NAME).addColumn("deleted_at", "datetime").execute();
+
+         return config;
+      }
+   },
+   {
+      version: 4,
+      up: async (config, { db }) => {
+         return {
+            ...config,
+            auth: {
+               ...config.auth,
+               basepath: "/api/auth2"
+            }
+         };
+      }
+   },
+   {
+      version: 5,
+      up: async (config, { db }) => {
+         //console.log("config", _jsonp(config));
+         const cors = config.server.cors?.allow_methods ?? [];
+         set(config.server, "cors.allow_methods", [...new Set([...cors, "PATCH"])]);
+         return config;
+      }
+   },
+   {
+      version: 6,
+      up: async (config, { db }) => {
+         return config;
+      }
+   }
+];
+
+export const CURRENT_VERSION = migrations[migrations.length - 1]?.version ?? 0;
+export const TABLE_NAME = "__bknd";
+
+export async function migrateTo(
+   current: number,
+   to: number,
+   config: GenericConfigObject,
+   ctx: MigrationContext
+): Promise<[number, GenericConfigObject]> {
+   //console.log("migrating from", current, "to", CURRENT_VERSION, config);
+   const todo = migrations.filter((m) => m.version > current && m.version <= to);
+   //console.log("todo", todo.length);
+   let updated = Object.assign({}, config);
+
+   let i = 0;
+   let version = current;
+   for (const migration of todo) {
+      //console.log("-- running migration", i + 1, "of", todo.length, { version: migration.version });
+      try {
+         updated = await migration.up(updated, ctx);
+         version = migration.version;
+         i++;
+      } catch (e: any) {
+         console.error(e);
+         throw new Error(`Migration ${migration.version} failed: ${e.message}`);
+      }
+   }
+
+   return [version, updated];
+}
+
+export async function migrateSchema(to: number, ctx: MigrationContext, current: number = 0) {
+   console.log("migrating SCHEMA to", to, "from", current);
+   const todo = migrations.filter((m) => m.version > current && m.version <= to && m.schema);
+   console.log("todo", todo.length);
+
+   let i = 0;
+   let version = 0;
+   for (const migration of todo) {
+      console.log("-- running migration", i + 1, "of", todo.length);
+      try {
+         await migration.up({}, ctx);
+         version = migration.version;
+         i++;
+      } catch (e: any) {
+         console.error(e);
+         throw new Error(`Migration ${migration.version} failed: ${e.message}`);
+      }
+   }
+
+   return version;
+}
+
+export async function migrate(
+   current: number,
+   config: GenericConfigObject,
+   ctx: MigrationContext
+): Promise<[number, GenericConfigObject]> {
+   return migrateTo(current, CURRENT_VERSION, config, ctx);
+}

app/src/modules/permissions/index.ts

@@ -0,0 +1,7 @@
+import { Permission } from "core";
+
+export const configRead = new Permission("system.config.read");
+export const configReadSecrets = new Permission("system.config.read.secrets");
+export const configWrite = new Permission("system.config.write");
+export const schemaRead = new Permission("system.schema.read");
+export const build = new Permission("system.build");

app/src/modules/registries.ts

@@ -0,0 +1,7 @@
+import { MediaAdapterRegistry } from "media";
+
+const registries = {
+   media: MediaAdapterRegistry
+} as const;
+
+export { registries };

app/src/modules/server/AppController.ts

@@ -0,0 +1,110 @@
+import type { ClassController } from "core";
+import { SimpleRenderer } from "core";
+import { FetchTask, Flow, LogTask } from "flows";
+import { Hono } from "hono";
+import { endTime, startTime } from "hono/timing";
+import type { App } from "../../App";
+
+export class AppController implements ClassController {
+   constructor(
+      private readonly app: App,
+      private config: any = {}
+   ) {}
+
+   getController(): Hono {
+      const hono = new Hono();
+
+      // @todo: add test endpoints
+
+      hono
+         .get("/config", (c) => {
+            return c.json(this.app.toJSON());
+         })
+         .get("/ping", (c) => {
+            //console.log("c", c);
+            try {
+               // @ts-ignore @todo: fix with env
+               const context: any = c.req.raw.cf ? c.req.raw.cf : c.env.cf;
+               const cf = {
+                  colo: context.colo,
+                  city: context.city,
+                  postal: context.postalCode,
+                  region: context.region,
+                  regionCode: context.regionCode,
+                  continent: context.continent,
+                  country: context.country,
+                  eu: context.isEUCountry,
+                  lat: context.latitude,
+                  lng: context.longitude,
+                  timezone: context.timezone
+               };
+               return c.json({ pong: true, cf, another: 6 });
+            } catch (e) {
+               return c.json({ pong: true, cf: null });
+            }
+         });
+
+      // test endpoints
+      if (this.config?.registerTest) {
+         hono.get("/test/kv", async (c) => {
+            // @ts-ignore
+            const cache = c.env!.CACHE as KVNamespace;
+            startTime(c, "kv-get");
+            const value: any = await cache.get("count");
+            endTime(c, "kv-get");
+            console.log("value", value);
+            startTime(c, "kv-put");
+            if (!value) {
+               await cache.put("count", "1");
+            } else {
+               await cache.put("count", (Number(value) + 1).toString());
+            }
+            endTime(c, "kv-put");
+
+            let cf: any = {};
+            // @ts-ignore
+            if ("cf" in c.req.raw) {
+               cf = {
+                  // @ts-ignore
+                  colo: c.req.raw.cf?.colo
+               };
+            }
+
+            return c.json({ pong: true, value, cf });
+         });
+
+         hono.get("/test/flow", async (c) => {
+            const first = new LogTask("Task 0");
+            const second = new LogTask("Task 1");
+            const third = new LogTask("Task 2", { delay: 250 });
+            const fourth = new FetchTask("Fetch Something", {
+               url: "https://jsonplaceholder.typicode.com/todos/1"
+            });
+            const fifth = new LogTask("Task 4"); // without connection
+
+            const flow = new Flow("flow", [first, second, third, fourth, fifth]);
+            flow.task(first).asInputFor(second);
+            flow.task(first).asInputFor(third);
+            flow.task(fourth).asOutputFor(third);
+
+            flow.setRespondingTask(fourth);
+
+            const execution = flow.createExecution();
+            await execution.start();
+
+            const results = flow.tasks.map((t) => t.toJSON());
+
+            return c.json({ results, response: execution.getResponse() });
+         });
+
+         hono.get("/test/template", async (c) => {
+            const renderer = new SimpleRenderer({ var: 123 });
+            const template = "Variable: {{ var }}";
+
+            return c.text(await renderer.render(template));
+         });
+      }
+
+      return hono;
+   }
+}

app/src/modules/server/AppServer.ts

@@ -0,0 +1,143 @@
+import { Exception, isDebug } from "core";
+import { type Static, StringEnum, Type } from "core/utils";
+import { Hono } from "hono";
+import { cors } from "hono/cors";
+import { timing } from "hono/timing";
+import { Module } from "modules/Module";
+
+const serverMethods = ["GET", "POST", "PATCH", "PUT", "DELETE"];
+export const serverConfigSchema = Type.Object(
+   {
+      admin: Type.Object(
+         {
+            basepath: Type.Optional(Type.String({ default: "", pattern: "^(/.+)?$" })),
+            color_scheme: Type.Optional(StringEnum(["dark", "light"], { default: "light" })),
+            logo_return_path: Type.Optional(
+               Type.String({
+                  default: "/",
+                  description: "Path to return to after *clicking* the logo"
+               })
+            )
+         },
+         { default: {}, additionalProperties: false }
+      ),
+      cors: Type.Object(
+         {
+            origin: Type.String({ default: "*" }),
+            allow_methods: Type.Array(StringEnum(serverMethods), {
+               default: serverMethods,
+               uniqueItems: true
+            }),
+            allow_headers: Type.Array(Type.String(), {
+               default: ["Content-Type", "Content-Length", "Authorization", "Accept"]
+            })
+         },
+         { default: {}, additionalProperties: false }
+      )
+   },
+   {
+      additionalProperties: false
+   }
+);
+
+export type AppServerConfig = Static<typeof serverConfigSchema>;
+
+/*declare global {
+   interface Request {
+      cf: IncomingRequestCfProperties;
+   }
+}*/
+
+export class AppServer extends Module<typeof serverConfigSchema> {
+   private admin_html?: string;
+
+   override getRestrictedPaths() {
+      return [];
+   }
+
+   get client() {
+      return this.ctx.server;
+   }
+
+   getSchema() {
+      return serverConfigSchema;
+   }
+
+   override async build() {
+      //this.client.use(timing());
+
+      /*this.client.use("*", async (c, next) => {
+         console.log(`[${c.req.method}] ${c.req.url}`);
+         await next();
+      });*/
+      this.client.use(
+         "*",
+         cors({
+            origin: this.config.cors.origin,
+            allowMethods: this.config.cors.allow_methods,
+            allowHeaders: this.config.cors.allow_headers
+         })
+      );
+
+      /*this.client.use(async (c, next) => {
+         c.res.headers.set("X-Powered-By", "BKND");
+         try {
+            c.res.headers.set("X-Colo", c.req.raw.cf.colo);
+         } catch (e) {}
+         await next();
+      });
+      this.client.use(async (c, next) => {
+         console.log(`[${c.req.method}] ${c.req.url}`);
+         await next();
+      });*/
+
+      this.client.onError((err, c) => {
+         //throw err;
+         console.error(err);
+
+         if (err instanceof Response) {
+            return err;
+         }
+
+         /*if (isDebug()) {
+            console.log("accept", c.req.header("Accept"));
+            if (c.req.header("Accept") === "application/json") {
+               const stack = err.stack;
+
+               if ("toJSON" in err && typeof err.toJSON === "function") {
+                  return c.json({ ...err.toJSON(), stack }, 500);
+               }
+
+               return c.json({ message: String(err), stack }, 500);
+            } else {
+               throw err;
+            }
+         }*/
+
+         if (err instanceof Exception) {
+            console.log("---is exception", err.code);
+            return c.json(err.toJSON(), err.code as any);
+         }
+
+         return c.json({ error: err.message }, 500);
+      });
+      this.setBuilt();
+   }
+
+   setAdminHtml(html: string) {
+      this.admin_html = html;
+      const basepath = (String(this.config.admin.basepath) + "/").replace(/\/+$/, "/");
+
+      this.client.get(basepath + "*", async (c, next) => {
+         return c.html(this.admin_html!);
+      });
+   }
+
+   getAdminHtml() {
+      return this.admin_html;
+   }
+
+   override toJSON(secrets?: boolean) {
+      return this.config;
+   }
+}

app/src/modules/server/SystemController.ts

@@ -0,0 +1,311 @@
+/// <reference types="@cloudflare/workers-types" />
+
+import type { ClassController } from "core";
+import { tbValidator as tb } from "core";
+import { StringEnum, Type, TypeInvalidError } from "core/utils";
+import { type Context, Hono } from "hono";
+import { MODULE_NAMES, type ModuleKey, getDefaultConfig } from "modules/ModuleManager";
+import * as SystemPermissions from "modules/permissions";
+import { generateOpenAPI } from "modules/server/openapi";
+import type { App } from "../../App";
+
+const booleanLike = Type.Transform(Type.String())
+   .Decode((v) => v === "1")
+   .Encode((v) => (v ? "1" : "0"));
+
+export class SystemController implements ClassController {
+   constructor(private readonly app: App) {}
+
+   get ctx() {
+      return this.app.modules.ctx();
+   }
+
+   private registerConfigController(client: Hono<any>): void {
+      const hono = new Hono();
+
+      /*hono.use("*", async (c, next) => {
+         //this.ctx.guard.throwUnlessGranted(SystemPermissions.configRead);
+         console.log("perm?", this.ctx.guard.hasPermission(SystemPermissions.configRead));
+         return next();
+      });*/
+
+      hono.get(
+         "/:module?",
+         tb("param", Type.Object({ module: Type.Optional(StringEnum(MODULE_NAMES)) })),
+         tb(
+            "query",
+            Type.Object({
+               secrets: Type.Optional(booleanLike)
+            })
+         ),
+         async (c) => {
+            // @todo: allow secrets if authenticated user is admin
+            const { secrets } = c.req.valid("query");
+            const { module } = c.req.valid("param");
+
+            secrets && this.ctx.guard.throwUnlessGranted(SystemPermissions.configReadSecrets);
+
+            const config = this.app.toJSON(secrets);
+
+            return c.json(
+               module
+                  ? {
+                       version: this.app.version(),
+                       module,
+                       config: config[module]
+                    }
+                  : config
+            );
+         }
+      );
+
+      async function handleConfigUpdateResponse(c: Context<any>, cb: () => Promise<object>) {
+         try {
+            return c.json(await cb(), { status: 202 });
+         } catch (e) {
+            console.error(e);
+
+            if (e instanceof TypeInvalidError) {
+               return c.json({ success: false, errors: e.errors }, { status: 400 });
+            }
+
+            return c.json({ success: false }, { status: 500 });
+         }
+      }
+
+      hono.post(
+         "/set/:module",
+         tb(
+            "query",
+            Type.Object({
+               force: Type.Optional(booleanLike)
+            })
+         ),
+         async (c) => {
+            const module = c.req.param("module") as any;
+            const { force } = c.req.valid("query");
+            const value = await c.req.json();
+
+            this.ctx.guard.throwUnlessGranted(SystemPermissions.configWrite);
+
+            return await handleConfigUpdateResponse(c, async () => {
+               // you must explicitly set force to override existing values
+               // because omitted values gets removed
+               if (force === true) {
+                  await this.app.mutateConfig(module).set(value);
+               } else {
+                  await this.app.mutateConfig(module).patch("", value);
+               }
+               return {
+                  success: true,
+                  module,
+                  config: this.app.module[module].config
+               };
+            });
+         }
+      );
+
+      hono.post("/add/:module/:path", async (c) => {
+         // @todo: require auth (admin)
+         const module = c.req.param("module") as any;
+         const value = await c.req.json();
+         const path = c.req.param("path") as string;
+
+         this.ctx.guard.throwUnlessGranted(SystemPermissions.configWrite);
+
+         const moduleConfig = this.app.mutateConfig(module);
+         if (moduleConfig.has(path)) {
+            return c.json({ success: false, path, error: "Path already exists" }, { status: 400 });
+         }
+         console.log("-- add", module, path, value);
+
+         return await handleConfigUpdateResponse(c, async () => {
+            await moduleConfig.patch(path, value);
+            return {
+               success: true,
+               module,
+               config: this.app.module[module].config
+            };
+         });
+      });
+
+      hono.patch("/patch/:module/:path", async (c) => {
+         // @todo: require auth (admin)
+         const module = c.req.param("module") as any;
+         const value = await c.req.json();
+         const path = c.req.param("path");
+
+         this.ctx.guard.throwUnlessGranted(SystemPermissions.configWrite);
+
+         return await handleConfigUpdateResponse(c, async () => {
+            await this.app.mutateConfig(module).patch(path, value);
+            return {
+               success: true,
+               module,
+               config: this.app.module[module].config
+            };
+         });
+      });
+
+      hono.put("/overwrite/:module/:path", async (c) => {
+         // @todo: require auth (admin)
+         const module = c.req.param("module") as any;
+         const value = await c.req.json();
+         const path = c.req.param("path");
+
+         this.ctx.guard.throwUnlessGranted(SystemPermissions.configWrite);
+
+         return await handleConfigUpdateResponse(c, async () => {
+            await this.app.mutateConfig(module).overwrite(path, value);
+            return {
+               success: true,
+               module,
+               config: this.app.module[module].config
+            };
+         });
+      });
+
+      hono.delete("/remove/:module/:path", async (c) => {
+         // @todo: require auth (admin)
+         const module = c.req.param("module") as any;
+         const path = c.req.param("path")!;
+
+         this.ctx.guard.throwUnlessGranted(SystemPermissions.configWrite);
+
+         return await handleConfigUpdateResponse(c, async () => {
+            await this.app.mutateConfig(module).remove(path);
+            return {
+               success: true,
+               module,
+               config: this.app.module[module].config
+            };
+         });
+      });
+
+      client.route("/config", hono);
+   }
+
+   getController(): Hono {
+      const hono = new Hono();
+
+      this.registerConfigController(hono);
+
+      hono.get(
+         "/schema/:module?",
+         tb(
+            "query",
+            Type.Object({
+               config: Type.Optional(booleanLike),
+               secrets: Type.Optional(booleanLike)
+            })
+         ),
+         async (c) => {
+            const module = c.req.param("module") as ModuleKey | undefined;
+            const { config, secrets } = c.req.valid("query");
+            this.ctx.guard.throwUnlessGranted(SystemPermissions.schemaRead);
+            config && this.ctx.guard.throwUnlessGranted(SystemPermissions.configRead);
+            secrets && this.ctx.guard.throwUnlessGranted(SystemPermissions.configReadSecrets);
+
+            const { version, ...schema } = this.app.getSchema();
+
+            if (module) {
+               return c.json({
+                  module,
+                  version,
+                  schema: schema[module],
+                  config: config ? this.app.module[module].toJSON(secrets) : undefined
+               });
+            }
+
+            return c.json({
+               module,
+               version,
+               schema,
+               config: config ? this.app.toJSON(secrets) : undefined,
+               permissions: this.app.modules.ctx().guard.getPermissionNames()
+            });
+         }
+      );
+
+      hono.post(
+         "/build",
+         tb(
+            "query",
+            Type.Object({
+               sync: Type.Optional(booleanLike),
+               drop: Type.Optional(booleanLike),
+               save: Type.Optional(booleanLike)
+            })
+         ),
+         async (c) => {
+            const { sync, drop, save } = c.req.valid("query") as Record<string, boolean>;
+            this.ctx.guard.throwUnlessGranted(SystemPermissions.build);
+
+            await this.app.build({ sync, drop, save });
+            return c.json({ success: true, options: { sync, drop, save } });
+         }
+      );
+
+      hono.get("/ping", async (c) => {
+         //console.log("c", c);
+         try {
+            // @ts-ignore @todo: fix with env
+            const context: any = c.req.raw.cf ? c.req.raw.cf : c.env.cf;
+            const cf = {
+               colo: context.colo,
+               city: context.city,
+               postal: context.postalCode,
+               region: context.region,
+               regionCode: context.regionCode,
+               continent: context.continent,
+               country: context.country,
+               eu: context.isEUCountry,
+               lat: context.latitude,
+               lng: context.longitude,
+               timezone: context.timezone
+            };
+            return c.json({ pong: true });
+         } catch (e) {
+            return c.json({ pong: true });
+         }
+      });
+
+      hono.get("/info", async (c) => {
+         return c.json({
+            version: this.app.version(),
+            test: 2,
+            // @ts-ignore
+            app: !!c.var.app
+         });
+      });
+
+      hono.get("/openapi.json", async (c) => {
+         //const config = this.app.toJSON();
+         const config = JSON.parse(getDefaultConfig() as any);
+         return c.json(generateOpenAPI(config));
+      });
+
+      /*hono.get("/test/sql", async (c) => {
+         // @ts-ignore
+         const ai = c.env?.AI as Ai;
+         const messages = [
+            { role: "system", content: "You are a friendly assistant" },
+            {
+               role: "user",
+               content: "just say hello"
+            }
+         ];
+
+         const stream = await ai.run("@cf/meta/llama-3.1-8b-instruct", {
+            messages,
+            stream: true
+         });
+
+         return new Response(stream, {
+            headers: { "content-type": "text/event-stream" }
+         });
+      });*/
+
+      return hono;
+   }
+}

app/src/modules/server/openapi.ts

@@ -0,0 +1,312 @@
+import { Type } from "core/utils";
+import type { ModuleConfigs } from "modules/ModuleManager";
+import type { OpenAPIV3 as OAS } from "openapi-types";
+
+function prefixPaths(paths: OAS.PathsObject, prefix: string): OAS.PathsObject {
+   const result: OAS.PathsObject = {};
+   for (const [path, pathItem] of Object.entries(paths)) {
+      result[`${prefix}${path}`] = pathItem;
+   }
+   return result;
+}
+
+function systemRoutes(config: ModuleConfigs): { paths: OAS.Document["paths"] } {
+   const tags = ["system"];
+   const paths: OAS.PathsObject = {
+      "/ping": {
+         get: {
+            summary: "Ping",
+            responses: {
+               "200": {
+                  description: "Pong",
+                  content: {
+                     "application/json": {
+                        schema: Type.Object({
+                           pong: Type.Boolean({ default: true })
+                        })
+                     }
+                  }
+               }
+            },
+            tags
+         }
+      },
+      "/config": {
+         get: {
+            summary: "Get config",
+            responses: {
+               "200": {
+                  description: "Config",
+                  content: {
+                     "application/json": {
+                        schema: Type.Object({
+                           version: Type.Number() as any,
+                           server: Type.Object({}),
+                           data: Type.Object({}),
+                           auth: Type.Object({}),
+                           flows: Type.Object({}),
+                           media: Type.Object({})
+                        })
+                     }
+                  }
+               }
+            },
+            tags
+         }
+      },
+      "/schema": {
+         get: {
+            summary: "Get config",
+            responses: {
+               "200": {
+                  description: "Config",
+                  content: {
+                     "application/json": {
+                        schema: Type.Object({
+                           version: Type.Number() as any,
+                           schema: Type.Object({
+                              server: Type.Object({}),
+                              data: Type.Object({}),
+                              auth: Type.Object({}),
+                              flows: Type.Object({}),
+                              media: Type.Object({})
+                           })
+                        })
+                     }
+                  }
+               }
+            },
+            tags
+         }
+      }
+   };
+
+   return { paths: prefixPaths(paths, "/api/system") };
+}
+
+function dataRoutes(config: ModuleConfigs): { paths: OAS.Document["paths"] } {
+   const schemas = {
+      entityData: Type.Object({
+         id: Type.Number() as any
+      })
+   };
+   const repoManyResponses: OAS.ResponsesObject = {
+      "200": {
+         description: "List of entities",
+         content: {
+            "application/json": {
+               schema: Type.Array(schemas.entityData)
+            }
+         }
+      }
+   };
+   const repoSingleResponses: OAS.ResponsesObject = {
+      "200": {
+         description: "Entity",
+         content: {
+            "application/json": {
+               schema: schemas.entityData
+            }
+         }
+      }
+   };
+   const params = {
+      entity: {
+         name: "entity",
+         in: "path",
+         required: true,
+         schema: Type.String()
+      },
+      entityId: {
+         name: "id",
+         in: "path",
+         required: true,
+         schema: Type.Number() as any
+      }
+   };
+
+   const tags = ["data"];
+   const paths: OAS.PathsObject = {
+      "/{entity}": {
+         get: {
+            summary: "List entities",
+            parameters: [params.entity],
+            responses: repoManyResponses,
+            tags
+         },
+         post: {
+            summary: "Create entity",
+            parameters: [params.entity],
+            requestBody: {
+               content: {
+                  "application/json": {
+                     schema: Type.Object({})
+                  }
+               }
+            },
+            responses: repoSingleResponses,
+            tags
+         }
+      },
+      "/{entity}/{id}": {
+         get: {
+            summary: "Get entity",
+            parameters: [params.entity, params.entityId],
+            responses: repoSingleResponses,
+            tags
+         },
+         patch: {
+            summary: "Update entity",
+            parameters: [params.entity, params.entityId],
+            requestBody: {
+               content: {
+                  "application/json": {
+                     schema: Type.Object({})
+                  }
+               }
+            },
+            responses: repoSingleResponses,
+            tags
+         },
+         delete: {
+            summary: "Delete entity",
+            parameters: [params.entity, params.entityId],
+            responses: {
+               "200": {
+                  description: "Entity deleted"
+               }
+            },
+            tags
+         }
+      }
+   };
+
+   return { paths: prefixPaths(paths, config.data.basepath!) };
+}
+
+function authRoutes(config: ModuleConfigs): { paths: OAS.Document["paths"] } {
+   const schemas = {
+      user: Type.Object({
+         id: Type.String(),
+         email: Type.String(),
+         name: Type.String()
+      })
+   };
+
+   const tags = ["auth"];
+   const paths: OAS.PathsObject = {
+      "/password/login": {
+         post: {
+            summary: "Login",
+            requestBody: {
+               content: {
+                  "application/json": {
+                     schema: Type.Object({
+                        email: Type.String(),
+                        password: Type.String()
+                     })
+                  }
+               }
+            },
+            responses: {
+               "200": {
+                  description: "User",
+                  content: {
+                     "application/json": {
+                        schema: Type.Object({
+                           user: schemas.user
+                        })
+                     }
+                  }
+               }
+            },
+            tags
+         }
+      },
+      "/password/register": {
+         post: {
+            summary: "Register",
+            requestBody: {
+               content: {
+                  "application/json": {
+                     schema: Type.Object({
+                        email: Type.String(),
+                        password: Type.String()
+                     })
+                  }
+               }
+            },
+            responses: {
+               "200": {
+                  description: "User",
+                  content: {
+                     "application/json": {
+                        schema: Type.Object({
+                           user: schemas.user
+                        })
+                     }
+                  }
+               }
+            },
+            tags
+         }
+      },
+      "/me": {
+         get: {
+            summary: "Get me",
+            responses: {
+               "200": {
+                  description: "User",
+                  content: {
+                     "application/json": {
+                        schema: Type.Object({
+                           user: schemas.user
+                        })
+                     }
+                  }
+               }
+            },
+            tags
+         }
+      },
+      "/strategies": {
+         get: {
+            summary: "Get auth strategies",
+            responses: {
+               "200": {
+                  description: "Strategies",
+                  content: {
+                     "application/json": {
+                        schema: Type.Object({
+                           strategies: Type.Object({})
+                        })
+                     }
+                  }
+               }
+            },
+            tags
+         }
+      }
+   };
+
+   return { paths: prefixPaths(paths, config.auth.basepath!) };
+}
+
+export function generateOpenAPI(config: ModuleConfigs): OAS.Document {
+   const system = systemRoutes(config);
+   const data = dataRoutes(config);
+   const auth = authRoutes(config);
+
+   return {
+      openapi: "3.1.0",
+      info: {
+         title: "bknd API",
+         version: "0.0.0"
+      },
+      paths: {
+         ...system.paths,
+         ...data.paths,
+         ...auth.paths
+      }
+   };
+}