module manager: switched config table to use entity

app/__test__/ModuleManager.spec.ts

@@ -1,7 +1,7 @@
 import { describe, expect, test } from "bun:test";
 import { mark, stripMark } from "../src/core/utils";
 import { ModuleManager } from "../src/modules/ModuleManager";
-import { CURRENT_VERSION, TABLE_NAME, migrateSchema } from "../src/modules/migrations";
+import { CURRENT_VERSION, TABLE_NAME } from "../src/modules/migrations";
 import { getDummyConnection } from "./helper";
 
 describe("ModuleManager", async () => {
@@ -34,13 +34,13 @@ describe("ModuleManager", async () => {
 
       const c2 = getDummyConnection();
       const db = c2.dummyConnection.kysely;
-      await migrateSchema(CURRENT_VERSION, { db });
+      const mm2 = new ModuleManager(c2.dummyConnection, { initial: { version, ...json } });
+      await mm2.syncConfigTable();
       await db
          .updateTable(TABLE_NAME)
          .set({ json: JSON.stringify(json), version: CURRENT_VERSION })
          .execute();
 
-      const mm2 = new ModuleManager(c2.dummyConnection, { initial: { version, ...json } });
       await mm2.build();
 
       expect(json).toEqual(mm2.configs());
@@ -52,21 +52,19 @@ describe("ModuleManager", async () => {
       await mm.build();
       const version = mm.version();
       const json = mm.configs();
-      //const { version, ...json } = mm.toJSON() as any;
 
       const c2 = getDummyConnection();
       const db = c2.dummyConnection.kysely;
-      console.log("here2");
-      await migrateSchema(CURRENT_VERSION, { db });
-      await db
-         .updateTable(TABLE_NAME)
-         .set({ json: JSON.stringify(json), version: CURRENT_VERSION - 1 })
-         .execute();
-
       const mm2 = new ModuleManager(c2.dummyConnection, {
          initial: { version: version - 1, ...json }
       });
-      console.log("here3");
+      await mm2.syncConfigTable();
+
+      await db
+         .insertInto(TABLE_NAME)
+         .values({ json: JSON.stringify(json), type: "config", version: CURRENT_VERSION - 1 })
+         .execute();
+
       await mm2.build();
    });
 
@@ -80,15 +78,15 @@ describe("ModuleManager", async () => {
 
       const c2 = getDummyConnection();
       const db = c2.dummyConnection.kysely;
-      await migrateSchema(CURRENT_VERSION, { db });
-      await db
-         .updateTable(TABLE_NAME)
-         .set({ json: JSON.stringify(json), version: CURRENT_VERSION })
-         .execute();
 
       const mm2 = new ModuleManager(c2.dummyConnection, {
          initial: { version: version - 1, ...json }
       });
+      await mm2.syncConfigTable();
+      await db
+         .insertInto(TABLE_NAME)
+         .values({ type: "config", json: JSON.stringify(json), version: CURRENT_VERSION })
+         .execute();
 
       expect(mm2.build()).rejects.toThrow(/version.*do not match/);
    });
@@ -102,7 +100,9 @@ describe("ModuleManager", async () => {
 
       const c2 = getDummyConnection();
       const db = c2.dummyConnection.kysely;
-      await migrateSchema(CURRENT_VERSION, { db });
+
+      const mm2 = new ModuleManager(c2.dummyConnection);
+      await mm2.syncConfigTable();
 
       const config = {
          ...json,
@@ -112,12 +112,11 @@ describe("ModuleManager", async () => {
          }
       };
       await db
-         .updateTable(TABLE_NAME)
+         .insertInto(TABLE_NAME)
-         .set({ json: JSON.stringify(config), version: CURRENT_VERSION })
+         .values({ type: "config", json: JSON.stringify(config), version: CURRENT_VERSION })
          .execute();
 
       // run without config given
-      const mm2 = new ModuleManager(c2.dummyConnection);
       await mm2.build();
 
       expect(mm2.configs().data.basepath).toBe("/api/data2");
@@ -175,7 +174,15 @@ describe("ModuleManager", async () => {
 
       const c2 = getDummyConnection();
       const db = c2.dummyConnection.kysely;
-      await migrateSchema(CURRENT_VERSION, { db });
+
+      const mm2 = new ModuleManager(c2.dummyConnection, {
+         initial: {
+            auth: {
+               basepath: "/shouldnt/take/this"
+            }
+         }
+      });
+      await mm2.syncConfigTable();
       const payload = {
          ...json,
          auth: {
@@ -185,20 +192,13 @@ describe("ModuleManager", async () => {
          }
       };
       await db
-         .updateTable(TABLE_NAME)
+         .insertInto(TABLE_NAME)
-         .set({
+         .values({
+            type: "config",
             json: JSON.stringify(payload),
             version: CURRENT_VERSION
          })
          .execute();
-
-      const mm2 = new ModuleManager(c2.dummyConnection, {
-         initial: {
-            auth: {
-               basepath: "/shouldnt/take/this"
-            }
-         }
-      });
       await mm2.build();
       expect(mm2.configs().auth.basepath).toBe("/api/auth2");
    });

app/__test__/data/mutation.simple.test.ts

@@ -132,14 +132,47 @@ describe("Mutator simple", async () => {
       const data = (await em.repository(items).findMany()).data;
       //console.log(data);
 
-      await em.mutator(items).deleteMany({ label: "delete" });
+      await em.mutator(items).deleteWhere({ label: "delete" });
 
       expect((await em.repository(items).findMany()).data.length).toBe(data.length - 2);
       //console.log((await em.repository(items).findMany()).data);
 
-      await em.mutator(items).deleteMany();
+      await em.mutator(items).deleteWhere();
       expect((await em.repository(items).findMany()).data.length).toBe(0);
 
       //expect(res.data.count).toBe(0);
    });
+
+   test("updateMany", async () => {
+      await em.mutator(items).insertOne({ label: "update", count: 1 });
+      await em.mutator(items).insertOne({ label: "update too", count: 1 });
+      await em.mutator(items).insertOne({ label: "keep" });
+
+      // expect no update
+      await em.mutator(items).updateWhere(
+         { count: 2 },
+         {
+            count: 10
+         }
+      );
+      expect((await em.repository(items).findMany()).data).toEqual([
+         { id: 6, label: "update", count: 1 },
+         { id: 7, label: "update too", count: 1 },
+         { id: 8, label: "keep", count: 0 }
+      ]);
+
+      // expect 2 to be updated
+      await em.mutator(items).updateWhere(
+         { count: 2 },
+         {
+            count: 1
+         }
+      );
+
+      expect((await em.repository(items).findMany()).data).toEqual([
+         { id: 6, label: "update", count: 2 },
+         { id: 7, label: "update too", count: 2 },
+         { id: 8, label: "keep", count: 0 }
+      ]);
+   });
 });

app/src/core/errors.ts

@@ -27,6 +27,10 @@ export class BkndError extends Error {
       super(message);
    }
 
+   static with(message: string, details?: Record<string, any>, type?: string) {
+      throw new BkndError(message, details, type);
+   }
+
    toJSON() {
       return {
          type: this.type ?? "unknown",

app/src/data/entities/Mutator.ts

@@ -250,7 +250,7 @@ export class Mutator<DB> implements EmitsEvents {
    }
 
    // @todo: decide whether entries should be deleted all at once or one by one (for events)
-   async deleteMany(where?: RepoQuery["where"]): Promise<MutatorResponse<EntityData>> {
+   async deleteWhere(where?: RepoQuery["where"]): Promise<MutatorResponse<EntityData>> {
       const entity = this.entity;
 
       const qb = this.appendWhere(this.conn.deleteFrom(entity.name), where).returning(
@@ -267,4 +267,30 @@ export class Mutator<DB> implements EmitsEvents {
 
       return res;
    }
+
+   async updateWhere(
+      data: EntityData,
+      where?: RepoQuery["where"]
+   ): Promise<MutatorResponse<EntityData>> {
+      const entity = this.entity;
+
+      const validatedData = await this.getValidatedData(data, "update");
+
+      /*await this.emgr.emit(
+         new Mutator.Events.MutatorUpdateBefore({ entity, entityId: id, data: validatedData })
+      );*/
+
+      const query = this.appendWhere(this.conn.updateTable(entity.name), where)
+         .set(validatedData)
+         //.where(entity.id().name, "=", id)
+         .returning(entity.getSelect());
+
+      const res = await this.many(query);
+
+      /*await this.emgr.emit(
+         new Mutator.Events.MutatorUpdateAfter({ entity, entityId: id, data: res.data })
+      );*/
+
+      return res;
+   }
 }

app/src/media/api/MediaController.ts

@@ -181,7 +181,7 @@ export class MediaController implements ClassController {
             if (ids_to_delete.length > 0) {
                await this.media.em
                   .mutator(mediaEntity)
-                  .deleteMany({ [id_field]: { $in: ids_to_delete } });
+                  .deleteWhere({ [id_field]: { $in: ids_to_delete } });
             }
 
             return c.json({ ok: true, result: result.data, ...info });

app/src/modules/ModuleManager.ts

@@ -1,13 +1,22 @@
 import { Diff } from "@sinclair/typebox/value";
 import { Guard } from "auth";
-import { DebugLogger, isDebug } from "core";
+import { BkndError, DebugLogger, Exception, isDebug } from "core";
 import { EventManager } from "core/events";
 import { Default, type Static, objectEach, transformObject } from "core/utils";
-import { type Connection, EntityManager } from "data";
+import {
+   type Connection,
+   EntityManager,
+   type Schema,
+   datetime,
+   entity,
+   enumm,
+   json,
+   number
+} from "data";
 import { Hono } from "hono";
 import { type Kysely, sql } from "kysely";
 import { mergeWith } from "lodash-es";
-import { CURRENT_VERSION, TABLE_NAME, migrate, migrateSchema } from "modules/migrations";
+import { CURRENT_VERSION, TABLE_NAME, migrate } from "modules/migrations";
 import { AppServer } from "modules/server/AppServer";
 import { AppAuth } from "../auth/AppAuth";
 import { AppData } from "../data/AppData";
@@ -64,8 +73,23 @@ type ConfigTable<Json = ModuleConfigs> = {
    updated_at?: Date;
 };
 
+const __bknd = entity(TABLE_NAME, {
+   version: number().required(),
+   type: enumm({ enum: ["config", "diff", "backup"] }).required(),
+   json: json().required(),
+   created_at: datetime(),
+   updated_at: datetime()
+});
+type ConfigTable2 = Schema<typeof __bknd>;
+type T_INTERNAL_EM = {
+   __bknd: ConfigTable2;
+};
+
 export class ModuleManager {
    private modules: Modules;
+   // internal em for __bknd config table
+   __em!: EntityManager<T_INTERNAL_EM>;
+   // ctx for modules
    em!: EntityManager<any>;
    server!: Hono;
    emgr!: EventManager;
@@ -78,12 +102,13 @@ export class ModuleManager {
    // @todo: keep? not doing anything with it
    private readonly _booted_with?: "provided" | "partial";
 
-   private logger = new DebugLogger(isDebug() && false);
+   private logger = new DebugLogger(false);
 
    constructor(
       private readonly connection: Connection,
       private options?: Partial<ModuleManagerOptions>
    ) {
+      this.__em = new EntityManager([__bknd], this.connection);
       this.modules = {} as Modules;
       this.emgr = new EventManager();
       const context = this.ctx(true);
@@ -126,6 +151,22 @@ export class ModuleManager {
       }
    }
 
+   private repo() {
+      return this.__em.repo(__bknd);
+   }
+
+   private mutator() {
+      return this.__em.mutator(__bknd);
+   }
+
+   private get db() {
+      return this.connection.kysely as Kysely<{ table: ConfigTable }>;
+   }
+
+   async syncConfigTable() {
+      return await this.__em.schema().sync({ force: true });
+   }
+
    private rebuildServer() {
       this.server = new Hono();
       if (this.options?.basePath) {
@@ -159,27 +200,22 @@ export class ModuleManager {
       };
    }
 
-   private get db() {
-      return this.connection.kysely as Kysely<{ table: ConfigTable }>;
-   }
-
-   get table() {
-      return TABLE_NAME as "table";
-   }
-
    private async fetch(): Promise<ConfigTable> {
       this.logger.context("fetch").log("fetching");
 
       const startTime = performance.now();
-      const result = await this.db
+      const { data: result } = await this.repo().findOne(
-         .selectFrom(this.table)
+         { type: "config" },
-         .selectAll()
+         {
-         .where("type", "=", "config")
+            sort: { by: "version", dir: "desc" }
-         .orderBy("version", "desc")
+         }
-         .executeTakeFirstOrThrow();
+      );
+      if (!result) {
+         throw BkndError.with("no config");
+      }
 
       this.logger.log("took", performance.now() - startTime, "ms", result).clear();
-      return result;
+      return result as ConfigTable;
    }
 
    async save() {
@@ -188,64 +224,65 @@ export class ModuleManager {
       const version = this.version();
 
       const json = JSON.stringify(configs) as any;
-      const state = await this.fetch();
 
-      if (state.version !== version) {
+      try {
-         // @todo: mark all others as "backup"
+         const state = await this.fetch();
-         this.logger.log("version conflict, storing new version", state.version, version);
+
-         await this.db
+         if (state.version !== version) {
-            .insertInto(this.table)
+            // @todo: mark all others as "backup"
-            .values({
+            this.logger.log("version conflict, storing new version", state.version, version);
+            await this.mutator().insertOne({
                version,
-               type: "config",
+               type: "backup",
                json
-            })
+            });
-            .execute();
+         } else {
-      } else {
+            this.logger.log("version matches");
-         this.logger.log("version matches");
 
-         const diff = Diff(state.json, JSON.parse(json));
+            const diff = Diff(state.json, JSON.parse(json));
-         this.logger.log("checking diff", diff);
+            this.logger.log("checking diff", diff);
 
-         if (diff.length > 0) {
+            if (diff.length > 0) {
-            // store diff
+               // store diff
-            await this.db
+               await this.mutator().insertOne({
-               .insertInto(this.table)
-               .values({
                   version,
                   type: "diff",
                   json: JSON.stringify(diff) as any
-               })
+               });
-               .execute();
+               // store new version
-
+               // @todo: maybe by id?
-            await this.db
+               await this.mutator().updateWhere(
-               .updateTable(this.table)
+                  {
-               .set({ version, json, updated_at: sql`CURRENT_TIMESTAMP` })
+                     version,
-               .where((eb) => eb.and([eb("type", "=", "config"), eb("version", "=", version)]))
+                     json,
-               .execute();
+                     updated_at: new Date()
+                  },
+                  {
+                     type: "config",
+                     version
+                  }
+               );
+            } else {
+               this.logger.log("no diff, not saving");
+            }
+         }
+      } catch (e) {
+         if (e instanceof BkndError) {
+            // no config, just save
+            await this.mutator().insertOne({
+               type: "config",
+               version,
+               json,
+               created_at: new Date(),
+               updated_at: new Date()
+            });
          } else {
-            this.logger.log("no diff, not saving");
+            console.error("Aborting");
+            throw e;
          }
       }
 
-      // cleanup
+      // @todo: cleanup old versions?
-      /*this.logger.log("cleaning up");
-      const result = await this.db
-         .deleteFrom(this.table)
-         .where((eb) =>
-            eb.or([
-               // empty migrations
-               eb.and([
-                  eb("type", "=", "config"),
-                  eb("version", "<", version),
-                  eb("json", "is", null)
-               ]),
-               // past diffs
-               eb.and([eb("type", "=", "diff"), eb("version", "<", version)])
-            ])
-         )
-         .executeTakeFirst();
-      this.logger.log("cleaned up", result.numDeletedRows);*/
 
       this.logger.clear();
       return this;
@@ -256,6 +293,8 @@ export class ModuleManager {
 
       if (this.version() < CURRENT_VERSION) {
          this.logger.log("there are migrations, verify version");
+         // sync __bknd table
+         await this.syncConfigTable();
 
          // modules must be built before migration
          await this.buildModules({ graceful: true });
@@ -270,14 +309,7 @@ export class ModuleManager {
             }
          } catch (e: any) {
             this.logger.clear(); // fetch couldn't clear
-
+            throw new Error(`Version is ${this.version()}, fetch failed: ${e.message}`);
-            // if table doesn't exist, migrate schema to version
-            if (e.message.includes("no such table")) {
-               this.logger.log("table has to created, migrating schema up to", this.version());
-               await migrateSchema(this.version(), { db: this.db });
-            } else {
-               throw new Error(`Version is ${this.version()}, fetch failed: ${e.message}`);
-            }
          }
 
          this.logger.log("now migrating");
@@ -345,6 +377,10 @@ export class ModuleManager {
       this.logger.context("build").log("version", this.version());
       this.logger.log("booted with", this._booted_with);
 
+      if (this.version() !== CURRENT_VERSION) {
+         await this.syncConfigTable();
+      }
+
       // if no config provided, try fetch from db
       if (this.version() === 0) {
          this.logger.context("no version").log("version is 0");
@@ -358,23 +394,15 @@ export class ModuleManager {
             this.logger.clear(); // fetch couldn't clear
 
             this.logger.context("error handler").log("fetch failed", e.message);
-            // if table doesn't exist, migrate schema, set default config and latest version
-            if (e.message.includes("no such table")) {
-               this.logger.log("migrate schema to", CURRENT_VERSION);
-               await migrateSchema(CURRENT_VERSION, { db: this.db });
-               this._version = CURRENT_VERSION;
 
-               // we can safely build modules, since config version is up to date
+            // we can safely build modules, since config version is up to date
-               // it's up to date because we use default configs (no fetch result)
+            // it's up to date because we use default configs (no fetch result)
-               await this.buildModules();
+            this._version = CURRENT_VERSION;
-               await this.save();
+            await this.buildModules();
+            await this.save();
 
-               this.logger.clear();
+            this.logger.clear();
-               return this;
+            return this;
-            } else {
-               throw e;
-               //throw new Error("Issues connecting to the database. Reason: " + e.message);
-            }
          }
          this.logger.clear();
       }

app/src/modules/migrations.ts

@@ -16,26 +16,8 @@ export type Migration = {
 export const migrations: Migration[] = [
    {
       version: 1,
-      schema: true,
+      //schema: true,
-      up: async (config, { db }) => {
+      up: async (config) => config
-         //console.log("config given", config);
-         await db.schema
-            .createTable(TABLE_NAME)
-            .addColumn("id", "integer", (col) => col.primaryKey().notNull().autoIncrement())
-            .addColumn("version", "integer", (col) => col.notNull())
-            .addColumn("type", "text", (col) => col.notNull())
-            .addColumn("json", "text")
-            .addColumn("created_at", "datetime", (col) => col.defaultTo(sql`CURRENT_TIMESTAMP`))
-            .addColumn("updated_at", "datetime", (col) => col.defaultTo(sql`CURRENT_TIMESTAMP`))
-            .execute();
-
-         await db
-            .insertInto(TABLE_NAME)
-            .values({ version: 1, type: "config", json: null })
-            .execute();
-
-         return config;
-      }
    },
    {
       version: 2,
@@ -45,12 +27,8 @@ export const migrations: Migration[] = [
    },
    {
       version: 3,
-      schema: true,
+      //schema: true,
-      up: async (config, { db }) => {
+      up: async (config) => config
-         await db.schema.alterTable(TABLE_NAME).addColumn("deleted_at", "datetime").execute();
-
-         return config;
-      }
    },
    {
       version: 4,
@@ -127,28 +105,6 @@ export async function migrateTo(
    return [version, updated];
 }
 
-export async function migrateSchema(to: number, ctx: MigrationContext, current: number = 0) {
-   console.log("migrating SCHEMA to", to, "from", current);
-   const todo = migrations.filter((m) => m.version > current && m.version <= to && m.schema);
-   console.log("todo", todo.length);
-
-   let i = 0;
-   let version = 0;
-   for (const migration of todo) {
-      console.log("-- running migration", i + 1, "of", todo.length);
-      try {
-         await migration.up({}, ctx);
-         version = migration.version;
-         i++;
-      } catch (e: any) {
-         console.error(e);
-         throw new Error(`Migration ${migration.version} failed: ${e.message}`);
-      }
-   }
-
-   return version;
-}
-
 export async function migrate(
    current: number,
    config: GenericConfigObject,