Merge remote-tracking branch 'origin/refactor/module-manager' into release/0.2.3

app/__test__/ModuleManager.spec.ts

@@ -1,7 +1,8 @@
 import { describe, expect, test } from "bun:test";
 import { mark, stripMark } from "../src/core/utils";
-import { ModuleManager } from "../src/modules/ModuleManager";
+import { entity, text } from "../src/data";
-import { CURRENT_VERSION, TABLE_NAME, migrateSchema } from "../src/modules/migrations";
+import { ModuleManager, getDefaultConfig } from "../src/modules/ModuleManager";
+import { CURRENT_VERSION, TABLE_NAME } from "../src/modules/migrations";
 import { getDummyConnection } from "./helper";
 
 describe("ModuleManager", async () => {
@@ -29,21 +30,68 @@ describe("ModuleManager", async () => {
       const mm = new ModuleManager(c.dummyConnection);
       await mm.build();
       const version = mm.version();
-      const json = mm.configs();
+      const configs = mm.configs();
+      const json = stripMark({
+         ...configs,
+         data: {
+            ...configs.data,
+            basepath: "/api/data2",
+            entities: {
+               test: entity("test", {
+                  content: text()
+               }).toJSON()
+            }
+         }
+      });
       //const { version, ...json } = mm.toJSON() as any;
 
       const c2 = getDummyConnection();
       const db = c2.dummyConnection.kysely;
-      await migrateSchema(CURRENT_VERSION, { db });
+      const mm2 = new ModuleManager(c2.dummyConnection, { initial: { version, ...json } });
+      await mm2.syncConfigTable();
       await db
-         .updateTable(TABLE_NAME)
+         .insertInto(TABLE_NAME)
-         .set({ json: JSON.stringify(json), version: CURRENT_VERSION })
+         .values({ type: "config", json: JSON.stringify(json), version: CURRENT_VERSION })
          .execute();
 
-      const mm2 = new ModuleManager(c2.dummyConnection, { initial: { version, ...json } });
       await mm2.build();
 
-      expect(json).toEqual(mm2.configs());
+      expect(json).toEqual(stripMark(mm2.configs()));
+   });
+
+   test("s3.1: (fetch) config given, table exists, version matches", async () => {
+      const configs = getDefaultConfig();
+      const json = {
+         ...configs,
+         data: {
+            ...configs.data,
+            basepath: "/api/data2",
+            entities: {
+               test: entity("test", {
+                  content: text()
+               }).toJSON()
+            }
+         }
+      };
+      //const { version, ...json } = mm.toJSON() as any;
+
+      const { dummyConnection } = getDummyConnection();
+      const db = dummyConnection.kysely;
+      const mm2 = new ModuleManager(dummyConnection);
+      await mm2.syncConfigTable();
+      // assume an initial version
+      await db.insertInto(TABLE_NAME).values({ type: "config", json: null, version: 1 }).execute();
+      await db
+         .insertInto(TABLE_NAME)
+         .values({ type: "config", json: JSON.stringify(json), version: CURRENT_VERSION })
+         .execute();
+
+      await mm2.build();
+
+      expect(stripMark(json)).toEqual(stripMark(mm2.configs()));
+      expect(mm2.configs().data.entities.test).toBeDefined();
+      expect(mm2.configs().data.entities.test.fields.content).toBeDefined();
+      expect(mm2.get("data").toJSON().entities.test.fields.content).toBeDefined();
    });
 
    test("s4: config given, table exists, version outdated, migrate", async () => {
@@ -52,21 +100,19 @@ describe("ModuleManager", async () => {
       await mm.build();
       const version = mm.version();
       const json = mm.configs();
-      //const { version, ...json } = mm.toJSON() as any;
 
       const c2 = getDummyConnection();
       const db = c2.dummyConnection.kysely;
-      console.log("here2");
-      await migrateSchema(CURRENT_VERSION, { db });
-      await db
-         .updateTable(TABLE_NAME)
-         .set({ json: JSON.stringify(json), version: CURRENT_VERSION - 1 })
-         .execute();
-
       const mm2 = new ModuleManager(c2.dummyConnection, {
          initial: { version: version - 1, ...json }
       });
-      console.log("here3");
+      await mm2.syncConfigTable();
+
+      await db
+         .insertInto(TABLE_NAME)
+         .values({ json: JSON.stringify(json), type: "config", version: CURRENT_VERSION - 1 })
+         .execute();
+
       await mm2.build();
    });
 
@@ -80,15 +126,15 @@ describe("ModuleManager", async () => {
 
       const c2 = getDummyConnection();
       const db = c2.dummyConnection.kysely;
-      await migrateSchema(CURRENT_VERSION, { db });
-      await db
-         .updateTable(TABLE_NAME)
-         .set({ json: JSON.stringify(json), version: CURRENT_VERSION })
-         .execute();
 
       const mm2 = new ModuleManager(c2.dummyConnection, {
          initial: { version: version - 1, ...json }
       });
+      await mm2.syncConfigTable();
+      await db
+         .insertInto(TABLE_NAME)
+         .values({ type: "config", json: JSON.stringify(json), version: CURRENT_VERSION })
+         .execute();
 
       expect(mm2.build()).rejects.toThrow(/version.*do not match/);
    });
@@ -102,7 +148,9 @@ describe("ModuleManager", async () => {
 
       const c2 = getDummyConnection();
       const db = c2.dummyConnection.kysely;
-      await migrateSchema(CURRENT_VERSION, { db });
+
+      const mm2 = new ModuleManager(c2.dummyConnection);
+      await mm2.syncConfigTable();
 
       const config = {
          ...json,
@@ -112,12 +160,11 @@ describe("ModuleManager", async () => {
          }
       };
       await db
-         .updateTable(TABLE_NAME)
+         .insertInto(TABLE_NAME)
-         .set({ json: JSON.stringify(config), version: CURRENT_VERSION })
+         .values({ type: "config", json: JSON.stringify(config), version: CURRENT_VERSION })
          .execute();
 
       // run without config given
-      const mm2 = new ModuleManager(c2.dummyConnection);
       await mm2.build();
 
       expect(mm2.configs().data.basepath).toBe("/api/data2");
@@ -148,50 +195,61 @@ describe("ModuleManager", async () => {
       });
    });
 
-   // @todo: check what happens here
+   test("partial config given", async () => {
-   /*test("blank app, modify deep config", async () => {
       const { dummyConnection } = getDummyConnection();
 
-      const mm = new ModuleManager(dummyConnection);
+      const partial = {
-      await mm.build();
+         auth: {
-
+            enabled: true
-      /!* await mm
-         .get("data")
-         .schema()
-         .patch("entities.test", {
-            fields: {
-               content: {
-                  type: "text"
-               }
          }
+      };
+      const mm = new ModuleManager(dummyConnection, {
+         initial: partial
       });
       await mm.build();
 
-      expect(mm.configs().data.entities?.users?.fields?.email.type).toBe("text");
+      expect(mm.version()).toBe(CURRENT_VERSION);
-
+      expect(mm.built()).toBe(true);
-      expect(
+      expect(mm.configs().auth.enabled).toBe(true);
-         mm.get("data").schema().patch("desc", "entities.users.config.sort_dir")
+      expect(mm.configs().data.entities.users).toBeDefined();
-      ).rejects.toThrow();
+   });
-      await mm.build();*!/
+
-      expect(mm.configs().data.entities?.users?.fields?.email.type).toBe("text");
+   test("partial config given, but db version exists", async () => {
-      console.log("here", mm.configs());
+      const c = getDummyConnection();
-      await mm
+      const mm = new ModuleManager(c.dummyConnection);
-         .get("data")
+      await mm.build();
-         .schema()
+      const json = mm.configs();
-         .patch("entities.users", { config: { sort_dir: "desc" } });
+
-      await mm.build();
+      const c2 = getDummyConnection();
-      expect(mm.toJSON());
+      const db = c2.dummyConnection.kysely;
-
+
-      //console.log(_jsonp(mm.toJSON().data));
+      const mm2 = new ModuleManager(c2.dummyConnection, {
-      /!*expect(mm.configs().data.entities!.test!.fields!.content.type).toBe("text");
+         initial: {
-      expect(mm.configs().data.entities!.users!.config!.sort_dir).toBe("desc");*!/
+            auth: {
-   });*/
+               basepath: "/shouldnt/take/this"
-
+            }
-   /*test("accessing modules", async () => {
+         }
-      const { dummyConnection } = getDummyConnection();
+      });
-
+      await mm2.syncConfigTable();
-      const mm = new ModuleManager(dummyConnection);
+      const payload = {
-
+         ...json,
-      //mm.get("auth").mutate().set({});
+         auth: {
-   });*/
+            ...json.auth,
+            enabled: true,
+            basepath: "/api/auth2"
+         }
+      };
+      await db
+         .insertInto(TABLE_NAME)
+         .values({
+            type: "config",
+            json: JSON.stringify(payload),
+            version: CURRENT_VERSION
+         })
+         .execute();
+      await mm2.build();
+      expect(mm2.configs().auth.basepath).toBe("/api/auth2");
+   });
+
+   // @todo: add tests for migrations (check "backup" and new version)
 });

app/__test__/core/object/diff.test.ts

@@ -0,0 +1,443 @@
+import { describe, expect, it, test } from "bun:test";
+import { apply, diff, revert } from "../../../src/core/object/diff";
+
+describe("diff", () => {
+   it("should detect added properties", () => {
+      const oldObj = { a: 1 };
+      const newObj = { a: 1, b: 2 };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "a",
+            p: ["b"],
+            o: undefined,
+            n: 2
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should detect removed properties", () => {
+      const oldObj = { a: 1, b: 2 };
+      const newObj = { a: 1 };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "r",
+            p: ["b"],
+            o: 2,
+            n: undefined
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should detect edited properties", () => {
+      const oldObj = { a: 1 };
+      const newObj = { a: 2 };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "e",
+            p: ["a"],
+            o: 1,
+            n: 2
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should detect changes in nested objects", () => {
+      const oldObj = { a: { b: 1 } };
+      const newObj = { a: { b: 2 } };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "e",
+            p: ["a", "b"],
+            o: 1,
+            n: 2
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should detect changes in arrays", () => {
+      const oldObj = { a: [1, 2, 3] };
+      const newObj = { a: [1, 4, 3, 5] };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "e",
+            p: ["a", 1],
+            o: 2,
+            n: 4
+         },
+         {
+            t: "a",
+            p: ["a", 3],
+            o: undefined,
+            n: 5
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should handle adding elements to an empty array", () => {
+      const oldObj = { a: [] };
+      const newObj = { a: [1, 2, 3] };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "a",
+            p: ["a", 0],
+            o: undefined,
+            n: 1
+         },
+         {
+            t: "a",
+            p: ["a", 1],
+            o: undefined,
+            n: 2
+         },
+         {
+            t: "a",
+            p: ["a", 2],
+            o: undefined,
+            n: 3
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should handle removing elements from an array", () => {
+      const oldObj = { a: [1, 2, 3] };
+      const newObj = { a: [1, 3] };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "e",
+            p: ["a", 1],
+            o: 2,
+            n: 3
+         },
+         {
+            t: "r",
+            p: ["a", 2],
+            o: 3,
+            n: undefined
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should handle complex nested changes", () => {
+      const oldObj = {
+         a: {
+            b: [1, 2, { c: 3 }]
+         }
+      };
+
+      const newObj = {
+         a: {
+            b: [1, 2, { c: 4 }, 5]
+         }
+      };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "e",
+            p: ["a", "b", 2, "c"],
+            o: 3,
+            n: 4
+         },
+         {
+            t: "a",
+            p: ["a", "b", 3],
+            o: undefined,
+            n: 5
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should handle undefined and null values", () => {
+      const oldObj = { a: undefined, b: null };
+      const newObj = { a: null, b: undefined };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "e",
+            p: ["a"],
+            o: undefined,
+            n: null
+         },
+         {
+            t: "e",
+            p: ["b"],
+            o: null,
+            n: undefined
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should handle type changes", () => {
+      const oldObj = { a: 1 };
+      const newObj = { a: "1" };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "e",
+            p: ["a"],
+            o: 1,
+            n: "1"
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should handle properties added and removed simultaneously", () => {
+      const oldObj = { a: 1, b: 2 };
+      const newObj = { a: 1, c: 3 };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "r",
+            p: ["b"],
+            o: 2,
+            n: undefined
+         },
+         {
+            t: "a",
+            p: ["c"],
+            o: undefined,
+            n: 3
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should handle arrays replaced with objects", () => {
+      const oldObj = { a: [1, 2, 3] };
+      const newObj = { a: { b: 4 } };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "e",
+            p: ["a"],
+            o: [1, 2, 3],
+            n: { b: 4 }
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should handle objects replaced with primitives", () => {
+      const oldObj = { a: { b: 1 } };
+      const newObj = { a: 2 };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "e",
+            p: ["a"],
+            o: { b: 1 },
+            n: 2
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should handle root object changes", () => {
+      const oldObj = { a: 1 };
+      const newObj = { b: 2 };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "r",
+            p: ["a"],
+            o: 1,
+            n: undefined
+         },
+         {
+            t: "a",
+            p: ["b"],
+            o: undefined,
+            n: 2
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should handle identical objects", () => {
+      const oldObj = { a: 1, b: { c: 2 } };
+      const newObj = { a: 1, b: { c: 2 } };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should handle empty objects", () => {
+      const oldObj = {};
+      const newObj = {};
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should handle changes from empty object to non-empty object", () => {
+      const oldObj = {};
+      const newObj = { a: 1 };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "a",
+            p: ["a"],
+            o: undefined,
+            n: 1
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should handle changes from non-empty object to empty object", () => {
+      const oldObj = { a: 1 };
+      const newObj = {};
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "r",
+            p: ["a"],
+            o: 1,
+            n: undefined
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+});

app/__test__/data/mutation.simple.test.ts

@@ -132,14 +132,47 @@ describe("Mutator simple", async () => {
       const data = (await em.repository(items).findMany()).data;
       //console.log(data);
 
-      await em.mutator(items).deleteMany({ label: "delete" });
+      await em.mutator(items).deleteWhere({ label: "delete" });
 
       expect((await em.repository(items).findMany()).data.length).toBe(data.length - 2);
       //console.log((await em.repository(items).findMany()).data);
 
-      await em.mutator(items).deleteMany();
+      await em.mutator(items).deleteWhere();
       expect((await em.repository(items).findMany()).data.length).toBe(0);
 
       //expect(res.data.count).toBe(0);
    });
+
+   test("updateMany", async () => {
+      await em.mutator(items).insertOne({ label: "update", count: 1 });
+      await em.mutator(items).insertOne({ label: "update too", count: 1 });
+      await em.mutator(items).insertOne({ label: "keep" });
+
+      // expect no update
+      await em.mutator(items).updateWhere(
+         { count: 2 },
+         {
+            count: 10
+         }
+      );
+      expect((await em.repository(items).findMany()).data).toEqual([
+         { id: 6, label: "update", count: 1 },
+         { id: 7, label: "update too", count: 1 },
+         { id: 8, label: "keep", count: 0 }
+      ]);
+
+      // expect 2 to be updated
+      await em.mutator(items).updateWhere(
+         { count: 2 },
+         {
+            count: 1
+         }
+      );
+
+      expect((await em.repository(items).findMany()).data).toEqual([
+         { id: 6, label: "update", count: 2 },
+         { id: 7, label: "update too", count: 2 },
+         { id: 8, label: "keep", count: 0 }
+      ]);
+   });
 });

app/src/core/errors.ts

@@ -27,6 +27,10 @@ export class BkndError extends Error {
       super(message);
    }
 
+   static with(message: string, details?: Record<string, any>, type?: string) {
+      throw new BkndError(message, details, type);
+   }
+
    toJSON() {
       return {
          type: this.type ?? "unknown",

app/src/core/object/diff.ts

@@ -0,0 +1,181 @@
+enum Change {
+   Add = "a",
+   Remove = "r",
+   Edit = "e"
+}
+
+type Object = object;
+type Primitive = string | number | boolean | null | object | any[] | undefined;
+
+interface DiffEntry {
+   t: Change | string;
+   p: (string | number)[];
+   o: Primitive;
+   n: Primitive;
+}
+
+function isObject(value: any): value is Object {
+   return value !== null && value.constructor.name === "Object";
+}
+function isPrimitive(value: any): value is Primitive {
+   try {
+      return (
+         value === null ||
+         value === undefined ||
+         typeof value === "string" ||
+         typeof value === "number" ||
+         typeof value === "boolean" ||
+         Array.isArray(value) ||
+         isObject(value)
+      );
+   } catch (e) {
+      return false;
+   }
+}
+
+function diff(oldObj: Object, newObj: Object): DiffEntry[] {
+   const diffs: DiffEntry[] = [];
+
+   function recurse(oldValue: Primitive, newValue: Primitive, path: (string | number)[]) {
+      if (!isPrimitive(oldValue) || !isPrimitive(newValue)) {
+         throw new Error("Diff: Only primitive types are supported");
+      }
+
+      if (oldValue === newValue) {
+         return;
+      }
+
+      if (typeof oldValue !== typeof newValue) {
+         diffs.push({
+            t: Change.Edit,
+            p: path,
+            o: oldValue,
+            n: newValue
+         });
+      } else if (Array.isArray(oldValue) && Array.isArray(newValue)) {
+         const maxLength = Math.max(oldValue.length, newValue.length);
+         for (let i = 0; i < maxLength; i++) {
+            if (i >= oldValue.length) {
+               diffs.push({
+                  t: Change.Add,
+                  p: [...path, i],
+                  o: undefined,
+                  n: newValue[i]
+               });
+            } else if (i >= newValue.length) {
+               diffs.push({
+                  t: Change.Remove,
+                  p: [...path, i],
+                  o: oldValue[i],
+                  n: undefined
+               });
+            } else {
+               recurse(oldValue[i], newValue[i], [...path, i]);
+            }
+         }
+      } else if (isObject(oldValue) && isObject(newValue)) {
+         const oKeys = Object.keys(oldValue);
+         const nKeys = Object.keys(newValue);
+         const allKeys = new Set([...oKeys, ...nKeys]);
+         for (const key of allKeys) {
+            if (!(key in oldValue)) {
+               diffs.push({
+                  t: Change.Add,
+                  p: [...path, key],
+                  o: undefined,
+                  n: newValue[key]
+               });
+            } else if (!(key in newValue)) {
+               diffs.push({
+                  t: Change.Remove,
+                  p: [...path, key],
+                  o: oldValue[key],
+                  n: undefined
+               });
+            } else {
+               recurse(oldValue[key], newValue[key], [...path, key]);
+            }
+         }
+      } else {
+         diffs.push({
+            t: Change.Edit,
+            p: path,
+            o: oldValue,
+            n: newValue
+         });
+      }
+   }
+
+   recurse(oldObj, newObj, []);
+   return diffs;
+}
+
+function apply(obj: Object, diffs: DiffEntry[]): any {
+   const clonedObj = clone(obj);
+
+   for (const diff of diffs) {
+      applyChange(clonedObj, diff);
+   }
+
+   return clonedObj;
+}
+
+function revert(obj: Object, diffs: DiffEntry[]): any {
+   const clonedObj = clone(obj);
+   const reversedDiffs = diffs.slice().reverse();
+
+   for (const diff of reversedDiffs) {
+      revertChange(clonedObj, diff);
+   }
+
+   return clonedObj;
+}
+
+function applyChange(obj: Object, diff: DiffEntry) {
+   const { p: path, t: type, n: newValue } = diff;
+   const parent = getParent(obj, path.slice(0, -1));
+   const key = path[path.length - 1]!;
+
+   if (type === Change.Add || type === Change.Edit) {
+      parent[key] = newValue;
+   } else if (type === Change.Remove) {
+      if (Array.isArray(parent)) {
+         parent.splice(key as number, 1);
+      } else {
+         delete parent[key];
+      }
+   }
+}
+
+function revertChange(obj: Object, diff: DiffEntry) {
+   const { p: path, t: type, o: oldValue } = diff;
+   const parent = getParent(obj, path.slice(0, -1));
+   const key = path[path.length - 1]!;
+
+   if (type === Change.Add) {
+      if (Array.isArray(parent)) {
+         parent.splice(key as number, 1);
+      } else {
+         delete parent[key];
+      }
+   } else if (type === Change.Remove || type === Change.Edit) {
+      parent[key] = oldValue;
+   }
+}
+
+function getParent(obj: Object, path: (string | number)[]): any {
+   let current = obj;
+   for (const key of path) {
+      if (current[key] === undefined) {
+         current[key] = typeof key === "number" ? [] : {};
+      }
+      current = current[key];
+   }
+   return current;
+}
+
+function clone<In extends Object>(obj: In): In {
+   return JSON.parse(JSON.stringify(obj));
+}
+
+export { diff, apply, revert, clone };

app/src/core/utils/typebox/index.ts

@@ -72,10 +72,10 @@ export class TypeInvalidError extends Error {
    }
 }
 
-export function stripMark(obj: any) {
+export function stripMark<O = any>(obj: O) {
    const newObj = cloneDeep(obj);
    mark(newObj, false);
-   return newObj;
+   return newObj as O;
 }
 
 export function mark(obj: any, validated = true) {

app/src/data/entities/Mutator.ts

@@ -250,7 +250,7 @@ export class Mutator<DB> implements EmitsEvents {
    }
 
    // @todo: decide whether entries should be deleted all at once or one by one (for events)
-   async deleteMany(where?: RepoQuery["where"]): Promise<MutatorResponse<EntityData>> {
+   async deleteWhere(where?: RepoQuery["where"]): Promise<MutatorResponse<EntityData>> {
       const entity = this.entity;
 
       const qb = this.appendWhere(this.conn.deleteFrom(entity.name), where).returning(
@@ -267,4 +267,30 @@ export class Mutator<DB> implements EmitsEvents {
 
       return res;
    }
+
+   async updateWhere(
+      data: EntityData,
+      where?: RepoQuery["where"]
+   ): Promise<MutatorResponse<EntityData>> {
+      const entity = this.entity;
+
+      const validatedData = await this.getValidatedData(data, "update");
+
+      /*await this.emgr.emit(
+         new Mutator.Events.MutatorUpdateBefore({ entity, entityId: id, data: validatedData })
+      );*/
+
+      const query = this.appendWhere(this.conn.updateTable(entity.name), where)
+         .set(validatedData)
+         //.where(entity.id().name, "=", id)
+         .returning(entity.getSelect());
+
+      const res = await this.many(query);
+
+      /*await this.emgr.emit(
+         new Mutator.Events.MutatorUpdateAfter({ entity, entityId: id, data: res.data })
+      );*/
+
+      return res;
+   }
 }

app/src/data/entities/query/Repository.ts

@@ -162,8 +162,7 @@ export class Repository<DB = any, TB extends keyof DB = any> implements EmitsEve
    protected async performQuery(qb: RepositoryQB): Promise<RepositoryResponse> {
       const entity = this.entity;
       const compiled = qb.compile();
-      /*const { sql, parameters } = qb.compile();
+      //console.log("performQuery", compiled.sql, compiled.parameters);
-      console.log("many", sql, parameters);*/
 
       const start = performance.now();
       const selector = (as = "count") => this.conn.fn.countAll<number>().as(as);
@@ -263,6 +262,7 @@ export class Repository<DB = any, TB extends keyof DB = any> implements EmitsEve
          qb = qb.orderBy(aliased(options.sort.by), options.sort.dir);
       }
 
+      //console.log("options", { _options, options, exclude_options });
       return { qb, options };
    }
 
@@ -286,14 +286,11 @@ export class Repository<DB = any, TB extends keyof DB = any> implements EmitsEve
       where: RepoQuery["where"],
       _options?: Partial<Omit<RepoQuery, "where" | "limit" | "offset">>
    ): Promise<RepositoryResponse<DB[TB] | undefined>> {
-      const { qb, options } = this.buildQuery(
+      const { qb, options } = this.buildQuery({
-         {
          ..._options,
          where,
          limit: 1
-         },
+      });
-         ["offset", "sort"]
-      );
 
       return this.single(qb, options) as any;
    }

app/src/data/entities/query/WhereBuilder.ts

@@ -15,7 +15,6 @@ import type {
    SelectQueryBuilder,
    UpdateQueryBuilder
 } from "kysely";
-import type { RepositoryQB } from "./Repository";
 
 type Builder = ExpressionBuilder<any, any>;
 type Wrapper = ExpressionWrapper<any, any, any>;

app/src/data/fields/Field.ts

@@ -236,8 +236,8 @@ export abstract class Field<
 
    toJSON() {
       return {
-         //name: this.name,
+         // @todo: current workaround because of fixed string type
-         type: this.type,
+         type: this.type as any,
          config: this.config
       };
    }

app/src/data/fields/JsonSchemaField.ts

@@ -54,7 +54,7 @@ export class JsonSchemaField<
 
       if (parentValid) {
          // already checked in parent
-         if (!value || typeof value !== "object") {
+         if (!this.isRequired() && (!value || typeof value !== "object")) {
             //console.log("jsonschema:valid: not checking", this.name, value, context);
             return true;
          }
@@ -65,6 +65,7 @@ export class JsonSchemaField<
       } else {
          //console.log("jsonschema:invalid", this.name, value, context);
       }
+      //console.log("jsonschema:invalid:fromParent", this.name, value, context);
 
       return false;
    }
@@ -110,9 +111,13 @@ export class JsonSchemaField<
    ): Promise<string | undefined> {
       const value = await super.transformPersist(_value, em, context);
       if (this.nullish(value)) return value;
+      //console.log("jsonschema:transformPersist", this.name, _value, context);
 
       if (!this.isValid(value)) {
+         //console.error("jsonschema:transformPersist:invalid", this.name, value);
          throw new TransformPersistFailedException(this.name, value);
+      } else {
+         //console.log("jsonschema:transformPersist:valid", this.name, value);
       }
 
       if (!value || typeof value !== "object") return this.getDefault();

app/src/media/api/MediaController.ts

@@ -181,7 +181,7 @@ export class MediaController implements ClassController {
             if (ids_to_delete.length > 0) {
                await this.media.em
                   .mutator(mediaEntity)
-                  .deleteMany({ [id_field]: { $in: ids_to_delete } });
+                  .deleteWhere({ [id_field]: { $in: ids_to_delete } });
             }
 
             return c.json({ ok: true, result: result.data, ...info });

app/src/modules/ModuleManager.ts

@@ -1,12 +1,31 @@
-import { Diff } from "@sinclair/typebox/value";
 import { Guard } from "auth";
-import { DebugLogger, isDebug } from "core";
+import { BkndError, DebugLogger, Exception, isDebug } from "core";
 import { EventManager } from "core/events";
-import { Default, type Static, objectEach, transformObject } from "core/utils";
+import { clone, diff } from "core/object/diff";
-import { type Connection, EntityManager } from "data";
+import {
+   Default,
+   type Static,
+   StringEnum,
+   Type,
+   objectEach,
+   stripMark,
+   transformObject
+} from "core/utils";
+import {
+   type Connection,
+   EntityManager,
+   type Schema,
+   datetime,
+   entity,
+   enumm,
+   jsonSchema,
+   number
+} from "data";
+import { TransformPersistFailedException } from "data/errors";
 import { Hono } from "hono";
 import { type Kysely, sql } from "kysely";
-import { CURRENT_VERSION, TABLE_NAME, migrate, migrateSchema } from "modules/migrations";
+import { mergeWith } from "lodash-es";
+import { CURRENT_VERSION, TABLE_NAME, migrate } from "modules/migrations";
 import { AppServer } from "modules/server/AppServer";
 import { AppAuth } from "../auth/AppAuth";
 import { AppData } from "../data/AppData";
@@ -37,10 +56,13 @@ export type ModuleSchemas = {
 export type ModuleConfigs = {
    [K in keyof ModuleSchemas]: Static<ModuleSchemas[K]>;
 };
+type PartialRec<T> = { [P in keyof T]?: PartialRec<T[P]> };
 
-export type InitialModuleConfigs = {
+export type InitialModuleConfigs =
+   | ({
         version: number;
-} & Partial<ModuleConfigs>;
+     } & ModuleConfigs)
+   | PartialRec<ModuleConfigs>;
 
 export type ModuleManagerOptions = {
    initial?: InitialModuleConfigs;
@@ -61,8 +83,36 @@ type ConfigTable<Json = ModuleConfigs> = {
    updated_at?: Date;
 };
 
+const configJsonSchema = Type.Union([
+   getDefaultSchema(),
+   Type.Array(
+      Type.Object({
+         t: StringEnum(["a", "r", "e"]),
+         p: Type.Array(Type.Union([Type.String(), Type.Number()])),
+         o: Type.Optional(Type.Any()),
+         n: Type.Optional(Type.Any())
+      })
+   )
+]);
+const __bknd = entity(TABLE_NAME, {
+   version: number().required(),
+   type: enumm({ enum: ["config", "diff", "backup"] }).required(),
+   json: jsonSchema({ schema: configJsonSchema }).required(),
+   created_at: datetime(),
+   updated_at: datetime()
+});
+type ConfigTable2 = Schema<typeof __bknd>;
+type T_INTERNAL_EM = {
+   __bknd: ConfigTable2;
+};
+
+// @todo: cleanup old diffs on upgrade
+// @todo: cleanup multiple backups on upgrade
 export class ModuleManager {
    private modules: Modules;
+   // internal em for __bknd config table
+   __em!: EntityManager<T_INTERNAL_EM>;
+   // ctx for modules
    em!: EntityManager<any>;
    server!: Hono;
    emgr!: EventManager;
@@ -71,28 +121,32 @@ export class ModuleManager {
    private _version: number = 0;
    private _built = false;
    private _fetched = false;
-   private readonly _provided;
 
-   private logger = new DebugLogger(isDebug() && false);
+   // @todo: keep? not doing anything with it
+   private readonly _booted_with?: "provided" | "partial";
+
+   private logger = new DebugLogger(false);
 
    constructor(
       private readonly connection: Connection,
       private options?: Partial<ModuleManagerOptions>
    ) {
+      this.__em = new EntityManager([__bknd], this.connection);
       this.modules = {} as Modules;
       this.emgr = new EventManager();
       const context = this.ctx(true);
       let initial = {} as Partial<ModuleConfigs>;
 
       if (options?.initial) {
+         if ("version" in options.initial) {
             const { version, ...initialConfig } = options.initial;
-         if (version && initialConfig) {
             this._version = version;
-            initial = initialConfig;
+            initial = stripMark(initialConfig);
 
-            this._provided = true;
+            this._booted_with = "provided";
          } else {
-            throw new Error("Initial was provided, but it needs a version!");
+            initial = mergeWith(getDefaultConfig(), options.initial);
+            this._booted_with = "partial";
          }
       }
 
@@ -120,6 +174,22 @@ export class ModuleManager {
       }
    }
 
+   private repo() {
+      return this.__em.repo(__bknd);
+   }
+
+   private mutator() {
+      return this.__em.mutator(__bknd);
+   }
+
+   private get db() {
+      return this.connection.kysely as Kysely<{ table: ConfigTable }>;
+   }
+
+   async syncConfigTable() {
+      return await this.__em.schema().sync({ force: true });
+   }
+
    private rebuildServer() {
       this.server = new Hono();
       if (this.options?.basePath) {
@@ -153,27 +223,22 @@ export class ModuleManager {
       };
    }
 
-   private get db() {
-      return this.connection.kysely as Kysely<{ table: ConfigTable }>;
-   }
-
-   get table() {
-      return TABLE_NAME as "table";
-   }
-
    private async fetch(): Promise<ConfigTable> {
       this.logger.context("fetch").log("fetching");
 
       const startTime = performance.now();
-      const result = await this.db
+      const { data: result } = await this.repo().findOne(
-         .selectFrom(this.table)
+         { type: "config" },
-         .selectAll()
+         {
-         .where("type", "=", "config")
+            sort: { by: "version", dir: "desc" }
-         .orderBy("version", "desc")
+         }
-         .executeTakeFirstOrThrow();
+      );
+      if (!result) {
+         throw BkndError.with("no config");
+      }
 
       this.logger.log("took", performance.now() - startTime, "ms", result).clear();
-      return result;
+      return result as ConfigTable;
    }
 
    async save() {
@@ -181,65 +246,75 @@ export class ModuleManager {
       const configs = this.configs();
       const version = this.version();
 
-      const json = JSON.stringify(configs) as any;
+      try {
          const state = await this.fetch();
+         this.logger.log("fetched version", state.version);
 
          if (state.version !== version) {
             // @todo: mark all others as "backup"
             this.logger.log("version conflict, storing new version", state.version, version);
-         await this.db
+            await this.mutator().insertOne({
-            .insertInto(this.table)
+               version: state.version,
-            .values({
+               type: "backup",
-               version,
+               json: configs
+            });
+            await this.mutator().insertOne({
+               version: version,
                type: "config",
-               json
+               json: configs
-            })
+            });
-            .execute();
          } else {
             this.logger.log("version matches");
 
-         const diff = Diff(state.json, JSON.parse(json));
+            // clean configs because of Diff() function
-         this.logger.log("checking diff", diff);
+            const diffs = diff(state.json, clone(configs));
+            this.logger.log("checking diff", diffs);
 
             if (diff.length > 0) {
                // store diff
-            await this.db
+               await this.mutator().insertOne({
-               .insertInto(this.table)
-               .values({
                   version,
                   type: "diff",
-                  json: JSON.stringify(diff) as any
+                  json: clone(diffs)
-               })
+               });
-               .execute();
 
-            await this.db
+               // store new version
-               .updateTable(this.table)
+               await this.mutator().updateWhere(
-               .set({ version, json, updated_at: sql`CURRENT_TIMESTAMP` })
+                  {
-               .where((eb) => eb.and([eb("type", "=", "config"), eb("version", "=", version)]))
+                     version,
-               .execute();
+                     json: configs,
+                     updated_at: new Date()
+                  },
+                  {
+                     type: "config",
+                     version
+                  }
+               );
             } else {
                this.logger.log("no diff, not saving");
             }
          }
+      } catch (e) {
+         if (e instanceof BkndError) {
+            this.logger.log("no config, just save fresh");
+            // no config, just save
+            await this.mutator().insertOne({
+               type: "config",
+               version,
+               json: configs,
+               created_at: new Date(),
+               updated_at: new Date()
+            });
+         } else if (e instanceof TransformPersistFailedException) {
+            console.error("Cannot save invalid config");
+            throw e;
+         } else {
+            console.error("Aborting");
+            throw e;
+         }
+      }
 
-      // cleanup
+      // @todo: cleanup old versions?
-      /*this.logger.log("cleaning up");
-      const result = await this.db
-         .deleteFrom(this.table)
-         .where((eb) =>
-            eb.or([
-               // empty migrations
-               eb.and([
-                  eb("type", "=", "config"),
-                  eb("version", "<", version),
-                  eb("json", "is", null)
-               ]),
-               // past diffs
-               eb.and([eb("type", "=", "diff"), eb("version", "<", version)])
-            ])
-         )
-         .executeTakeFirst();
-      this.logger.log("cleaned up", result.numDeletedRows);*/
 
       this.logger.clear();
       return this;
@@ -250,6 +325,8 @@ export class ModuleManager {
 
       if (this.version() < CURRENT_VERSION) {
          this.logger.log("there are migrations, verify version");
+         // sync __bknd table
+         await this.syncConfigTable();
 
          // modules must be built before migration
          await this.buildModules({ graceful: true });
@@ -264,15 +341,8 @@ export class ModuleManager {
             }
          } catch (e: any) {
             this.logger.clear(); // fetch couldn't clear
-
-            // if table doesn't exist, migrate schema to version
-            if (e.message.includes("no such table")) {
-               this.logger.log("table has to created, migrating schema up to", this.version());
-               await migrateSchema(this.version(), { db: this.db });
-            } else {
             throw new Error(`Version is ${this.version()}, fetch failed: ${e.message}`);
          }
-         }
 
          this.logger.log("now migrating");
          let version = this.version();
@@ -337,10 +407,16 @@ export class ModuleManager {
 
    async build() {
       this.logger.context("build").log("version", this.version());
+      this.logger.log("booted with", this._booted_with);
+
+      // @todo: check this, because you could start without an initial config
+      if (this.version() !== CURRENT_VERSION) {
+         await this.syncConfigTable();
+      }
 
       // if no config provided, try fetch from db
       if (this.version() === 0) {
-         this.logger.context("build no config").log("version is 0");
+         this.logger.context("no version").log("version is 0");
          try {
             const result = await this.fetch();
 
@@ -351,23 +427,15 @@ export class ModuleManager {
             this.logger.clear(); // fetch couldn't clear
 
             this.logger.context("error handler").log("fetch failed", e.message);
-            // if table doesn't exist, migrate schema, set default config and latest version
-            if (e.message.includes("no such table")) {
-               this.logger.log("migrate schema to", CURRENT_VERSION);
-               await migrateSchema(CURRENT_VERSION, { db: this.db });
-               this._version = CURRENT_VERSION;
 
             // we can safely build modules, since config version is up to date
             // it's up to date because we use default configs (no fetch result)
+            this._version = CURRENT_VERSION;
             await this.buildModules();
             await this.save();
 
             this.logger.clear();
             return this;
-            } else {
-               throw e;
-               //throw new Error("Issues connecting to the database. Reason: " + e.message);
-            }
          }
          this.logger.clear();
       }

app/src/modules/migrations.ts

@@ -16,26 +16,8 @@ export type Migration = {
 export const migrations: Migration[] = [
    {
       version: 1,
-      schema: true,
+      //schema: true,
-      up: async (config, { db }) => {
+      up: async (config) => config
-         //console.log("config given", config);
-         await db.schema
-            .createTable(TABLE_NAME)
-            .addColumn("id", "integer", (col) => col.primaryKey().notNull().autoIncrement())
-            .addColumn("version", "integer", (col) => col.notNull())
-            .addColumn("type", "text", (col) => col.notNull())
-            .addColumn("json", "text")
-            .addColumn("created_at", "datetime", (col) => col.defaultTo(sql`CURRENT_TIMESTAMP`))
-            .addColumn("updated_at", "datetime", (col) => col.defaultTo(sql`CURRENT_TIMESTAMP`))
-            .execute();
-
-         await db
-            .insertInto(TABLE_NAME)
-            .values({ version: 1, type: "config", json: null })
-            .execute();
-
-         return config;
-      }
    },
    {
       version: 2,
@@ -45,12 +27,8 @@ export const migrations: Migration[] = [
    },
    {
       version: 3,
-      schema: true,
+      //schema: true,
-      up: async (config, { db }) => {
+      up: async (config) => config
-         await db.schema.alterTable(TABLE_NAME).addColumn("deleted_at", "datetime").execute();
-
-         return config;
-      }
    },
    {
       version: 4,
@@ -94,6 +72,13 @@ export const migrations: Migration[] = [
          };
       }
    }
+   /*{
+      version: 8,
+      up: async (config, { db }) => {
+         await db.deleteFrom(TABLE_NAME).where("type", "=", "diff").execute();
+         return config;
+      }
+   }*/
 ];
 
 export const CURRENT_VERSION = migrations[migrations.length - 1]?.version ?? 0;
@@ -127,28 +112,6 @@ export async function migrateTo(
    return [version, updated];
 }
 
-export async function migrateSchema(to: number, ctx: MigrationContext, current: number = 0) {
-   console.log("migrating SCHEMA to", to, "from", current);
-   const todo = migrations.filter((m) => m.version > current && m.version <= to && m.schema);
-   console.log("todo", todo.length);
-
-   let i = 0;
-   let version = 0;
-   for (const migration of todo) {
-      console.log("-- running migration", i + 1, "of", todo.length);
-      try {
-         await migration.up({}, ctx);
-         version = migration.version;
-         i++;
-      } catch (e: any) {
-         console.error(e);
-         throw new Error(`Migration ${migration.version} failed: ${e.message}`);
-      }
-   }
-
-   return version;
-}
-
 export async function migrate(
    current: number,
    config: GenericConfigObject,