added new diffing method to module manager

Signed-off-by: dswbx <dennis.senn@gmx.ch>

app/__test__/ModuleManager.spec.ts

@@ -250,4 +250,6 @@ describe("ModuleManager", async () => {
       await mm2.build();
       expect(mm2.configs().auth.basepath).toBe("/api/auth2");
    });
+
+   // @todo: add tests for migrations (check "backup" and new version)
 });

app/__test__/core/object/diff.test.ts

@@ -0,0 +1,443 @@
+import { describe, expect, it, test } from "bun:test";
+import { apply, diff, revert } from "../../../src/core/object/diff";
+
+describe("diff", () => {
+   it("should detect added properties", () => {
+      const oldObj = { a: 1 };
+      const newObj = { a: 1, b: 2 };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "a",
+            p: ["b"],
+            o: undefined,
+            n: 2
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should detect removed properties", () => {
+      const oldObj = { a: 1, b: 2 };
+      const newObj = { a: 1 };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "r",
+            p: ["b"],
+            o: 2,
+            n: undefined
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should detect edited properties", () => {
+      const oldObj = { a: 1 };
+      const newObj = { a: 2 };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "e",
+            p: ["a"],
+            o: 1,
+            n: 2
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should detect changes in nested objects", () => {
+      const oldObj = { a: { b: 1 } };
+      const newObj = { a: { b: 2 } };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "e",
+            p: ["a", "b"],
+            o: 1,
+            n: 2
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should detect changes in arrays", () => {
+      const oldObj = { a: [1, 2, 3] };
+      const newObj = { a: [1, 4, 3, 5] };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "e",
+            p: ["a", 1],
+            o: 2,
+            n: 4
+         },
+         {
+            t: "a",
+            p: ["a", 3],
+            o: undefined,
+            n: 5
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should handle adding elements to an empty array", () => {
+      const oldObj = { a: [] };
+      const newObj = { a: [1, 2, 3] };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "a",
+            p: ["a", 0],
+            o: undefined,
+            n: 1
+         },
+         {
+            t: "a",
+            p: ["a", 1],
+            o: undefined,
+            n: 2
+         },
+         {
+            t: "a",
+            p: ["a", 2],
+            o: undefined,
+            n: 3
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should handle removing elements from an array", () => {
+      const oldObj = { a: [1, 2, 3] };
+      const newObj = { a: [1, 3] };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "e",
+            p: ["a", 1],
+            o: 2,
+            n: 3
+         },
+         {
+            t: "r",
+            p: ["a", 2],
+            o: 3,
+            n: undefined
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should handle complex nested changes", () => {
+      const oldObj = {
+         a: {
+            b: [1, 2, { c: 3 }]
+         }
+      };
+
+      const newObj = {
+         a: {
+            b: [1, 2, { c: 4 }, 5]
+         }
+      };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "e",
+            p: ["a", "b", 2, "c"],
+            o: 3,
+            n: 4
+         },
+         {
+            t: "a",
+            p: ["a", "b", 3],
+            o: undefined,
+            n: 5
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should handle undefined and null values", () => {
+      const oldObj = { a: undefined, b: null };
+      const newObj = { a: null, b: undefined };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "e",
+            p: ["a"],
+            o: undefined,
+            n: null
+         },
+         {
+            t: "e",
+            p: ["b"],
+            o: null,
+            n: undefined
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should handle type changes", () => {
+      const oldObj = { a: 1 };
+      const newObj = { a: "1" };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "e",
+            p: ["a"],
+            o: 1,
+            n: "1"
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should handle properties added and removed simultaneously", () => {
+      const oldObj = { a: 1, b: 2 };
+      const newObj = { a: 1, c: 3 };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "r",
+            p: ["b"],
+            o: 2,
+            n: undefined
+         },
+         {
+            t: "a",
+            p: ["c"],
+            o: undefined,
+            n: 3
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should handle arrays replaced with objects", () => {
+      const oldObj = { a: [1, 2, 3] };
+      const newObj = { a: { b: 4 } };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "e",
+            p: ["a"],
+            o: [1, 2, 3],
+            n: { b: 4 }
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should handle objects replaced with primitives", () => {
+      const oldObj = { a: { b: 1 } };
+      const newObj = { a: 2 };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "e",
+            p: ["a"],
+            o: { b: 1 },
+            n: 2
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should handle root object changes", () => {
+      const oldObj = { a: 1 };
+      const newObj = { b: 2 };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "r",
+            p: ["a"],
+            o: 1,
+            n: undefined
+         },
+         {
+            t: "a",
+            p: ["b"],
+            o: undefined,
+            n: 2
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should handle identical objects", () => {
+      const oldObj = { a: 1, b: { c: 2 } };
+      const newObj = { a: 1, b: { c: 2 } };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should handle empty objects", () => {
+      const oldObj = {};
+      const newObj = {};
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should handle changes from empty object to non-empty object", () => {
+      const oldObj = {};
+      const newObj = { a: 1 };
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "a",
+            p: ["a"],
+            o: undefined,
+            n: 1
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+
+   it("should handle changes from non-empty object to empty object", () => {
+      const oldObj = { a: 1 };
+      const newObj = {};
+
+      const diffs = diff(oldObj, newObj);
+
+      expect(diffs).toEqual([
+         {
+            t: "r",
+            p: ["a"],
+            o: 1,
+            n: undefined
+         }
+      ]);
+
+      const appliedObj = apply(oldObj, diffs);
+      expect(appliedObj).toEqual(newObj);
+
+      const revertedObj = revert(newObj, diffs);
+      expect(revertedObj).toEqual(oldObj);
+   });
+});

app/src/core/object/diff.ts

@@ -0,0 +1,181 @@
+enum Change {
+   Add = "a",
+   Remove = "r",
+   Edit = "e"
+}
+
+type Object = object;
+type Primitive = string | number | boolean | null | object | any[] | undefined;
+
+interface DiffEntry {
+   t: Change | string;
+   p: (string | number)[];
+   o: Primitive;
+   n: Primitive;
+}
+
+function isObject(value: any): value is Object {
+   return value !== null && value.constructor.name === "Object";
+}
+function isPrimitive(value: any): value is Primitive {
+   try {
+      return (
+         value === null ||
+         value === undefined ||
+         typeof value === "string" ||
+         typeof value === "number" ||
+         typeof value === "boolean" ||
+         Array.isArray(value) ||
+         isObject(value)
+      );
+   } catch (e) {
+      return false;
+   }
+}
+
+function diff(oldObj: Object, newObj: Object): DiffEntry[] {
+   const diffs: DiffEntry[] = [];
+
+   function recurse(oldValue: Primitive, newValue: Primitive, path: (string | number)[]) {
+      if (!isPrimitive(oldValue) || !isPrimitive(newValue)) {
+         throw new Error("Diff: Only primitive types are supported");
+      }
+
+      if (oldValue === newValue) {
+         return;
+      }
+
+      if (typeof oldValue !== typeof newValue) {
+         diffs.push({
+            t: Change.Edit,
+            p: path,
+            o: oldValue,
+            n: newValue
+         });
+      } else if (Array.isArray(oldValue) && Array.isArray(newValue)) {
+         const maxLength = Math.max(oldValue.length, newValue.length);
+         for (let i = 0; i < maxLength; i++) {
+            if (i >= oldValue.length) {
+               diffs.push({
+                  t: Change.Add,
+                  p: [...path, i],
+                  o: undefined,
+                  n: newValue[i]
+               });
+            } else if (i >= newValue.length) {
+               diffs.push({
+                  t: Change.Remove,
+                  p: [...path, i],
+                  o: oldValue[i],
+                  n: undefined
+               });
+            } else {
+               recurse(oldValue[i], newValue[i], [...path, i]);
+            }
+         }
+      } else if (isObject(oldValue) && isObject(newValue)) {
+         const oKeys = Object.keys(oldValue);
+         const nKeys = Object.keys(newValue);
+         const allKeys = new Set([...oKeys, ...nKeys]);
+         for (const key of allKeys) {
+            if (!(key in oldValue)) {
+               diffs.push({
+                  t: Change.Add,
+                  p: [...path, key],
+                  o: undefined,
+                  n: newValue[key]
+               });
+            } else if (!(key in newValue)) {
+               diffs.push({
+                  t: Change.Remove,
+                  p: [...path, key],
+                  o: oldValue[key],
+                  n: undefined
+               });
+            } else {
+               recurse(oldValue[key], newValue[key], [...path, key]);
+            }
+         }
+      } else {
+         diffs.push({
+            t: Change.Edit,
+            p: path,
+            o: oldValue,
+            n: newValue
+         });
+      }
+   }
+
+   recurse(oldObj, newObj, []);
+   return diffs;
+}
+
+function apply(obj: Object, diffs: DiffEntry[]): any {
+   const clonedObj = clone(obj);
+
+   for (const diff of diffs) {
+      applyChange(clonedObj, diff);
+   }
+
+   return clonedObj;
+}
+
+function revert(obj: Object, diffs: DiffEntry[]): any {
+   const clonedObj = clone(obj);
+   const reversedDiffs = diffs.slice().reverse();
+
+   for (const diff of reversedDiffs) {
+      revertChange(clonedObj, diff);
+   }
+
+   return clonedObj;
+}
+
+function applyChange(obj: Object, diff: DiffEntry) {
+   const { p: path, t: type, n: newValue } = diff;
+   const parent = getParent(obj, path.slice(0, -1));
+   const key = path[path.length - 1]!;
+
+   if (type === Change.Add || type === Change.Edit) {
+      parent[key] = newValue;
+   } else if (type === Change.Remove) {
+      if (Array.isArray(parent)) {
+         parent.splice(key as number, 1);
+      } else {
+         delete parent[key];
+      }
+   }
+}
+
+function revertChange(obj: Object, diff: DiffEntry) {
+   const { p: path, t: type, o: oldValue } = diff;
+   const parent = getParent(obj, path.slice(0, -1));
+   const key = path[path.length - 1]!;
+
+   if (type === Change.Add) {
+      if (Array.isArray(parent)) {
+         parent.splice(key as number, 1);
+      } else {
+         delete parent[key];
+      }
+   } else if (type === Change.Remove || type === Change.Edit) {
+      parent[key] = oldValue;
+   }
+}
+
+function getParent(obj: Object, path: (string | number)[]): any {
+   let current = obj;
+   for (const key of path) {
+      if (current[key] === undefined) {
+         current[key] = typeof key === "number" ? [] : {};
+      }
+      current = current[key];
+   }
+   return current;
+}
+
+function clone<In extends Object>(obj: In): In {
+   return JSON.parse(JSON.stringify(obj));
+}
+
+export { diff, apply, revert, clone };

app/src/data/entities/query/WhereBuilder.ts

@@ -15,7 +15,6 @@ import type {
    SelectQueryBuilder,
    UpdateQueryBuilder
 } from "kysely";
-import type { RepositoryQB } from "./Repository";
 
 type Builder = ExpressionBuilder<any, any>;
 type Wrapper = ExpressionWrapper<any, any, any>;

app/src/modules/ModuleManager.ts

@@ -1,7 +1,7 @@
-import { Diff } from "@sinclair/typebox/value";
 import { Guard } from "auth";
 import { BkndError, DebugLogger, Exception, isDebug } from "core";
 import { EventManager } from "core/events";
+import { clone, diff } from "core/object/diff";
 import {
    Default,
    type Static,
@@ -18,7 +18,6 @@ import {
    datetime,
    entity,
    enumm,
-   json,
    jsonSchema,
    number
 } from "data";
@@ -87,9 +86,10 @@ const configJsonSchema = Type.Union([
    getDefaultSchema(),
    Type.Array(
       Type.Object({
-         type: StringEnum(["insert", "update", "delete"]),
+         t: StringEnum(["a", "r", "e"]),
-         value: Type.Any(),
+         p: Type.Array(Type.Union([Type.String(), Type.Number()])),
-         path: Type.Optional(Type.String())
+         o: Type.Optional(Type.Any()),
+         n: Type.Optional(Type.Any())
       })
    )
 ]);
@@ -105,6 +105,8 @@ type T_INTERNAL_EM = {
    __bknd: ConfigTable2;
 };
 
+// @todo: cleanup old diffs on upgrade
+// @todo: cleanup multiple backups on upgrade
 export class ModuleManager {
    private modules: Modules;
    // internal em for __bknd config table
@@ -251,26 +253,31 @@ export class ModuleManager {
             // @todo: mark all others as "backup"
             this.logger.log("version conflict, storing new version", state.version, version);
             await this.mutator().insertOne({
-               version,
+               version: state.version,
                type: "backup",
                json: configs
             });
+            await this.mutator().insertOne({
+               version: version,
+               type: "config",
+               json: configs
+            });
          } else {
             this.logger.log("version matches");
 
             // clean configs because of Diff() function
-            const diff = Diff(state.json, JSON.parse(JSON.stringify(configs)));
+            const diffs = diff(state.json, clone(configs));
-            this.logger.log("checking diff", diff);
+            this.logger.log("checking diff", diffs);
 
             if (diff.length > 0) {
                // store diff
                await this.mutator().insertOne({
                   version,
                   type: "diff",
-                  json: diff
+                  json: clone(diffs)
                });
+
                // store new version
-               // @todo: maybe by id?
                await this.mutator().updateWhere(
                   {
                      version,

app/src/modules/migrations.ts

@@ -72,6 +72,13 @@ export const migrations: Migration[] = [
          };
       }
    }
+   /*{
+      version: 8,
+      up: async (config, { db }) => {
+         await db.deleteFrom(TABLE_NAME).where("type", "=", "diff").execute();
+         return config;
+      }
+   }*/
 ];
 
 export const CURRENT_VERSION = migrations[migrations.length - 1]?.version ?? 0;