From 6c2f7b32e5268581bd144c1e1ef424ee86efbf07 Mon Sep 17 00:00:00 2001 From: dswbx Date: Thu, 5 Dec 2024 08:12:46 +0100 Subject: [PATCH 1/6] module manager: allow initial config without config as fallback --- app/__test__/ModuleManager.spec.ts | 90 ++++++++++++++++-------------- app/src/modules/ModuleManager.ts | 25 ++++++--- 2 files changed, 65 insertions(+), 50 deletions(-) diff --git a/app/__test__/ModuleManager.spec.ts b/app/__test__/ModuleManager.spec.ts index d58b98a..d79bf27 100644 --- a/app/__test__/ModuleManager.spec.ts +++ b/app/__test__/ModuleManager.spec.ts @@ -148,50 +148,58 @@ describe("ModuleManager", async () => { }); }); - // @todo: check what happens here - /*test("blank app, modify deep config", async () => { + test("partial config given", async () => { const { dummyConnection } = getDummyConnection(); - const mm = new ModuleManager(dummyConnection); + const partial = { + auth: { + enabled: true + } + }; + const mm = new ModuleManager(dummyConnection, { + initial: partial + }); await mm.build(); - /!* await mm - .get("data") - .schema() - .patch("entities.test", { - fields: { - content: { - type: "text" - } + expect(mm.version()).toBe(CURRENT_VERSION); + expect(mm.built()).toBe(true); + expect(mm.configs().auth.enabled).toBe(true); + expect(mm.configs().data.entities.users).toBeDefined(); + }); + + test("partial config given, but db version exists", async () => { + const c = getDummyConnection(); + const mm = new ModuleManager(c.dummyConnection); + await mm.build(); + const json = mm.configs(); + + const c2 = getDummyConnection(); + const db = c2.dummyConnection.kysely; + await migrateSchema(CURRENT_VERSION, { db }); + const payload = { + ...json, + auth: { + ...json.auth, + enabled: true, + basepath: "/api/auth2" + } + }; + await db + .updateTable(TABLE_NAME) + .set({ + json: JSON.stringify(payload), + version: CURRENT_VERSION + }) + .execute(); + + const mm2 = new ModuleManager(c2.dummyConnection, { + initial: { + auth: { + basepath: "/shouldnt/take/this" } - }); - await mm.build(); - - expect(mm.configs().data.entities?.users?.fields?.email.type).toBe("text"); - - expect( - mm.get("data").schema().patch("desc", "entities.users.config.sort_dir") - ).rejects.toThrow(); - await mm.build();*!/ - expect(mm.configs().data.entities?.users?.fields?.email.type).toBe("text"); - console.log("here", mm.configs()); - await mm - .get("data") - .schema() - .patch("entities.users", { config: { sort_dir: "desc" } }); - await mm.build(); - expect(mm.toJSON()); - - //console.log(_jsonp(mm.toJSON().data)); - /!*expect(mm.configs().data.entities!.test!.fields!.content.type).toBe("text"); - expect(mm.configs().data.entities!.users!.config!.sort_dir).toBe("desc");*!/ - });*/ - - /*test("accessing modules", async () => { - const { dummyConnection } = getDummyConnection(); - - const mm = new ModuleManager(dummyConnection); - - //mm.get("auth").mutate().set({}); - });*/ + } + }); + await mm2.build(); + expect(mm2.configs().auth.basepath).toBe("/api/auth2"); + }); }); diff --git a/app/src/modules/ModuleManager.ts b/app/src/modules/ModuleManager.ts index 51a1768..b77e207 100644 --- a/app/src/modules/ModuleManager.ts +++ b/app/src/modules/ModuleManager.ts @@ -6,6 +6,7 @@ import { Default, type Static, objectEach, transformObject } from "core/utils"; import { type Connection, EntityManager } from "data"; import { Hono } from "hono"; import { type Kysely, sql } from "kysely"; +import { mergeWith } from "lodash-es"; import { CURRENT_VERSION, TABLE_NAME, migrate, migrateSchema } from "modules/migrations"; import { AppServer } from "modules/server/AppServer"; import { AppAuth } from "../auth/AppAuth"; @@ -38,9 +39,11 @@ export type ModuleConfigs = { [K in keyof ModuleSchemas]: Static; }; -export type InitialModuleConfigs = { - version: number; -} & Partial; +export type InitialModuleConfigs = + | ({ + version: number; + } & ModuleConfigs) + | Partial; export type ModuleManagerOptions = { initial?: InitialModuleConfigs; @@ -71,7 +74,9 @@ export class ModuleManager { private _version: number = 0; private _built = false; private _fetched = false; - private readonly _provided; + + // @todo: keep? not doing anything with it + private readonly _booted_with?: "provided" | "partial"; private logger = new DebugLogger(isDebug() && false); @@ -85,14 +90,15 @@ export class ModuleManager { let initial = {} as Partial; if (options?.initial) { - const { version, ...initialConfig } = options.initial; - if (version && initialConfig) { + if ("version" in options.initial) { + const { version, ...initialConfig } = options.initial; this._version = version; initial = initialConfig; - this._provided = true; + this._booted_with = "provided"; } else { - throw new Error("Initial was provided, but it needs a version!"); + initial = mergeWith(getDefaultConfig(), options.initial); + this._booted_with = "partial"; } } @@ -337,10 +343,11 @@ export class ModuleManager { async build() { this.logger.context("build").log("version", this.version()); + this.logger.log("booted with", this._booted_with); // if no config provided, try fetch from db if (this.version() === 0) { - this.logger.context("build no config").log("version is 0"); + this.logger.context("no version").log("version is 0"); try { const result = await this.fetch(); From 3757157a06fca709750b1a7ef631777cd961c797 Mon Sep 17 00:00:00 2001 From: dswbx Date: Thu, 5 Dec 2024 09:23:34 +0100 Subject: [PATCH 2/6] module manager: switched config table to use entity --- app/__test__/ModuleManager.spec.ts | 64 +++---- app/__test__/data/mutation.simple.test.ts | 37 +++- app/src/core/errors.ts | 4 + app/src/data/entities/Mutator.ts | 28 ++- app/src/media/api/MediaController.ts | 2 +- app/src/modules/ModuleManager.ts | 204 ++++++++++++---------- app/src/modules/migrations.ts | 52 +----- 7 files changed, 219 insertions(+), 172 deletions(-) diff --git a/app/__test__/ModuleManager.spec.ts b/app/__test__/ModuleManager.spec.ts index d79bf27..38e1670 100644 --- a/app/__test__/ModuleManager.spec.ts +++ b/app/__test__/ModuleManager.spec.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from "bun:test"; import { mark, stripMark } from "../src/core/utils"; import { ModuleManager } from "../src/modules/ModuleManager"; -import { CURRENT_VERSION, TABLE_NAME, migrateSchema } from "../src/modules/migrations"; +import { CURRENT_VERSION, TABLE_NAME } from "../src/modules/migrations"; import { getDummyConnection } from "./helper"; describe("ModuleManager", async () => { @@ -34,13 +34,13 @@ describe("ModuleManager", async () => { const c2 = getDummyConnection(); const db = c2.dummyConnection.kysely; - await migrateSchema(CURRENT_VERSION, { db }); + const mm2 = new ModuleManager(c2.dummyConnection, { initial: { version, ...json } }); + await mm2.syncConfigTable(); await db .updateTable(TABLE_NAME) .set({ json: JSON.stringify(json), version: CURRENT_VERSION }) .execute(); - const mm2 = new ModuleManager(c2.dummyConnection, { initial: { version, ...json } }); await mm2.build(); expect(json).toEqual(mm2.configs()); @@ -52,21 +52,19 @@ describe("ModuleManager", async () => { await mm.build(); const version = mm.version(); const json = mm.configs(); - //const { version, ...json } = mm.toJSON() as any; const c2 = getDummyConnection(); const db = c2.dummyConnection.kysely; - console.log("here2"); - await migrateSchema(CURRENT_VERSION, { db }); - await db - .updateTable(TABLE_NAME) - .set({ json: JSON.stringify(json), version: CURRENT_VERSION - 1 }) - .execute(); - const mm2 = new ModuleManager(c2.dummyConnection, { initial: { version: version - 1, ...json } }); - console.log("here3"); + await mm2.syncConfigTable(); + + await db + .insertInto(TABLE_NAME) + .values({ json: JSON.stringify(json), type: "config", version: CURRENT_VERSION - 1 }) + .execute(); + await mm2.build(); }); @@ -80,15 +78,15 @@ describe("ModuleManager", async () => { const c2 = getDummyConnection(); const db = c2.dummyConnection.kysely; - await migrateSchema(CURRENT_VERSION, { db }); - await db - .updateTable(TABLE_NAME) - .set({ json: JSON.stringify(json), version: CURRENT_VERSION }) - .execute(); const mm2 = new ModuleManager(c2.dummyConnection, { initial: { version: version - 1, ...json } }); + await mm2.syncConfigTable(); + await db + .insertInto(TABLE_NAME) + .values({ type: "config", json: JSON.stringify(json), version: CURRENT_VERSION }) + .execute(); expect(mm2.build()).rejects.toThrow(/version.*do not match/); }); @@ -102,7 +100,9 @@ describe("ModuleManager", async () => { const c2 = getDummyConnection(); const db = c2.dummyConnection.kysely; - await migrateSchema(CURRENT_VERSION, { db }); + + const mm2 = new ModuleManager(c2.dummyConnection); + await mm2.syncConfigTable(); const config = { ...json, @@ -112,12 +112,11 @@ describe("ModuleManager", async () => { } }; await db - .updateTable(TABLE_NAME) - .set({ json: JSON.stringify(config), version: CURRENT_VERSION }) + .insertInto(TABLE_NAME) + .values({ type: "config", json: JSON.stringify(config), version: CURRENT_VERSION }) .execute(); // run without config given - const mm2 = new ModuleManager(c2.dummyConnection); await mm2.build(); expect(mm2.configs().data.basepath).toBe("/api/data2"); @@ -175,7 +174,15 @@ describe("ModuleManager", async () => { const c2 = getDummyConnection(); const db = c2.dummyConnection.kysely; - await migrateSchema(CURRENT_VERSION, { db }); + + const mm2 = new ModuleManager(c2.dummyConnection, { + initial: { + auth: { + basepath: "/shouldnt/take/this" + } + } + }); + await mm2.syncConfigTable(); const payload = { ...json, auth: { @@ -185,20 +192,13 @@ describe("ModuleManager", async () => { } }; await db - .updateTable(TABLE_NAME) - .set({ + .insertInto(TABLE_NAME) + .values({ + type: "config", json: JSON.stringify(payload), version: CURRENT_VERSION }) .execute(); - - const mm2 = new ModuleManager(c2.dummyConnection, { - initial: { - auth: { - basepath: "/shouldnt/take/this" - } - } - }); await mm2.build(); expect(mm2.configs().auth.basepath).toBe("/api/auth2"); }); diff --git a/app/__test__/data/mutation.simple.test.ts b/app/__test__/data/mutation.simple.test.ts index b3f0c77..dd385af 100644 --- a/app/__test__/data/mutation.simple.test.ts +++ b/app/__test__/data/mutation.simple.test.ts @@ -132,14 +132,47 @@ describe("Mutator simple", async () => { const data = (await em.repository(items).findMany()).data; //console.log(data); - await em.mutator(items).deleteMany({ label: "delete" }); + await em.mutator(items).deleteWhere({ label: "delete" }); expect((await em.repository(items).findMany()).data.length).toBe(data.length - 2); //console.log((await em.repository(items).findMany()).data); - await em.mutator(items).deleteMany(); + await em.mutator(items).deleteWhere(); expect((await em.repository(items).findMany()).data.length).toBe(0); //expect(res.data.count).toBe(0); }); + + test("updateMany", async () => { + await em.mutator(items).insertOne({ label: "update", count: 1 }); + await em.mutator(items).insertOne({ label: "update too", count: 1 }); + await em.mutator(items).insertOne({ label: "keep" }); + + // expect no update + await em.mutator(items).updateWhere( + { count: 2 }, + { + count: 10 + } + ); + expect((await em.repository(items).findMany()).data).toEqual([ + { id: 6, label: "update", count: 1 }, + { id: 7, label: "update too", count: 1 }, + { id: 8, label: "keep", count: 0 } + ]); + + // expect 2 to be updated + await em.mutator(items).updateWhere( + { count: 2 }, + { + count: 1 + } + ); + + expect((await em.repository(items).findMany()).data).toEqual([ + { id: 6, label: "update", count: 2 }, + { id: 7, label: "update too", count: 2 }, + { id: 8, label: "keep", count: 0 } + ]); + }); }); diff --git a/app/src/core/errors.ts b/app/src/core/errors.ts index ce63ed9..860bd9d 100644 --- a/app/src/core/errors.ts +++ b/app/src/core/errors.ts @@ -27,6 +27,10 @@ export class BkndError extends Error { super(message); } + static with(message: string, details?: Record, type?: string) { + throw new BkndError(message, details, type); + } + toJSON() { return { type: this.type ?? "unknown", diff --git a/app/src/data/entities/Mutator.ts b/app/src/data/entities/Mutator.ts index aef2bf1..ed7f9ef 100644 --- a/app/src/data/entities/Mutator.ts +++ b/app/src/data/entities/Mutator.ts @@ -250,7 +250,7 @@ export class Mutator implements EmitsEvents { } // @todo: decide whether entries should be deleted all at once or one by one (for events) - async deleteMany(where?: RepoQuery["where"]): Promise> { + async deleteWhere(where?: RepoQuery["where"]): Promise> { const entity = this.entity; const qb = this.appendWhere(this.conn.deleteFrom(entity.name), where).returning( @@ -267,4 +267,30 @@ export class Mutator implements EmitsEvents { return res; } + + async updateWhere( + data: EntityData, + where?: RepoQuery["where"] + ): Promise> { + const entity = this.entity; + + const validatedData = await this.getValidatedData(data, "update"); + + /*await this.emgr.emit( + new Mutator.Events.MutatorUpdateBefore({ entity, entityId: id, data: validatedData }) + );*/ + + const query = this.appendWhere(this.conn.updateTable(entity.name), where) + .set(validatedData) + //.where(entity.id().name, "=", id) + .returning(entity.getSelect()); + + const res = await this.many(query); + + /*await this.emgr.emit( + new Mutator.Events.MutatorUpdateAfter({ entity, entityId: id, data: res.data }) + );*/ + + return res; + } } diff --git a/app/src/media/api/MediaController.ts b/app/src/media/api/MediaController.ts index 59665fa..9597759 100644 --- a/app/src/media/api/MediaController.ts +++ b/app/src/media/api/MediaController.ts @@ -181,7 +181,7 @@ export class MediaController implements ClassController { if (ids_to_delete.length > 0) { await this.media.em .mutator(mediaEntity) - .deleteMany({ [id_field]: { $in: ids_to_delete } }); + .deleteWhere({ [id_field]: { $in: ids_to_delete } }); } return c.json({ ok: true, result: result.data, ...info }); diff --git a/app/src/modules/ModuleManager.ts b/app/src/modules/ModuleManager.ts index b77e207..4774ac2 100644 --- a/app/src/modules/ModuleManager.ts +++ b/app/src/modules/ModuleManager.ts @@ -1,13 +1,22 @@ import { Diff } from "@sinclair/typebox/value"; import { Guard } from "auth"; -import { DebugLogger, isDebug } from "core"; +import { BkndError, DebugLogger, Exception, isDebug } from "core"; import { EventManager } from "core/events"; import { Default, type Static, objectEach, transformObject } from "core/utils"; -import { type Connection, EntityManager } from "data"; +import { + type Connection, + EntityManager, + type Schema, + datetime, + entity, + enumm, + json, + number +} from "data"; import { Hono } from "hono"; import { type Kysely, sql } from "kysely"; import { mergeWith } from "lodash-es"; -import { CURRENT_VERSION, TABLE_NAME, migrate, migrateSchema } from "modules/migrations"; +import { CURRENT_VERSION, TABLE_NAME, migrate } from "modules/migrations"; import { AppServer } from "modules/server/AppServer"; import { AppAuth } from "../auth/AppAuth"; import { AppData } from "../data/AppData"; @@ -64,8 +73,23 @@ type ConfigTable = { updated_at?: Date; }; +const __bknd = entity(TABLE_NAME, { + version: number().required(), + type: enumm({ enum: ["config", "diff", "backup"] }).required(), + json: json().required(), + created_at: datetime(), + updated_at: datetime() +}); +type ConfigTable2 = Schema; +type T_INTERNAL_EM = { + __bknd: ConfigTable2; +}; + export class ModuleManager { private modules: Modules; + // internal em for __bknd config table + __em!: EntityManager; + // ctx for modules em!: EntityManager; server!: Hono; emgr!: EventManager; @@ -78,12 +102,13 @@ export class ModuleManager { // @todo: keep? not doing anything with it private readonly _booted_with?: "provided" | "partial"; - private logger = new DebugLogger(isDebug() && false); + private logger = new DebugLogger(false); constructor( private readonly connection: Connection, private options?: Partial ) { + this.__em = new EntityManager([__bknd], this.connection); this.modules = {} as Modules; this.emgr = new EventManager(); const context = this.ctx(true); @@ -126,6 +151,22 @@ export class ModuleManager { } } + private repo() { + return this.__em.repo(__bknd); + } + + private mutator() { + return this.__em.mutator(__bknd); + } + + private get db() { + return this.connection.kysely as Kysely<{ table: ConfigTable }>; + } + + async syncConfigTable() { + return await this.__em.schema().sync({ force: true }); + } + private rebuildServer() { this.server = new Hono(); if (this.options?.basePath) { @@ -159,27 +200,22 @@ export class ModuleManager { }; } - private get db() { - return this.connection.kysely as Kysely<{ table: ConfigTable }>; - } - - get table() { - return TABLE_NAME as "table"; - } - private async fetch(): Promise { this.logger.context("fetch").log("fetching"); const startTime = performance.now(); - const result = await this.db - .selectFrom(this.table) - .selectAll() - .where("type", "=", "config") - .orderBy("version", "desc") - .executeTakeFirstOrThrow(); + const { data: result } = await this.repo().findOne( + { type: "config" }, + { + sort: { by: "version", dir: "desc" } + } + ); + if (!result) { + throw BkndError.with("no config"); + } this.logger.log("took", performance.now() - startTime, "ms", result).clear(); - return result; + return result as ConfigTable; } async save() { @@ -188,64 +224,65 @@ export class ModuleManager { const version = this.version(); const json = JSON.stringify(configs) as any; - const state = await this.fetch(); - if (state.version !== version) { - // @todo: mark all others as "backup" - this.logger.log("version conflict, storing new version", state.version, version); - await this.db - .insertInto(this.table) - .values({ + try { + const state = await this.fetch(); + + if (state.version !== version) { + // @todo: mark all others as "backup" + this.logger.log("version conflict, storing new version", state.version, version); + await this.mutator().insertOne({ version, - type: "config", + type: "backup", json - }) - .execute(); - } else { - this.logger.log("version matches"); + }); + } else { + this.logger.log("version matches"); - const diff = Diff(state.json, JSON.parse(json)); - this.logger.log("checking diff", diff); + const diff = Diff(state.json, JSON.parse(json)); + this.logger.log("checking diff", diff); - if (diff.length > 0) { - // store diff - await this.db - .insertInto(this.table) - .values({ + if (diff.length > 0) { + // store diff + await this.mutator().insertOne({ version, type: "diff", json: JSON.stringify(diff) as any - }) - .execute(); - - await this.db - .updateTable(this.table) - .set({ version, json, updated_at: sql`CURRENT_TIMESTAMP` }) - .where((eb) => eb.and([eb("type", "=", "config"), eb("version", "=", version)])) - .execute(); + }); + // store new version + // @todo: maybe by id? + await this.mutator().updateWhere( + { + version, + json, + updated_at: new Date() + }, + { + type: "config", + version + } + ); + } else { + this.logger.log("no diff, not saving"); + } + } + } catch (e) { + if (e instanceof BkndError) { + // no config, just save + await this.mutator().insertOne({ + type: "config", + version, + json, + created_at: new Date(), + updated_at: new Date() + }); } else { - this.logger.log("no diff, not saving"); + console.error("Aborting"); + throw e; } } - // cleanup - /*this.logger.log("cleaning up"); - const result = await this.db - .deleteFrom(this.table) - .where((eb) => - eb.or([ - // empty migrations - eb.and([ - eb("type", "=", "config"), - eb("version", "<", version), - eb("json", "is", null) - ]), - // past diffs - eb.and([eb("type", "=", "diff"), eb("version", "<", version)]) - ]) - ) - .executeTakeFirst(); - this.logger.log("cleaned up", result.numDeletedRows);*/ + // @todo: cleanup old versions? this.logger.clear(); return this; @@ -256,6 +293,8 @@ export class ModuleManager { if (this.version() < CURRENT_VERSION) { this.logger.log("there are migrations, verify version"); + // sync __bknd table + await this.syncConfigTable(); // modules must be built before migration await this.buildModules({ graceful: true }); @@ -270,14 +309,7 @@ export class ModuleManager { } } catch (e: any) { this.logger.clear(); // fetch couldn't clear - - // if table doesn't exist, migrate schema to version - if (e.message.includes("no such table")) { - this.logger.log("table has to created, migrating schema up to", this.version()); - await migrateSchema(this.version(), { db: this.db }); - } else { - throw new Error(`Version is ${this.version()}, fetch failed: ${e.message}`); - } + throw new Error(`Version is ${this.version()}, fetch failed: ${e.message}`); } this.logger.log("now migrating"); @@ -345,6 +377,10 @@ export class ModuleManager { this.logger.context("build").log("version", this.version()); this.logger.log("booted with", this._booted_with); + if (this.version() !== CURRENT_VERSION) { + await this.syncConfigTable(); + } + // if no config provided, try fetch from db if (this.version() === 0) { this.logger.context("no version").log("version is 0"); @@ -358,23 +394,15 @@ export class ModuleManager { this.logger.clear(); // fetch couldn't clear this.logger.context("error handler").log("fetch failed", e.message); - // if table doesn't exist, migrate schema, set default config and latest version - if (e.message.includes("no such table")) { - this.logger.log("migrate schema to", CURRENT_VERSION); - await migrateSchema(CURRENT_VERSION, { db: this.db }); - this._version = CURRENT_VERSION; - // we can safely build modules, since config version is up to date - // it's up to date because we use default configs (no fetch result) - await this.buildModules(); - await this.save(); + // we can safely build modules, since config version is up to date + // it's up to date because we use default configs (no fetch result) + this._version = CURRENT_VERSION; + await this.buildModules(); + await this.save(); - this.logger.clear(); - return this; - } else { - throw e; - //throw new Error("Issues connecting to the database. Reason: " + e.message); - } + this.logger.clear(); + return this; } this.logger.clear(); } diff --git a/app/src/modules/migrations.ts b/app/src/modules/migrations.ts index 9cf0beb..1d4834a 100644 --- a/app/src/modules/migrations.ts +++ b/app/src/modules/migrations.ts @@ -16,26 +16,8 @@ export type Migration = { export const migrations: Migration[] = [ { version: 1, - schema: true, - up: async (config, { db }) => { - //console.log("config given", config); - await db.schema - .createTable(TABLE_NAME) - .addColumn("id", "integer", (col) => col.primaryKey().notNull().autoIncrement()) - .addColumn("version", "integer", (col) => col.notNull()) - .addColumn("type", "text", (col) => col.notNull()) - .addColumn("json", "text") - .addColumn("created_at", "datetime", (col) => col.defaultTo(sql`CURRENT_TIMESTAMP`)) - .addColumn("updated_at", "datetime", (col) => col.defaultTo(sql`CURRENT_TIMESTAMP`)) - .execute(); - - await db - .insertInto(TABLE_NAME) - .values({ version: 1, type: "config", json: null }) - .execute(); - - return config; - } + //schema: true, + up: async (config) => config }, { version: 2, @@ -45,12 +27,8 @@ export const migrations: Migration[] = [ }, { version: 3, - schema: true, - up: async (config, { db }) => { - await db.schema.alterTable(TABLE_NAME).addColumn("deleted_at", "datetime").execute(); - - return config; - } + //schema: true, + up: async (config) => config }, { version: 4, @@ -127,28 +105,6 @@ export async function migrateTo( return [version, updated]; } -export async function migrateSchema(to: number, ctx: MigrationContext, current: number = 0) { - console.log("migrating SCHEMA to", to, "from", current); - const todo = migrations.filter((m) => m.version > current && m.version <= to && m.schema); - console.log("todo", todo.length); - - let i = 0; - let version = 0; - for (const migration of todo) { - console.log("-- running migration", i + 1, "of", todo.length); - try { - await migration.up({}, ctx); - version = migration.version; - i++; - } catch (e: any) { - console.error(e); - throw new Error(`Migration ${migration.version} failed: ${e.message}`); - } - } - - return version; -} - export async function migrate( current: number, config: GenericConfigObject, From a3348122e69f665d8d10ef15d253f28151c11946 Mon Sep 17 00:00:00 2001 From: dswbx Date: Thu, 5 Dec 2024 09:50:25 +0100 Subject: [PATCH 3/6] module manager: use json schema field for additional validation --- app/src/data/fields/JsonSchemaField.ts | 7 +++++- app/src/modules/ModuleManager.ts | 34 +++++++++++++++++++------- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/app/src/data/fields/JsonSchemaField.ts b/app/src/data/fields/JsonSchemaField.ts index 5f4e2c4..b414866 100644 --- a/app/src/data/fields/JsonSchemaField.ts +++ b/app/src/data/fields/JsonSchemaField.ts @@ -54,7 +54,7 @@ export class JsonSchemaField< if (parentValid) { // already checked in parent - if (!value || typeof value !== "object") { + if (!this.isRequired() && (!value || typeof value !== "object")) { //console.log("jsonschema:valid: not checking", this.name, value, context); return true; } @@ -65,6 +65,7 @@ export class JsonSchemaField< } else { //console.log("jsonschema:invalid", this.name, value, context); } + //console.log("jsonschema:invalid:fromParent", this.name, value, context); return false; } @@ -110,9 +111,13 @@ export class JsonSchemaField< ): Promise { const value = await super.transformPersist(_value, em, context); if (this.nullish(value)) return value; + //console.log("jsonschema:transformPersist", this.name, _value, context); if (!this.isValid(value)) { + //console.error("jsonschema:transformPersist:invalid", this.name, value); throw new TransformPersistFailedException(this.name, value); + } else { + //console.log("jsonschema:transformPersist:valid", this.name, value); } if (!value || typeof value !== "object") return this.getDefault(); diff --git a/app/src/modules/ModuleManager.ts b/app/src/modules/ModuleManager.ts index 4774ac2..b365fdd 100644 --- a/app/src/modules/ModuleManager.ts +++ b/app/src/modules/ModuleManager.ts @@ -2,7 +2,7 @@ import { Diff } from "@sinclair/typebox/value"; import { Guard } from "auth"; import { BkndError, DebugLogger, Exception, isDebug } from "core"; import { EventManager } from "core/events"; -import { Default, type Static, objectEach, transformObject } from "core/utils"; +import { Default, type Static, StringEnum, Type, objectEach, transformObject } from "core/utils"; import { type Connection, EntityManager, @@ -11,8 +11,10 @@ import { entity, enumm, json, + jsonSchema, number } from "data"; +import { TransformPersistFailedException } from "data/errors"; import { Hono } from "hono"; import { type Kysely, sql } from "kysely"; import { mergeWith } from "lodash-es"; @@ -73,10 +75,20 @@ type ConfigTable = { updated_at?: Date; }; +const configJsonSchema = Type.Union([ + getDefaultSchema(), + Type.Array( + Type.Object({ + type: StringEnum(["insert", "update", "delete"]), + value: Type.Any(), + path: Type.Optional(Type.String()) + }) + ) +]); const __bknd = entity(TABLE_NAME, { version: number().required(), type: enumm({ enum: ["config", "diff", "backup"] }).required(), - json: json().required(), + json: jsonSchema({ schema: configJsonSchema }).required(), created_at: datetime(), updated_at: datetime() }); @@ -223,10 +235,9 @@ export class ModuleManager { const configs = this.configs(); const version = this.version(); - const json = JSON.stringify(configs) as any; - try { const state = await this.fetch(); + this.logger.log("fetched version", state.version); if (state.version !== version) { // @todo: mark all others as "backup" @@ -234,12 +245,13 @@ export class ModuleManager { await this.mutator().insertOne({ version, type: "backup", - json + json: configs }); } else { this.logger.log("version matches"); - const diff = Diff(state.json, JSON.parse(json)); + // clean configs because of Diff() function + const diff = Diff(state.json, JSON.parse(JSON.stringify(configs))); this.logger.log("checking diff", diff); if (diff.length > 0) { @@ -247,14 +259,14 @@ export class ModuleManager { await this.mutator().insertOne({ version, type: "diff", - json: JSON.stringify(diff) as any + json: diff }); // store new version // @todo: maybe by id? await this.mutator().updateWhere( { version, - json, + json: configs, updated_at: new Date() }, { @@ -268,14 +280,18 @@ export class ModuleManager { } } catch (e) { if (e instanceof BkndError) { + this.logger.log("no config, just save fresh"); // no config, just save await this.mutator().insertOne({ type: "config", version, - json, + json: configs, created_at: new Date(), updated_at: new Date() }); + } else if (e instanceof TransformPersistFailedException) { + console.error("Cannot save invalid config"); + throw e; } else { console.error("Aborting"); throw e; From 77a6b6e7f5e8b78d58494825a0b5e34242b8e142 Mon Sep 17 00:00:00 2001 From: dswbx Date: Thu, 5 Dec 2024 10:37:05 +0100 Subject: [PATCH 4/6] =?UTF-8?q?fix=20repository's=20findOne=20to=20allow?= =?UTF-8?q?=20offset=20and=20sort=20=E2=80=93=20fixes=20module=20manager's?= =?UTF-8?q?=20config=20retrieval?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/__test__/ModuleManager.spec.ts | 58 +++++++++++++++++++++-- app/src/core/utils/typebox/index.ts | 4 +- app/src/data/entities/query/Repository.ts | 17 +++---- app/src/modules/ModuleManager.ts | 13 ++++- 4 files changed, 73 insertions(+), 19 deletions(-) diff --git a/app/__test__/ModuleManager.spec.ts b/app/__test__/ModuleManager.spec.ts index 38e1670..26b2b9a 100644 --- a/app/__test__/ModuleManager.spec.ts +++ b/app/__test__/ModuleManager.spec.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from "bun:test"; import { mark, stripMark } from "../src/core/utils"; -import { ModuleManager } from "../src/modules/ModuleManager"; +import { entity, text } from "../src/data"; +import { ModuleManager, getDefaultConfig } from "../src/modules/ModuleManager"; import { CURRENT_VERSION, TABLE_NAME } from "../src/modules/migrations"; import { getDummyConnection } from "./helper"; @@ -29,7 +30,19 @@ describe("ModuleManager", async () => { const mm = new ModuleManager(c.dummyConnection); await mm.build(); const version = mm.version(); - const json = mm.configs(); + const configs = mm.configs(); + const json = stripMark({ + ...configs, + data: { + ...configs.data, + basepath: "/api/data2", + entities: { + test: entity("test", { + content: text() + }).toJSON() + } + } + }); //const { version, ...json } = mm.toJSON() as any; const c2 = getDummyConnection(); @@ -37,13 +50,48 @@ describe("ModuleManager", async () => { const mm2 = new ModuleManager(c2.dummyConnection, { initial: { version, ...json } }); await mm2.syncConfigTable(); await db - .updateTable(TABLE_NAME) - .set({ json: JSON.stringify(json), version: CURRENT_VERSION }) + .insertInto(TABLE_NAME) + .values({ type: "config", json: JSON.stringify(json), version: CURRENT_VERSION }) .execute(); await mm2.build(); - expect(json).toEqual(mm2.configs()); + expect(json).toEqual(stripMark(mm2.configs())); + }); + + test("s3.1: (fetch) config given, table exists, version matches", async () => { + const configs = getDefaultConfig(); + const json = { + ...configs, + data: { + ...configs.data, + basepath: "/api/data2", + entities: { + test: entity("test", { + content: text() + }).toJSON() + } + } + }; + //const { version, ...json } = mm.toJSON() as any; + + const { dummyConnection } = getDummyConnection(); + const db = dummyConnection.kysely; + const mm2 = new ModuleManager(dummyConnection); + await mm2.syncConfigTable(); + // assume an initial version + await db.insertInto(TABLE_NAME).values({ type: "config", json: null, version: 1 }).execute(); + await db + .insertInto(TABLE_NAME) + .values({ type: "config", json: JSON.stringify(json), version: CURRENT_VERSION }) + .execute(); + + await mm2.build(); + + expect(stripMark(json)).toEqual(stripMark(mm2.configs())); + expect(mm2.configs().data.entities.test).toBeDefined(); + expect(mm2.configs().data.entities.test.fields.content).toBeDefined(); + expect(mm2.get("data").toJSON().entities.test.fields.content).toBeDefined(); }); test("s4: config given, table exists, version outdated, migrate", async () => { diff --git a/app/src/core/utils/typebox/index.ts b/app/src/core/utils/typebox/index.ts index 2b923c1..2e08d7a 100644 --- a/app/src/core/utils/typebox/index.ts +++ b/app/src/core/utils/typebox/index.ts @@ -72,10 +72,10 @@ export class TypeInvalidError extends Error { } } -export function stripMark(obj: any) { +export function stripMark(obj: O) { const newObj = cloneDeep(obj); mark(newObj, false); - return newObj; + return newObj as O; } export function mark(obj: any, validated = true) { diff --git a/app/src/data/entities/query/Repository.ts b/app/src/data/entities/query/Repository.ts index f296adf..8156869 100644 --- a/app/src/data/entities/query/Repository.ts +++ b/app/src/data/entities/query/Repository.ts @@ -162,8 +162,7 @@ export class Repository implements EmitsEve protected async performQuery(qb: RepositoryQB): Promise { const entity = this.entity; const compiled = qb.compile(); - /*const { sql, parameters } = qb.compile(); - console.log("many", sql, parameters);*/ + //console.log("performQuery", compiled.sql, compiled.parameters); const start = performance.now(); const selector = (as = "count") => this.conn.fn.countAll().as(as); @@ -263,6 +262,7 @@ export class Repository implements EmitsEve qb = qb.orderBy(aliased(options.sort.by), options.sort.dir); } + //console.log("options", { _options, options, exclude_options }); return { qb, options }; } @@ -286,14 +286,11 @@ export class Repository implements EmitsEve where: RepoQuery["where"], _options?: Partial> ): Promise> { - const { qb, options } = this.buildQuery( - { - ..._options, - where, - limit: 1 - }, - ["offset", "sort"] - ); + const { qb, options } = this.buildQuery({ + ..._options, + where, + limit: 1 + }); return this.single(qb, options) as any; } diff --git a/app/src/modules/ModuleManager.ts b/app/src/modules/ModuleManager.ts index b365fdd..5da0dc2 100644 --- a/app/src/modules/ModuleManager.ts +++ b/app/src/modules/ModuleManager.ts @@ -2,7 +2,15 @@ import { Diff } from "@sinclair/typebox/value"; import { Guard } from "auth"; import { BkndError, DebugLogger, Exception, isDebug } from "core"; import { EventManager } from "core/events"; -import { Default, type Static, StringEnum, Type, objectEach, transformObject } from "core/utils"; +import { + Default, + type Static, + StringEnum, + Type, + objectEach, + stripMark, + transformObject +} from "core/utils"; import { type Connection, EntityManager, @@ -130,7 +138,7 @@ export class ModuleManager { if ("version" in options.initial) { const { version, ...initialConfig } = options.initial; this._version = version; - initial = initialConfig; + initial = stripMark(initialConfig); this._booted_with = "provided"; } else { @@ -393,6 +401,7 @@ export class ModuleManager { this.logger.context("build").log("version", this.version()); this.logger.log("booted with", this._booted_with); + // @todo: check this, because you could start without an initial config if (this.version() !== CURRENT_VERSION) { await this.syncConfigTable(); } From 7e990feb993bc9e61c6119f761c102b770b3e13e Mon Sep 17 00:00:00 2001 From: dswbx Date: Thu, 5 Dec 2024 20:23:51 +0100 Subject: [PATCH 5/6] added new diffing method to module manager Signed-off-by: dswbx --- app/__test__/ModuleManager.spec.ts | 2 + app/__test__/core/object/diff.test.ts | 443 ++++++++++++++++++++ app/src/core/object/diff.ts | 181 ++++++++ app/src/data/entities/query/WhereBuilder.ts | 1 - app/src/modules/ModuleManager.ts | 27 +- app/src/modules/migrations.ts | 7 + 6 files changed, 650 insertions(+), 11 deletions(-) create mode 100644 app/__test__/core/object/diff.test.ts create mode 100644 app/src/core/object/diff.ts diff --git a/app/__test__/ModuleManager.spec.ts b/app/__test__/ModuleManager.spec.ts index 26b2b9a..2e928d6 100644 --- a/app/__test__/ModuleManager.spec.ts +++ b/app/__test__/ModuleManager.spec.ts @@ -250,4 +250,6 @@ describe("ModuleManager", async () => { await mm2.build(); expect(mm2.configs().auth.basepath).toBe("/api/auth2"); }); + + // @todo: add tests for migrations (check "backup" and new version) }); diff --git a/app/__test__/core/object/diff.test.ts b/app/__test__/core/object/diff.test.ts new file mode 100644 index 0000000..b6ac6e4 --- /dev/null +++ b/app/__test__/core/object/diff.test.ts @@ -0,0 +1,443 @@ +import { describe, expect, it, test } from "bun:test"; +import { apply, diff, revert } from "../../../src/core/object/diff"; + +describe("diff", () => { + it("should detect added properties", () => { + const oldObj = { a: 1 }; + const newObj = { a: 1, b: 2 }; + + const diffs = diff(oldObj, newObj); + + expect(diffs).toEqual([ + { + t: "a", + p: ["b"], + o: undefined, + n: 2 + } + ]); + + const appliedObj = apply(oldObj, diffs); + expect(appliedObj).toEqual(newObj); + + const revertedObj = revert(newObj, diffs); + expect(revertedObj).toEqual(oldObj); + }); + + it("should detect removed properties", () => { + const oldObj = { a: 1, b: 2 }; + const newObj = { a: 1 }; + + const diffs = diff(oldObj, newObj); + + expect(diffs).toEqual([ + { + t: "r", + p: ["b"], + o: 2, + n: undefined + } + ]); + + const appliedObj = apply(oldObj, diffs); + expect(appliedObj).toEqual(newObj); + + const revertedObj = revert(newObj, diffs); + expect(revertedObj).toEqual(oldObj); + }); + + it("should detect edited properties", () => { + const oldObj = { a: 1 }; + const newObj = { a: 2 }; + + const diffs = diff(oldObj, newObj); + + expect(diffs).toEqual([ + { + t: "e", + p: ["a"], + o: 1, + n: 2 + } + ]); + + const appliedObj = apply(oldObj, diffs); + expect(appliedObj).toEqual(newObj); + + const revertedObj = revert(newObj, diffs); + expect(revertedObj).toEqual(oldObj); + }); + + it("should detect changes in nested objects", () => { + const oldObj = { a: { b: 1 } }; + const newObj = { a: { b: 2 } }; + + const diffs = diff(oldObj, newObj); + + expect(diffs).toEqual([ + { + t: "e", + p: ["a", "b"], + o: 1, + n: 2 + } + ]); + + const appliedObj = apply(oldObj, diffs); + expect(appliedObj).toEqual(newObj); + + const revertedObj = revert(newObj, diffs); + expect(revertedObj).toEqual(oldObj); + }); + + it("should detect changes in arrays", () => { + const oldObj = { a: [1, 2, 3] }; + const newObj = { a: [1, 4, 3, 5] }; + + const diffs = diff(oldObj, newObj); + + expect(diffs).toEqual([ + { + t: "e", + p: ["a", 1], + o: 2, + n: 4 + }, + { + t: "a", + p: ["a", 3], + o: undefined, + n: 5 + } + ]); + + const appliedObj = apply(oldObj, diffs); + expect(appliedObj).toEqual(newObj); + + const revertedObj = revert(newObj, diffs); + expect(revertedObj).toEqual(oldObj); + }); + + it("should handle adding elements to an empty array", () => { + const oldObj = { a: [] }; + const newObj = { a: [1, 2, 3] }; + + const diffs = diff(oldObj, newObj); + + expect(diffs).toEqual([ + { + t: "a", + p: ["a", 0], + o: undefined, + n: 1 + }, + { + t: "a", + p: ["a", 1], + o: undefined, + n: 2 + }, + { + t: "a", + p: ["a", 2], + o: undefined, + n: 3 + } + ]); + + const appliedObj = apply(oldObj, diffs); + expect(appliedObj).toEqual(newObj); + + const revertedObj = revert(newObj, diffs); + expect(revertedObj).toEqual(oldObj); + }); + + it("should handle removing elements from an array", () => { + const oldObj = { a: [1, 2, 3] }; + const newObj = { a: [1, 3] }; + + const diffs = diff(oldObj, newObj); + + expect(diffs).toEqual([ + { + t: "e", + p: ["a", 1], + o: 2, + n: 3 + }, + { + t: "r", + p: ["a", 2], + o: 3, + n: undefined + } + ]); + + const appliedObj = apply(oldObj, diffs); + expect(appliedObj).toEqual(newObj); + + const revertedObj = revert(newObj, diffs); + expect(revertedObj).toEqual(oldObj); + }); + + it("should handle complex nested changes", () => { + const oldObj = { + a: { + b: [1, 2, { c: 3 }] + } + }; + + const newObj = { + a: { + b: [1, 2, { c: 4 }, 5] + } + }; + + const diffs = diff(oldObj, newObj); + + expect(diffs).toEqual([ + { + t: "e", + p: ["a", "b", 2, "c"], + o: 3, + n: 4 + }, + { + t: "a", + p: ["a", "b", 3], + o: undefined, + n: 5 + } + ]); + + const appliedObj = apply(oldObj, diffs); + expect(appliedObj).toEqual(newObj); + + const revertedObj = revert(newObj, diffs); + expect(revertedObj).toEqual(oldObj); + }); + + it("should handle undefined and null values", () => { + const oldObj = { a: undefined, b: null }; + const newObj = { a: null, b: undefined }; + + const diffs = diff(oldObj, newObj); + + expect(diffs).toEqual([ + { + t: "e", + p: ["a"], + o: undefined, + n: null + }, + { + t: "e", + p: ["b"], + o: null, + n: undefined + } + ]); + + const appliedObj = apply(oldObj, diffs); + expect(appliedObj).toEqual(newObj); + + const revertedObj = revert(newObj, diffs); + expect(revertedObj).toEqual(oldObj); + }); + + it("should handle type changes", () => { + const oldObj = { a: 1 }; + const newObj = { a: "1" }; + + const diffs = diff(oldObj, newObj); + + expect(diffs).toEqual([ + { + t: "e", + p: ["a"], + o: 1, + n: "1" + } + ]); + + const appliedObj = apply(oldObj, diffs); + expect(appliedObj).toEqual(newObj); + + const revertedObj = revert(newObj, diffs); + expect(revertedObj).toEqual(oldObj); + }); + + it("should handle properties added and removed simultaneously", () => { + const oldObj = { a: 1, b: 2 }; + const newObj = { a: 1, c: 3 }; + + const diffs = diff(oldObj, newObj); + + expect(diffs).toEqual([ + { + t: "r", + p: ["b"], + o: 2, + n: undefined + }, + { + t: "a", + p: ["c"], + o: undefined, + n: 3 + } + ]); + + const appliedObj = apply(oldObj, diffs); + expect(appliedObj).toEqual(newObj); + + const revertedObj = revert(newObj, diffs); + expect(revertedObj).toEqual(oldObj); + }); + + it("should handle arrays replaced with objects", () => { + const oldObj = { a: [1, 2, 3] }; + const newObj = { a: { b: 4 } }; + + const diffs = diff(oldObj, newObj); + + expect(diffs).toEqual([ + { + t: "e", + p: ["a"], + o: [1, 2, 3], + n: { b: 4 } + } + ]); + + const appliedObj = apply(oldObj, diffs); + expect(appliedObj).toEqual(newObj); + + const revertedObj = revert(newObj, diffs); + expect(revertedObj).toEqual(oldObj); + }); + + it("should handle objects replaced with primitives", () => { + const oldObj = { a: { b: 1 } }; + const newObj = { a: 2 }; + + const diffs = diff(oldObj, newObj); + + expect(diffs).toEqual([ + { + t: "e", + p: ["a"], + o: { b: 1 }, + n: 2 + } + ]); + + const appliedObj = apply(oldObj, diffs); + expect(appliedObj).toEqual(newObj); + + const revertedObj = revert(newObj, diffs); + expect(revertedObj).toEqual(oldObj); + }); + + it("should handle root object changes", () => { + const oldObj = { a: 1 }; + const newObj = { b: 2 }; + + const diffs = diff(oldObj, newObj); + + expect(diffs).toEqual([ + { + t: "r", + p: ["a"], + o: 1, + n: undefined + }, + { + t: "a", + p: ["b"], + o: undefined, + n: 2 + } + ]); + + const appliedObj = apply(oldObj, diffs); + expect(appliedObj).toEqual(newObj); + + const revertedObj = revert(newObj, diffs); + expect(revertedObj).toEqual(oldObj); + }); + + it("should handle identical objects", () => { + const oldObj = { a: 1, b: { c: 2 } }; + const newObj = { a: 1, b: { c: 2 } }; + + const diffs = diff(oldObj, newObj); + + expect(diffs).toEqual([]); + + const appliedObj = apply(oldObj, diffs); + expect(appliedObj).toEqual(newObj); + + const revertedObj = revert(newObj, diffs); + expect(revertedObj).toEqual(oldObj); + }); + + it("should handle empty objects", () => { + const oldObj = {}; + const newObj = {}; + + const diffs = diff(oldObj, newObj); + + expect(diffs).toEqual([]); + + const appliedObj = apply(oldObj, diffs); + expect(appliedObj).toEqual(newObj); + + const revertedObj = revert(newObj, diffs); + expect(revertedObj).toEqual(oldObj); + }); + + it("should handle changes from empty object to non-empty object", () => { + const oldObj = {}; + const newObj = { a: 1 }; + + const diffs = diff(oldObj, newObj); + + expect(diffs).toEqual([ + { + t: "a", + p: ["a"], + o: undefined, + n: 1 + } + ]); + + const appliedObj = apply(oldObj, diffs); + expect(appliedObj).toEqual(newObj); + + const revertedObj = revert(newObj, diffs); + expect(revertedObj).toEqual(oldObj); + }); + + it("should handle changes from non-empty object to empty object", () => { + const oldObj = { a: 1 }; + const newObj = {}; + + const diffs = diff(oldObj, newObj); + + expect(diffs).toEqual([ + { + t: "r", + p: ["a"], + o: 1, + n: undefined + } + ]); + + const appliedObj = apply(oldObj, diffs); + expect(appliedObj).toEqual(newObj); + + const revertedObj = revert(newObj, diffs); + expect(revertedObj).toEqual(oldObj); + }); +}); diff --git a/app/src/core/object/diff.ts b/app/src/core/object/diff.ts new file mode 100644 index 0000000..9a182bd --- /dev/null +++ b/app/src/core/object/diff.ts @@ -0,0 +1,181 @@ +enum Change { + Add = "a", + Remove = "r", + Edit = "e" +} + +type Object = object; +type Primitive = string | number | boolean | null | object | any[] | undefined; + +interface DiffEntry { + t: Change | string; + p: (string | number)[]; + o: Primitive; + n: Primitive; +} + +function isObject(value: any): value is Object { + return value !== null && value.constructor.name === "Object"; +} +function isPrimitive(value: any): value is Primitive { + try { + return ( + value === null || + value === undefined || + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" || + Array.isArray(value) || + isObject(value) + ); + } catch (e) { + return false; + } +} + +function diff(oldObj: Object, newObj: Object): DiffEntry[] { + const diffs: DiffEntry[] = []; + + function recurse(oldValue: Primitive, newValue: Primitive, path: (string | number)[]) { + if (!isPrimitive(oldValue) || !isPrimitive(newValue)) { + throw new Error("Diff: Only primitive types are supported"); + } + + if (oldValue === newValue) { + return; + } + + if (typeof oldValue !== typeof newValue) { + diffs.push({ + t: Change.Edit, + p: path, + o: oldValue, + n: newValue + }); + } else if (Array.isArray(oldValue) && Array.isArray(newValue)) { + const maxLength = Math.max(oldValue.length, newValue.length); + for (let i = 0; i < maxLength; i++) { + if (i >= oldValue.length) { + diffs.push({ + t: Change.Add, + p: [...path, i], + o: undefined, + n: newValue[i] + }); + } else if (i >= newValue.length) { + diffs.push({ + t: Change.Remove, + p: [...path, i], + o: oldValue[i], + n: undefined + }); + } else { + recurse(oldValue[i], newValue[i], [...path, i]); + } + } + } else if (isObject(oldValue) && isObject(newValue)) { + const oKeys = Object.keys(oldValue); + const nKeys = Object.keys(newValue); + const allKeys = new Set([...oKeys, ...nKeys]); + for (const key of allKeys) { + if (!(key in oldValue)) { + diffs.push({ + t: Change.Add, + p: [...path, key], + o: undefined, + n: newValue[key] + }); + } else if (!(key in newValue)) { + diffs.push({ + t: Change.Remove, + p: [...path, key], + o: oldValue[key], + n: undefined + }); + } else { + recurse(oldValue[key], newValue[key], [...path, key]); + } + } + } else { + diffs.push({ + t: Change.Edit, + p: path, + o: oldValue, + n: newValue + }); + } + } + + recurse(oldObj, newObj, []); + return diffs; +} + +function apply(obj: Object, diffs: DiffEntry[]): any { + const clonedObj = clone(obj); + + for (const diff of diffs) { + applyChange(clonedObj, diff); + } + + return clonedObj; +} + +function revert(obj: Object, diffs: DiffEntry[]): any { + const clonedObj = clone(obj); + const reversedDiffs = diffs.slice().reverse(); + + for (const diff of reversedDiffs) { + revertChange(clonedObj, diff); + } + + return clonedObj; +} + +function applyChange(obj: Object, diff: DiffEntry) { + const { p: path, t: type, n: newValue } = diff; + const parent = getParent(obj, path.slice(0, -1)); + const key = path[path.length - 1]!; + + if (type === Change.Add || type === Change.Edit) { + parent[key] = newValue; + } else if (type === Change.Remove) { + if (Array.isArray(parent)) { + parent.splice(key as number, 1); + } else { + delete parent[key]; + } + } +} + +function revertChange(obj: Object, diff: DiffEntry) { + const { p: path, t: type, o: oldValue } = diff; + const parent = getParent(obj, path.slice(0, -1)); + const key = path[path.length - 1]!; + + if (type === Change.Add) { + if (Array.isArray(parent)) { + parent.splice(key as number, 1); + } else { + delete parent[key]; + } + } else if (type === Change.Remove || type === Change.Edit) { + parent[key] = oldValue; + } +} + +function getParent(obj: Object, path: (string | number)[]): any { + let current = obj; + for (const key of path) { + if (current[key] === undefined) { + current[key] = typeof key === "number" ? [] : {}; + } + current = current[key]; + } + return current; +} + +function clone(obj: In): In { + return JSON.parse(JSON.stringify(obj)); +} + +export { diff, apply, revert, clone }; diff --git a/app/src/data/entities/query/WhereBuilder.ts b/app/src/data/entities/query/WhereBuilder.ts index 455ecf4..5168d0e 100644 --- a/app/src/data/entities/query/WhereBuilder.ts +++ b/app/src/data/entities/query/WhereBuilder.ts @@ -15,7 +15,6 @@ import type { SelectQueryBuilder, UpdateQueryBuilder } from "kysely"; -import type { RepositoryQB } from "./Repository"; type Builder = ExpressionBuilder; type Wrapper = ExpressionWrapper; diff --git a/app/src/modules/ModuleManager.ts b/app/src/modules/ModuleManager.ts index 5da0dc2..f0c4013 100644 --- a/app/src/modules/ModuleManager.ts +++ b/app/src/modules/ModuleManager.ts @@ -1,7 +1,7 @@ -import { Diff } from "@sinclair/typebox/value"; import { Guard } from "auth"; import { BkndError, DebugLogger, Exception, isDebug } from "core"; import { EventManager } from "core/events"; +import { clone, diff } from "core/object/diff"; import { Default, type Static, @@ -18,7 +18,6 @@ import { datetime, entity, enumm, - json, jsonSchema, number } from "data"; @@ -87,9 +86,10 @@ const configJsonSchema = Type.Union([ getDefaultSchema(), Type.Array( Type.Object({ - type: StringEnum(["insert", "update", "delete"]), - value: Type.Any(), - path: Type.Optional(Type.String()) + t: StringEnum(["a", "r", "e"]), + p: Type.Array(Type.Union([Type.String(), Type.Number()])), + o: Type.Optional(Type.Any()), + n: Type.Optional(Type.Any()) }) ) ]); @@ -105,6 +105,8 @@ type T_INTERNAL_EM = { __bknd: ConfigTable2; }; +// @todo: cleanup old diffs on upgrade +// @todo: cleanup multiple backups on upgrade export class ModuleManager { private modules: Modules; // internal em for __bknd config table @@ -251,26 +253,31 @@ export class ModuleManager { // @todo: mark all others as "backup" this.logger.log("version conflict, storing new version", state.version, version); await this.mutator().insertOne({ - version, + version: state.version, type: "backup", json: configs }); + await this.mutator().insertOne({ + version: version, + type: "config", + json: configs + }); } else { this.logger.log("version matches"); // clean configs because of Diff() function - const diff = Diff(state.json, JSON.parse(JSON.stringify(configs))); - this.logger.log("checking diff", diff); + const diffs = diff(state.json, clone(configs)); + this.logger.log("checking diff", diffs); if (diff.length > 0) { // store diff await this.mutator().insertOne({ version, type: "diff", - json: diff + json: clone(diffs) }); + // store new version - // @todo: maybe by id? await this.mutator().updateWhere( { version, diff --git a/app/src/modules/migrations.ts b/app/src/modules/migrations.ts index 1d4834a..b85f01a 100644 --- a/app/src/modules/migrations.ts +++ b/app/src/modules/migrations.ts @@ -72,6 +72,13 @@ export const migrations: Migration[] = [ }; } } + /*{ + version: 8, + up: async (config, { db }) => { + await db.deleteFrom(TABLE_NAME).where("type", "=", "diff").execute(); + return config; + } + }*/ ]; export const CURRENT_VERSION = migrations[migrations.length - 1]?.version ?? 0; From ba517feab5dd80682fc59ed3326ac2d29a773382 Mon Sep 17 00:00:00 2001 From: dswbx Date: Thu, 5 Dec 2024 20:48:49 +0100 Subject: [PATCH 6/6] fix typings for initial partial config Signed-off-by: dswbx --- app/src/data/fields/Field.ts | 4 ++-- app/src/modules/ModuleManager.ts | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/src/data/fields/Field.ts b/app/src/data/fields/Field.ts index b5e332d..5260d61 100644 --- a/app/src/data/fields/Field.ts +++ b/app/src/data/fields/Field.ts @@ -236,8 +236,8 @@ export abstract class Field< toJSON() { return { - //name: this.name, - type: this.type, + // @todo: current workaround because of fixed string type + type: this.type as any, config: this.config }; } diff --git a/app/src/modules/ModuleManager.ts b/app/src/modules/ModuleManager.ts index f0c4013..e43dfa7 100644 --- a/app/src/modules/ModuleManager.ts +++ b/app/src/modules/ModuleManager.ts @@ -56,12 +56,13 @@ export type ModuleSchemas = { export type ModuleConfigs = { [K in keyof ModuleSchemas]: Static; }; +type PartialRec = { [P in keyof T]?: PartialRec }; export type InitialModuleConfigs = | ({ version: number; } & ModuleConfigs) - | Partial; + | PartialRec; export type ModuleManagerOptions = { initial?: InitialModuleConfigs;