diff --git a/app/__test__/ModuleManager.spec.ts b/app/__test__/ModuleManager.spec.ts index d79bf27..38e1670 100644 --- a/app/__test__/ModuleManager.spec.ts +++ b/app/__test__/ModuleManager.spec.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from "bun:test"; import { mark, stripMark } from "../src/core/utils"; import { ModuleManager } from "../src/modules/ModuleManager"; -import { CURRENT_VERSION, TABLE_NAME, migrateSchema } from "../src/modules/migrations"; +import { CURRENT_VERSION, TABLE_NAME } from "../src/modules/migrations"; import { getDummyConnection } from "./helper"; describe("ModuleManager", async () => { @@ -34,13 +34,13 @@ describe("ModuleManager", async () => { const c2 = getDummyConnection(); const db = c2.dummyConnection.kysely; - await migrateSchema(CURRENT_VERSION, { db }); + const mm2 = new ModuleManager(c2.dummyConnection, { initial: { version, ...json } }); + await mm2.syncConfigTable(); await db .updateTable(TABLE_NAME) .set({ json: JSON.stringify(json), version: CURRENT_VERSION }) .execute(); - const mm2 = new ModuleManager(c2.dummyConnection, { initial: { version, ...json } }); await mm2.build(); expect(json).toEqual(mm2.configs()); @@ -52,21 +52,19 @@ describe("ModuleManager", async () => { await mm.build(); const version = mm.version(); const json = mm.configs(); - //const { version, ...json } = mm.toJSON() as any; const c2 = getDummyConnection(); const db = c2.dummyConnection.kysely; - console.log("here2"); - await migrateSchema(CURRENT_VERSION, { db }); - await db - .updateTable(TABLE_NAME) - .set({ json: JSON.stringify(json), version: CURRENT_VERSION - 1 }) - .execute(); - const mm2 = new ModuleManager(c2.dummyConnection, { initial: { version: version - 1, ...json } }); - console.log("here3"); + await mm2.syncConfigTable(); + + await db + .insertInto(TABLE_NAME) + .values({ json: JSON.stringify(json), type: "config", version: CURRENT_VERSION - 1 }) + .execute(); + await mm2.build(); }); @@ -80,15 +78,15 @@ describe("ModuleManager", async () => { const c2 = getDummyConnection(); const db = c2.dummyConnection.kysely; - await migrateSchema(CURRENT_VERSION, { db }); - await db - .updateTable(TABLE_NAME) - .set({ json: JSON.stringify(json), version: CURRENT_VERSION }) - .execute(); const mm2 = new ModuleManager(c2.dummyConnection, { initial: { version: version - 1, ...json } }); + await mm2.syncConfigTable(); + await db + .insertInto(TABLE_NAME) + .values({ type: "config", json: JSON.stringify(json), version: CURRENT_VERSION }) + .execute(); expect(mm2.build()).rejects.toThrow(/version.*do not match/); }); @@ -102,7 +100,9 @@ describe("ModuleManager", async () => { const c2 = getDummyConnection(); const db = c2.dummyConnection.kysely; - await migrateSchema(CURRENT_VERSION, { db }); + + const mm2 = new ModuleManager(c2.dummyConnection); + await mm2.syncConfigTable(); const config = { ...json, @@ -112,12 +112,11 @@ describe("ModuleManager", async () => { } }; await db - .updateTable(TABLE_NAME) - .set({ json: JSON.stringify(config), version: CURRENT_VERSION }) + .insertInto(TABLE_NAME) + .values({ type: "config", json: JSON.stringify(config), version: CURRENT_VERSION }) .execute(); // run without config given - const mm2 = new ModuleManager(c2.dummyConnection); await mm2.build(); expect(mm2.configs().data.basepath).toBe("/api/data2"); @@ -175,7 +174,15 @@ describe("ModuleManager", async () => { const c2 = getDummyConnection(); const db = c2.dummyConnection.kysely; - await migrateSchema(CURRENT_VERSION, { db }); + + const mm2 = new ModuleManager(c2.dummyConnection, { + initial: { + auth: { + basepath: "/shouldnt/take/this" + } + } + }); + await mm2.syncConfigTable(); const payload = { ...json, auth: { @@ -185,20 +192,13 @@ describe("ModuleManager", async () => { } }; await db - .updateTable(TABLE_NAME) - .set({ + .insertInto(TABLE_NAME) + .values({ + type: "config", json: JSON.stringify(payload), version: CURRENT_VERSION }) .execute(); - - const mm2 = new ModuleManager(c2.dummyConnection, { - initial: { - auth: { - basepath: "/shouldnt/take/this" - } - } - }); await mm2.build(); expect(mm2.configs().auth.basepath).toBe("/api/auth2"); }); diff --git a/app/__test__/data/mutation.simple.test.ts b/app/__test__/data/mutation.simple.test.ts index b3f0c77..dd385af 100644 --- a/app/__test__/data/mutation.simple.test.ts +++ b/app/__test__/data/mutation.simple.test.ts @@ -132,14 +132,47 @@ describe("Mutator simple", async () => { const data = (await em.repository(items).findMany()).data; //console.log(data); - await em.mutator(items).deleteMany({ label: "delete" }); + await em.mutator(items).deleteWhere({ label: "delete" }); expect((await em.repository(items).findMany()).data.length).toBe(data.length - 2); //console.log((await em.repository(items).findMany()).data); - await em.mutator(items).deleteMany(); + await em.mutator(items).deleteWhere(); expect((await em.repository(items).findMany()).data.length).toBe(0); //expect(res.data.count).toBe(0); }); + + test("updateMany", async () => { + await em.mutator(items).insertOne({ label: "update", count: 1 }); + await em.mutator(items).insertOne({ label: "update too", count: 1 }); + await em.mutator(items).insertOne({ label: "keep" }); + + // expect no update + await em.mutator(items).updateWhere( + { count: 2 }, + { + count: 10 + } + ); + expect((await em.repository(items).findMany()).data).toEqual([ + { id: 6, label: "update", count: 1 }, + { id: 7, label: "update too", count: 1 }, + { id: 8, label: "keep", count: 0 } + ]); + + // expect 2 to be updated + await em.mutator(items).updateWhere( + { count: 2 }, + { + count: 1 + } + ); + + expect((await em.repository(items).findMany()).data).toEqual([ + { id: 6, label: "update", count: 2 }, + { id: 7, label: "update too", count: 2 }, + { id: 8, label: "keep", count: 0 } + ]); + }); }); diff --git a/app/src/core/errors.ts b/app/src/core/errors.ts index ce63ed9..860bd9d 100644 --- a/app/src/core/errors.ts +++ b/app/src/core/errors.ts @@ -27,6 +27,10 @@ export class BkndError extends Error { super(message); } + static with(message: string, details?: Record, type?: string) { + throw new BkndError(message, details, type); + } + toJSON() { return { type: this.type ?? "unknown", diff --git a/app/src/data/entities/Mutator.ts b/app/src/data/entities/Mutator.ts index aef2bf1..ed7f9ef 100644 --- a/app/src/data/entities/Mutator.ts +++ b/app/src/data/entities/Mutator.ts @@ -250,7 +250,7 @@ export class Mutator implements EmitsEvents { } // @todo: decide whether entries should be deleted all at once or one by one (for events) - async deleteMany(where?: RepoQuery["where"]): Promise> { + async deleteWhere(where?: RepoQuery["where"]): Promise> { const entity = this.entity; const qb = this.appendWhere(this.conn.deleteFrom(entity.name), where).returning( @@ -267,4 +267,30 @@ export class Mutator implements EmitsEvents { return res; } + + async updateWhere( + data: EntityData, + where?: RepoQuery["where"] + ): Promise> { + const entity = this.entity; + + const validatedData = await this.getValidatedData(data, "update"); + + /*await this.emgr.emit( + new Mutator.Events.MutatorUpdateBefore({ entity, entityId: id, data: validatedData }) + );*/ + + const query = this.appendWhere(this.conn.updateTable(entity.name), where) + .set(validatedData) + //.where(entity.id().name, "=", id) + .returning(entity.getSelect()); + + const res = await this.many(query); + + /*await this.emgr.emit( + new Mutator.Events.MutatorUpdateAfter({ entity, entityId: id, data: res.data }) + );*/ + + return res; + } } diff --git a/app/src/media/api/MediaController.ts b/app/src/media/api/MediaController.ts index 59665fa..9597759 100644 --- a/app/src/media/api/MediaController.ts +++ b/app/src/media/api/MediaController.ts @@ -181,7 +181,7 @@ export class MediaController implements ClassController { if (ids_to_delete.length > 0) { await this.media.em .mutator(mediaEntity) - .deleteMany({ [id_field]: { $in: ids_to_delete } }); + .deleteWhere({ [id_field]: { $in: ids_to_delete } }); } return c.json({ ok: true, result: result.data, ...info }); diff --git a/app/src/modules/ModuleManager.ts b/app/src/modules/ModuleManager.ts index b77e207..4774ac2 100644 --- a/app/src/modules/ModuleManager.ts +++ b/app/src/modules/ModuleManager.ts @@ -1,13 +1,22 @@ import { Diff } from "@sinclair/typebox/value"; import { Guard } from "auth"; -import { DebugLogger, isDebug } from "core"; +import { BkndError, DebugLogger, Exception, isDebug } from "core"; import { EventManager } from "core/events"; import { Default, type Static, objectEach, transformObject } from "core/utils"; -import { type Connection, EntityManager } from "data"; +import { + type Connection, + EntityManager, + type Schema, + datetime, + entity, + enumm, + json, + number +} from "data"; import { Hono } from "hono"; import { type Kysely, sql } from "kysely"; import { mergeWith } from "lodash-es"; -import { CURRENT_VERSION, TABLE_NAME, migrate, migrateSchema } from "modules/migrations"; +import { CURRENT_VERSION, TABLE_NAME, migrate } from "modules/migrations"; import { AppServer } from "modules/server/AppServer"; import { AppAuth } from "../auth/AppAuth"; import { AppData } from "../data/AppData"; @@ -64,8 +73,23 @@ type ConfigTable = { updated_at?: Date; }; +const __bknd = entity(TABLE_NAME, { + version: number().required(), + type: enumm({ enum: ["config", "diff", "backup"] }).required(), + json: json().required(), + created_at: datetime(), + updated_at: datetime() +}); +type ConfigTable2 = Schema; +type T_INTERNAL_EM = { + __bknd: ConfigTable2; +}; + export class ModuleManager { private modules: Modules; + // internal em for __bknd config table + __em!: EntityManager; + // ctx for modules em!: EntityManager; server!: Hono; emgr!: EventManager; @@ -78,12 +102,13 @@ export class ModuleManager { // @todo: keep? not doing anything with it private readonly _booted_with?: "provided" | "partial"; - private logger = new DebugLogger(isDebug() && false); + private logger = new DebugLogger(false); constructor( private readonly connection: Connection, private options?: Partial ) { + this.__em = new EntityManager([__bknd], this.connection); this.modules = {} as Modules; this.emgr = new EventManager(); const context = this.ctx(true); @@ -126,6 +151,22 @@ export class ModuleManager { } } + private repo() { + return this.__em.repo(__bknd); + } + + private mutator() { + return this.__em.mutator(__bknd); + } + + private get db() { + return this.connection.kysely as Kysely<{ table: ConfigTable }>; + } + + async syncConfigTable() { + return await this.__em.schema().sync({ force: true }); + } + private rebuildServer() { this.server = new Hono(); if (this.options?.basePath) { @@ -159,27 +200,22 @@ export class ModuleManager { }; } - private get db() { - return this.connection.kysely as Kysely<{ table: ConfigTable }>; - } - - get table() { - return TABLE_NAME as "table"; - } - private async fetch(): Promise { this.logger.context("fetch").log("fetching"); const startTime = performance.now(); - const result = await this.db - .selectFrom(this.table) - .selectAll() - .where("type", "=", "config") - .orderBy("version", "desc") - .executeTakeFirstOrThrow(); + const { data: result } = await this.repo().findOne( + { type: "config" }, + { + sort: { by: "version", dir: "desc" } + } + ); + if (!result) { + throw BkndError.with("no config"); + } this.logger.log("took", performance.now() - startTime, "ms", result).clear(); - return result; + return result as ConfigTable; } async save() { @@ -188,64 +224,65 @@ export class ModuleManager { const version = this.version(); const json = JSON.stringify(configs) as any; - const state = await this.fetch(); - if (state.version !== version) { - // @todo: mark all others as "backup" - this.logger.log("version conflict, storing new version", state.version, version); - await this.db - .insertInto(this.table) - .values({ + try { + const state = await this.fetch(); + + if (state.version !== version) { + // @todo: mark all others as "backup" + this.logger.log("version conflict, storing new version", state.version, version); + await this.mutator().insertOne({ version, - type: "config", + type: "backup", json - }) - .execute(); - } else { - this.logger.log("version matches"); + }); + } else { + this.logger.log("version matches"); - const diff = Diff(state.json, JSON.parse(json)); - this.logger.log("checking diff", diff); + const diff = Diff(state.json, JSON.parse(json)); + this.logger.log("checking diff", diff); - if (diff.length > 0) { - // store diff - await this.db - .insertInto(this.table) - .values({ + if (diff.length > 0) { + // store diff + await this.mutator().insertOne({ version, type: "diff", json: JSON.stringify(diff) as any - }) - .execute(); - - await this.db - .updateTable(this.table) - .set({ version, json, updated_at: sql`CURRENT_TIMESTAMP` }) - .where((eb) => eb.and([eb("type", "=", "config"), eb("version", "=", version)])) - .execute(); + }); + // store new version + // @todo: maybe by id? + await this.mutator().updateWhere( + { + version, + json, + updated_at: new Date() + }, + { + type: "config", + version + } + ); + } else { + this.logger.log("no diff, not saving"); + } + } + } catch (e) { + if (e instanceof BkndError) { + // no config, just save + await this.mutator().insertOne({ + type: "config", + version, + json, + created_at: new Date(), + updated_at: new Date() + }); } else { - this.logger.log("no diff, not saving"); + console.error("Aborting"); + throw e; } } - // cleanup - /*this.logger.log("cleaning up"); - const result = await this.db - .deleteFrom(this.table) - .where((eb) => - eb.or([ - // empty migrations - eb.and([ - eb("type", "=", "config"), - eb("version", "<", version), - eb("json", "is", null) - ]), - // past diffs - eb.and([eb("type", "=", "diff"), eb("version", "<", version)]) - ]) - ) - .executeTakeFirst(); - this.logger.log("cleaned up", result.numDeletedRows);*/ + // @todo: cleanup old versions? this.logger.clear(); return this; @@ -256,6 +293,8 @@ export class ModuleManager { if (this.version() < CURRENT_VERSION) { this.logger.log("there are migrations, verify version"); + // sync __bknd table + await this.syncConfigTable(); // modules must be built before migration await this.buildModules({ graceful: true }); @@ -270,14 +309,7 @@ export class ModuleManager { } } catch (e: any) { this.logger.clear(); // fetch couldn't clear - - // if table doesn't exist, migrate schema to version - if (e.message.includes("no such table")) { - this.logger.log("table has to created, migrating schema up to", this.version()); - await migrateSchema(this.version(), { db: this.db }); - } else { - throw new Error(`Version is ${this.version()}, fetch failed: ${e.message}`); - } + throw new Error(`Version is ${this.version()}, fetch failed: ${e.message}`); } this.logger.log("now migrating"); @@ -345,6 +377,10 @@ export class ModuleManager { this.logger.context("build").log("version", this.version()); this.logger.log("booted with", this._booted_with); + if (this.version() !== CURRENT_VERSION) { + await this.syncConfigTable(); + } + // if no config provided, try fetch from db if (this.version() === 0) { this.logger.context("no version").log("version is 0"); @@ -358,23 +394,15 @@ export class ModuleManager { this.logger.clear(); // fetch couldn't clear this.logger.context("error handler").log("fetch failed", e.message); - // if table doesn't exist, migrate schema, set default config and latest version - if (e.message.includes("no such table")) { - this.logger.log("migrate schema to", CURRENT_VERSION); - await migrateSchema(CURRENT_VERSION, { db: this.db }); - this._version = CURRENT_VERSION; - // we can safely build modules, since config version is up to date - // it's up to date because we use default configs (no fetch result) - await this.buildModules(); - await this.save(); + // we can safely build modules, since config version is up to date + // it's up to date because we use default configs (no fetch result) + this._version = CURRENT_VERSION; + await this.buildModules(); + await this.save(); - this.logger.clear(); - return this; - } else { - throw e; - //throw new Error("Issues connecting to the database. Reason: " + e.message); - } + this.logger.clear(); + return this; } this.logger.clear(); } diff --git a/app/src/modules/migrations.ts b/app/src/modules/migrations.ts index 9cf0beb..1d4834a 100644 --- a/app/src/modules/migrations.ts +++ b/app/src/modules/migrations.ts @@ -16,26 +16,8 @@ export type Migration = { export const migrations: Migration[] = [ { version: 1, - schema: true, - up: async (config, { db }) => { - //console.log("config given", config); - await db.schema - .createTable(TABLE_NAME) - .addColumn("id", "integer", (col) => col.primaryKey().notNull().autoIncrement()) - .addColumn("version", "integer", (col) => col.notNull()) - .addColumn("type", "text", (col) => col.notNull()) - .addColumn("json", "text") - .addColumn("created_at", "datetime", (col) => col.defaultTo(sql`CURRENT_TIMESTAMP`)) - .addColumn("updated_at", "datetime", (col) => col.defaultTo(sql`CURRENT_TIMESTAMP`)) - .execute(); - - await db - .insertInto(TABLE_NAME) - .values({ version: 1, type: "config", json: null }) - .execute(); - - return config; - } + //schema: true, + up: async (config) => config }, { version: 2, @@ -45,12 +27,8 @@ export const migrations: Migration[] = [ }, { version: 3, - schema: true, - up: async (config, { db }) => { - await db.schema.alterTable(TABLE_NAME).addColumn("deleted_at", "datetime").execute(); - - return config; - } + //schema: true, + up: async (config) => config }, { version: 4, @@ -127,28 +105,6 @@ export async function migrateTo( return [version, updated]; } -export async function migrateSchema(to: number, ctx: MigrationContext, current: number = 0) { - console.log("migrating SCHEMA to", to, "from", current); - const todo = migrations.filter((m) => m.version > current && m.version <= to && m.schema); - console.log("todo", todo.length); - - let i = 0; - let version = 0; - for (const migration of todo) { - console.log("-- running migration", i + 1, "of", todo.length); - try { - await migration.up({}, ctx); - version = migration.version; - i++; - } catch (e: any) { - console.error(e); - throw new Error(`Migration ${migration.version} failed: ${e.message}`); - } - } - - return version; -} - export async function migrate( current: number, config: GenericConfigObject,