shishantbiswas/bknd
1
0
Fork 0
mirror of https://github.com/shishantbiswas/bknd.git synced 2026-03-16 04:27:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

module manager: switched config table to use entity

Browse Source
This commit is contained in:
dswbx
2024-12-05 09:23:34 +01:00
parent 6c2f7b32e5
commit 3757157a06
7 changed files with 219 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
app/__test__/ModuleManager.spec.ts
View File

@@ -1,7 +1,7 @@
import { describe, expect, test } from "bun:test";
import { mark, stripMark } from "../src/core/utils";
import { ModuleManager } from "../src/modules/ModuleManager";
import { CURRENT_VERSION, TABLE_NAME, migrateSchema } from "../src/modules/migrations";
import { CURRENT_VERSION, TABLE_NAME } from "../src/modules/migrations";
import { getDummyConnection } from "./helper";
describe("ModuleManager", async () => {
@@ -34,13 +34,13 @@ describe("ModuleManager", async () => {
const c2 = getDummyConnection();
const db = c2.dummyConnection.kysely;
await migrateSchema(CURRENT_VERSION, { db });
const mm2 = new ModuleManager(c2.dummyConnection, { initial: { version, ...json } });
await mm2.syncConfigTable();
await db
.updateTable(TABLE_NAME)
.set({ json: JSON.stringify(json), version: CURRENT_VERSION })
.execute();
const mm2 = new ModuleManager(c2.dummyConnection, { initial: { version, ...json } });
await mm2.build();
expect(json).toEqual(mm2.configs());
@@ -52,21 +52,19 @@ describe("ModuleManager", async () => {
await mm.build();
const version = mm.version();
const json = mm.configs();
//const { version, ...json } = mm.toJSON() as any;
const c2 = getDummyConnection();
const db = c2.dummyConnection.kysely;
console.log("here2");
await migrateSchema(CURRENT_VERSION, { db });
await db
.updateTable(TABLE_NAME)
.set({ json: JSON.stringify(json), version: CURRENT_VERSION - 1 })
.execute();
const mm2 = new ModuleManager(c2.dummyConnection, {
initial: { version: version - 1, ...json }
});
console.log("here3");
await mm2.syncConfigTable();
await db
.insertInto(TABLE_NAME)
.values({ json: JSON.stringify(json), type: "config", version: CURRENT_VERSION - 1 })
.execute();
await mm2.build();
});
@@ -80,15 +78,15 @@ describe("ModuleManager", async () => {
const c2 = getDummyConnection();
const db = c2.dummyConnection.kysely;
await migrateSchema(CURRENT_VERSION, { db });
await db
.updateTable(TABLE_NAME)
.set({ json: JSON.stringify(json), version: CURRENT_VERSION })
.execute();
const mm2 = new ModuleManager(c2.dummyConnection, {
initial: { version: version - 1, ...json }
});
await mm2.syncConfigTable();
await db
.insertInto(TABLE_NAME)
.values({ type: "config", json: JSON.stringify(json), version: CURRENT_VERSION })
.execute();
expect(mm2.build()).rejects.toThrow(/version.*do not match/);
});
@@ -102,7 +100,9 @@ describe("ModuleManager", async () => {
const c2 = getDummyConnection();
const db = c2.dummyConnection.kysely;
await migrateSchema(CURRENT_VERSION, { db });
const mm2 = new ModuleManager(c2.dummyConnection);
await mm2.syncConfigTable();
const config = {
...json,
@@ -112,12 +112,11 @@ describe("ModuleManager", async () => {
}
};
await db
.updateTable(TABLE_NAME)
.set({ json: JSON.stringify(config), version: CURRENT_VERSION })
.insertInto(TABLE_NAME)
.values({ type: "config", json: JSON.stringify(config), version: CURRENT_VERSION })
.execute();
// run without config given
const mm2 = new ModuleManager(c2.dummyConnection);
await mm2.build();
expect(mm2.configs().data.basepath).toBe("/api/data2");
@@ -175,7 +174,15 @@ describe("ModuleManager", async () => {
const c2 = getDummyConnection();
const db = c2.dummyConnection.kysely;
await migrateSchema(CURRENT_VERSION, { db });
const mm2 = new ModuleManager(c2.dummyConnection, {
initial: {
auth: {
basepath: "/shouldnt/take/this"
}
}
});
await mm2.syncConfigTable();
const payload = {
...json,
auth: {
@@ -185,20 +192,13 @@ describe("ModuleManager", async () => {
}
};
await db
.updateTable(TABLE_NAME)
.set({
.insertInto(TABLE_NAME)
.values({
type: "config",
json: JSON.stringify(payload),
version: CURRENT_VERSION
})
.execute();
const mm2 = new ModuleManager(c2.dummyConnection, {
initial: {
auth: {
basepath: "/shouldnt/take/this"
}
}
});
await mm2.build();
expect(mm2.configs().auth.basepath).toBe("/api/auth2");
});

37
app/__test__/data/mutation.simple.test.ts
View File

@@ -132,14 +132,47 @@ describe("Mutator simple", async () => {
const data = (await em.repository(items).findMany()).data;
//console.log(data);
await em.mutator(items).deleteMany({ label: "delete" });
await em.mutator(items).deleteWhere({ label: "delete" });
expect((await em.repository(items).findMany()).data.length).toBe(data.length - 2);
//console.log((await em.repository(items).findMany()).data);
await em.mutator(items).deleteMany();
await em.mutator(items).deleteWhere();
expect((await em.repository(items).findMany()).data.length).toBe(0);
//expect(res.data.count).toBe(0);
});
test("updateMany", async () => {
await em.mutator(items).insertOne({ label: "update", count: 1 });
await em.mutator(items).insertOne({ label: "update too", count: 1 });
await em.mutator(items).insertOne({ label: "keep" });
// expect no update
await em.mutator(items).updateWhere(
{ count: 2 },
{
count: 10
}
);
expect((await em.repository(items).findMany()).data).toEqual([
{ id: 6, label: "update", count: 1 },
{ id: 7, label: "update too", count: 1 },
{ id: 8, label: "keep", count: 0 }
]);
// expect 2 to be updated
await em.mutator(items).updateWhere(
{ count: 2 },
{
count: 1
}
);
expect((await em.repository(items).findMany()).data).toEqual([
{ id: 6, label: "update", count: 2 },
{ id: 7, label: "update too", count: 2 },
{ id: 8, label: "keep", count: 0 }
]);
});
});

4
app/src/core/errors.ts
View File

@@ -27,6 +27,10 @@ export class BkndError extends Error {
super(message);
}
static with(message: string, details?: Record<string, any>, type?: string) {
throw new BkndError(message, details, type);
}
toJSON() {
return {
type: this.type ?? "unknown",

28
app/src/data/entities/Mutator.ts
View File

@@ -250,7 +250,7 @@ export class Mutator<DB> implements EmitsEvents {
}
// @todo: decide whether entries should be deleted all at once or one by one (for events)
async deleteMany(where?: RepoQuery["where"]): Promise<MutatorResponse<EntityData>> {
async deleteWhere(where?: RepoQuery["where"]): Promise<MutatorResponse<EntityData>> {
const entity = this.entity;
const qb = this.appendWhere(this.conn.deleteFrom(entity.name), where).returning(
@@ -267,4 +267,30 @@ export class Mutator<DB> implements EmitsEvents {
return res;
}
async updateWhere(
data: EntityData,
where?: RepoQuery["where"]
): Promise<MutatorResponse<EntityData>> {
const entity = this.entity;
const validatedData = await this.getValidatedData(data, "update");
/*await this.emgr.emit(
new Mutator.Events.MutatorUpdateBefore({ entity, entityId: id, data: validatedData })
);*/
const query = this.appendWhere(this.conn.updateTable(entity.name), where)
.set(validatedData)
//.where(entity.id().name, "=", id)
.returning(entity.getSelect());
const res = await this.many(query);
/*await this.emgr.emit(
new Mutator.Events.MutatorUpdateAfter({ entity, entityId: id, data: res.data })
);*/
return res;
}
}

2
app/src/media/api/MediaController.ts
View File

@@ -181,7 +181,7 @@ export class MediaController implements ClassController {
if (ids_to_delete.length > 0) {
await this.media.em
.mutator(mediaEntity)
.deleteMany({ [id_field]: { $in: ids_to_delete } });
.deleteWhere({ [id_field]: { $in: ids_to_delete } });
}
return c.json({ ok: true, result: result.data, ...info });

204
app/src/modules/ModuleManager.ts
View File

@@ -1,13 +1,22 @@
import { Diff } from "@sinclair/typebox/value";
import { Guard } from "auth";
import { DebugLogger, isDebug } from "core";
import { BkndError, DebugLogger, Exception, isDebug } from "core";
import { EventManager } from "core/events";
import { Default, type Static, objectEach, transformObject } from "core/utils";
import { type Connection, EntityManager } from "data";
import {
type Connection,
EntityManager,
type Schema,
datetime,
entity,
enumm,
json,
number
} from "data";
import { Hono } from "hono";
import { type Kysely, sql } from "kysely";
import { mergeWith } from "lodash-es";
import { CURRENT_VERSION, TABLE_NAME, migrate, migrateSchema } from "modules/migrations";
import { CURRENT_VERSION, TABLE_NAME, migrate } from "modules/migrations";
import { AppServer } from "modules/server/AppServer";
import { AppAuth } from "../auth/AppAuth";
import { AppData } from "../data/AppData";
@@ -64,8 +73,23 @@ type ConfigTable<Json = ModuleConfigs> = {
updated_at?: Date;
};
const __bknd = entity(TABLE_NAME, {
version: number().required(),
type: enumm({ enum: ["config", "diff", "backup"] }).required(),
json: json().required(),
created_at: datetime(),
updated_at: datetime()
});
type ConfigTable2 = Schema<typeof __bknd>;
type T_INTERNAL_EM = {
__bknd: ConfigTable2;
};
export class ModuleManager {
private modules: Modules;
// internal em for __bknd config table
__em!: EntityManager<T_INTERNAL_EM>;
// ctx for modules
em!: EntityManager<any>;
server!: Hono;
emgr!: EventManager;
@@ -78,12 +102,13 @@ export class ModuleManager {
// @todo: keep? not doing anything with it
private readonly _booted_with?: "provided" | "partial";
private logger = new DebugLogger(isDebug() && false);
private logger = new DebugLogger(false);
constructor(
private readonly connection: Connection,
private options?: Partial<ModuleManagerOptions>
) {
this.__em = new EntityManager([__bknd], this.connection);
this.modules = {} as Modules;
this.emgr = new EventManager();
const context = this.ctx(true);
@@ -126,6 +151,22 @@ export class ModuleManager {
}
}
private repo() {
return this.__em.repo(__bknd);
}
private mutator() {
return this.__em.mutator(__bknd);
}
private get db() {
return this.connection.kysely as Kysely<{ table: ConfigTable }>;
}
async syncConfigTable() {
return await this.__em.schema().sync({ force: true });
}
private rebuildServer() {
this.server = new Hono();
if (this.options?.basePath) {
@@ -159,27 +200,22 @@ export class ModuleManager {
};
}
private get db() {
return this.connection.kysely as Kysely<{ table: ConfigTable }>;
}
get table() {
return TABLE_NAME as "table";
}
private async fetch(): Promise<ConfigTable> {
this.logger.context("fetch").log("fetching");
const startTime = performance.now();
const result = await this.db
.selectFrom(this.table)
.selectAll()
.where("type", "=", "config")
.orderBy("version", "desc")
.executeTakeFirstOrThrow();
const { data: result } = await this.repo().findOne(
{ type: "config" },
{
sort: { by: "version", dir: "desc" }
}
);
if (!result) {
throw BkndError.with("no config");
}
this.logger.log("took", performance.now() - startTime, "ms", result).clear();
return result;
return result as ConfigTable;
}
async save() {
@@ -188,64 +224,65 @@ export class ModuleManager {
const version = this.version();
const json = JSON.stringify(configs) as any;
const state = await this.fetch();
if (state.version !== version) {
// @todo: mark all others as "backup"
this.logger.log("version conflict, storing new version", state.version, version);
await this.db
.insertInto(this.table)
.values({
try {
const state = await this.fetch();
if (state.version !== version) {
// @todo: mark all others as "backup"
this.logger.log("version conflict, storing new version", state.version, version);
await this.mutator().insertOne({
version,
type: "config",
type: "backup",
json
})
.execute();
} else {
this.logger.log("version matches");
});
} else {
this.logger.log("version matches");
const diff = Diff(state.json, JSON.parse(json));
this.logger.log("checking diff", diff);
const diff = Diff(state.json, JSON.parse(json));
this.logger.log("checking diff", diff);
if (diff.length > 0) {
// store diff
await this.db
.insertInto(this.table)
.values({
if (diff.length > 0) {
// store diff
await this.mutator().insertOne({
version,
type: "diff",
json: JSON.stringify(diff) as any
})
.execute();
await this.db
.updateTable(this.table)
.set({ version, json, updated_at: sql`CURRENT_TIMESTAMP` })
.where((eb) => eb.and([eb("type", "=", "config"), eb("version", "=", version)]))
.execute();
});
// store new version
// @todo: maybe by id?
await this.mutator().updateWhere(
{
version,
json,
updated_at: new Date()
},
{
type: "config",
version
}
);
} else {
this.logger.log("no diff, not saving");
}
}
} catch (e) {
if (e instanceof BkndError) {
// no config, just save
await this.mutator().insertOne({
type: "config",
version,
json,
created_at: new Date(),
updated_at: new Date()
});
} else {
this.logger.log("no diff, not saving");
console.error("Aborting");
throw e;
}
}
// cleanup
/*this.logger.log("cleaning up");
const result = await this.db
.deleteFrom(this.table)
.where((eb) =>
eb.or([
// empty migrations
eb.and([
eb("type", "=", "config"),
eb("version", "<", version),
eb("json", "is", null)
]),
// past diffs
eb.and([eb("type", "=", "diff"), eb("version", "<", version)])
])
)
.executeTakeFirst();
this.logger.log("cleaned up", result.numDeletedRows);*/
// @todo: cleanup old versions?
this.logger.clear();
return this;
@@ -256,6 +293,8 @@ export class ModuleManager {
if (this.version() < CURRENT_VERSION) {
this.logger.log("there are migrations, verify version");
// sync __bknd table
await this.syncConfigTable();
// modules must be built before migration
await this.buildModules({ graceful: true });
@@ -270,14 +309,7 @@ export class ModuleManager {
}
} catch (e: any) {
this.logger.clear(); // fetch couldn't clear
// if table doesn't exist, migrate schema to version
if (e.message.includes("no such table")) {
this.logger.log("table has to created, migrating schema up to", this.version());
await migrateSchema(this.version(), { db: this.db });
} else {
throw new Error(`Version is ${this.version()}, fetch failed: ${e.message}`);
}
throw new Error(`Version is ${this.version()}, fetch failed: ${e.message}`);
}
this.logger.log("now migrating");
@@ -345,6 +377,10 @@ export class ModuleManager {
this.logger.context("build").log("version", this.version());
this.logger.log("booted with", this._booted_with);
if (this.version() !== CURRENT_VERSION) {
await this.syncConfigTable();
}
// if no config provided, try fetch from db
if (this.version() === 0) {
this.logger.context("no version").log("version is 0");
@@ -358,23 +394,15 @@ export class ModuleManager {
this.logger.clear(); // fetch couldn't clear
this.logger.context("error handler").log("fetch failed", e.message);
// if table doesn't exist, migrate schema, set default config and latest version
if (e.message.includes("no such table")) {
this.logger.log("migrate schema to", CURRENT_VERSION);
await migrateSchema(CURRENT_VERSION, { db: this.db });
this._version = CURRENT_VERSION;
// we can safely build modules, since config version is up to date
// it's up to date because we use default configs (no fetch result)
await this.buildModules();
await this.save();
// we can safely build modules, since config version is up to date
// it's up to date because we use default configs (no fetch result)
this._version = CURRENT_VERSION;
await this.buildModules();
await this.save();
this.logger.clear();
return this;
} else {
throw e;
//throw new Error("Issues connecting to the database. Reason: " + e.message);
}
this.logger.clear();
return this;
}
this.logger.clear();
}

52
app/src/modules/migrations.ts
View File

@@ -16,26 +16,8 @@ export type Migration = {
export const migrations: Migration[] = [
{
version: 1,
schema: true,
up: async (config, { db }) => {
//console.log("config given", config);
await db.schema
.createTable(TABLE_NAME)
.addColumn("id", "integer", (col) => col.primaryKey().notNull().autoIncrement())
.addColumn("version", "integer", (col) => col.notNull())
.addColumn("type", "text", (col) => col.notNull())
.addColumn("json", "text")
.addColumn("created_at", "datetime", (col) => col.defaultTo(sql`CURRENT_TIMESTAMP`))
.addColumn("updated_at", "datetime", (col) => col.defaultTo(sql`CURRENT_TIMESTAMP`))
.execute();
await db
.insertInto(TABLE_NAME)
.values({ version: 1, type: "config", json: null })
.execute();
return config;
}
//schema: true,
up: async (config) => config
},
{
version: 2,
@@ -45,12 +27,8 @@ export const migrations: Migration[] = [
},
{
version: 3,
schema: true,
up: async (config, { db }) => {
await db.schema.alterTable(TABLE_NAME).addColumn("deleted_at", "datetime").execute();
return config;
}
//schema: true,
up: async (config) => config
},
{
version: 4,
@@ -127,28 +105,6 @@ export async function migrateTo(
return [version, updated];
}
export async function migrateSchema(to: number, ctx: MigrationContext, current: number = 0) {
console.log("migrating SCHEMA to", to, "from", current);
const todo = migrations.filter((m) => m.version > current && m.version <= to && m.schema);
console.log("todo", todo.length);
let i = 0;
let version = 0;
for (const migration of todo) {
console.log("-- running migration", i + 1, "of", todo.length);
try {
await migration.up({}, ctx);
version = migration.version;
i++;
} catch (e: any) {
console.error(e);
throw new Error(`Migration ${migration.version} failed: ${e.message}`);
}
}
return version;
}
export async function migrate(
current: number,
config: GenericConfigObject,