shishantbiswas/bknd
1
0
Fork 0
mirror of https://github.com/shishantbiswas/bknd.git synced 2026-03-16 04:27:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/refactor/module-manager' into release/0.2.3

Browse Source
This commit is contained in:
dswbx
2024-12-07 09:29:31 +01:00
parent 067213ff60 ba517feab5
commit c7c6aa7df8
14 changed files with 1013 additions and 236 deletions
Download Patch File Download Diff File Expand all files Collapse all files

190
app/__test__/ModuleManager.spec.ts
View File

@@ -1,7 +1,8 @@
import { describe, expect, test } from "bun:test";
import { mark, stripMark } from "../src/core/utils";
import { ModuleManager } from "../src/modules/ModuleManager";
import { CURRENT_VERSION, TABLE_NAME, migrateSchema } from "../src/modules/migrations";
import { entity, text } from "../src/data";
import { ModuleManager, getDefaultConfig } from "../src/modules/ModuleManager";
import { CURRENT_VERSION, TABLE_NAME } from "../src/modules/migrations";
import { getDummyConnection } from "./helper";
describe("ModuleManager", async () => {
@@ -29,21 +30,68 @@ describe("ModuleManager", async () => {
const mm = new ModuleManager(c.dummyConnection);
await mm.build();
const version = mm.version();
const json = mm.configs();
const configs = mm.configs();
const json = stripMark({
...configs,
data: {
...configs.data,
basepath: "/api/data2",
entities: {
test: entity("test", {
content: text()
}).toJSON()
}
}
});
//const { version, ...json } = mm.toJSON() as any;
const c2 = getDummyConnection();
const db = c2.dummyConnection.kysely;
await migrateSchema(CURRENT_VERSION, { db });
const mm2 = new ModuleManager(c2.dummyConnection, { initial: { version, ...json } });
await mm2.syncConfigTable();
await db
.updateTable(TABLE_NAME)
.set({ json: JSON.stringify(json), version: CURRENT_VERSION })
.insertInto(TABLE_NAME)
.values({ type: "config", json: JSON.stringify(json), version: CURRENT_VERSION })
.execute();
const mm2 = new ModuleManager(c2.dummyConnection, { initial: { version, ...json } });
await mm2.build();
expect(json).toEqual(mm2.configs());
expect(json).toEqual(stripMark(mm2.configs()));
});
test("s3.1: (fetch) config given, table exists, version matches", async () => {
const configs = getDefaultConfig();
const json = {
...configs,
data: {
...configs.data,
basepath: "/api/data2",
entities: {
test: entity("test", {
content: text()
}).toJSON()
}
}
};
//const { version, ...json } = mm.toJSON() as any;
const { dummyConnection } = getDummyConnection();
const db = dummyConnection.kysely;
const mm2 = new ModuleManager(dummyConnection);
await mm2.syncConfigTable();
// assume an initial version
await db.insertInto(TABLE_NAME).values({ type: "config", json: null, version: 1 }).execute();
await db
.insertInto(TABLE_NAME)
.values({ type: "config", json: JSON.stringify(json), version: CURRENT_VERSION })
.execute();
await mm2.build();
expect(stripMark(json)).toEqual(stripMark(mm2.configs()));
expect(mm2.configs().data.entities.test).toBeDefined();
expect(mm2.configs().data.entities.test.fields.content).toBeDefined();
expect(mm2.get("data").toJSON().entities.test.fields.content).toBeDefined();
});
test("s4: config given, table exists, version outdated, migrate", async () => {
@@ -52,21 +100,19 @@ describe("ModuleManager", async () => {
await mm.build();
const version = mm.version();
const json = mm.configs();
//const { version, ...json } = mm.toJSON() as any;
const c2 = getDummyConnection();
const db = c2.dummyConnection.kysely;
console.log("here2");
await migrateSchema(CURRENT_VERSION, { db });
await db
.updateTable(TABLE_NAME)
.set({ json: JSON.stringify(json), version: CURRENT_VERSION - 1 })
.execute();
const mm2 = new ModuleManager(c2.dummyConnection, {
initial: { version: version - 1, ...json }
});
console.log("here3");
await mm2.syncConfigTable();
await db
.insertInto(TABLE_NAME)
.values({ json: JSON.stringify(json), type: "config", version: CURRENT_VERSION - 1 })
.execute();
await mm2.build();
});
@@ -80,15 +126,15 @@ describe("ModuleManager", async () => {
const c2 = getDummyConnection();
const db = c2.dummyConnection.kysely;
await migrateSchema(CURRENT_VERSION, { db });
await db
.updateTable(TABLE_NAME)
.set({ json: JSON.stringify(json), version: CURRENT_VERSION })
.execute();
const mm2 = new ModuleManager(c2.dummyConnection, {
initial: { version: version - 1, ...json }
});
await mm2.syncConfigTable();
await db
.insertInto(TABLE_NAME)
.values({ type: "config", json: JSON.stringify(json), version: CURRENT_VERSION })
.execute();
expect(mm2.build()).rejects.toThrow(/version.*do not match/);
});
@@ -102,7 +148,9 @@ describe("ModuleManager", async () => {
const c2 = getDummyConnection();
const db = c2.dummyConnection.kysely;
await migrateSchema(CURRENT_VERSION, { db });
const mm2 = new ModuleManager(c2.dummyConnection);
await mm2.syncConfigTable();
const config = {
...json,
@@ -112,12 +160,11 @@ describe("ModuleManager", async () => {
}
};
await db
.updateTable(TABLE_NAME)
.set({ json: JSON.stringify(config), version: CURRENT_VERSION })
.insertInto(TABLE_NAME)
.values({ type: "config", json: JSON.stringify(config), version: CURRENT_VERSION })
.execute();
// run without config given
const mm2 = new ModuleManager(c2.dummyConnection);
await mm2.build();
expect(mm2.configs().data.basepath).toBe("/api/data2");
@@ -148,50 +195,61 @@ describe("ModuleManager", async () => {
});
});
// @todo: check what happens here
/*test("blank app, modify deep config", async () => {
test("partial config given", async () => {
const { dummyConnection } = getDummyConnection();
const mm = new ModuleManager(dummyConnection);
await mm.build();
/!* await mm
.get("data")
.schema()
.patch("entities.test", {
fields: {
content: {
type: "text"
}
const partial = {
auth: {
enabled: true
}
};
const mm = new ModuleManager(dummyConnection, {
initial: partial
});
await mm.build();
expect(mm.configs().data.entities?.users?.fields?.email.type).toBe("text");
expect(
mm.get("data").schema().patch("desc", "entities.users.config.sort_dir")
).rejects.toThrow();
await mm.build();*!/
expect(mm.configs().data.entities?.users?.fields?.email.type).toBe("text");
console.log("here", mm.configs());
await mm
.get("data")
.schema()
.patch("entities.users", { config: { sort_dir: "desc" } });
await mm.build();
expect(mm.toJSON());
//console.log(_jsonp(mm.toJSON().data));
/!*expect(mm.configs().data.entities!.test!.fields!.content.type).toBe("text");
expect(mm.configs().data.entities!.users!.config!.sort_dir).toBe("desc");*!/
});*/
/*test("accessing modules", async () => {
const { dummyConnection } = getDummyConnection();
const mm = new ModuleManager(dummyConnection);
//mm.get("auth").mutate().set({});
});*/
expect(mm.version()).toBe(CURRENT_VERSION);
expect(mm.built()).toBe(true);
expect(mm.configs().auth.enabled).toBe(true);
expect(mm.configs().data.entities.users).toBeDefined();
});
test("partial config given, but db version exists", async () => {
const c = getDummyConnection();
const mm = new ModuleManager(c.dummyConnection);
await mm.build();
const json = mm.configs();
const c2 = getDummyConnection();
const db = c2.dummyConnection.kysely;
const mm2 = new ModuleManager(c2.dummyConnection, {
initial: {
auth: {
basepath: "/shouldnt/take/this"
}
}
});
await mm2.syncConfigTable();
const payload = {
...json,
auth: {
...json.auth,
enabled: true,
basepath: "/api/auth2"
}
};
await db
.insertInto(TABLE_NAME)
.values({
type: "config",
json: JSON.stringify(payload),
version: CURRENT_VERSION
})
.execute();
await mm2.build();
expect(mm2.configs().auth.basepath).toBe("/api/auth2");
});
// @todo: add tests for migrations (check "backup" and new version)
});

443
app/__test__/core/object/diff.test.ts Normal file
View File

@@ -0,0 +1,443 @@
import { describe, expect, it, test } from "bun:test";
import { apply, diff, revert } from "../../../src/core/object/diff";
describe("diff", () => {
it("should detect added properties", () => {
const oldObj = { a: 1 };
const newObj = { a: 1, b: 2 };
const diffs = diff(oldObj, newObj);
expect(diffs).toEqual([
{
t: "a",
p: ["b"],
o: undefined,
n: 2
}
]);
const appliedObj = apply(oldObj, diffs);
expect(appliedObj).toEqual(newObj);
const revertedObj = revert(newObj, diffs);
expect(revertedObj).toEqual(oldObj);
});
it("should detect removed properties", () => {
const oldObj = { a: 1, b: 2 };
const newObj = { a: 1 };
const diffs = diff(oldObj, newObj);
expect(diffs).toEqual([
{
t: "r",
p: ["b"],
o: 2,
n: undefined
}
]);
const appliedObj = apply(oldObj, diffs);
expect(appliedObj).toEqual(newObj);
const revertedObj = revert(newObj, diffs);
expect(revertedObj).toEqual(oldObj);
});
it("should detect edited properties", () => {
const oldObj = { a: 1 };
const newObj = { a: 2 };
const diffs = diff(oldObj, newObj);
expect(diffs).toEqual([
{
t: "e",
p: ["a"],
o: 1,
n: 2
}
]);
const appliedObj = apply(oldObj, diffs);
expect(appliedObj).toEqual(newObj);
const revertedObj = revert(newObj, diffs);
expect(revertedObj).toEqual(oldObj);
});
it("should detect changes in nested objects", () => {
const oldObj = { a: { b: 1 } };
const newObj = { a: { b: 2 } };
const diffs = diff(oldObj, newObj);
expect(diffs).toEqual([
{
t: "e",
p: ["a", "b"],
o: 1,
n: 2
}
]);
const appliedObj = apply(oldObj, diffs);
expect(appliedObj).toEqual(newObj);
const revertedObj = revert(newObj, diffs);
expect(revertedObj).toEqual(oldObj);
});
it("should detect changes in arrays", () => {
const oldObj = { a: [1, 2, 3] };
const newObj = { a: [1, 4, 3, 5] };
const diffs = diff(oldObj, newObj);
expect(diffs).toEqual([
{
t: "e",
p: ["a", 1],
o: 2,
n: 4
},
{
t: "a",
p: ["a", 3],
o: undefined,
n: 5
}
]);
const appliedObj = apply(oldObj, diffs);
expect(appliedObj).toEqual(newObj);
const revertedObj = revert(newObj, diffs);
expect(revertedObj).toEqual(oldObj);
});
it("should handle adding elements to an empty array", () => {
const oldObj = { a: [] };
const newObj = { a: [1, 2, 3] };
const diffs = diff(oldObj, newObj);
expect(diffs).toEqual([
{
t: "a",
p: ["a", 0],
o: undefined,
n: 1
},
{
t: "a",
p: ["a", 1],
o: undefined,
n: 2
},
{
t: "a",
p: ["a", 2],
o: undefined,
n: 3
}
]);
const appliedObj = apply(oldObj, diffs);
expect(appliedObj).toEqual(newObj);
const revertedObj = revert(newObj, diffs);
expect(revertedObj).toEqual(oldObj);
});
it("should handle removing elements from an array", () => {
const oldObj = { a: [1, 2, 3] };
const newObj = { a: [1, 3] };
const diffs = diff(oldObj, newObj);
expect(diffs).toEqual([
{
t: "e",
p: ["a", 1],
o: 2,
n: 3
},
{
t: "r",
p: ["a", 2],
o: 3,
n: undefined
}
]);
const appliedObj = apply(oldObj, diffs);
expect(appliedObj).toEqual(newObj);
const revertedObj = revert(newObj, diffs);
expect(revertedObj).toEqual(oldObj);
});
it("should handle complex nested changes", () => {
const oldObj = {
a: {
b: [1, 2, { c: 3 }]
}
};
const newObj = {
a: {
b: [1, 2, { c: 4 }, 5]
}
};
const diffs = diff(oldObj, newObj);
expect(diffs).toEqual([
{
t: "e",
p: ["a", "b", 2, "c"],
o: 3,
n: 4
},
{
t: "a",
p: ["a", "b", 3],
o: undefined,
n: 5
}
]);
const appliedObj = apply(oldObj, diffs);
expect(appliedObj).toEqual(newObj);
const revertedObj = revert(newObj, diffs);
expect(revertedObj).toEqual(oldObj);
});
it("should handle undefined and null values", () => {
const oldObj = { a: undefined, b: null };
const newObj = { a: null, b: undefined };
const diffs = diff(oldObj, newObj);
expect(diffs).toEqual([
{
t: "e",
p: ["a"],
o: undefined,
n: null
},
{
t: "e",
p: ["b"],
o: null,
n: undefined
}
]);
const appliedObj = apply(oldObj, diffs);
expect(appliedObj).toEqual(newObj);
const revertedObj = revert(newObj, diffs);
expect(revertedObj).toEqual(oldObj);
});
it("should handle type changes", () => {
const oldObj = { a: 1 };
const newObj = { a: "1" };
const diffs = diff(oldObj, newObj);
expect(diffs).toEqual([
{
t: "e",
p: ["a"],
o: 1,
n: "1"
}
]);
const appliedObj = apply(oldObj, diffs);
expect(appliedObj).toEqual(newObj);
const revertedObj = revert(newObj, diffs);
expect(revertedObj).toEqual(oldObj);
});
it("should handle properties added and removed simultaneously", () => {
const oldObj = { a: 1, b: 2 };
const newObj = { a: 1, c: 3 };
const diffs = diff(oldObj, newObj);
expect(diffs).toEqual([
{
t: "r",
p: ["b"],
o: 2,
n: undefined
},
{
t: "a",
p: ["c"],
o: undefined,
n: 3
}
]);
const appliedObj = apply(oldObj, diffs);
expect(appliedObj).toEqual(newObj);
const revertedObj = revert(newObj, diffs);
expect(revertedObj).toEqual(oldObj);
});
it("should handle arrays replaced with objects", () => {
const oldObj = { a: [1, 2, 3] };
const newObj = { a: { b: 4 } };
const diffs = diff(oldObj, newObj);
expect(diffs).toEqual([
{
t: "e",
p: ["a"],
o: [1, 2, 3],
n: { b: 4 }
}
]);
const appliedObj = apply(oldObj, diffs);
expect(appliedObj).toEqual(newObj);
const revertedObj = revert(newObj, diffs);
expect(revertedObj).toEqual(oldObj);
});
it("should handle objects replaced with primitives", () => {
const oldObj = { a: { b: 1 } };
const newObj = { a: 2 };
const diffs = diff(oldObj, newObj);
expect(diffs).toEqual([
{
t: "e",
p: ["a"],
o: { b: 1 },
n: 2
}
]);
const appliedObj = apply(oldObj, diffs);
expect(appliedObj).toEqual(newObj);
const revertedObj = revert(newObj, diffs);
expect(revertedObj).toEqual(oldObj);
});
it("should handle root object changes", () => {
const oldObj = { a: 1 };
const newObj = { b: 2 };
const diffs = diff(oldObj, newObj);
expect(diffs).toEqual([
{
t: "r",
p: ["a"],
o: 1,
n: undefined
},
{
t: "a",
p: ["b"],
o: undefined,
n: 2
}
]);
const appliedObj = apply(oldObj, diffs);
expect(appliedObj).toEqual(newObj);
const revertedObj = revert(newObj, diffs);
expect(revertedObj).toEqual(oldObj);
});
it("should handle identical objects", () => {
const oldObj = { a: 1, b: { c: 2 } };
const newObj = { a: 1, b: { c: 2 } };
const diffs = diff(oldObj, newObj);
expect(diffs).toEqual([]);
const appliedObj = apply(oldObj, diffs);
expect(appliedObj).toEqual(newObj);
const revertedObj = revert(newObj, diffs);
expect(revertedObj).toEqual(oldObj);
});
it("should handle empty objects", () => {
const oldObj = {};
const newObj = {};
const diffs = diff(oldObj, newObj);
expect(diffs).toEqual([]);
const appliedObj = apply(oldObj, diffs);
expect(appliedObj).toEqual(newObj);
const revertedObj = revert(newObj, diffs);
expect(revertedObj).toEqual(oldObj);
});
it("should handle changes from empty object to non-empty object", () => {
const oldObj = {};
const newObj = { a: 1 };
const diffs = diff(oldObj, newObj);
expect(diffs).toEqual([
{
t: "a",
p: ["a"],
o: undefined,
n: 1
}
]);
const appliedObj = apply(oldObj, diffs);
expect(appliedObj).toEqual(newObj);
const revertedObj = revert(newObj, diffs);
expect(revertedObj).toEqual(oldObj);
});
it("should handle changes from non-empty object to empty object", () => {
const oldObj = { a: 1 };
const newObj = {};
const diffs = diff(oldObj, newObj);
expect(diffs).toEqual([
{
t: "r",
p: ["a"],
o: 1,
n: undefined
}
]);
const appliedObj = apply(oldObj, diffs);
expect(appliedObj).toEqual(newObj);
const revertedObj = revert(newObj, diffs);
expect(revertedObj).toEqual(oldObj);
});
});

37
app/__test__/data/mutation.simple.test.ts
View File

@@ -132,14 +132,47 @@ describe("Mutator simple", async () => {
const data = (await em.repository(items).findMany()).data;
//console.log(data);
await em.mutator(items).deleteMany({ label: "delete" });
await em.mutator(items).deleteWhere({ label: "delete" });
expect((await em.repository(items).findMany()).data.length).toBe(data.length - 2);
//console.log((await em.repository(items).findMany()).data);
await em.mutator(items).deleteMany();
await em.mutator(items).deleteWhere();
expect((await em.repository(items).findMany()).data.length).toBe(0);
//expect(res.data.count).toBe(0);
});
test("updateMany", async () => {
await em.mutator(items).insertOne({ label: "update", count: 1 });
await em.mutator(items).insertOne({ label: "update too", count: 1 });
await em.mutator(items).insertOne({ label: "keep" });
// expect no update
await em.mutator(items).updateWhere(
{ count: 2 },
{
count: 10
}
);
expect((await em.repository(items).findMany()).data).toEqual([
{ id: 6, label: "update", count: 1 },
{ id: 7, label: "update too", count: 1 },
{ id: 8, label: "keep", count: 0 }
]);
// expect 2 to be updated
await em.mutator(items).updateWhere(
{ count: 2 },
{
count: 1
}
);
expect((await em.repository(items).findMany()).data).toEqual([
{ id: 6, label: "update", count: 2 },
{ id: 7, label: "update too", count: 2 },
{ id: 8, label: "keep", count: 0 }
]);
});
});

4
app/src/core/errors.ts
View File

@@ -27,6 +27,10 @@ export class BkndError extends Error {
super(message);
}
static with(message: string, details?: Record<string, any>, type?: string) {
throw new BkndError(message, details, type);
}
toJSON() {
return {
type: this.type ?? "unknown",

181
app/src/core/object/diff.ts Normal file
View File

@@ -0,0 +1,181 @@
enum Change {
Add = "a",
Remove = "r",
Edit = "e"
}
type Object = object;
type Primitive = string | number | boolean | null | object | any[] | undefined;
interface DiffEntry {
t: Change | string;
p: (string | number)[];
o: Primitive;
n: Primitive;
}
function isObject(value: any): value is Object {
return value !== null && value.constructor.name === "Object";
}
function isPrimitive(value: any): value is Primitive {
try {
return (
value === null ||
value === undefined ||
typeof value === "string" ||
typeof value === "number" ||
typeof value === "boolean" ||
Array.isArray(value) ||
isObject(value)
);
} catch (e) {
return false;
}
}
function diff(oldObj: Object, newObj: Object): DiffEntry[] {
const diffs: DiffEntry[] = [];
function recurse(oldValue: Primitive, newValue: Primitive, path: (string | number)[]) {
if (!isPrimitive(oldValue) || !isPrimitive(newValue)) {
throw new Error("Diff: Only primitive types are supported");
}
if (oldValue === newValue) {
return;
}
if (typeof oldValue !== typeof newValue) {
diffs.push({
t: Change.Edit,
p: path,
o: oldValue,
n: newValue
});
} else if (Array.isArray(oldValue) && Array.isArray(newValue)) {
const maxLength = Math.max(oldValue.length, newValue.length);
for (let i = 0; i < maxLength; i++) {
if (i >= oldValue.length) {
diffs.push({
t: Change.Add,
p: [...path, i],
o: undefined,
n: newValue[i]
});
} else if (i >= newValue.length) {
diffs.push({
t: Change.Remove,
p: [...path, i],
o: oldValue[i],
n: undefined
});
} else {
recurse(oldValue[i], newValue[i], [...path, i]);
}
}
} else if (isObject(oldValue) && isObject(newValue)) {
const oKeys = Object.keys(oldValue);
const nKeys = Object.keys(newValue);
const allKeys = new Set([...oKeys, ...nKeys]);
for (const key of allKeys) {
if (!(key in oldValue)) {
diffs.push({
t: Change.Add,
p: [...path, key],
o: undefined,
n: newValue[key]
});
} else if (!(key in newValue)) {
diffs.push({
t: Change.Remove,
p: [...path, key],
o: oldValue[key],
n: undefined
});
} else {
recurse(oldValue[key], newValue[key], [...path, key]);
}
}
} else {
diffs.push({
t: Change.Edit,
p: path,
o: oldValue,
n: newValue
});
}
}
recurse(oldObj, newObj, []);
return diffs;
}
function apply(obj: Object, diffs: DiffEntry[]): any {
const clonedObj = clone(obj);
for (const diff of diffs) {
applyChange(clonedObj, diff);
}
return clonedObj;
}
function revert(obj: Object, diffs: DiffEntry[]): any {
const clonedObj = clone(obj);
const reversedDiffs = diffs.slice().reverse();
for (const diff of reversedDiffs) {
revertChange(clonedObj, diff);
}
return clonedObj;
}
function applyChange(obj: Object, diff: DiffEntry) {
const { p: path, t: type, n: newValue } = diff;
const parent = getParent(obj, path.slice(0, -1));
const key = path[path.length - 1]!;
if (type === Change.Add || type === Change.Edit) {
parent[key] = newValue;
} else if (type === Change.Remove) {
if (Array.isArray(parent)) {
parent.splice(key as number, 1);
} else {
delete parent[key];
}
}
}
function revertChange(obj: Object, diff: DiffEntry) {
const { p: path, t: type, o: oldValue } = diff;
const parent = getParent(obj, path.slice(0, -1));
const key = path[path.length - 1]!;
if (type === Change.Add) {
if (Array.isArray(parent)) {
parent.splice(key as number, 1);
} else {
delete parent[key];
}
} else if (type === Change.Remove || type === Change.Edit) {
parent[key] = oldValue;
}
}
function getParent(obj: Object, path: (string | number)[]): any {
let current = obj;
for (const key of path) {
if (current[key] === undefined) {
current[key] = typeof key === "number" ? [] : {};
}
current = current[key];
}
return current;
}
function clone<In extends Object>(obj: In): In {
return JSON.parse(JSON.stringify(obj));
}
export { diff, apply, revert, clone };

4
app/src/core/utils/typebox/index.ts
View File

@@ -72,10 +72,10 @@ export class TypeInvalidError extends Error {
}
}
export function stripMark(obj: any) {
export function stripMark<O = any>(obj: O) {
const newObj = cloneDeep(obj);
mark(newObj, false);
return newObj;
return newObj as O;
}
export function mark(obj: any, validated = true) {

28
app/src/data/entities/Mutator.ts
View File

@@ -250,7 +250,7 @@ export class Mutator<DB> implements EmitsEvents {
}
// @todo: decide whether entries should be deleted all at once or one by one (for events)
async deleteMany(where?: RepoQuery["where"]): Promise<MutatorResponse<EntityData>> {
async deleteWhere(where?: RepoQuery["where"]): Promise<MutatorResponse<EntityData>> {
const entity = this.entity;
const qb = this.appendWhere(this.conn.deleteFrom(entity.name), where).returning(
@@ -267,4 +267,30 @@ export class Mutator<DB> implements EmitsEvents {
return res;
}
async updateWhere(
data: EntityData,
where?: RepoQuery["where"]
): Promise<MutatorResponse<EntityData>> {
const entity = this.entity;
const validatedData = await this.getValidatedData(data, "update");
/*await this.emgr.emit(
new Mutator.Events.MutatorUpdateBefore({ entity, entityId: id, data: validatedData })
);*/
const query = this.appendWhere(this.conn.updateTable(entity.name), where)
.set(validatedData)
//.where(entity.id().name, "=", id)
.returning(entity.getSelect());
const res = await this.many(query);
/*await this.emgr.emit(
new Mutator.Events.MutatorUpdateAfter({ entity, entityId: id, data: res.data })
);*/
return res;
}
}

11
app/src/data/entities/query/Repository.ts
View File

@@ -162,8 +162,7 @@ export class Repository<DB = any, TB extends keyof DB = any> implements EmitsEve
protected async performQuery(qb: RepositoryQB): Promise<RepositoryResponse> {
const entity = this.entity;
const compiled = qb.compile();
/*const { sql, parameters } = qb.compile();
console.log("many", sql, parameters);*/
//console.log("performQuery", compiled.sql, compiled.parameters);
const start = performance.now();
const selector = (as = "count") => this.conn.fn.countAll<number>().as(as);
@@ -263,6 +262,7 @@ export class Repository<DB = any, TB extends keyof DB = any> implements EmitsEve
qb = qb.orderBy(aliased(options.sort.by), options.sort.dir);
}
//console.log("options", { _options, options, exclude_options });
return { qb, options };
}
@@ -286,14 +286,11 @@ export class Repository<DB = any, TB extends keyof DB = any> implements EmitsEve
where: RepoQuery["where"],
_options?: Partial<Omit<RepoQuery, "where" | "limit" | "offset">>
): Promise<RepositoryResponse<DB[TB] | undefined>> {
const { qb, options } = this.buildQuery(
{
const { qb, options } = this.buildQuery({
..._options,
where,
limit: 1
},
["offset", "sort"]
);
});
return this.single(qb, options) as any;
}

1
app/src/data/entities/query/WhereBuilder.ts
View File

@@ -15,7 +15,6 @@ import type {
SelectQueryBuilder,
UpdateQueryBuilder
} from "kysely";
import type { RepositoryQB } from "./Repository";
type Builder = ExpressionBuilder<any, any>;
type Wrapper = ExpressionWrapper<any, any, any>;

4
app/src/data/fields/Field.ts
View File

@@ -236,8 +236,8 @@ export abstract class Field<
toJSON() {
return {
//name: this.name,
type: this.type,
// @todo: current workaround because of fixed string type
type: this.type as any,
config: this.config
};
}

7
app/src/data/fields/JsonSchemaField.ts
View File

@@ -54,7 +54,7 @@ export class JsonSchemaField<
if (parentValid) {
// already checked in parent
if (!value || typeof value !== "object") {
if (!this.isRequired() && (!value || typeof value !== "object")) {
//console.log("jsonschema:valid: not checking", this.name, value, context);
return true;
}
@@ -65,6 +65,7 @@ export class JsonSchemaField<
} else {
//console.log("jsonschema:invalid", this.name, value, context);
}
//console.log("jsonschema:invalid:fromParent", this.name, value, context);
return false;
}
@@ -110,9 +111,13 @@ export class JsonSchemaField<
): Promise<string | undefined> {
const value = await super.transformPersist(_value, em, context);
if (this.nullish(value)) return value;
//console.log("jsonschema:transformPersist", this.name, _value, context);
if (!this.isValid(value)) {
//console.error("jsonschema:transformPersist:invalid", this.name, value);
throw new TransformPersistFailedException(this.name, value);
} else {
//console.log("jsonschema:transformPersist:valid", this.name, value);
}
if (!value || typeof value !== "object") return this.getDefault();

2
app/src/media/api/MediaController.ts
View File

@@ -181,7 +181,7 @@ export class MediaController implements ClassController {
if (ids_to_delete.length > 0) {
await this.media.em
.mutator(mediaEntity)
.deleteMany({ [id_field]: { $in: ids_to_delete } });
.deleteWhere({ [id_field]: { $in: ids_to_delete } });
}
return c.json({ ok: true, result: result.data, ...info });

236
app/src/modules/ModuleManager.ts
View File

@@ -1,12 +1,31 @@
import { Diff } from "@sinclair/typebox/value";
import { Guard } from "auth";
import { DebugLogger, isDebug } from "core";
import { BkndError, DebugLogger, Exception, isDebug } from "core";
import { EventManager } from "core/events";
import { Default, type Static, objectEach, transformObject } from "core/utils";
import { type Connection, EntityManager } from "data";
import { clone, diff } from "core/object/diff";
import {
Default,
type Static,
StringEnum,
Type,
objectEach,
stripMark,
transformObject
} from "core/utils";
import {
type Connection,
EntityManager,
type Schema,
datetime,
entity,
enumm,
jsonSchema,
number
} from "data";
import { TransformPersistFailedException } from "data/errors";
import { Hono } from "hono";
import { type Kysely, sql } from "kysely";
import { CURRENT_VERSION, TABLE_NAME, migrate, migrateSchema } from "modules/migrations";
import { mergeWith } from "lodash-es";
import { CURRENT_VERSION, TABLE_NAME, migrate } from "modules/migrations";
import { AppServer } from "modules/server/AppServer";
import { AppAuth } from "../auth/AppAuth";
import { AppData } from "../data/AppData";
@@ -37,10 +56,13 @@ export type ModuleSchemas = {
export type ModuleConfigs = {
[K in keyof ModuleSchemas]: Static<ModuleSchemas[K]>;
};
type PartialRec<T> = { [P in keyof T]?: PartialRec<T[P]> };
export type InitialModuleConfigs = {
export type InitialModuleConfigs =
| ({
version: number;
} & Partial<ModuleConfigs>;
} & ModuleConfigs)
| PartialRec<ModuleConfigs>;
export type ModuleManagerOptions = {
initial?: InitialModuleConfigs;
@@ -61,8 +83,36 @@ type ConfigTable<Json = ModuleConfigs> = {
updated_at?: Date;
};
const configJsonSchema = Type.Union([
getDefaultSchema(),
Type.Array(
Type.Object({
t: StringEnum(["a", "r", "e"]),
p: Type.Array(Type.Union([Type.String(), Type.Number()])),
o: Type.Optional(Type.Any()),
n: Type.Optional(Type.Any())
})
)
]);
const __bknd = entity(TABLE_NAME, {
version: number().required(),
type: enumm({ enum: ["config", "diff", "backup"] }).required(),
json: jsonSchema({ schema: configJsonSchema }).required(),
created_at: datetime(),
updated_at: datetime()
});
type ConfigTable2 = Schema<typeof __bknd>;
type T_INTERNAL_EM = {
__bknd: ConfigTable2;
};
// @todo: cleanup old diffs on upgrade
// @todo: cleanup multiple backups on upgrade
export class ModuleManager {
private modules: Modules;
// internal em for __bknd config table
__em!: EntityManager<T_INTERNAL_EM>;
// ctx for modules
em!: EntityManager<any>;
server!: Hono;
emgr!: EventManager;
@@ -71,28 +121,32 @@ export class ModuleManager {
private _version: number = 0;
private _built = false;
private _fetched = false;
private readonly _provided;
private logger = new DebugLogger(isDebug() && false);
// @todo: keep? not doing anything with it
private readonly _booted_with?: "provided" | "partial";
private logger = new DebugLogger(false);
constructor(
private readonly connection: Connection,
private options?: Partial<ModuleManagerOptions>
) {
this.__em = new EntityManager([__bknd], this.connection);
this.modules = {} as Modules;
this.emgr = new EventManager();
const context = this.ctx(true);
let initial = {} as Partial<ModuleConfigs>;
if (options?.initial) {
if ("version" in options.initial) {
const { version, ...initialConfig } = options.initial;
if (version && initialConfig) {
this._version = version;
initial = initialConfig;
initial = stripMark(initialConfig);
this._provided = true;
this._booted_with = "provided";
} else {
throw new Error("Initial was provided, but it needs a version!");
initial = mergeWith(getDefaultConfig(), options.initial);
this._booted_with = "partial";
}
}
@@ -120,6 +174,22 @@ export class ModuleManager {
}
}
private repo() {
return this.__em.repo(__bknd);
}
private mutator() {
return this.__em.mutator(__bknd);
}
private get db() {
return this.connection.kysely as Kysely<{ table: ConfigTable }>;
}
async syncConfigTable() {
return await this.__em.schema().sync({ force: true });
}
private rebuildServer() {
this.server = new Hono();
if (this.options?.basePath) {
@@ -153,27 +223,22 @@ export class ModuleManager {
};
}
private get db() {
return this.connection.kysely as Kysely<{ table: ConfigTable }>;
}
get table() {
return TABLE_NAME as "table";
}
private async fetch(): Promise<ConfigTable> {
this.logger.context("fetch").log("fetching");
const startTime = performance.now();
const result = await this.db
.selectFrom(this.table)
.selectAll()
.where("type", "=", "config")
.orderBy("version", "desc")
.executeTakeFirstOrThrow();
const { data: result } = await this.repo().findOne(
{ type: "config" },
{
sort: { by: "version", dir: "desc" }
}
);
if (!result) {
throw BkndError.with("no config");
}
this.logger.log("took", performance.now() - startTime, "ms", result).clear();
return result;
return result as ConfigTable;
}
async save() {
@@ -181,65 +246,75 @@ export class ModuleManager {
const configs = this.configs();
const version = this.version();
const json = JSON.stringify(configs) as any;
try {
const state = await this.fetch();
this.logger.log("fetched version", state.version);
if (state.version !== version) {
// @todo: mark all others as "backup"
this.logger.log("version conflict, storing new version", state.version, version);
await this.db
.insertInto(this.table)
.values({
version,
await this.mutator().insertOne({
version: state.version,
type: "backup",
json: configs
});
await this.mutator().insertOne({
version: version,
type: "config",
json
})
.execute();
json: configs
});
} else {
this.logger.log("version matches");
const diff = Diff(state.json, JSON.parse(json));
this.logger.log("checking diff", diff);
// clean configs because of Diff() function
const diffs = diff(state.json, clone(configs));
this.logger.log("checking diff", diffs);
if (diff.length > 0) {
// store diff
await this.db
.insertInto(this.table)
.values({
await this.mutator().insertOne({
version,
type: "diff",
json: JSON.stringify(diff) as any
})
.execute();
json: clone(diffs)
});
await this.db
.updateTable(this.table)
.set({ version, json, updated_at: sql`CURRENT_TIMESTAMP` })
.where((eb) => eb.and([eb("type", "=", "config"), eb("version", "=", version)]))
.execute();
// store new version
await this.mutator().updateWhere(
{
version,
json: configs,
updated_at: new Date()
},
{
type: "config",
version
}
);
} else {
this.logger.log("no diff, not saving");
}
}
} catch (e) {
if (e instanceof BkndError) {
this.logger.log("no config, just save fresh");
// no config, just save
await this.mutator().insertOne({
type: "config",
version,
json: configs,
created_at: new Date(),
updated_at: new Date()
});
} else if (e instanceof TransformPersistFailedException) {
console.error("Cannot save invalid config");
throw e;
} else {
console.error("Aborting");
throw e;
}
}
// cleanup
/*this.logger.log("cleaning up");
const result = await this.db
.deleteFrom(this.table)
.where((eb) =>
eb.or([
// empty migrations
eb.and([
eb("type", "=", "config"),
eb("version", "<", version),
eb("json", "is", null)
]),
// past diffs
eb.and([eb("type", "=", "diff"), eb("version", "<", version)])
])
)
.executeTakeFirst();
this.logger.log("cleaned up", result.numDeletedRows);*/
// @todo: cleanup old versions?
this.logger.clear();
return this;
@@ -250,6 +325,8 @@ export class ModuleManager {
if (this.version() < CURRENT_VERSION) {
this.logger.log("there are migrations, verify version");
// sync __bknd table
await this.syncConfigTable();
// modules must be built before migration
await this.buildModules({ graceful: true });
@@ -264,15 +341,8 @@ export class ModuleManager {
}
} catch (e: any) {
this.logger.clear(); // fetch couldn't clear
// if table doesn't exist, migrate schema to version
if (e.message.includes("no such table")) {
this.logger.log("table has to created, migrating schema up to", this.version());
await migrateSchema(this.version(), { db: this.db });
} else {
throw new Error(`Version is ${this.version()}, fetch failed: ${e.message}`);
}
}
this.logger.log("now migrating");
let version = this.version();
@@ -337,10 +407,16 @@ export class ModuleManager {
async build() {
this.logger.context("build").log("version", this.version());
this.logger.log("booted with", this._booted_with);
// @todo: check this, because you could start without an initial config
if (this.version() !== CURRENT_VERSION) {
await this.syncConfigTable();
}
// if no config provided, try fetch from db
if (this.version() === 0) {
this.logger.context("build no config").log("version is 0");
this.logger.context("no version").log("version is 0");
try {
const result = await this.fetch();
@@ -351,23 +427,15 @@ export class ModuleManager {
this.logger.clear(); // fetch couldn't clear
this.logger.context("error handler").log("fetch failed", e.message);
// if table doesn't exist, migrate schema, set default config and latest version
if (e.message.includes("no such table")) {
this.logger.log("migrate schema to", CURRENT_VERSION);
await migrateSchema(CURRENT_VERSION, { db: this.db });
this._version = CURRENT_VERSION;
// we can safely build modules, since config version is up to date
// it's up to date because we use default configs (no fetch result)
this._version = CURRENT_VERSION;
await this.buildModules();
await this.save();
this.logger.clear();
return this;
} else {
throw e;
//throw new Error("Issues connecting to the database. Reason: " + e.message);
}
}
this.logger.clear();
}

59
app/src/modules/migrations.ts
View File

@@ -16,26 +16,8 @@ export type Migration = {
export const migrations: Migration[] = [
{
version: 1,
schema: true,
up: async (config, { db }) => {
//console.log("config given", config);
await db.schema
.createTable(TABLE_NAME)
.addColumn("id", "integer", (col) => col.primaryKey().notNull().autoIncrement())
.addColumn("version", "integer", (col) => col.notNull())
.addColumn("type", "text", (col) => col.notNull())
.addColumn("json", "text")
.addColumn("created_at", "datetime", (col) => col.defaultTo(sql`CURRENT_TIMESTAMP`))
.addColumn("updated_at", "datetime", (col) => col.defaultTo(sql`CURRENT_TIMESTAMP`))
.execute();
await db
.insertInto(TABLE_NAME)
.values({ version: 1, type: "config", json: null })
.execute();
return config;
}
//schema: true,
up: async (config) => config
},
{
version: 2,
@@ -45,12 +27,8 @@ export const migrations: Migration[] = [
},
{
version: 3,
schema: true,
up: async (config, { db }) => {
await db.schema.alterTable(TABLE_NAME).addColumn("deleted_at", "datetime").execute();
return config;
}
//schema: true,
up: async (config) => config
},
{
version: 4,
@@ -94,6 +72,13 @@ export const migrations: Migration[] = [
};
}
}
/*{
version: 8,
up: async (config, { db }) => {
await db.deleteFrom(TABLE_NAME).where("type", "=", "diff").execute();
return config;
}
}*/
];
export const CURRENT_VERSION = migrations[migrations.length - 1]?.version ?? 0;
@@ -127,28 +112,6 @@ export async function migrateTo(
return [version, updated];
}
export async function migrateSchema(to: number, ctx: MigrationContext, current: number = 0) {
console.log("migrating SCHEMA to", to, "from", current);
const todo = migrations.filter((m) => m.version > current && m.version <= to && m.schema);
console.log("todo", todo.length);
let i = 0;
let version = 0;
for (const migration of todo) {
console.log("-- running migration", i + 1, "of", todo.length);
try {
await migration.up({}, ctx);
version = migration.version;
i++;
} catch (e: any) {
console.error(e);
throw new Error(`Migration ${migration.version} failed: ${e.message}`);
}
}
return version;
}
export async function migrate(
current: number,
config: GenericConfigObject,