shishantbiswas/bknd
1
0
Fork 0
mirror of https://github.com/shishantbiswas/bknd.git synced 2026-03-16 04:27:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor module schema handling and add sync mechanism

Browse Source 
Redesigned entity and index management with methods to streamline schema updates and added a sync flag to signal required DB syncs post-build. Enhanced test coverage and functionality for schema modifications, including support for additional fields.
This commit is contained in:
dswbx
2025-01-10 14:43:39 +01:00
parent 475563b5e1
commit a8c20d3675
11 changed files with 413 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
app/__test__/Module.spec.ts
View File

@@ -1,38 +0,0 @@
import { describe, expect, test } from "bun:test";
import { type TSchema, Type, stripMark } from "../src/core/utils";
import { Module } from "../src/modules/Module";
function createModule<Schema extends TSchema>(schema: Schema) {
class TestModule extends Module<typeof schema> {
getSchema() {
return schema;
}
toJSON() {
return this.config;
}
useForceParse() {
return true;
}
}
return TestModule;
}
describe("Module", async () => {
test("basic", async () => {});
test("listener", async () => {
let result: any;
const module = createModule(Type.Object({ a: Type.String() }));
const m = new module({ a: "test" });
await m.schema().set({ a: "test2" });
m.setListener(async (c) => {
await new Promise((r) => setTimeout(r, 10));
result = stripMark(c);
});
await m.schema().set({ a: "test3" });
expect(result).toEqual({ a: "test3" });
});
});

30
app/__test__/modules/AppAuth.spec.ts
View File

@@ -1,6 +1,7 @@
import { afterAll, beforeAll, beforeEach, describe, expect, spyOn, test } from "bun:test";
import { createApp } from "../../src";
import { AuthController } from "../../src/auth/api/AuthController";
import { em, entity, text } from "../../src/data";
import { AppAuth, type ModuleBuildContext } from "../../src/modules";
import { disableConsoleLog, enableConsoleLog } from "../helper";
import { makeCtx, moduleTestSuite } from "./module-test-suite";
@@ -102,4 +103,33 @@ describe("AppAuth", () => {
expect(spy.mock.calls.length).toBe(2);
});
test("should allow additional user fields", async () => {
const app = createApp({
initialConfig: {
auth: {
entity_name: "users",
enabled: true
},
data: em({
users: entity("users", {
additional: text()
})
}).toJSON()
}
});
await app.build();
const userfields = app.modules.em.entity("users").fields.map((f) => f.name);
expect(userfields).toContain("additional");
expect(userfields).toEqual([
"id",
"additional",
"email",
"strategy",
"strategy_value",
"role"
]);
});
});

48
app/__test__/modules/AppMedia.spec.ts
View File

@@ -1,7 +1,53 @@
import { describe } from "bun:test";
import { describe, expect, test } from "bun:test";
import { createApp, registries } from "../../src";
import { em, entity, text } from "../../src/data";
import { StorageLocalAdapter } from "../../src/media/storage/adapters/StorageLocalAdapter";
import { AppMedia } from "../../src/modules";
import { moduleTestSuite } from "./module-test-suite";
describe("AppMedia", () => {
moduleTestSuite(AppMedia);
test("should allow additional fields", async () => {
registries.media.register("local", StorageLocalAdapter);
const app = createApp({
initialConfig: {
media: {
entity_name: "media",
enabled: true,
adapter: {
type: "local",
config: {
path: "./"
}
}
},
data: em({
media: entity("media", {
additional: text()
})
}).toJSON()
}
});
await app.build();
const fields = app.modules.em.entity("media").fields.map((f) => f.name);
expect(fields).toContain("additional");
expect(fields).toEqual([
"id",
"additional",
"path",
"folder",
"mime_type",
"size",
"scope",
"etag",
"modified_at",
"reference",
"entity_id",
"metadata"
]);
});
});

200
app/__test__/modules/Module.spec.ts Normal file
View File

@@ -0,0 +1,200 @@
import { describe, expect, test } from "bun:test";
import { type TSchema, Type, stripMark } from "../../src/core/utils";
import { EntityManager, em, entity, index, text } from "../../src/data";
import { DummyConnection } from "../../src/data/connection/DummyConnection";
import { Module } from "../../src/modules/Module";
function createModule<Schema extends TSchema>(schema: Schema) {
class TestModule extends Module<typeof schema> {
getSchema() {
return schema;
}
toJSON() {
return this.config;
}
useForceParse() {
return true;
}
}
return TestModule;
}
describe("Module", async () => {
describe("basic", () => {
test("listener", async () => {
let result: any;
const module = createModule(Type.Object({ a: Type.String() }));
const m = new module({ a: "test" });
await m.schema().set({ a: "test2" });
m.setListener(async (c) => {
await new Promise((r) => setTimeout(r, 10));
result = stripMark(c);
});
await m.schema().set({ a: "test3" });
expect(result).toEqual({ a: "test3" });
});
});
describe("db schema", () => {
class M extends Module {
override getSchema() {
return Type.Object({});
}
prt = {
ensureEntity: this.ensureEntity.bind(this),
ensureIndex: this.ensureIndex.bind(this),
ensureSchema: this.ensureSchema.bind(this)
};
get em() {
return this.ctx.em;
}
}
function make(_em: ReturnType<typeof em>) {
const em = new EntityManager(
Object.values(_em.entities),
new DummyConnection(),
_em.relations,
_em.indices
);
return new M({} as any, { em, flags: Module.ctx_flags } as any);
}
function flat(_em: EntityManager) {
return {
entities: _em.entities.map((e) => ({
name: e.name,
fields: e.fields.map((f) => f.name)
})),
indices: _em.indices.map((i) => ({
name: i.name,
entity: i.entity.name,
fields: i.fields.map((f) => f.name),
unique: i.unique
}))
};
}
test("no change", () => {
const initial = em({});
const m = make(initial);
expect(m.ctx.flags.sync_required).toBe(false);
expect(flat(make(initial).em)).toEqual({
entities: [],
indices: []
});
});
test("init", () => {
const initial = em({
users: entity("u", {
name: text()
})
});
const m = make(initial);
expect(m.ctx.flags.sync_required).toBe(false);
expect(flat(m.em)).toEqual({
entities: [
{
name: "u",
fields: ["id", "name"]
}
],
indices: []
});
});
test("ensure entity", () => {
const initial = em({
users: entity("u", {
name: text()
})
});
const m = make(initial);
expect(flat(m.em)).toEqual({
entities: [
{
name: "u",
fields: ["id", "name"]
}
],
indices: []
});
// this should add a new entity
m.prt.ensureEntity(
entity("p", {
title: text()
})
);
// this should only add the field "important"
m.prt.ensureEntity(
entity("u", {
important: text()
})
);
expect(m.ctx.flags.sync_required).toBe(true);
expect(flat(m.em)).toEqual({
entities: [
{
name: "u",
fields: ["id", "name", "important"]
},
{
name: "p",
fields: ["id", "title"]
}
],
indices: []
});
});
test("ensure index", () => {
const users = entity("u", {
name: text(),
title: text()
});
const initial = em({ users }, ({ index }, { users }) => {
index(users).on(["title"]);
});
const m = make(initial);
m.prt.ensureIndex(index(users).on(["name"]));
expect(m.ctx.flags.sync_required).toBe(true);
expect(flat(m.em)).toEqual({
entities: [
{
name: "u",
fields: ["id", "name", "title"]
}
],
indices: [
{
name: "idx_u_title",
entity: "u",
fields: ["title"],
unique: false
},
{
name: "idx_u_name",
entity: "u",
fields: ["name"],
unique: false
}
]
});
});
});
});

10
app/__test__/ModuleManager.spec.ts → app/__test__/modules/ModuleManager.spec.ts
View File

@@ -1,9 +1,9 @@
import { describe, expect, test } from "bun:test";
import { mark, stripMark } from "../src/core/utils";
import { entity, text } from "../src/data";
import { ModuleManager, getDefaultConfig } from "../src/modules/ModuleManager";
import { CURRENT_VERSION, TABLE_NAME } from "../src/modules/migrations";
import { getDummyConnection } from "./helper";
import { stripMark } from "../../src/core/utils";
import { entity, text } from "../../src/data";
import { ModuleManager, getDefaultConfig } from "../../src/modules/ModuleManager";
import { CURRENT_VERSION, TABLE_NAME } from "../../src/modules/migrations";
import { getDummyConnection } from "../helper";
describe("ModuleManager", async () => {
test("s1: no config, no build", async () => {

43
app/src/auth/AppAuth.ts
View File

@@ -4,7 +4,7 @@ import { auth } from "auth/middlewares";
import { type DB, Exception, type PrimaryFieldType } from "core";
import { type Static, secureRandomString, transformObject } from "core/utils";
import { type Entity, EntityIndex, type EntityManager } from "data";
import { type FieldSchema, entity, enumm, make, text } from "data/prototype";
import { type FieldSchema, em, entity, enumm, make, text } from "data/prototype";
import type { Hono } from "hono";
import { pick } from "lodash-es";
import { Module } from "modules/Module";
@@ -250,43 +250,30 @@ export class AppAuth extends Module<typeof authConfigSchema> {
};
registerEntities() {
const users = this.getUsersEntity();
if (!this.em.hasEntity(users.name)) {
this.em.addEntity(users);
} else {
// if exists, check all fields required are there
// @todo: add to context: "needs sync" flag
const _entity = this.getUsersEntity(true);
for (const field of _entity.fields) {
const _field = users.field(field.name);
if (!_field) {
users.addField(field);
const name = this.config.entity_name as "users";
const {
entities: { users }
} = this.ensureSchema(
em(
{
[name]: entity(name, AppAuth.usersFields)
},
({ index }, { users }) => {
index(users).on(["email"], true).on(["strategy"]).on(["strategy_value"]);
}
}
}
const indices = [
new EntityIndex(users, [users.field("email")!], true),
new EntityIndex(users, [users.field("strategy")!]),
new EntityIndex(users, [users.field("strategy_value")!])
];
indices.forEach((index) => {
if (!this.em.hasIndex(index)) {
this.em.addIndex(index);
}
});
)
);
try {
const roles = Object.keys(this.config.roles ?? {});
const field = make("role", enumm({ enum: roles }));
this.em.entity(users.name).__experimental_replaceField("role", field);
users.__experimental_replaceField("role", field);
} catch (e) {}
try {
const strategies = Object.keys(this.config.strategies ?? {});
const field = make("strategy", enumm({ enum: strategies }));
this.em.entity(users.name).__experimental_replaceField("strategy", field);
users.__experimental_replaceField("strategy", field);
} catch (e) {}
}

16
app/src/data/prototype/index.ts
View File

@@ -272,18 +272,22 @@ class EntityManagerPrototype<Entities extends Record<string, Entity>> extends En
}
}
type Chained<Fn extends (...args: any[]) => any, Rt = ReturnType<Fn>> = <E extends Entity>(
e: E
) => {
[K in keyof Rt]: Rt[K] extends (...args: any[]) => any
? (...args: Parameters<Rt[K]>) => Rt
type Chained<R extends Record<string, (...args: any[]) => any>> = {
[K in keyof R]: R[K] extends (...args: any[]) => any
? (...args: Parameters<R[K]>) => Chained<R>
: never;
};
type ChainedFn<
Fn extends (...args: any[]) => Record<string, (...args: any[]) => any>,
Return extends ReturnType<Fn> = ReturnType<Fn>
> = (e: Entity) => {
[K in keyof Return]: (...args: Parameters<Return[K]>) => Chained<Return>;
};
export function em<Entities extends Record<string, Entity>>(
entities: Entities,
schema?: (
fns: { relation: Chained<typeof relation>; index: Chained<typeof index> },
fns: { relation: ChainedFn<typeof relation>; index: ChainedFn<typeof index> },
entities: Entities
) => void
) {

41
app/src/media/AppMedia.ts
View File

@@ -1,8 +1,17 @@
import type { PrimaryFieldType } from "core";
import { EntityIndex, type EntityManager } from "data";
import { type Entity, EntityIndex, type EntityManager } from "data";
import { type FileUploadedEventData, Storage, type StorageAdapter } from "media";
import { Module } from "modules/Module";
import { type FieldSchema, boolean, datetime, entity, json, number, text } from "../data/prototype";
import {
type FieldSchema,
boolean,
datetime,
em,
entity,
json,
number,
text
} from "../data/prototype";
import { MediaController } from "./api/MediaController";
import { ADAPTERS, buildMediaSchema, type mediaConfigSchema, registry } from "./media-schema";
@@ -17,6 +26,7 @@ export class AppMedia extends Module<typeof mediaConfigSchema> {
private _storage?: Storage;
override async build() {
console.log("building");
if (!this.config.enabled) {
this.setBuilt();
return;
@@ -38,18 +48,13 @@ export class AppMedia extends Module<typeof mediaConfigSchema> {
this.setupListeners();
this.ctx.server.route(this.basepath, new MediaController(this).getController());
// @todo: add check for media entity
const mediaEntity = this.getMediaEntity();
if (!this.ctx.em.hasEntity(mediaEntity)) {
this.ctx.em.addEntity(mediaEntity);
}
const pathIndex = new EntityIndex(mediaEntity, [mediaEntity.field("path")!], true);
if (!this.ctx.em.hasIndex(pathIndex)) {
this.ctx.em.addIndex(pathIndex);
}
// @todo: check indices
const mediaEntity = this.getMediaEntity(true);
const name = mediaEntity.name as "media";
this.ensureSchema(
em({ [name]: mediaEntity }, ({ index }, { media }) => {
index(media).on(["path"], true).on(["reference"]);
})
);
} catch (e) {
console.error(e);
throw new Error(
@@ -94,13 +99,13 @@ export class AppMedia extends Module<typeof mediaConfigSchema> {
metadata: json()
};
getMediaEntity() {
getMediaEntity(forceCreate?: boolean): Entity<"media", typeof AppMedia.mediaFields> {
const entity_name = this.config.entity_name;
if (!this.em.hasEntity(entity_name)) {
return entity(entity_name, AppMedia.mediaFields, undefined, "system");
if (forceCreate || !this.em.hasEntity(entity_name)) {
return entity(entity_name as "media", AppMedia.mediaFields, undefined, "system");
}
return this.em.entity(entity_name);
return this.em.entity(entity_name) as any;
}
get em(): EntityManager {

47
app/src/modules/Module.ts
View File

@@ -3,7 +3,7 @@ import type { Guard } from "auth";
import { SchemaObject } from "core";
import type { EventManager } from "core/events";
import type { Static, TSchema } from "core/utils";
import type { Connection, EntityManager } from "data";
import type { Connection, Entity, EntityIndex, EntityManager, em as prototypeEm } from "data";
import type { Hono } from "hono";
export type ServerEnv = {
@@ -21,6 +21,7 @@ export type ModuleBuildContext = {
em: EntityManager;
emgr: EventManager<any>;
guard: Guard;
flags: (typeof Module)["ctx_flags"];
};
export abstract class Module<Schema extends TSchema = TSchema, ConfigSchema = Static<Schema>> {
@@ -43,6 +44,13 @@ export abstract class Module<Schema extends TSchema = TSchema, ConfigSchema = St
});
}
static ctx_flags = {
sync_required: false
} as {
// signal that a sync is required at the end of build
sync_required: boolean;
};
onBeforeUpdate(from: ConfigSchema, to: ConfigSchema): ConfigSchema | Promise<ConfigSchema> {
return to;
}
@@ -129,4 +137,41 @@ export abstract class Module<Schema extends TSchema = TSchema, ConfigSchema = St
toJSON(secrets?: boolean): Static<ReturnType<(typeof this)["getSchema"]>> {
return this.config;
}
// @todo: add a method to signal the requirement of database sync!!!
protected ensureEntity(entity: Entity) {
// check fields
if (!this.ctx.em.hasEntity(entity.name)) {
this.ctx.em.addEntity(entity);
this.ctx.flags.sync_required = true;
return;
}
const instance = this.ctx.em.entity(entity.name);
// if exists, check all fields required are there
// @todo: check if the field also equal
for (const field of entity.fields) {
const _field = instance.field(field.name);
if (!_field) {
instance.addField(field);
this.ctx.flags.sync_required = true;
}
}
}
protected ensureIndex(index: EntityIndex) {
if (!this.ctx.em.hasIndex(index)) {
this.ctx.em.addIndex(index);
this.ctx.flags.sync_required = true;
}
}
protected ensureSchema<Schema extends ReturnType<typeof prototypeEm>>(schema: Schema): Schema {
Object.values(schema.entities ?? {}).forEach(this.ensureEntity.bind(this));
schema.indices?.forEach(this.ensureIndex.bind(this));
return schema;
}
}

15
app/src/modules/ModuleManager.ts
View File

@@ -1,4 +1,3 @@
import type { App } from "App";
import { Guard } from "auth";
import { BkndError, DebugLogger } from "core";
import { EventManager } from "core/events";
@@ -34,7 +33,7 @@ import { AppAuth } from "../auth/AppAuth";
import { AppData } from "../data/AppData";
import { AppFlows } from "../flows/AppFlows";
import { AppMedia } from "../media/AppMedia";
import type { Module, ModuleBuildContext, ServerEnv } from "./Module";
import { Module, type ModuleBuildContext, type ServerEnv } from "./Module";
export type { ModuleBuildContext };
@@ -230,7 +229,8 @@ export class ModuleManager {
server: this.server,
em: this.em,
emgr: this.emgr,
guard: this.guard
guard: this.guard,
flags: Module.ctx_flags
};
}
@@ -415,7 +415,14 @@ export class ModuleManager {
}
this._built = true;
this.logger.log("modules built");
this.logger.log("modules built", ctx.flags);
if (ctx.flags.sync_required) {
this.logger.log("db sync requested");
await ctx.em.schema().sync({ force: true });
await this.save();
ctx.flags.sync_required = false; // reset
}
}
async build() {

34
app/vite.dev.ts
View File

@@ -1,4 +1,10 @@
import { serve } from "./src/adapter/vite";
import { serveStatic } from "@hono/node-server/serve-static";
import { createClient } from "@libsql/client/node";
import { App, registries } from "./src";
import { LibsqlConnection } from "./src/data";
import { StorageLocalAdapter } from "./src/media/storage/adapters/StorageLocalAdapter";
registries.media.register("local", StorageLocalAdapter);
const credentials = {
url: import.meta.env.VITE_DB_URL!,
@@ -8,10 +14,22 @@ if (!credentials.url) {
throw new Error("Missing VITE_DB_URL env variable. Add it to .env file");
}
export default serve({
connection: {
type: "libsql",
config: credentials
},
forceDev: true
});
const connection = new LibsqlConnection(createClient(credentials));
export default {
async fetch(request: Request) {
const app = App.create({ connection });
app.emgr.onEvent(
App.Events.AppBuiltEvent,
async () => {
app.registerAdminController({ forceDev: true });
app.module.server.client.get("/assets/*", serveStatic({ root: "./" }));
},
"sync"
);
await app.build();
return app.fetch(request);
}
};