diff --git a/app/__test__/App.spec.ts b/app/__test__/App.spec.ts index 0b456e1..57b5eeb 100644 --- a/app/__test__/App.spec.ts +++ b/app/__test__/App.spec.ts @@ -19,7 +19,7 @@ describe("App tests", async () => { test("plugins", async () => { const called: string[] = []; const app = createApp({ - initialConfig: { + config: { auth: { enabled: true, }, diff --git a/app/__test__/adapter/adapter.test.ts b/app/__test__/adapter/adapter.test.ts index ee9a87e..1644c89 100644 --- a/app/__test__/adapter/adapter.test.ts +++ b/app/__test__/adapter/adapter.test.ts @@ -19,52 +19,16 @@ describe("adapter", () => { expect( omitKeys( await adapter.makeConfig( - { app: (a) => ({ initialConfig: { server: { cors: { origin: a.env.TEST } } } }) }, + { app: (a) => ({ config: { server: { cors: { origin: a.env.TEST } } } }) }, { env: { TEST: "test" } }, ), ["connection"], ), ).toEqual({ - initialConfig: { server: { cors: { origin: "test" } } }, + config: { server: { cors: { origin: "test" } } }, }); }); - /* it.only("...", async () => { - const app = await adapter.createAdapterApp(); - }); */ - - it("reuses apps correctly", async () => { - const id = crypto.randomUUID(); - - const first = await adapter.createAdapterApp( - { - initialConfig: { server: { cors: { origin: "random" } } }, - }, - undefined, - { id }, - ); - const second = await adapter.createAdapterApp(); - const third = await adapter.createAdapterApp(undefined, undefined, { id }); - - await first.build(); - await second.build(); - await third.build(); - - expect(first.toJSON().server.cors.origin).toEqual("random"); - expect(first).toBe(third); - expect(first).not.toBe(second); - expect(second).not.toBe(third); - expect(second.toJSON().server.cors.origin).toEqual("*"); - - // recreate the first one - const first2 = await adapter.createAdapterApp(undefined, undefined, { id, force: true }); - await first2.build(); - expect(first2).not.toBe(first); - expect(first2).not.toBe(third); - expect(first2).not.toBe(second); - expect(first2.toJSON().server.cors.origin).toEqual("*"); - }); - adapterTestSuite(bunTestRunner, { makeApp: adapter.createFrameworkApp, label: "framework app", diff --git a/app/__test__/app/App.spec.ts b/app/__test__/app/App.spec.ts index aff3e53..fab85b8 100644 --- a/app/__test__/app/App.spec.ts +++ b/app/__test__/app/App.spec.ts @@ -1,9 +1,19 @@ import { describe, expect, mock, test } from "bun:test"; import type { ModuleBuildContext } from "../../src"; import { App, createApp } from "core/test/utils"; -import * as proto from "../../src/data/prototype"; +import * as proto from "data/prototype"; +import { DbModuleManager } from "modules/db/DbModuleManager"; describe("App", () => { + test("use db mode by default", async () => { + const app = createApp(); + await app.build(); + + expect(app.mode).toBe("db"); + expect(app.isReadOnly()).toBe(false); + expect(app.modules instanceof DbModuleManager).toBe(true); + }); + test("seed includes ctx and app", async () => { const called = mock(() => null); await createApp({ @@ -29,7 +39,7 @@ describe("App", () => { expect(called).toHaveBeenCalled(); const app = createApp({ - initialConfig: { + config: { data: proto .em({ todos: proto.entity("todos", { @@ -139,7 +149,7 @@ describe("App", () => { test("getMcpClient", async () => { const app = createApp({ - initialConfig: { + config: { server: { mcp: { enabled: true, diff --git a/app/__test__/app/mcp/mcp.auth.test.ts b/app/__test__/app/mcp/mcp.auth.test.ts index ea02274..4d20d40 100644 --- a/app/__test__/app/mcp/mcp.auth.test.ts +++ b/app/__test__/app/mcp/mcp.auth.test.ts @@ -29,7 +29,7 @@ describe("mcp auth", async () => { let server: McpServer; beforeEach(async () => { app = createApp({ - initialConfig: { + config: { auth: { enabled: true, jwt: { diff --git a/app/__test__/app/mcp/mcp.base.test.ts b/app/__test__/app/mcp/mcp.base.test.ts index df8bdeb..e861595 100644 --- a/app/__test__/app/mcp/mcp.base.test.ts +++ b/app/__test__/app/mcp/mcp.base.test.ts @@ -8,7 +8,7 @@ describe("mcp", () => { registries.media.register("local", StorageLocalAdapter); const app = createApp({ - initialConfig: { + config: { auth: { enabled: true, }, diff --git a/app/__test__/app/mcp/mcp.data.test.ts b/app/__test__/app/mcp/mcp.data.test.ts index 69b5106..4a962b3 100644 --- a/app/__test__/app/mcp/mcp.data.test.ts +++ b/app/__test__/app/mcp/mcp.data.test.ts @@ -41,7 +41,7 @@ describe("mcp data", async () => { beforeEach(async () => { const time = performance.now(); app = createApp({ - initialConfig: { + config: { server: { mcp: { enabled: true, diff --git a/app/__test__/app/mcp/mcp.media.test.ts b/app/__test__/app/mcp/mcp.media.test.ts index c10e8bb..ff47ddd 100644 --- a/app/__test__/app/mcp/mcp.media.test.ts +++ b/app/__test__/app/mcp/mcp.media.test.ts @@ -21,7 +21,7 @@ describe("mcp media", async () => { beforeEach(async () => { registries.media.register("local", StorageLocalAdapter); app = createApp({ - initialConfig: { + config: { media: { enabled: true, adapter: { diff --git a/app/__test__/app/mcp/mcp.server.test.ts b/app/__test__/app/mcp/mcp.server.test.ts index 3ada557..f9eb729 100644 --- a/app/__test__/app/mcp/mcp.server.test.ts +++ b/app/__test__/app/mcp/mcp.server.test.ts @@ -11,7 +11,7 @@ describe("mcp system", async () => { let server: McpServer; beforeAll(async () => { app = createApp({ - initialConfig: { + config: { server: { mcp: { enabled: true, diff --git a/app/__test__/app/mcp/mcp.system.test.ts b/app/__test__/app/mcp/mcp.system.test.ts index 6b08628..d20765f 100644 --- a/app/__test__/app/mcp/mcp.system.test.ts +++ b/app/__test__/app/mcp/mcp.system.test.ts @@ -14,7 +14,7 @@ describe("mcp system", async () => { let server: McpServer; beforeAll(async () => { app = createApp({ - initialConfig: { + config: { server: { mcp: { enabled: true, diff --git a/app/__test__/app/repro.spec.ts b/app/__test__/app/repro.spec.ts index be2da38..b18497f 100644 --- a/app/__test__/app/repro.spec.ts +++ b/app/__test__/app/repro.spec.ts @@ -88,7 +88,7 @@ describe("repros", async () => { fns.relation(schema.product_likes).manyToOne(schema.users); }, ); - const app = createApp({ initialConfig: { data: schema.toJSON() } }); + const app = createApp({ config: { data: schema.toJSON() } }); await app.build(); const info = (await (await app.server.request("/api/data/info/products")).json()) as any; diff --git a/app/__test__/integration/auth.integration.test.ts b/app/__test__/integration/auth.integration.test.ts index 298ad31..cf71f05 100644 --- a/app/__test__/integration/auth.integration.test.ts +++ b/app/__test__/integration/auth.integration.test.ts @@ -1,13 +1,9 @@ import { afterAll, afterEach, beforeAll, describe, expect, it } from "bun:test"; -import { App, createApp } from "../../src"; -import type { AuthResponse } from "../../src/auth"; +import { App, createApp, type AuthResponse } from "../../src"; import { auth } from "../../src/auth/middlewares"; import { randomString, secureRandomString, withDisabledConsole } from "../../src/core/utils"; import { disableConsoleLog, enableConsoleLog, getDummyConnection } from "../helper"; -const { dummyConnection, afterAllCleanup } = getDummyConnection(); -afterEach(afterAllCleanup); - beforeAll(disableConsoleLog); afterAll(enableConsoleLog); @@ -66,9 +62,10 @@ const configs = { }; function createAuthApp() { + const { dummyConnection } = getDummyConnection(); const app = createApp({ connection: dummyConnection, - initialConfig: { + config: { auth: configs.auth, }, }); diff --git a/app/__test__/media/MediaController.spec.ts b/app/__test__/media/MediaController.spec.ts index 7c9ae9f..f3aeebd 100644 --- a/app/__test__/media/MediaController.spec.ts +++ b/app/__test__/media/MediaController.spec.ts @@ -16,7 +16,7 @@ const path = `${assetsPath}/image.png`; async function makeApp(mediaOverride: Partial = {}) { const app = createApp({ - initialConfig: { + config: { media: mergeObject( { enabled: true, diff --git a/app/__test__/modules/AppAuth.spec.ts b/app/__test__/modules/AppAuth.spec.ts index e523fbc..5ea58ac 100644 --- a/app/__test__/modules/AppAuth.spec.ts +++ b/app/__test__/modules/AppAuth.spec.ts @@ -147,7 +147,7 @@ describe("AppAuth", () => { test("registers auth middleware for bknd routes only", async () => { const app = createApp({ - initialConfig: { + config: { auth: { enabled: true, jwt: { @@ -177,7 +177,7 @@ describe("AppAuth", () => { test("should allow additional user fields", async () => { const app = createApp({ - initialConfig: { + config: { auth: { entity_name: "users", enabled: true, @@ -201,7 +201,7 @@ describe("AppAuth", () => { test("ensure user field configs is always correct", async () => { const app = createApp({ - initialConfig: { + config: { auth: { enabled: true, }, diff --git a/app/__test__/modules/AppMedia.spec.ts b/app/__test__/modules/AppMedia.spec.ts index d09041b..ff86c86 100644 --- a/app/__test__/modules/AppMedia.spec.ts +++ b/app/__test__/modules/AppMedia.spec.ts @@ -18,7 +18,7 @@ describe("AppMedia", () => { registries.media.register("local", StorageLocalAdapter); const app = createApp({ - initialConfig: { + config: { media: { entity_name: "media", enabled: true, diff --git a/app/__test__/modules/DbModuleManager.spec.ts b/app/__test__/modules/DbModuleManager.spec.ts new file mode 100644 index 0000000..a148192 --- /dev/null +++ b/app/__test__/modules/DbModuleManager.spec.ts @@ -0,0 +1,22 @@ +import { it, expect, describe } from "bun:test"; +import { DbModuleManager } from "modules/db/DbModuleManager"; +import { getDummyConnection } from "../helper"; + +describe("DbModuleManager", () => { + it("should extract secrets", async () => { + const { dummyConnection } = getDummyConnection(false); + const m = new DbModuleManager(dummyConnection, { + initial: { + auth: { + enabled: true, + jwt: { + secret: "test", + }, + }, + }, + }); + await m.build(); + expect(m.toJSON(true).auth.jwt.secret).toBe("test"); + await m.save(); + }); +}); diff --git a/app/__test__/modules/ModuleManager.spec.ts b/app/__test__/modules/ModuleManager.spec.ts index 9c24de0..66f2380 100644 --- a/app/__test__/modules/ModuleManager.spec.ts +++ b/app/__test__/modules/ModuleManager.spec.ts @@ -2,8 +2,10 @@ import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test"; import { disableConsoleLog, enableConsoleLog } from "core/utils"; import { Module } from "modules/Module"; -import { type ConfigTable, getDefaultConfig, ModuleManager } from "modules/ModuleManager"; -import { CURRENT_VERSION, TABLE_NAME } from "modules/migrations"; +import { getDefaultConfig } from "modules/ModuleManager"; +import { type ConfigTable, DbModuleManager as ModuleManager } from "modules/db/DbModuleManager"; + +import { CURRENT_VERSION, TABLE_NAME } from "modules/db/migrations"; import { getDummyConnection } from "../helper"; import { s, stripMark } from "core/utils/schema"; import { Connection } from "data/connection/Connection"; diff --git a/app/internal/docs.build-assets.ts b/app/internal/docs.build-assets.ts index 127def7..5ad062c 100644 --- a/app/internal/docs.build-assets.ts +++ b/app/internal/docs.build-assets.ts @@ -3,7 +3,7 @@ import { createApp } from "bknd/adapter/bun"; async function generate() { console.info("Generating MCP documentation..."); const app = await createApp({ - initialConfig: { + config: { server: { mcp: { enabled: true, diff --git a/app/src/App.ts b/app/src/App.ts index bd62092..4712c55 100644 --- a/app/src/App.ts +++ b/app/src/App.ts @@ -5,13 +5,14 @@ import type { em as prototypeEm } from "data/prototype"; import { Connection } from "data/connection/Connection"; import type { Hono } from "hono"; import { - ModuleManager, type InitialModuleConfigs, - type ModuleBuildContext, type ModuleConfigs, - type ModuleManagerOptions, type Modules, + ModuleManager, + type ModuleBuildContext, + type ModuleManagerOptions, } from "modules/ModuleManager"; +import { DbModuleManager } from "modules/db/DbModuleManager"; import * as SystemPermissions from "modules/permissions"; import { AdminController, type AdminControllerOptions } from "modules/server/AdminController"; import { SystemController } from "modules/server/SystemController"; @@ -93,17 +94,19 @@ export type AppOptions = { email?: IEmailDriver; cache?: ICacheDriver; }; + mode?: "db" | "code"; + readonly?: boolean; }; export type CreateAppConfig = { /** * bla */ connection?: Connection | { url: string }; - initialConfig?: InitialModuleConfigs; + config?: InitialModuleConfigs; options?: AppOptions; }; -export type AppConfig = InitialModuleConfigs; +export type AppConfig = { version: number } & ModuleConfigs; export type LocalApiOptions = Request | ApiOptions; export class App { @@ -121,8 +124,8 @@ export class App(module: Module) { - return this.modules.mutateConfigSafe(module); - } - get server() { return this.modules.server; } @@ -377,5 +388,5 @@ export function createApp(config: CreateAppConfig = {}) { throw new Error("Invalid connection"); } - return new App(config.connection, config.initialConfig, config.options); + return new App(config.connection, config.config, config.options); } diff --git a/app/src/adapter/adapter-test-suite.ts b/app/src/adapter/adapter-test-suite.ts index dba432b..8ba2db3 100644 --- a/app/src/adapter/adapter-test-suite.ts +++ b/app/src/adapter/adapter-test-suite.ts @@ -1,5 +1,5 @@ import type { TestRunner } from "core/test"; -import type { BkndConfig, DefaultArgs, FrameworkOptions, RuntimeOptions } from "./index"; +import type { BkndConfig, DefaultArgs } from "./index"; import type { App } from "App"; export function adapterTestSuite< @@ -13,16 +13,8 @@ export function adapterTestSuite< label = "app", overrides = {}, }: { - makeApp: ( - config: Config, - args?: Args, - opts?: RuntimeOptions | FrameworkOptions, - ) => Promise; - makeHandler?: ( - config?: Config, - args?: Args, - opts?: RuntimeOptions | FrameworkOptions, - ) => (request: Request) => Promise; + makeApp: (config: Config, args?: Args) => Promise; + makeHandler?: (config?: Config, args?: Args) => (request: Request) => Promise; label?: string; overrides?: { dbUrl?: string; @@ -30,7 +22,6 @@ export function adapterTestSuite< }, ) { const { test, expect, mock } = testRunner; - const id = crypto.randomUUID(); test(`creates ${label}`, async () => { const beforeBuild = mock(async () => null) as any; @@ -39,7 +30,7 @@ export function adapterTestSuite< const config = { app: (env) => ({ connection: { url: env.url }, - initialConfig: { + config: { server: { cors: { origin: env.origin } }, }, }), @@ -53,7 +44,6 @@ export function adapterTestSuite< url: overrides.dbUrl ?? ":memory:", origin: "localhost", } as any, - { force: false, id }, ); expect(app).toBeDefined(); expect(app.toJSON().server.cors.origin).toEqual("localhost"); @@ -68,8 +58,8 @@ export function adapterTestSuite< return { res, data }; }; - test("responds with the same app id", async () => { - const fetcher = makeHandler(undefined, undefined, { force: false, id }); + /* test.skip("responds with the same app id", async () => { + const fetcher = makeHandler(undefined, undefined, { id }); const { res, data } = await getConfig(fetcher); expect(res.ok).toBe(true); @@ -77,14 +67,14 @@ export function adapterTestSuite< expect(data.server.cors.origin).toEqual("localhost"); }); - test("creates fresh & responds to api config", async () => { + test.skip("creates fresh & responds to api config", async () => { // set the same id, but force recreate - const fetcher = makeHandler(undefined, undefined, { id, force: true }); + const fetcher = makeHandler(undefined, undefined, { id }); const { res, data } = await getConfig(fetcher); expect(res.ok).toBe(true); expect(res.status).toBe(200); expect(data.server.cors.origin).toEqual("*"); - }); + }); */ } } diff --git a/app/src/adapter/astro/astro.adapter.spec.ts b/app/src/adapter/astro/astro.adapter.spec.ts index 3f3d1e8..54f3eb2 100644 --- a/app/src/adapter/astro/astro.adapter.spec.ts +++ b/app/src/adapter/astro/astro.adapter.spec.ts @@ -10,6 +10,6 @@ afterAll(enableConsoleLog); describe("astro adapter", () => { adapterTestSuite(bunTestRunner, { makeApp: astro.getApp, - makeHandler: (c, a, o) => (request: Request) => astro.serve(c, a, o)({ request }), + makeHandler: (c, a) => (request: Request) => astro.serve(c, a)({ request }), }); }); diff --git a/app/src/adapter/astro/astro.adapter.ts b/app/src/adapter/astro/astro.adapter.ts index a684f73..7f24923 100644 --- a/app/src/adapter/astro/astro.adapter.ts +++ b/app/src/adapter/astro/astro.adapter.ts @@ -1,4 +1,4 @@ -import { type FrameworkBkndConfig, createFrameworkApp, type FrameworkOptions } from "bknd/adapter"; +import { type FrameworkBkndConfig, createFrameworkApp } from "bknd/adapter"; type AstroEnv = NodeJS.ProcessEnv; type TAstro = { @@ -9,17 +9,12 @@ export type AstroBkndConfig = FrameworkBkndConfig; export async function getApp( config: AstroBkndConfig = {}, args: Env = {} as Env, - opts: FrameworkOptions = {}, ) { - return await createFrameworkApp(config, args ?? import.meta.env, opts); + return await createFrameworkApp(config, args ?? import.meta.env); } -export function serve( - config: AstroBkndConfig = {}, - args: Env = {} as Env, - opts?: FrameworkOptions, -) { +export function serve(config: AstroBkndConfig = {}, args: Env = {} as Env) { return async (fnArgs: TAstro) => { - return (await getApp(config, args, opts)).fetch(fnArgs.request); + return (await getApp(config, args)).fetch(fnArgs.request); }; } diff --git a/app/src/adapter/aws/aws-lambda.adapter.ts b/app/src/adapter/aws/aws-lambda.adapter.ts index ad19047..8c43b3d 100644 --- a/app/src/adapter/aws/aws-lambda.adapter.ts +++ b/app/src/adapter/aws/aws-lambda.adapter.ts @@ -1,7 +1,7 @@ import type { App } from "bknd"; import { handle } from "hono/aws-lambda"; import { serveStatic } from "@hono/node-server/serve-static"; -import { type RuntimeBkndConfig, createRuntimeApp, type RuntimeOptions } from "bknd/adapter"; +import { type RuntimeBkndConfig, createRuntimeApp } from "bknd/adapter"; type AwsLambdaEnv = object; export type AwsLambdaBkndConfig = @@ -20,7 +20,6 @@ export type AwsLambdaBkndConfig = export async function createApp( { adminOptions = false, assets, ...config }: AwsLambdaBkndConfig = {}, args: Env = {} as Env, - opts?: RuntimeOptions, ): Promise { let additional: Partial = { adminOptions, @@ -57,17 +56,15 @@ export async function createApp( ...additional, }, args ?? process.env, - opts, ); } export function serve( config: AwsLambdaBkndConfig = {}, args: Env = {} as Env, - opts?: RuntimeOptions, ) { return async (event) => { - const app = await createApp(config, args, opts); + const app = await createApp(config, args); return await handle(app.server)(event); }; } diff --git a/app/src/adapter/aws/aws.adapter.spec.ts b/app/src/adapter/aws/aws.adapter.spec.ts index e6873d8..e3007e4 100644 --- a/app/src/adapter/aws/aws.adapter.spec.ts +++ b/app/src/adapter/aws/aws.adapter.spec.ts @@ -11,8 +11,8 @@ describe("aws adapter", () => { adapterTestSuite(bunTestRunner, { makeApp: awsLambda.createApp, // @todo: add a request to lambda event translator? - makeHandler: (c, a, o) => async (request: Request) => { - const app = await awsLambda.createApp(c, a, o); + makeHandler: (c, a) => async (request: Request) => { + const app = await awsLambda.createApp(c, a); return app.fetch(request); }, }); diff --git a/app/src/adapter/bun/bun.adapter.ts b/app/src/adapter/bun/bun.adapter.ts index 03689d5..00b61b5 100644 --- a/app/src/adapter/bun/bun.adapter.ts +++ b/app/src/adapter/bun/bun.adapter.ts @@ -1,7 +1,7 @@ /// import path from "node:path"; -import { type RuntimeBkndConfig, createRuntimeApp, type RuntimeOptions } from "bknd/adapter"; +import { type RuntimeBkndConfig, createRuntimeApp } from "bknd/adapter"; import { registerLocalMediaAdapter } from "."; import { config, type App } from "bknd"; import type { ServeOptions } from "bun"; @@ -13,7 +13,6 @@ export type BunBkndConfig = RuntimeBkndConfig & Omit( { distPath, serveStatic: _serveStatic, ...config }: BunBkndConfig = {}, args: Env = {} as Env, - opts?: RuntimeOptions, ) { const root = path.resolve(distPath ?? "./node_modules/bknd/dist", "static"); registerLocalMediaAdapter(); @@ -28,19 +27,17 @@ export async function createApp( ...config, }, args ?? (process.env as Env), - opts, ); } export function createHandler( config: BunBkndConfig = {}, args: Env = {} as Env, - opts?: RuntimeOptions, ) { let app: App | undefined; return async (req: Request) => { if (!app) { - app = await createApp(config, args ?? (process.env as Env), opts); + app = await createApp(config, args ?? (process.env as Env)); } return app.fetch(req); }; @@ -50,7 +47,7 @@ export function serve( { distPath, connection, - initialConfig, + config: _config, options, port = config.server.default_port, onBuilt, @@ -60,7 +57,6 @@ export function serve( ...serveOptions }: BunBkndConfig = {}, args: Env = {} as Env, - opts?: RuntimeOptions, ) { Bun.serve({ ...serveOptions, @@ -68,7 +64,7 @@ export function serve( fetch: createHandler( { connection, - initialConfig, + config: _config, options, onBuilt, buildConfig, @@ -77,7 +73,6 @@ export function serve( serveStatic, }, args, - opts, ), }); diff --git a/app/src/adapter/cloudflare/cloudflare-workers.adapter.spec.ts b/app/src/adapter/cloudflare/cloudflare-workers.adapter.spec.ts index 64ba65b..65477b6 100644 --- a/app/src/adapter/cloudflare/cloudflare-workers.adapter.spec.ts +++ b/app/src/adapter/cloudflare/cloudflare-workers.adapter.spec.ts @@ -5,8 +5,8 @@ import { adapterTestSuite } from "adapter/adapter-test-suite"; import { bunTestRunner } from "adapter/bun/test"; import { type CloudflareBkndConfig, createApp } from "./cloudflare-workers.adapter"; -beforeAll(disableConsoleLog); -afterAll(enableConsoleLog); +/* beforeAll(disableConsoleLog); +afterAll(enableConsoleLog); */ describe("cf adapter", () => { const DB_URL = ":memory:"; @@ -20,31 +20,31 @@ describe("cf adapter", () => { const staticConfig = await makeConfig( { connection: { url: DB_URL }, - initialConfig: { data: { basepath: DB_URL } }, + config: { data: { basepath: DB_URL } }, }, $ctx({ DB_URL }), ); - expect(staticConfig.initialConfig).toEqual({ data: { basepath: DB_URL } }); + expect(staticConfig.config).toEqual({ data: { basepath: DB_URL } }); expect(staticConfig.connection).toBeDefined(); const dynamicConfig = await makeConfig( { app: (env) => ({ - initialConfig: { data: { basepath: env.DB_URL } }, + config: { data: { basepath: env.DB_URL } }, connection: { url: env.DB_URL }, }), }, $ctx({ DB_URL }), ); - expect(dynamicConfig.initialConfig).toEqual({ data: { basepath: DB_URL } }); + expect(dynamicConfig.config).toEqual({ data: { basepath: DB_URL } }); expect(dynamicConfig.connection).toBeDefined(); }); adapterTestSuite>(bunTestRunner, { - makeApp: async (c, a, o) => { - return await createApp(c, { env: a } as any, o); + makeApp: async (c, a) => { + return await createApp(c, { env: a } as any); }, - makeHandler: (c, a, o) => { + makeHandler: (c, a) => { console.log("args", a); return async (request: any) => { const app = await createApp( @@ -53,7 +53,6 @@ describe("cf adapter", () => { connection: { url: DB_URL }, }, a as any, - o, ); return app.fetch(request); }; diff --git a/app/src/adapter/cloudflare/cloudflare-workers.adapter.ts b/app/src/adapter/cloudflare/cloudflare-workers.adapter.ts index 091dc30..fe278c4 100644 --- a/app/src/adapter/cloudflare/cloudflare-workers.adapter.ts +++ b/app/src/adapter/cloudflare/cloudflare-workers.adapter.ts @@ -5,7 +5,7 @@ import { Hono } from "hono"; import { serveStatic } from "hono/cloudflare-workers"; import type { MaybePromise } from "bknd"; import { $console } from "bknd/utils"; -import { createRuntimeApp, type RuntimeOptions } from "bknd/adapter"; +import { createRuntimeApp } from "bknd/adapter"; import { registerAsyncsExecutionContext, makeConfig, type CloudflareContext } from "./config"; declare global { @@ -34,12 +34,8 @@ export type CloudflareBkndConfig = RuntimeBkndConfig & }; export async function createApp( - config: CloudflareBkndConfig, + config: CloudflareBkndConfig = {}, ctx: Partial> = {}, - opts: RuntimeOptions = { - // by default, require the app to be rebuilt every time - force: true, - }, ) { const appConfig = await makeConfig( { @@ -53,7 +49,7 @@ export async function createApp( }, ctx, ); - return await createRuntimeApp(appConfig, ctx?.env, opts); + return await createRuntimeApp(appConfig, ctx?.env); } // compatiblity diff --git a/app/src/adapter/cloudflare/index.ts b/app/src/adapter/cloudflare/index.ts index 24f0459..a09da7b 100644 --- a/app/src/adapter/cloudflare/index.ts +++ b/app/src/adapter/cloudflare/index.ts @@ -3,6 +3,7 @@ import { d1Sqlite, type D1ConnectionConfig } from "./connection/D1Connection"; export { getFresh, createApp, + serve, type CloudflareEnv, type CloudflareBkndConfig, } from "./cloudflare-workers.adapter"; diff --git a/app/src/adapter/index.ts b/app/src/adapter/index.ts index 91ffcf7..fbb5e5c 100644 --- a/app/src/adapter/index.ts +++ b/app/src/adapter/index.ts @@ -21,13 +21,6 @@ export type BkndConfig = CreateAppConfig & { export type FrameworkBkndConfig = BkndConfig; -export type CreateAdapterAppOptions = { - force?: boolean; - id?: string; -}; -export type FrameworkOptions = CreateAdapterAppOptions; -export type RuntimeOptions = CreateAdapterAppOptions; - export type RuntimeBkndConfig = BkndConfig & { distPath?: string; serveStatic?: MiddlewareHandler | [string, MiddlewareHandler]; @@ -63,41 +56,29 @@ const apps = new Map(); export async function createAdapterApp( config: Config = {} as Config, args?: Args, - opts?: CreateAdapterAppOptions, ): Promise { - const id = opts?.id ?? "app"; - let app = apps.get(id); - if (!app || opts?.force) { - const appConfig = await makeConfig(config, args); - if (!appConfig.connection || !Connection.isConnection(appConfig.connection)) { - let connection: Connection | undefined; - if (Connection.isConnection(config.connection)) { - connection = config.connection; - } else { - const sqlite = (await import("bknd/adapter/sqlite")).sqlite; - const conf = appConfig.connection ?? { url: ":memory:" }; - connection = sqlite(conf) as any; - $console.info(`Using ${connection!.name} connection`, conf.url); - } - appConfig.connection = connection; - } - - app = App.create(appConfig); - - if (!opts?.force) { - apps.set(id, app); + const appConfig = await makeConfig(config, args); + if (!appConfig.connection || !Connection.isConnection(appConfig.connection)) { + let connection: Connection | undefined; + if (Connection.isConnection(config.connection)) { + connection = config.connection; + } else { + const sqlite = (await import("bknd/adapter/sqlite")).sqlite; + const conf = appConfig.connection ?? { url: ":memory:" }; + connection = sqlite(conf) as any; + $console.info(`Using ${connection!.name} connection`, conf.url); } + appConfig.connection = connection; } - return app; + return App.create(appConfig); } export async function createFrameworkApp( config: FrameworkBkndConfig = {}, args?: Args, - opts?: FrameworkOptions, ): Promise { - const app = await createAdapterApp(config, args, opts); + const app = await createAdapterApp(config, args); if (!app.isBuilt()) { if (config.onBuilt) { @@ -120,9 +101,8 @@ export async function createFrameworkApp( export async function createRuntimeApp( { serveStatic, adminOptions, ...config }: RuntimeBkndConfig = {}, args?: Args, - opts?: RuntimeOptions, ): Promise { - const app = await createAdapterApp(config, args, opts); + const app = await createAdapterApp(config, args); if (!app.isBuilt()) { app.emgr.onEvent( diff --git a/app/src/adapter/nextjs/nextjs.adapter.ts b/app/src/adapter/nextjs/nextjs.adapter.ts index bce7009..ba0953b 100644 --- a/app/src/adapter/nextjs/nextjs.adapter.ts +++ b/app/src/adapter/nextjs/nextjs.adapter.ts @@ -1,4 +1,4 @@ -import { createFrameworkApp, type FrameworkBkndConfig, type FrameworkOptions } from "bknd/adapter"; +import { createFrameworkApp, type FrameworkBkndConfig } from "bknd/adapter"; import { isNode } from "bknd/utils"; import type { NextApiRequest } from "next"; @@ -10,9 +10,8 @@ export type NextjsBkndConfig = FrameworkBkndConfig & { export async function getApp( config: NextjsBkndConfig, args: Env = {} as Env, - opts?: FrameworkOptions, ) { - return await createFrameworkApp(config, args ?? (process.env as Env), opts); + return await createFrameworkApp(config, args ?? (process.env as Env)); } function getCleanRequest(req: Request, cleanRequest: NextjsBkndConfig["cleanRequest"]) { @@ -41,10 +40,9 @@ function getCleanRequest(req: Request, cleanRequest: NextjsBkndConfig["cleanRequ export function serve( { cleanRequest, ...config }: NextjsBkndConfig = {}, args: Env = {} as Env, - opts?: FrameworkOptions, ) { return async (req: Request) => { - const app = await getApp(config, args, opts); + const app = await getApp(config, args); const request = getCleanRequest(req, cleanRequest); return app.fetch(request); }; diff --git a/app/src/adapter/node/node.adapter.ts b/app/src/adapter/node/node.adapter.ts index 5a2c058..fd96086 100644 --- a/app/src/adapter/node/node.adapter.ts +++ b/app/src/adapter/node/node.adapter.ts @@ -2,7 +2,7 @@ import path from "node:path"; import { serve as honoServe } from "@hono/node-server"; import { serveStatic } from "@hono/node-server/serve-static"; import { registerLocalMediaAdapter } from "adapter/node/storage"; -import { type RuntimeBkndConfig, createRuntimeApp, type RuntimeOptions } from "bknd/adapter"; +import { type RuntimeBkndConfig, createRuntimeApp } from "bknd/adapter"; import { config as $config, type App } from "bknd"; import { $console } from "bknd/utils"; @@ -18,7 +18,6 @@ export type NodeBkndConfig = RuntimeBkndConfig & { export async function createApp( { distPath, relativeDistPath, ...config }: NodeBkndConfig = {}, args: Env = {} as Env, - opts?: RuntimeOptions, ) { const root = path.relative( process.cwd(), @@ -36,19 +35,17 @@ export async function createApp( }, // @ts-ignore args ?? { env: process.env }, - opts, ); } export function createHandler( config: NodeBkndConfig = {}, args: Env = {} as Env, - opts?: RuntimeOptions, ) { let app: App | undefined; return async (req: Request) => { if (!app) { - app = await createApp(config, args ?? (process.env as Env), opts); + app = await createApp(config, args ?? (process.env as Env)); } return app.fetch(req); }; @@ -57,13 +54,12 @@ export function createHandler( export function serve( { port = $config.server.default_port, hostname, listener, ...config }: NodeBkndConfig = {}, args: Env = {} as Env, - opts?: RuntimeOptions, ) { honoServe( { port, hostname, - fetch: createHandler(config, args, opts), + fetch: createHandler(config, args), }, (connInfo) => { $console.log(`Server is running on http://localhost:${connInfo.port}`); diff --git a/app/src/adapter/react-router/react-router.adapter.spec.ts b/app/src/adapter/react-router/react-router.adapter.spec.ts index 25ef895..bb525c7 100644 --- a/app/src/adapter/react-router/react-router.adapter.spec.ts +++ b/app/src/adapter/react-router/react-router.adapter.spec.ts @@ -10,6 +10,6 @@ afterAll(enableConsoleLog); describe("react-router adapter", () => { adapterTestSuite(bunTestRunner, { makeApp: rr.getApp, - makeHandler: (c, a, o) => (request: Request) => rr.serve(c, a?.env, o)({ request }), + makeHandler: (c, a) => (request: Request) => rr.serve(c, a?.env)({ request }), }); }); diff --git a/app/src/adapter/react-router/react-router.adapter.ts b/app/src/adapter/react-router/react-router.adapter.ts index 4474509..f37260d 100644 --- a/app/src/adapter/react-router/react-router.adapter.ts +++ b/app/src/adapter/react-router/react-router.adapter.ts @@ -1,5 +1,4 @@ import { type FrameworkBkndConfig, createFrameworkApp } from "bknd/adapter"; -import type { FrameworkOptions } from "adapter"; type ReactRouterEnv = NodeJS.ProcessEnv; type ReactRouterFunctionArgs = { @@ -10,17 +9,15 @@ export type ReactRouterBkndConfig = FrameworkBkndConfig( config: ReactRouterBkndConfig, args: Env = {} as Env, - opts?: FrameworkOptions, ) { - return await createFrameworkApp(config, args ?? process.env, opts); + return await createFrameworkApp(config, args ?? process.env); } export function serve( config: ReactRouterBkndConfig = {}, args: Env = {} as Env, - opts?: FrameworkOptions, ) { return async (fnArgs: ReactRouterFunctionArgs) => { - return (await getApp(config, args, opts)).fetch(fnArgs.request); + return (await getApp(config, args)).fetch(fnArgs.request); }; } diff --git a/app/src/adapter/vite/vite.adapter.ts b/app/src/adapter/vite/vite.adapter.ts index c69bc1e..a4cb346 100644 --- a/app/src/adapter/vite/vite.adapter.ts +++ b/app/src/adapter/vite/vite.adapter.ts @@ -1,7 +1,7 @@ import { serveStatic } from "@hono/node-server/serve-static"; import { type DevServerOptions, default as honoViteDevServer } from "@hono/vite-dev-server"; import type { App } from "bknd"; -import { type RuntimeBkndConfig, createRuntimeApp, type FrameworkOptions } from "bknd/adapter"; +import { type RuntimeBkndConfig, createRuntimeApp } from "bknd/adapter"; import { registerLocalMediaAdapter } from "bknd/adapter/node"; import { devServerConfig } from "./dev-server-config"; import type { MiddlewareHandler } from "hono"; @@ -30,7 +30,6 @@ ${addBkndContext ? "" : ""} async function createApp( config: ViteBkndConfig = {}, env: ViteEnv = {} as ViteEnv, - opts: FrameworkOptions = {}, ): Promise { registerLocalMediaAdapter(); return await createRuntimeApp( @@ -47,18 +46,13 @@ async function createApp( ], }, env, - opts, ); } -export function serve( - config: ViteBkndConfig = {}, - args?: ViteEnv, - opts?: FrameworkOptions, -) { +export function serve(config: ViteBkndConfig = {}, args?: ViteEnv) { return { async fetch(request: Request, env: any, ctx: ExecutionContext) { - const app = await createApp(config, env, opts); + const app = await createApp(config, env); return app.fetch(request, env, ctx); }, }; diff --git a/app/src/core/types.ts b/app/src/core/types.ts index 1751766..03beae5 100644 --- a/app/src/core/types.ts +++ b/app/src/core/types.ts @@ -4,3 +4,5 @@ export interface Serializable { } export type MaybePromise = T | Promise; + +export type PartialRec = { [P in keyof T]?: PartialRec }; diff --git a/app/src/core/utils/objects.ts b/app/src/core/utils/objects.ts index 4a5e129..1c3cd82 100644 --- a/app/src/core/utils/objects.ts +++ b/app/src/core/utils/objects.ts @@ -396,6 +396,38 @@ export function getPath( } } +export function setPath(object: object, _path: string | (string | number)[], value: any) { + let path = _path; + // Optional string-path support. + // You can remove this `if` block if you don't need it. + if (typeof path === "string") { + const isQuoted = (str) => str[0] === '"' && str.at(-1) === '"'; + path = path + .split(/[.\[\]]+/) + .filter((x) => x) + .map((x) => (!Number.isNaN(Number(x)) ? Number(x) : x)) + .map((x) => (typeof x === "string" && isQuoted(x) ? x.slice(1, -1) : x)); + } + + if (path.length === 0) { + throw new Error("The path must have at least one entry in it"); + } + + const [head, ...tail] = path as any; + + if (tail.length === 0) { + object[head] = value; + return object; + } + + if (!(head in object)) { + object[head] = typeof tail[0] === "number" ? [] : {}; + } + + setPath(object[head], tail, value); + return object; +} + export function objectToJsLiteral(value: object, indent: number = 0, _level: number = 0): string { const nl = indent ? "\n" : ""; const pad = (lvl: number) => (indent ? " ".repeat(indent * lvl) : ""); diff --git a/app/src/data/api/DataController.ts b/app/src/data/api/DataController.ts index a0d7f02..18b26c2 100644 --- a/app/src/data/api/DataController.ts +++ b/app/src/data/api/DataController.ts @@ -51,6 +51,7 @@ export class DataController extends Controller { "/sync", permission(DataPermissions.databaseSync), mcpTool("data_sync", { + // @todo: should be removed if readonly annotations: { destructiveHint: true, }, diff --git a/app/src/data/connection/connection-test-suite.ts b/app/src/data/connection/connection-test-suite.ts index aed28aa..894f143 100644 --- a/app/src/data/connection/connection-test-suite.ts +++ b/app/src/data/connection/connection-test-suite.ts @@ -247,7 +247,7 @@ export function connectionTestSuite( const app = createApp({ connection: ctx.connection, - initialConfig: { + config: { data: schema.toJSON(), }, }); @@ -333,7 +333,7 @@ export function connectionTestSuite( const app = createApp({ connection: ctx.connection, - initialConfig: { + config: { data: schema.toJSON(), }, }); diff --git a/app/src/media/storage/adapters/cloudinary/StorageCloudinaryAdapter.ts b/app/src/media/storage/adapters/cloudinary/StorageCloudinaryAdapter.ts index 105dfef..490047b 100644 --- a/app/src/media/storage/adapters/cloudinary/StorageCloudinaryAdapter.ts +++ b/app/src/media/storage/adapters/cloudinary/StorageCloudinaryAdapter.ts @@ -1,12 +1,12 @@ -import { hash, pickHeaders, s, parse } from "bknd/utils"; +import { hash, pickHeaders, s, parse, secret } from "bknd/utils"; import type { FileBody, FileListObject, FileMeta } from "../../Storage"; import { StorageAdapter } from "../../StorageAdapter"; export const cloudinaryAdapterConfig = s.object( { cloud_name: s.string(), - api_key: s.string(), - api_secret: s.string(), + api_key: secret(), + api_secret: secret(), upload_preset: s.string().optional(), }, { title: "Cloudinary", description: "Cloudinary media storage" }, diff --git a/app/src/media/storage/adapters/s3/StorageS3Adapter.ts b/app/src/media/storage/adapters/s3/StorageS3Adapter.ts index bb89265..5926ebe 100644 --- a/app/src/media/storage/adapters/s3/StorageS3Adapter.ts +++ b/app/src/media/storage/adapters/s3/StorageS3Adapter.ts @@ -8,15 +8,15 @@ import type { } from "@aws-sdk/client-s3"; import { AwsClient } from "core/clients/aws/AwsClient"; import { isDebug } from "core/env"; -import { isFile, pickHeaders2, parse, s } from "bknd/utils"; +import { isFile, pickHeaders2, parse, s, secret } from "bknd/utils"; import { transform } from "lodash-es"; import type { FileBody, FileListObject } from "../../Storage"; import { StorageAdapter } from "../../StorageAdapter"; export const s3AdapterConfig = s.object( { - access_key: s.string(), - secret_access_key: s.string(), + access_key: secret(), + secret_access_key: secret(), url: s.string({ pattern: "^https?://(?:.*)?[^/.]+$", description: "URL to S3 compatible endpoint without trailing slash", diff --git a/app/src/modules/ModuleManager.ts b/app/src/modules/ModuleManager.ts index 2948b5c..7a49225 100644 --- a/app/src/modules/ModuleManager.ts +++ b/app/src/modules/ModuleManager.ts @@ -1,26 +1,20 @@ -import { mark, stripMark, $console, s, objectEach, transformObject, McpServer } from "bknd/utils"; +import { objectEach, transformObject, McpServer, type s } from "bknd/utils"; import { DebugLogger } from "core/utils/DebugLogger"; import { Guard } from "auth/authorize/Guard"; import { env } from "core/env"; -import { BkndError } from "core/errors"; import { EventManager, Event } from "core/events"; -import * as $diff from "core/object/diff"; import type { Connection } from "data/connection"; import { EntityManager } from "data/entities/EntityManager"; -import * as proto from "data/prototype"; -import { TransformPersistFailedException } from "data/errors"; import { Hono } from "hono"; -import type { Kysely } from "kysely"; -import { mergeWith } from "lodash-es"; -import { CURRENT_VERSION, TABLE_NAME, migrate } from "modules/migrations"; -import { AppServer } from "modules/server/AppServer"; -import { AppAuth } from "../auth/AppAuth"; -import { AppData } from "../data/AppData"; -import { AppFlows } from "../flows/AppFlows"; -import { AppMedia } from "../media/AppMedia"; import type { ServerEnv } from "./Controller"; import { Module, type ModuleBuildContext } from "./Module"; import { ModuleHelper } from "./ModuleHelper"; +import { AppServer } from "modules/server/AppServer"; +import { AppAuth } from "auth/AppAuth"; +import { AppData } from "data/AppData"; +import { AppFlows } from "flows/AppFlows"; +import { AppMedia } from "media/AppMedia"; +import type { PartialRec } from "core/types"; export type { ModuleBuildContext }; @@ -47,13 +41,8 @@ export type ModuleSchemas = { export type ModuleConfigs = { [K in keyof ModuleSchemas]: s.Static; }; -type PartialRec = { [P in keyof T]?: PartialRec }; -export type InitialModuleConfigs = - | ({ - version: number; - } & ModuleConfigs) - | PartialRec; +export type InitialModuleConfigs = { version?: number } & PartialRec; enum Verbosity { silent = 0, @@ -80,42 +69,14 @@ export type ModuleManagerOptions = { seed?: (ctx: ModuleBuildContext) => Promise; // called right after modules are built, before finish onModulesBuilt?: (ctx: ModuleBuildContext) => Promise; + // whether to store secrets in the database + storeSecrets?: boolean; + // provided secrets + secrets?: Record; /** @deprecated */ verbosity?: Verbosity; }; -export type ConfigTable = { - id?: number; - version: number; - type: "config" | "diff" | "backup"; - json: Json; - created_at?: Date; - updated_at?: Date; -}; - -const configJsonSchema = s.anyOf([ - getDefaultSchema(), - s.array( - s.strictObject({ - t: s.string({ enum: ["a", "r", "e"] }), - p: s.array(s.anyOf([s.string(), s.number()])), - o: s.any().optional(), - n: s.any().optional(), - }), - ), -]); -export const __bknd = proto.entity(TABLE_NAME, { - version: proto.number().required(), - type: proto.enumm({ enum: ["config", "diff", "backup"] }).required(), - json: proto.jsonSchema({ schema: configJsonSchema.toJSON() }).required(), - created_at: proto.datetime(), - updated_at: proto.datetime(), -}); -type ConfigTable2 = proto.Schema; -interface T_INTERNAL_EM { - __bknd: ConfigTable2; -} - const debug_modules = env("modules_debug"); abstract class ModuleManagerEvent extends Event<{ ctx: ModuleBuildContext } & A> {} @@ -127,8 +88,14 @@ export class ModuleManagerConfigUpdateEvent< }> { static override slug = "mm-config-update"; } +export class ModuleManagerSecretsExtractedEvent extends ModuleManagerEvent<{ + secrets: Record; +}> { + static override slug = "mm-secrets-extracted"; +} export const ModuleManagerEvents = { ModuleManagerConfigUpdateEvent, + ModuleManagerSecretsExtractedEvent, }; // @todo: cleanup old diffs on upgrade @@ -137,8 +104,6 @@ export class ModuleManager { static Events = ModuleManagerEvents; protected modules: Modules; - // internal em for __bknd config table - __em!: EntityManager; // ctx for modules em!: EntityManager; server!: Hono; @@ -146,42 +111,24 @@ export class ModuleManager { guard!: Guard; mcp!: ModuleBuildContext["mcp"]; - private _version: number = 0; - private _built = false; - private readonly _booted_with?: "provided" | "partial"; - private _stable_configs: ModuleConfigs | undefined; + protected _built = false; - private logger: DebugLogger; + protected logger: DebugLogger; constructor( - private readonly connection: Connection, - private options?: Partial, + protected readonly connection: Connection, + protected options?: Partial, ) { - this.__em = new EntityManager([__bknd], this.connection); this.modules = {} as Modules; this.emgr = new EventManager({ ...ModuleManagerEvents }); this.logger = new DebugLogger(debug_modules); - let initial = {} as Partial; - if (options?.initial) { - if ("version" in options.initial) { - const { version, ...initialConfig } = options.initial; - this._version = version; - initial = stripMark(initialConfig); - - this._booted_with = "provided"; - } else { - initial = mergeWith(getDefaultConfig(), options.initial); - this._booted_with = "partial"; - } - } - - this.logger.log("booted with", this._booted_with); - - this.createModules(initial); + this.createModules(options?.initial ?? {}); } - private createModules(initial: Partial) { + protected onModuleConfigUpdated(key: string, config: any) {} + + private createModules(initial: PartialRec) { this.logger.context("createModules").log("creating modules"); try { const context = this.ctx(true); @@ -211,46 +158,7 @@ export class ModuleManager { return this._built; } - /** - * This is set through module's setListener - * It's called everytime a module's config is updated in SchemaObject - * Needs to rebuild modules and save to database - */ - private async onModuleConfigUpdated(key: string, config: any) { - if (this.options?.onUpdated) { - await this.options.onUpdated(key as any, config); - } else { - await this.buildModules(); - } - } - - private repo() { - return this.__em.repo(__bknd, { - // to prevent exceptions when table doesn't exist - silent: true, - // disable counts for performance and compatibility - includeCounts: false, - }); - } - - private mutator() { - return this.__em.mutator(__bknd); - } - - private get db() { - // @todo: check why this is neccessary - return this.connection.kysely as unknown as Kysely<{ table: ConfigTable }>; - } - - // @todo: add indices for: version, type - async syncConfigTable() { - this.logger.context("sync").log("start"); - const result = await this.__em.schema().sync({ force: true }); - this.logger.log("done").clear(); - return result; - } - - private rebuildServer() { + protected rebuildServer() { this.server = new Hono(); if (this.options?.basePath) { this.server = this.server.basePath(this.options.basePath); @@ -299,252 +207,33 @@ export class ModuleManager { }; } - private async fetch(): Promise { - this.logger.context("fetch").log("fetching"); - const startTime = performance.now(); - - // disabling console log, because the table might not exist yet - const { data: result } = await this.repo().findOne( - { type: "config" }, - { - sort: { by: "version", dir: "desc" }, - }, - ); - - if (!result) { - this.logger.log("error fetching").clear(); - return undefined; - } - - this.logger - .log("took", performance.now() - startTime, "ms", { - version: result.version, - id: result.id, - }) - .clear(); - - return result as unknown as ConfigTable; - } - - async save() { - this.logger.context("save").log("saving version", this.version()); - const configs = this.configs(); - const version = this.version(); - - try { - const state = await this.fetch(); - if (!state) throw new BkndError("no config found"); - this.logger.log("fetched version", state.version); - - if (state.version !== version) { - // @todo: mark all others as "backup" - this.logger.log("version conflict, storing new version", state.version, version); - await this.mutator().insertOne({ - version: state.version, - type: "backup", - json: configs, - }); - await this.mutator().insertOne({ - version: version, - type: "config", - json: configs, - }); - } else { - this.logger.log("version matches", state.version); - - // clean configs because of Diff() function - const diffs = $diff.diff(state.json, $diff.clone(configs)); - this.logger.log("checking diff", [diffs.length]); - - if (diffs.length > 0) { - // validate diffs, it'll throw on invalid - this.validateDiffs(diffs); - - const date = new Date(); - // store diff - await this.mutator().insertOne({ - version, - type: "diff", - json: $diff.clone(diffs), - created_at: date, - updated_at: date, - }); - - // store new version - await this.mutator().updateWhere( - { - version, - json: configs, - updated_at: date, - } as any, - { - type: "config", - version, - }, - ); - } else { - this.logger.log("no diff, not saving"); - } - } - } catch (e) { - if (e instanceof BkndError && e.message === "no config found") { - this.logger.log("no config, just save fresh"); - // no config, just save - await this.mutator().insertOne({ - type: "config", - version, - json: configs, - created_at: new Date(), - updated_at: new Date(), - }); - } else if (e instanceof TransformPersistFailedException) { - $console.error("ModuleManager: Cannot save invalid config"); - this.revertModules(); - throw e; - } else { - $console.error("ModuleManager: Aborting"); - this.revertModules(); - throw e; - } - } - - // re-apply configs to all modules (important for system entities) - this.setConfigs(configs); - - // @todo: cleanup old versions? - - this.logger.clear(); - return this; - } - - private revertModules() { - if (this._stable_configs) { - $console.warn("ModuleManager: Reverting modules"); - this.setConfigs(this._stable_configs as any); - } else { - $console.error("ModuleManager: No stable configs to revert to"); - } - } - - /** - * Validates received diffs for an additional security control. - * Checks: - * - check if module is registered - * - run modules onBeforeUpdate() for added protection - * - * **Important**: Throw `Error` so it won't get catched. - * - * @param diffs - * @private - */ - private validateDiffs(diffs: $diff.DiffEntry[]): void { - // check top level paths, and only allow a single module to be modified in a single transaction - const modules = [...new Set(diffs.map((d) => d.p[0]))]; - if (modules.length === 0) { - return; - } - - for (const moduleName of modules) { - const name = moduleName as ModuleKey; - const module = this.get(name) as Module; - if (!module) { - const msg = "validateDiffs: module not registered"; - // biome-ignore format: ... - $console.error(msg, JSON.stringify({ module: name, diffs }, null, 2)); - throw new Error(msg); - } - - // pass diffs to the module to allow it to throw - if (this._stable_configs?.[name]) { - const current = $diff.clone(this._stable_configs?.[name]); - const modified = $diff.apply({ [name]: current }, diffs)[name]; - module.onBeforeUpdate(current, modified); - } - } - } - - private setConfigs(configs: ModuleConfigs): void { + protected async setConfigs(configs: ModuleConfigs): Promise { this.logger.log("setting configs"); - objectEach(configs, (config, key) => { + for await (const [key, config] of Object.entries(configs)) { try { // setting "noEmit" to true, to not force listeners to update - this.modules[key].schema().set(config as any, true); + const result = await this.modules[key].schema().set(config as any, true); } catch (e) { console.error(e); throw new Error( `Failed to set config for module ${key}: ${JSON.stringify(config, null, 2)}`, ); } - }); + } } - async build(opts?: { fetch?: boolean }) { - this.logger.context("build").log("version", this.version()); - await this.ctx().connection.init(); + async build(opts?: any) { + this.createModules(this.options?.initial ?? {}); + await this.buildModules(); - // if no config provided, try fetch from db - if (this.version() === 0 || opts?.fetch === true) { - if (opts?.fetch) { - this.logger.log("force fetch"); - } - - const result = await this.fetch(); - - // if no version, and nothing found, go with initial - if (!result) { - this.logger.log("nothing in database, go initial"); - await this.setupInitial(); - } else { - this.logger.log("db has", result.version); - // set version and config from fetched - this._version = result.version; - - if (this.options?.trustFetched === true) { - this.logger.log("trusting fetched config (mark)"); - mark(result.json); - } - - // if version doesn't match, migrate before building - if (this.version() !== CURRENT_VERSION) { - this.logger.log("now migrating"); - - await this.syncConfigTable(); - - const version_before = this.version(); - const [_version, _configs] = await migrate(version_before, result.json, { - db: this.db, - }); - - this._version = _version; - this.ctx().flags.sync_required = true; - - this.logger.log("migrated to", _version); - $console.log("Migrated config from", version_before, "to", this.version()); - - this.createModules(_configs); - await this.buildModules(); - } else { - this.logger.log("version is current", this.version()); - this.createModules(result.json); - await this.buildModules(); - } - } - } else { - if (this.version() !== CURRENT_VERSION) { - throw new Error( - `Given version (${this.version()}) and current version (${CURRENT_VERSION}) do not match.`, - ); - } - this.logger.log("current version is up to date", this.version()); - await this.buildModules(); - } - - this.logger.log("done"); - this.logger.clear(); return this; } - private async buildModules(options?: { graceful?: boolean; ignoreFlags?: boolean }) { + protected async buildModules(options?: { + graceful?: boolean; + ignoreFlags?: boolean; + drop?: boolean; + }) { const state = { built: false, modules: [] as ModuleKey[], @@ -580,12 +269,8 @@ export class ModuleManager { this.logger.log("db sync requested"); // sync db - await ctx.em.schema().sync({ force: true }); + await ctx.em.schema().sync({ force: true, drop: options?.drop }); state.synced = true; - - // save - await this.save(); - state.saved = true; } if (ctx.flags.ctx_reload_required) { @@ -601,92 +286,12 @@ export class ModuleManager { ctx.flags = Module.ctx_flags; // storing last stable config version - this._stable_configs = $diff.clone(this.configs()); + //this._stable_configs = $diff.clone(this.configs()); this.logger.clear(); return state; } - protected async setupInitial() { - this.logger.context("initial").log("start"); - this._version = CURRENT_VERSION; - await this.syncConfigTable(); - const state = await this.buildModules(); - if (!state.saved) { - await this.save(); - } - - const ctx = { - ...this.ctx(), - // disable events for initial setup - em: this.ctx().em.fork(), - }; - - // perform a sync - await ctx.em.schema().sync({ force: true }); - await this.options?.seed?.(ctx); - - // run first boot event - await this.options?.onFirstBoot?.(); - this.logger.clear(); - } - - mutateConfigSafe( - name: Module, - ): Pick, "set" | "patch" | "overwrite" | "remove"> { - const module = this.modules[name]; - - return new Proxy(module.schema(), { - get: (target, prop: string) => { - if (!["set", "patch", "overwrite", "remove"].includes(prop)) { - throw new Error(`Method ${prop} is not allowed`); - } - - return async (...args) => { - $console.log("[Safe Mutate]", name); - try { - // overwrite listener to run build inside this try/catch - module.setListener(async () => { - await this.emgr.emit( - new ModuleManagerConfigUpdateEvent({ - ctx: this.ctx(), - module: name, - config: module.config as any, - }), - ); - await this.buildModules(); - }); - - const result = await target[prop](...args); - - // revert to original listener - module.setListener(async (c) => { - await this.onModuleConfigUpdated(name, c); - }); - - // if there was an onUpdated listener, call it after success - // e.g. App uses it to register module routes - if (this.options?.onUpdated) { - await this.options.onUpdated(name, module.config as any); - } - - return result; - } catch (e) { - $console.error(`[Safe Mutate] failed "${name}":`, e); - - // revert to previous config & rebuild using original listener - this.revertModules(); - await this.onModuleConfigUpdated(name, module.config as any); - $console.warn(`[Safe Mutate] reverted "${name}":`); - - // make sure to throw the error - throw e; - } - }; - }, - }); - } - get(key: K): Modules[K] { if (!(key in this.modules)) { throw new Error(`Module "${key}" doesn't exist, cannot get`); @@ -695,7 +300,7 @@ export class ModuleManager { } version() { - return this._version; + return 0; } built() { diff --git a/app/src/modules/db/DbModuleManager.ts b/app/src/modules/db/DbModuleManager.ts new file mode 100644 index 0000000..bf69355 --- /dev/null +++ b/app/src/modules/db/DbModuleManager.ts @@ -0,0 +1,594 @@ +import { mark, stripMark, $console, s, SecretSchema, setPath } from "bknd/utils"; +import { BkndError } from "core/errors"; +import * as $diff from "core/object/diff"; +import type { Connection } from "data/connection"; +import type { EntityManager } from "data/entities/EntityManager"; +import * as proto from "data/prototype"; +import { TransformPersistFailedException } from "data/errors"; +import type { Kysely } from "kysely"; +import { mergeWith } from "lodash-es"; +import { CURRENT_VERSION, TABLE_NAME, migrate } from "./migrations"; +import { Module, type ModuleBuildContext } from "../Module"; +import { + type InitialModuleConfigs, + type ModuleConfigs, + type Modules, + type ModuleKey, + getDefaultSchema, + getDefaultConfig, + ModuleManager, + ModuleManagerConfigUpdateEvent, + type ModuleManagerOptions, + ModuleManagerSecretsExtractedEvent, +} from "../ModuleManager"; + +export type { ModuleBuildContext }; + +export type ConfigTable = { + id?: number; + version: number; + type: "config" | "diff" | "backup"; + json: Json; + created_at?: Date; + updated_at?: Date; +}; + +const configJsonSchema = s.anyOf([ + getDefaultSchema(), + s.array( + s.strictObject({ + t: s.string({ enum: ["a", "r", "e"] }), + p: s.array(s.anyOf([s.string(), s.number()])), + o: s.any().optional(), + n: s.any().optional(), + }), + ), +]); +export const __bknd = proto.entity(TABLE_NAME, { + version: proto.number().required(), + type: proto.enumm({ enum: ["config", "diff", "backup", "secrets"] }).required(), + json: proto.jsonSchema({ schema: configJsonSchema.toJSON() }).required(), + created_at: proto.datetime(), + updated_at: proto.datetime(), +}); +const __schema = proto.em({ __bknd }, ({ index }, { __bknd }) => { + index(__bknd).on(["version", "type"]); +}); + +type ConfigTable2 = proto.Schema; +interface T_INTERNAL_EM { + __bknd: ConfigTable2; +} + +// @todo: cleanup old diffs on upgrade +// @todo: cleanup multiple backups on upgrade +export class DbModuleManager extends ModuleManager { + // internal em for __bknd config table + __em!: EntityManager; + + private _version: number = 0; + private readonly _booted_with?: "provided" | "partial"; + private _stable_configs: ModuleConfigs | undefined; + + constructor(connection: Connection, options?: Partial) { + let initial = {} as InitialModuleConfigs; + let booted_with = "partial" as any; + let version = 0; + + if (options?.initial) { + if ("version" in options.initial && options.initial.version) { + const { version: _v, ...config } = options.initial; + version = _v as number; + initial = stripMark(config) as any; + + booted_with = "provided"; + } else { + initial = mergeWith(getDefaultConfig(), options.initial); + booted_with = "partial"; + } + } + + super(connection, { ...options, initial }); + + this.__em = __schema.proto.withConnection(this.connection) as any; + //this.__em = new EntityManager(__schema.entities, this.connection); + this._version = version; + this._booted_with = booted_with; + + this.logger.log("booted with", this._booted_with); + } + + /** + * This is set through module's setListener + * It's called everytime a module's config is updated in SchemaObject + * Needs to rebuild modules and save to database + */ + protected override async onModuleConfigUpdated(key: string, config: any) { + if (this.options?.onUpdated) { + await this.options.onUpdated(key as any, config); + } else { + await this.buildModules(); + } + } + + private repo() { + return this.__em.repo(__bknd, { + // to prevent exceptions when table doesn't exist + silent: true, + // disable counts for performance and compatibility + includeCounts: false, + }); + } + + private mutator() { + return this.__em.mutator(__bknd); + } + + private get db() { + // @todo: check why this is neccessary + return this.connection.kysely as unknown as Kysely<{ table: ConfigTable }>; + } + + // @todo: add indices for: version, type + async syncConfigTable() { + this.logger.context("sync").log("start"); + const result = await this.__em.schema().sync({ force: true }); + this.logger.log("done").clear(); + return result; + } + + private async fetch(): Promise<{ configs?: ConfigTable; secrets?: ConfigTable } | undefined> { + this.logger.context("fetch").log("fetching"); + const startTime = performance.now(); + + // disabling console log, because the table might not exist yet + const { data: result } = await this.repo().findMany({ + where: { type: { $in: ["config", "secrets"] } }, + sort: { by: "version", dir: "desc" }, + }); + + if (!result.length) { + this.logger.log("error fetching").clear(); + return undefined; + } + + const configs = result.filter((r) => r.type === "config")[0]; + const secrets = result.filter((r) => r.type === "secrets")[0]; + + this.logger + .log("took", performance.now() - startTime, "ms", { + version: result.version, + id: result.id, + }) + .clear(); + + return { configs, secrets }; + } + + extractSecrets() { + const moduleConfigs = structuredClone(this.configs()); + const secrets = this.options?.secrets || ({} as any); + + for (const [key, module] of Object.entries(this.modules)) { + const config = moduleConfigs[key]; + const schema = module.getSchema(); + + const extracted = [...schema.walk({ data: config })].filter( + (n) => n.schema instanceof SecretSchema, + ); + + //console.log("extracted", key, extracted, config); + for (const n of extracted) { + const path = [key, ...n.instancePath].join("."); + if (typeof n.data === "string" && n.data.length > 0) { + secrets[path] = n.data; + setPath(moduleConfigs, path, ""); + } + } + } + + return { + configs: moduleConfigs, + secrets, + }; + } + + async save() { + this.logger.context("save").log("saving version", this.version()); + const { configs, secrets } = this.extractSecrets(); + const version = this.version(); + const store_secrets = this.options?.storeSecrets !== false; + + await this.emgr.emit( + new ModuleManagerSecretsExtractedEvent({ + ctx: this.ctx(), + secrets, + }), + ); + + try { + const state = await this.fetch(); + if (!state || !state.configs) throw new BkndError("no config found"); + this.logger.log("fetched version", state.configs.version); + + if (state.configs.version !== version) { + // @todo: mark all others as "backup" + this.logger.log( + "version conflict, storing new version", + state.configs.version, + version, + ); + const updates = [ + { + version: state.configs.version, + type: "backup", + json: state.configs.json, + }, + { + version: version, + type: "config", + json: configs, + }, + ]; + if (store_secrets) { + updates.push({ + version: state.configs.version, + type: "secrets", + json: secrets, + }); + } + await this.mutator().insertMany(updates); + } else { + this.logger.log("version matches", state.configs.version); + + // clean configs because of Diff() function + const diffs = $diff.diff(state.configs.json, $diff.clone(configs)); + this.logger.log("checking diff", [diffs.length]); + const date = new Date(); + + if (diffs.length > 0) { + // validate diffs, it'll throw on invalid + this.validateDiffs(diffs); + + // store diff + await this.mutator().insertOne({ + version, + type: "diff", + json: $diff.clone(diffs), + created_at: date, + updated_at: date, + }); + + // store new version + await this.mutator().updateWhere( + { + version, + json: configs, + updated_at: date, + } as any, + { + type: "config", + version, + }, + ); + } else { + this.logger.log("no diff, not saving"); + } + + // store secrets + if (store_secrets) { + if (!state.secrets || state.secrets?.version !== version) { + await this.mutator().insertOne({ + version: state.configs.version, + type: "secrets", + json: secrets, + created_at: date, + updated_at: date, + }); + } else { + await this.mutator().updateOne(state.secrets.id!, { + version, + json: secrets, + updated_at: date, + } as any); + } + } + } + } catch (e) { + if (e instanceof BkndError && e.message === "no config found") { + this.logger.log("no config, just save fresh"); + // no config, just save + await this.mutator().insertOne({ + type: "config", + version, + json: configs, + created_at: new Date(), + updated_at: new Date(), + }); + } else if (e instanceof TransformPersistFailedException) { + $console.error("ModuleManager: Cannot save invalid config"); + this.revertModules(); + throw e; + } else { + $console.error("ModuleManager: Aborting"); + this.revertModules(); + throw e; + } + } + + // re-apply configs to all modules (important for system entities) + await this.setConfigs(this.configs()); + + // @todo: cleanup old versions? + + this.logger.clear(); + return this; + } + + private async revertModules() { + if (this._stable_configs) { + $console.warn("ModuleManager: Reverting modules"); + await this.setConfigs(this._stable_configs as any); + } else { + $console.error("ModuleManager: No stable configs to revert to"); + } + } + + /** + * Validates received diffs for an additional security control. + * Checks: + * - check if module is registered + * - run modules onBeforeUpdate() for added protection + * + * **Important**: Throw `Error` so it won't get catched. + * + * @param diffs + * @private + */ + private validateDiffs(diffs: $diff.DiffEntry[]): void { + // check top level paths, and only allow a single module to be modified in a single transaction + const modules = [...new Set(diffs.map((d) => d.p[0]))]; + if (modules.length === 0) { + return; + } + + for (const moduleName of modules) { + const name = moduleName as ModuleKey; + const module = this.get(name) as Module; + if (!module) { + const msg = "validateDiffs: module not registered"; + // biome-ignore format: ... + $console.error(msg, JSON.stringify({ module: name, diffs }, null, 2)); + throw new Error(msg); + } + + // pass diffs to the module to allow it to throw + if (this._stable_configs?.[name]) { + const current = $diff.clone(this._stable_configs?.[name]); + const modified = $diff.apply({ [name]: current }, diffs)[name]; + module.onBeforeUpdate(current, modified); + } + } + } + + override async build(opts?: { fetch?: boolean }) { + this.logger.context("build").log("version", this.version()); + await this.ctx().connection.init(); + + // if no config provided, try fetch from db + if (this.version() === 0 || opts?.fetch === true) { + if (opts?.fetch) { + this.logger.log("force fetch"); + } + + const result = await this.fetch(); + + // if no version, and nothing found, go with initial + if (!result?.configs) { + this.logger.log("nothing in database, go initial"); + await this.setupInitial(); + } else { + this.logger.log("db has", result.configs.version); + // set version and config from fetched + this._version = result.configs.version; + + if (result?.configs && result?.secrets) { + for (const [key, value] of Object.entries(result.secrets.json)) { + setPath(result.configs.json, key, value); + } + } + + if (this.options?.trustFetched === true) { + this.logger.log("trusting fetched config (mark)"); + mark(result.configs.json); + } + + // if version doesn't match, migrate before building + if (this.version() !== CURRENT_VERSION) { + this.logger.log("now migrating"); + + await this.syncConfigTable(); + + const version_before = this.version(); + const [_version, _configs] = await migrate(version_before, result.configs.json, { + db: this.db, + }); + + this._version = _version; + this.ctx().flags.sync_required = true; + + this.logger.log("migrated to", _version); + $console.log("Migrated config from", version_before, "to", this.version()); + + // @ts-expect-error + await this.setConfigs(_configs); + await this.buildModules(); + } else { + this.logger.log("version is current", this.version()); + + await this.setConfigs(result.configs.json); + await this.buildModules(); + } + } + } else { + if (this.version() !== CURRENT_VERSION) { + throw new Error( + `Given version (${this.version()}) and current version (${CURRENT_VERSION}) do not match.`, + ); + } + this.logger.log("current version is up to date", this.version()); + await this.buildModules(); + } + + this.logger.log("done"); + this.logger.clear(); + return this; + } + + protected override async buildModules(options?: { graceful?: boolean; ignoreFlags?: boolean }) { + const state = { + built: false, + modules: [] as ModuleKey[], + synced: false, + saved: false, + reloaded: false, + }; + + this.logger.context("buildModules").log("triggered", options, this._built); + if (options?.graceful && this._built) { + this.logger.log("skipping build (graceful)"); + return state; + } + + this.logger.log("building"); + const ctx = this.ctx(true); + for (const key in this.modules) { + await this.modules[key].setContext(ctx).build(); + this.logger.log("built", key); + state.modules.push(key as ModuleKey); + } + + this._built = state.built = true; + this.logger.log("modules built", ctx.flags); + + if (this.options?.onModulesBuilt) { + await this.options.onModulesBuilt(ctx); + } + + if (options?.ignoreFlags !== true) { + if (ctx.flags.sync_required) { + ctx.flags.sync_required = false; + this.logger.log("db sync requested"); + + // sync db + await ctx.em.schema().sync({ force: true }); + state.synced = true; + + // save + await this.save(); + state.saved = true; + } + + if (ctx.flags.ctx_reload_required) { + ctx.flags.ctx_reload_required = false; + this.logger.log("ctx reload requested"); + this.ctx(true); + state.reloaded = true; + } + } + + // reset all falgs + this.logger.log("resetting flags"); + ctx.flags = Module.ctx_flags; + + // storing last stable config version + this._stable_configs = $diff.clone(this.configs()); + + this.logger.clear(); + return state; + } + + protected async setupInitial() { + this.logger.context("initial").log("start"); + this._version = CURRENT_VERSION; + await this.syncConfigTable(); + const state = await this.buildModules(); + if (!state.saved) { + await this.save(); + } + + const ctx = { + ...this.ctx(), + // disable events for initial setup + em: this.ctx().em.fork(), + }; + + // perform a sync + await ctx.em.schema().sync({ force: true }); + await this.options?.seed?.(ctx); + + // run first boot event + await this.options?.onFirstBoot?.(); + this.logger.clear(); + } + + mutateConfigSafe( + name: Module, + ): Pick, "set" | "patch" | "overwrite" | "remove"> { + const module = this.modules[name]; + + return new Proxy(module.schema(), { + get: (target, prop: string) => { + if (!["set", "patch", "overwrite", "remove"].includes(prop)) { + throw new Error(`Method ${prop} is not allowed`); + } + + return async (...args) => { + $console.log("[Safe Mutate]", name); + try { + // overwrite listener to run build inside this try/catch + module.setListener(async () => { + await this.emgr.emit( + new ModuleManagerConfigUpdateEvent({ + ctx: this.ctx(), + module: name, + config: module.config as any, + }), + ); + await this.buildModules(); + }); + + const result = await target[prop](...args); + + // revert to original listener + module.setListener(async (c) => { + await this.onModuleConfigUpdated(name, c); + }); + + // if there was an onUpdated listener, call it after success + // e.g. App uses it to register module routes + if (this.options?.onUpdated) { + await this.options.onUpdated(name, module.config as any); + } + + return result; + } catch (e) { + $console.error(`[Safe Mutate] failed "${name}":`, e); + + // revert to previous config & rebuild using original listener + this.revertModules(); + await this.onModuleConfigUpdated(name, module.config as any); + $console.warn(`[Safe Mutate] reverted "${name}":`); + + // make sure to throw the error + throw e; + } + }; + }, + }); + } + + override version() { + return this._version; + } +} diff --git a/app/src/modules/migrations.ts b/app/src/modules/db/migrations.ts similarity index 100% rename from app/src/modules/migrations.ts rename to app/src/modules/db/migrations.ts diff --git a/app/src/modules/mcp/$object.ts b/app/src/modules/mcp/$object.ts index a57257b..f5fa6b4 100644 --- a/app/src/modules/mcp/$object.ts +++ b/app/src/modules/mcp/$object.ts @@ -6,7 +6,6 @@ import { type McpSchema, type SchemaWithMcpOptions, } from "./McpSchemaHelper"; -import type { Module } from "modules/Module"; export interface ObjectToolSchemaOptions extends s.IObjectOptions, SchemaWithMcpOptions {} @@ -79,6 +78,7 @@ export class ObjectToolSchema< private toolUpdate(node: s.Node) { const schema = this.mcp.cleanSchema; + return new Tool( [this.mcp.name, "update"].join("_"), { @@ -97,11 +97,12 @@ export class ObjectToolSchema< async (params, ctx: AppToolHandlerCtx) => { const { full, value, return_config } = params; const [module_name] = node.instancePath; + const manager = this.mcp.getManager(ctx); if (full) { - await ctx.context.app.mutateConfig(module_name as any).set(value); + await manager.mutateConfigSafe(module_name as any).set(value); } else { - await ctx.context.app.mutateConfig(module_name as any).patch("", value); + await manager.mutateConfigSafe(module_name as any).patch("", value); } let config: any = undefined; diff --git a/app/src/modules/mcp/$record.ts b/app/src/modules/mcp/$record.ts index a10054a..fc6dfaa 100644 --- a/app/src/modules/mcp/$record.ts +++ b/app/src/modules/mcp/$record.ts @@ -129,13 +129,14 @@ export class RecordToolSchema< const configs = ctx.context.app.toJSON(true); const config = getPath(configs, node.instancePath); const [module_name, ...rest] = node.instancePath; + const manager = this.mcp.getManager(ctx); if (params.key in config) { throw new Error(`Key "${params.key}" already exists in config`); } - await ctx.context.app - .mutateConfig(module_name as any) + await manager + .mutateConfigSafe(module_name as any) .patch([...rest, params.key], params.value); const newConfig = getPath(ctx.context.app.toJSON(), node.instancePath); @@ -175,13 +176,14 @@ export class RecordToolSchema< const configs = ctx.context.app.toJSON(true); const config = getPath(configs, node.instancePath); const [module_name, ...rest] = node.instancePath; + const manager = this.mcp.getManager(ctx); if (!(params.key in config)) { throw new Error(`Key "${params.key}" not found in config`); } - await ctx.context.app - .mutateConfig(module_name as any) + await manager + .mutateConfigSafe(module_name as any) .patch([...rest, params.key], params.value); const newConfig = getPath(ctx.context.app.toJSON(), node.instancePath); @@ -220,13 +222,14 @@ export class RecordToolSchema< const configs = ctx.context.app.toJSON(true); const config = getPath(configs, node.instancePath); const [module_name, ...rest] = node.instancePath; + const manager = this.mcp.getManager(ctx); if (!(params.key in config)) { throw new Error(`Key "${params.key}" not found in config`); } - await ctx.context.app - .mutateConfig(module_name as any) + await manager + .mutateConfigSafe(module_name as any) .remove([...rest, params.key].join(".")); const newConfig = getPath(ctx.context.app.toJSON(), node.instancePath); diff --git a/app/src/modules/mcp/$schema.ts b/app/src/modules/mcp/$schema.ts index 9c86d4a..c71424b 100644 --- a/app/src/modules/mcp/$schema.ts +++ b/app/src/modules/mcp/$schema.ts @@ -58,8 +58,9 @@ export const $schema = < async (params, ctx: AppToolHandlerCtx) => { const { value, return_config, secrets } = params; const [module_name, ...rest] = node.instancePath; + const manager = mcp.getManager(ctx); - await ctx.context.app.mutateConfig(module_name as any).overwrite(rest, value); + await manager.mutateConfigSafe(module_name as any).overwrite(rest, value); let config: any = undefined; if (return_config) { diff --git a/app/src/modules/mcp/McpSchemaHelper.ts b/app/src/modules/mcp/McpSchemaHelper.ts index 686e7ff..ecb16e4 100644 --- a/app/src/modules/mcp/McpSchemaHelper.ts +++ b/app/src/modules/mcp/McpSchemaHelper.ts @@ -10,6 +10,7 @@ import { } from "bknd/utils"; import type { ModuleBuildContext } from "modules"; import { excludePropertyTypes, rescursiveClean } from "./utils"; +import type { DbModuleManager } from "modules/db/DbModuleManager"; export const mcpSchemaSymbol = Symbol.for("bknd-mcp-schema"); @@ -74,4 +75,13 @@ export class McpSchemaHelper { }, }; } + + getManager(ctx: AppToolHandlerCtx): DbModuleManager { + const manager = ctx.context.app.modules; + if ("mutateConfigSafe" in manager) { + return manager as DbModuleManager; + } + + throw new Error("Manager not found"); + } } diff --git a/app/src/modules/mcp/system-mcp.ts b/app/src/modules/mcp/system-mcp.ts index bd278db..b1e7fa8 100644 --- a/app/src/modules/mcp/system-mcp.ts +++ b/app/src/modules/mcp/system-mcp.ts @@ -19,9 +19,11 @@ export function getSystemMcp(app: App) { ].sort((a, b) => a.name.localeCompare(b.name)); // tools from app schema - tools.push( - ...nodes.flatMap((n) => n.schema.getTools(n)).sort((a, b) => a.name.localeCompare(b.name)), - ); + if (!app.isReadOnly()) { + tools.push( + ...nodes.flatMap((n) => n.schema.getTools(n)).sort((a, b) => a.name.localeCompare(b.name)), + ); + } const resources = [...middlewareServer.resources, ...app.modules.ctx().mcp.resources]; diff --git a/app/src/modules/server/AppServer.ts b/app/src/modules/server/AppServer.ts index 57af316..eeaab9f 100644 --- a/app/src/modules/server/AppServer.ts +++ b/app/src/modules/server/AppServer.ts @@ -30,7 +30,7 @@ export const serverConfigSchema = $object( { description: "Server configuration", }, -); +).strict(); export type AppServerConfig = s.Static; diff --git a/app/src/modules/server/SystemController.ts b/app/src/modules/server/SystemController.ts index 7beda79..8db32e4 100644 --- a/app/src/modules/server/SystemController.ts +++ b/app/src/modules/server/SystemController.ts @@ -31,6 +31,7 @@ import * as SystemPermissions from "modules/permissions"; import { getVersion } from "core/env"; import type { Module } from "modules/Module"; import { getSystemMcp } from "modules/mcp/system-mcp"; +import type { DbModuleManager } from "modules/db/DbModuleManager"; export type ConfigUpdate = { success: true; @@ -43,6 +44,7 @@ export type ConfigUpdateResponse = export type SchemaResponse = { version: string; schema: ModuleSchemas; + readonly: boolean; config: ModuleConfigs; permissions: string[]; }; @@ -109,22 +111,163 @@ export class SystemController extends Controller { private registerConfigController(client: Hono): void { const { permission } = this.middlewares; // don't add auth again, it's already added in getController - const hono = this.create(); + const hono = this.create().use(permission(SystemPermissions.configRead)); - hono.use(permission(SystemPermissions.configRead)); + if (!this.app.isReadOnly()) { + const manager = this.app.modules as DbModuleManager; - hono.get( - "/raw", - describeRoute({ - summary: "Get the raw config", - tags: ["system"], - }), - permission([SystemPermissions.configReadSecrets]), - async (c) => { - // @ts-expect-error "fetch" is private - return c.json(await this.app.modules.fetch()); - }, - ); + hono.get( + "/raw", + describeRoute({ + summary: "Get the raw config", + tags: ["system"], + }), + permission([SystemPermissions.configReadSecrets]), + async (c) => { + // @ts-expect-error "fetch" is private + return c.json(await this.app.modules.fetch().then((r) => r?.configs)); + }, + ); + + async function handleConfigUpdateResponse( + c: Context, + cb: () => Promise, + ) { + try { + return c.json(await cb(), { status: 202 }); + } catch (e) { + $console.error("config update error", e); + + if (e instanceof InvalidSchemaError) { + return c.json( + { success: false, type: "type-invalid", errors: e.errors }, + { status: 400 }, + ); + } + if (e instanceof Error) { + return c.json( + { success: false, type: "error", error: e.message }, + { status: 500 }, + ); + } + + return c.json({ success: false, type: "unknown" }, { status: 500 }); + } + } + + hono.post( + "/set/:module", + permission(SystemPermissions.configWrite), + jsc("query", s.object({ force: s.boolean().optional() }), { skipOpenAPI: true }), + async (c) => { + const module = c.req.param("module") as any; + const { force } = c.req.valid("query"); + const value = await c.req.json(); + + return await handleConfigUpdateResponse(c, async () => { + // you must explicitly set force to override existing values + // because omitted values gets removed + if (force === true) { + // force overwrite defined keys + const newConfig = { + ...this.app.module[module].config, + ...value, + }; + await manager.mutateConfigSafe(module).set(newConfig); + } else { + await manager.mutateConfigSafe(module).patch("", value); + } + return { + success: true, + module, + config: this.app.module[module].config, + }; + }); + }, + ); + + hono.post("/add/:module/:path", permission(SystemPermissions.configWrite), async (c) => { + // @todo: require auth (admin) + const module = c.req.param("module") as any; + const value = await c.req.json(); + const path = c.req.param("path") as string; + + if (this.app.modules.get(module).schema().has(path)) { + return c.json( + { success: false, path, error: "Path already exists" }, + { status: 400 }, + ); + } + + return await handleConfigUpdateResponse(c, async () => { + await manager.mutateConfigSafe(module).patch(path, value); + return { + success: true, + module, + config: this.app.module[module].config, + }; + }); + }); + + hono.patch( + "/patch/:module/:path", + permission(SystemPermissions.configWrite), + async (c) => { + // @todo: require auth (admin) + const module = c.req.param("module") as any; + const value = await c.req.json(); + const path = c.req.param("path"); + + return await handleConfigUpdateResponse(c, async () => { + await manager.mutateConfigSafe(module).patch(path, value); + return { + success: true, + module, + config: this.app.module[module].config, + }; + }); + }, + ); + + hono.put( + "/overwrite/:module/:path", + permission(SystemPermissions.configWrite), + async (c) => { + // @todo: require auth (admin) + const module = c.req.param("module") as any; + const value = await c.req.json(); + const path = c.req.param("path"); + + return await handleConfigUpdateResponse(c, async () => { + await manager.mutateConfigSafe(module).overwrite(path, value); + return { + success: true, + module, + config: this.app.module[module].config, + }; + }); + }, + ); + + hono.delete( + "/remove/:module/:path", + permission(SystemPermissions.configWrite), + async (c) => { + // @todo: require auth (admin) + const module = c.req.param("module") as any; + const path = c.req.param("path")!; + + return await handleConfigUpdateResponse(c, async () => { + await manager.mutateConfigSafe(module).remove(path); + return { + success: true, + module, + config: this.app.module[module].config, + }; + }); + }, + ); + } hono.get( "/:module?", @@ -160,124 +303,6 @@ export class SystemController extends Controller { }, ); - async function handleConfigUpdateResponse(c: Context, cb: () => Promise) { - try { - return c.json(await cb(), { status: 202 }); - } catch (e) { - $console.error("config update error", e); - - if (e instanceof InvalidSchemaError) { - return c.json( - { success: false, type: "type-invalid", errors: e.errors }, - { status: 400 }, - ); - } - if (e instanceof Error) { - return c.json({ success: false, type: "error", error: e.message }, { status: 500 }); - } - - return c.json({ success: false, type: "unknown" }, { status: 500 }); - } - } - - hono.post( - "/set/:module", - permission(SystemPermissions.configWrite), - jsc("query", s.object({ force: s.boolean().optional() }), { skipOpenAPI: true }), - async (c) => { - const module = c.req.param("module") as any; - const { force } = c.req.valid("query"); - const value = await c.req.json(); - - return await handleConfigUpdateResponse(c, async () => { - // you must explicitly set force to override existing values - // because omitted values gets removed - if (force === true) { - // force overwrite defined keys - const newConfig = { - ...this.app.module[module].config, - ...value, - }; - await this.app.mutateConfig(module).set(newConfig); - } else { - await this.app.mutateConfig(module).patch("", value); - } - return { - success: true, - module, - config: this.app.module[module].config, - }; - }); - }, - ); - - hono.post("/add/:module/:path", permission(SystemPermissions.configWrite), async (c) => { - // @todo: require auth (admin) - const module = c.req.param("module") as any; - const value = await c.req.json(); - const path = c.req.param("path") as string; - - if (this.app.modules.get(module).schema().has(path)) { - return c.json({ success: false, path, error: "Path already exists" }, { status: 400 }); - } - - return await handleConfigUpdateResponse(c, async () => { - await this.app.mutateConfig(module).patch(path, value); - return { - success: true, - module, - config: this.app.module[module].config, - }; - }); - }); - - hono.patch("/patch/:module/:path", permission(SystemPermissions.configWrite), async (c) => { - // @todo: require auth (admin) - const module = c.req.param("module") as any; - const value = await c.req.json(); - const path = c.req.param("path"); - - return await handleConfigUpdateResponse(c, async () => { - await this.app.mutateConfig(module).patch(path, value); - return { - success: true, - module, - config: this.app.module[module].config, - }; - }); - }); - - hono.put("/overwrite/:module/:path", permission(SystemPermissions.configWrite), async (c) => { - // @todo: require auth (admin) - const module = c.req.param("module") as any; - const value = await c.req.json(); - const path = c.req.param("path"); - - return await handleConfigUpdateResponse(c, async () => { - await this.app.mutateConfig(module).overwrite(path, value); - return { - success: true, - module, - config: this.app.module[module].config, - }; - }); - }); - - hono.delete("/remove/:module/:path", permission(SystemPermissions.configWrite), async (c) => { - // @todo: require auth (admin) - const module = c.req.param("module") as any; - const path = c.req.param("path")!; - - return await handleConfigUpdateResponse(c, async () => { - await this.app.mutateConfig(module).remove(path); - return { - success: true, - module, - config: this.app.module[module].config, - }; - }); - }); - client.route("/config", hono); } @@ -307,6 +332,7 @@ export class SystemController extends Controller { async (c) => { const module = c.req.param("module") as ModuleKey | undefined; const { config, secrets, fresh } = c.req.valid("query"); + const readonly = this.app.isReadOnly(); config && this.ctx.guard.throwUnlessGranted(SystemPermissions.configRead, c); secrets && this.ctx.guard.throwUnlessGranted(SystemPermissions.configReadSecrets, c); @@ -321,6 +347,7 @@ export class SystemController extends Controller { if (module) { return c.json({ module, + readonly, version, schema: schema[module], config: config ? this.app.module[module].toJSON(secrets) : undefined, @@ -330,6 +357,7 @@ export class SystemController extends Controller { return c.json({ module, version, + readonly, schema, config: config ? this.app.toJSON(secrets) : undefined, permissions: this.app.modules.ctx().guard.getPermissionNames(), @@ -381,6 +409,8 @@ export class SystemController extends Controller { config: c.get("app")?.version(), bknd: getVersion(), }, + mode: this.app.mode, + readonly: this.app.isReadOnly(), runtime: getRuntimeKey(), connection: { name: this.app.em.connection.name, diff --git a/app/src/plugins/dev/sync-config.plugin.ts b/app/src/plugins/dev/sync-config.plugin.ts index 24d84d3..535e538 100644 --- a/app/src/plugins/dev/sync-config.plugin.ts +++ b/app/src/plugins/dev/sync-config.plugin.ts @@ -28,7 +28,7 @@ export function syncConfig({ if (firstBoot) { firstBoot = false; - await write?.(app.toJSON(true)); + await write?.(app.toJSON(includeSecrets)); } }, }); diff --git a/app/src/plugins/dev/sync-secrets.plugin.ts b/app/src/plugins/dev/sync-secrets.plugin.ts new file mode 100644 index 0000000..16e0357 --- /dev/null +++ b/app/src/plugins/dev/sync-secrets.plugin.ts @@ -0,0 +1,37 @@ +import { type App, ModuleManagerEvents, type AppPlugin } from "bknd"; +import type { DbModuleManager } from "modules/db/DbModuleManager"; + +export type SyncSecretsOptions = { + enabled?: boolean; + write: (secrets: Record) => Promise; +}; + +export function syncSecrets({ enabled = true, write }: SyncSecretsOptions): AppPlugin { + let firstBoot = true; + return (app: App) => ({ + name: "bknd-sync-secrets", + onBuilt: async () => { + if (!enabled) return; + const manager = app.modules as DbModuleManager; + + if (!("extractSecrets" in manager)) { + throw new Error("ModuleManager does not support secrets"); + } + + app.emgr.onEvent( + ModuleManagerEvents.ModuleManagerSecretsExtractedEvent, + async ({ params: { secrets } }) => { + await write?.(secrets); + }, + { + id: "sync-secrets", + }, + ); + + if (firstBoot) { + firstBoot = false; + await write?.(manager.extractSecrets().secrets); + } + }, + }); +} diff --git a/app/src/plugins/index.ts b/app/src/plugins/index.ts index eab3bf6..45db2d5 100644 --- a/app/src/plugins/index.ts +++ b/app/src/plugins/index.ts @@ -6,3 +6,4 @@ export { export { showRoutes, type ShowRoutesOptions } from "./dev/show-routes.plugin"; export { syncConfig, type SyncConfigOptions } from "./dev/sync-config.plugin"; export { syncTypes, type SyncTypesOptions } from "./dev/sync-types.plugin"; +export { syncSecrets, type SyncSecretsOptions } from "./dev/sync-secrets.plugin"; diff --git a/app/src/ui/client/BkndProvider.tsx b/app/src/ui/client/BkndProvider.tsx index 4e938d1..fa8ed35 100644 --- a/app/src/ui/client/BkndProvider.tsx +++ b/app/src/ui/client/BkndProvider.tsx @@ -1,6 +1,14 @@ import type { ModuleConfigs, ModuleSchemas } from "modules"; import { getDefaultConfig, getDefaultSchema } from "modules/ModuleManager"; -import { createContext, startTransition, useContext, useEffect, useRef, useState } from "react"; +import { + createContext, + startTransition, + useContext, + useEffect, + useRef, + useState, + type ReactNode, +} from "react"; import { useApi } from "ui/client"; import { type TSchemaActions, getSchemaActions } from "./schema/actions"; import { AppReduced } from "./utils/AppReduced"; @@ -15,6 +23,7 @@ export type BkndAdminOptions = { }; type BkndContext = { version: number; + readonly: boolean; schema: ModuleSchemas; config: ModuleConfigs; permissions: string[]; @@ -48,7 +57,12 @@ export function BkndProvider({ }) { const [withSecrets, setWithSecrets] = useState(includeSecrets); const [schema, setSchema] = - useState>(); + useState< + Pick< + BkndContext, + "version" | "schema" | "config" | "permissions" | "fallback" | "readonly" + > + >(); const [fetched, setFetched] = useState(false); const [error, setError] = useState(); const errorShown = useRef(false); @@ -97,6 +111,7 @@ export function BkndProvider({ ? res.body : ({ version: 0, + mode: "db", schema: getDefaultSchema(), config: getDefaultConfig(), permissions: [], @@ -173,3 +188,8 @@ export function useBkndOptions(): BkndAdminOptions { } ); } + +export function SchemaEditable({ children }: { children: ReactNode }) { + const { readonly } = useBknd(); + return !readonly ? children : null; +} diff --git a/app/src/ui/client/bknd.ts b/app/src/ui/client/bknd.ts index 55159a7..e384b07 100644 --- a/app/src/ui/client/bknd.ts +++ b/app/src/ui/client/bknd.ts @@ -1 +1 @@ -export { BkndProvider, type BkndAdminOptions, useBknd } from "./BkndProvider"; +export { BkndProvider, type BkndAdminOptions, useBknd, SchemaEditable } from "./BkndProvider"; diff --git a/app/src/ui/routes/data/_data.root.tsx b/app/src/ui/routes/data/_data.root.tsx index 344ef69..ce195ed 100644 --- a/app/src/ui/routes/data/_data.root.tsx +++ b/app/src/ui/routes/data/_data.root.tsx @@ -22,6 +22,7 @@ import { useBrowserTitle } from "ui/hooks/use-browser-title"; import * as AppShell from "ui/layouts/AppShell/AppShell"; import { routes, useNavigate, useRouteNavigate } from "ui/lib/routes"; import { testIds } from "ui/lib/config"; +import { SchemaEditable, useBknd } from "ui/client/bknd"; export function DataRoot({ children }) { // @todo: settings routes should be centralized @@ -73,9 +74,11 @@ export function DataRoot({ children }) { value={context} onChange={handleSegmentChange} /> - - - + + + + + } > @@ -254,11 +257,26 @@ export function DataEmpty() { useBrowserTitle(["Data"]); const [navigate] = useNavigate(); const { $data } = useBkndData(); + const { readonly } = useBknd(); function handleButtonClick() { navigate(routes.data.schema.root()); } + if (readonly) { + return ( + + ); + } + return ( - $data.modals.createRelation(entity.name), - }, - { - icon: TbPhoto, - label: "Add media", - onClick: () => $data.modals.createMedia(entity.name), - }, - () =>
, - { - icon: TbDatabasePlus, - label: "Create Entity", - onClick: () => $data.modals.createEntity(), - }, - ]} - position="bottom-end" - > - - + + $data.modals.createRelation(entity.name), + }, + { + icon: TbPhoto, + label: "Add media", + onClick: () => $data.modals.createMedia(entity.name), + }, + () =>
, + { + icon: TbDatabasePlus, + label: "Create Entity", + onClick: () => $data.modals.createEntity(), + }, + ]} + position="bottom-end" + > + + + } className="pl-3" @@ -149,6 +152,7 @@ const Fields = ({ entity }: { entity: Entity }) => { const [submitting, setSubmitting] = useState(false); const [updates, setUpdates] = useState(0); const { actions, $data, config } = useBkndData(); + const { readonly } = useBknd(); const [res, setRes] = useState(); const ref = useRef(null); async function handleUpdate() { @@ -169,7 +173,7 @@ const Fields = ({ entity }: { entity: Entity }) => { title="Fields" ActiveIcon={IconAlignJustified} renderHeaderRight={({ open }) => - open ? ( + open && !readonly ? ( @@ -181,11 +185,12 @@ const Fields = ({ entity }: { entity: Entity }) => {
)} ["relation", "media"].includes(f.type)) .map((i) => ({ @@ -205,7 +210,7 @@ const Fields = ({ entity }: { entity: Entity }) => { isNew={false} /> - {isDebug() && ( + {isDebug() && !readonly && (
@@ -278,6 +284,7 @@ const BasicSettings = ({ entity }: { entity: Entity }) => { formData={_config} onSubmit={console.log} className="legacy hide-required-mark fieldset-alternative mute-root" + readonly={readonly} />
diff --git a/app/src/ui/routes/data/data.schema.index.tsx b/app/src/ui/routes/data/data.schema.index.tsx index 2c01ac3..d8ccbde 100644 --- a/app/src/ui/routes/data/data.schema.index.tsx +++ b/app/src/ui/routes/data/data.schema.index.tsx @@ -1,4 +1,5 @@ import { Suspense, lazy } from "react"; +import { SchemaEditable } from "ui/client/bknd"; import { useBkndData } from "ui/client/schema/data/use-bknd-data"; import { Button } from "ui/components/buttons/Button"; import * as AppShell from "ui/layouts/AppShell/AppShell"; @@ -15,9 +16,11 @@ export function DataSchemaIndex() { <> - Create new - + + + } > Schema Overview diff --git a/app/src/ui/routes/data/forms/entity.fields.form.tsx b/app/src/ui/routes/data/forms/entity.fields.form.tsx index d8f7d53..4272f89 100644 --- a/app/src/ui/routes/data/forms/entity.fields.form.tsx +++ b/app/src/ui/routes/data/forms/entity.fields.form.tsx @@ -29,6 +29,7 @@ import { MantineSelect } from "ui/components/form/hook-form-mantine/MantineSelec import type { TPrimaryFieldFormat } from "data/fields/PrimaryField"; import { standardSchemaResolver } from "@hookform/resolvers/standard-schema"; import ErrorBoundary from "ui/components/display/ErrorBoundary"; +import { SchemaEditable } from "ui/client/bknd"; const fieldsSchemaObject = originalFieldsSchemaObject; const fieldsSchema = s.anyOf(Object.values(fieldsSchemaObject)); @@ -64,6 +65,7 @@ export type EntityFieldsFormProps = { routePattern?: string; defaultPrimaryFormat?: TPrimaryFieldFormat; isNew?: boolean; + readonly?: boolean; }; export type EntityFieldsFormRef = { @@ -76,7 +78,7 @@ export type EntityFieldsFormRef = { export const EntityFieldsForm = forwardRef( function EntityFieldsForm( - { fields: _fields, sortable, additionalFieldTypes, routePattern, isNew, ...props }, + { fields: _fields, sortable, additionalFieldTypes, routePattern, isNew, readonly, ...props }, ref, ) { const entityFields = Object.entries(_fields).map(([name, field]) => ({ @@ -162,6 +164,7 @@ export const EntityFieldsForm = forwardRef ( {fields.map((field, index) => ( )} - ( - { - handleAppend(type as any); - }} - /> - )} - > - - + + ( + { + handleAppend(type as any); + }} + /> + )} + > + + +
@@ -288,6 +294,7 @@ function EntityField({ dnd, routePattern, primary, + readonly, }: { field: FieldArrayWithId; index: number; @@ -303,6 +310,7 @@ function EntityField({ defaultFormat?: TPrimaryFieldFormat; editable?: boolean; }; + readonly?: boolean; }) { const prefix = `fields.${index}.field` as const; const type = field.field.type; @@ -393,6 +401,7 @@ function EntityField({ Required @@ -433,6 +442,7 @@ function EntityField({
@@ -440,11 +450,13 @@ function EntityField({