diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a807d56..bc89cce 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,7 +20,7 @@ jobs: - name: Setup Bun uses: oven-sh/setup-bun@v1 with: - bun-version: "1.2.19" + bun-version: "1.2.22" - name: Install dependencies working-directory: ./app diff --git a/app/__test__/App.spec.ts b/app/__test__/App.spec.ts index 0b456e1..7213216 100644 --- a/app/__test__/App.spec.ts +++ b/app/__test__/App.spec.ts @@ -1,12 +1,15 @@ -import { afterEach, describe, test, expect } from "bun:test"; +import { afterEach, describe, test, expect, beforeAll, afterAll } from "bun:test"; import { App, createApp } from "core/test/utils"; import { getDummyConnection } from "./helper"; import { Hono } from "hono"; import * as proto from "../src/data/prototype"; import { pick } from "lodash-es"; +import { disableConsoleLog, enableConsoleLog } from "core/utils/test"; + +beforeAll(() => disableConsoleLog()); const { dummyConnection, afterAllCleanup } = getDummyConnection(); -afterEach(afterAllCleanup); +afterEach(async () => (await afterAllCleanup()) && enableConsoleLog()); describe("App tests", async () => { test("boots and pongs", async () => { @@ -19,7 +22,7 @@ describe("App tests", async () => { test("plugins", async () => { const called: string[] = []; const app = createApp({ - initialConfig: { + config: { auth: { enabled: true, }, diff --git a/app/__test__/_assets/test.mp3 b/app/__test__/_assets/test.mp3 new file mode 100644 index 0000000..ab94045 Binary files /dev/null and b/app/__test__/_assets/test.mp3 differ diff --git a/app/__test__/_assets/test.txt b/app/__test__/_assets/test.txt new file mode 100644 index 0000000..b6fc4c6 --- /dev/null +++ b/app/__test__/_assets/test.txt @@ -0,0 +1 @@ +hello \ No newline at end of file diff --git a/app/__test__/adapter/adapter.test.ts b/app/__test__/adapter/adapter.test.ts index ee9a87e..8527b5d 100644 --- a/app/__test__/adapter/adapter.test.ts +++ b/app/__test__/adapter/adapter.test.ts @@ -1,4 +1,4 @@ -import { expect, describe, it, beforeAll, afterAll } from "bun:test"; +import { expect, describe, it, beforeAll, afterAll, mock } from "bun:test"; import * as adapter from "adapter"; import { disableConsoleLog, enableConsoleLog } from "core/utils"; import { adapterTestSuite } from "adapter/adapter-test-suite"; @@ -19,50 +19,39 @@ describe("adapter", () => { expect( omitKeys( await adapter.makeConfig( - { app: (a) => ({ initialConfig: { server: { cors: { origin: a.env.TEST } } } }) }, + { app: (a) => ({ config: { server: { cors: { origin: a.env.TEST } } } }) }, { env: { TEST: "test" } }, ), ["connection"], ), ).toEqual({ - initialConfig: { server: { cors: { origin: "test" } } }, + config: { server: { cors: { origin: "test" } } }, }); }); - /* it.only("...", async () => { - const app = await adapter.createAdapterApp(); - }); */ - - it("reuses apps correctly", async () => { - const id = crypto.randomUUID(); - - const first = await adapter.createAdapterApp( + it("allows all properties in app function", async () => { + const called = mock(() => null); + const config = await adapter.makeConfig( { - initialConfig: { server: { cors: { origin: "random" } } }, + app: (env) => ({ + connection: { url: "test" }, + config: { server: { cors: { origin: "test" } } }, + options: { + mode: "db", + }, + onBuilt: () => { + called(); + expect(env).toEqual({ foo: "bar" }); + }, + }), }, - undefined, - { id }, + { foo: "bar" }, ); - const second = await adapter.createAdapterApp(); - const third = await adapter.createAdapterApp(undefined, undefined, { id }); - - await first.build(); - await second.build(); - await third.build(); - - expect(first.toJSON().server.cors.origin).toEqual("random"); - expect(first).toBe(third); - expect(first).not.toBe(second); - expect(second).not.toBe(third); - expect(second.toJSON().server.cors.origin).toEqual("*"); - - // recreate the first one - const first2 = await adapter.createAdapterApp(undefined, undefined, { id, force: true }); - await first2.build(); - expect(first2).not.toBe(first); - expect(first2).not.toBe(third); - expect(first2).not.toBe(second); - expect(first2.toJSON().server.cors.origin).toEqual("*"); + expect(config.connection).toEqual({ url: "test" }); + expect(config.config).toEqual({ server: { cors: { origin: "test" } } }); + expect(config.options).toEqual({ mode: "db" }); + await config.onBuilt?.(null as any); + expect(called).toHaveBeenCalled(); }); adapterTestSuite(bunTestRunner, { diff --git a/app/__test__/api/Api.spec.ts b/app/__test__/api/Api.spec.ts index dee1e14..c1041d9 100644 --- a/app/__test__/api/Api.spec.ts +++ b/app/__test__/api/Api.spec.ts @@ -42,7 +42,6 @@ describe("Api", async () => { expect(api.isAuthVerified()).toBe(false); const params = api.getParams(); - console.log(params); expect(params.token).toBe(token); expect(params.token_transport).toBe("cookie"); expect(params.host).toBe("http://example.com"); diff --git a/app/__test__/app/App.spec.ts b/app/__test__/app/App.spec.ts index aff3e53..5bbee56 100644 --- a/app/__test__/app/App.spec.ts +++ b/app/__test__/app/App.spec.ts @@ -1,9 +1,23 @@ -import { describe, expect, mock, test } from "bun:test"; +import { afterAll, beforeAll, describe, expect, mock, test } from "bun:test"; import type { ModuleBuildContext } from "../../src"; import { App, createApp } from "core/test/utils"; -import * as proto from "../../src/data/prototype"; +import * as proto from "data/prototype"; +import { DbModuleManager } from "modules/db/DbModuleManager"; +import { disableConsoleLog, enableConsoleLog } from "core/utils/test"; + +beforeAll(disableConsoleLog); +afterAll(enableConsoleLog); describe("App", () => { + test("use db mode by default", async () => { + const app = createApp(); + await app.build(); + + expect(app.mode).toBe("db"); + expect(app.isReadOnly()).toBe(false); + expect(app.modules instanceof DbModuleManager).toBe(true); + }); + test("seed includes ctx and app", async () => { const called = mock(() => null); await createApp({ @@ -29,7 +43,7 @@ describe("App", () => { expect(called).toHaveBeenCalled(); const app = createApp({ - initialConfig: { + config: { data: proto .em({ todos: proto.entity("todos", { @@ -139,7 +153,7 @@ describe("App", () => { test("getMcpClient", async () => { const app = createApp({ - initialConfig: { + config: { server: { mcp: { enabled: true, diff --git a/app/__test__/app/AppServer.spec.ts b/app/__test__/app/AppServer.spec.ts index 46ad76e..1933787 100644 --- a/app/__test__/app/AppServer.spec.ts +++ b/app/__test__/app/AppServer.spec.ts @@ -16,6 +16,7 @@ describe("AppServer", () => { mcp: { enabled: false, path: "/api/system/mcp", + logLevel: "warning", }, }); } @@ -38,6 +39,7 @@ describe("AppServer", () => { mcp: { enabled: false, path: "/api/system/mcp", + logLevel: "warning", }, }); } diff --git a/app/__test__/app/code-only.test.ts b/app/__test__/app/code-only.test.ts new file mode 100644 index 0000000..26fb8e9 --- /dev/null +++ b/app/__test__/app/code-only.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, mock, test } from "bun:test"; +import { createApp as internalCreateApp, type CreateAppConfig } from "bknd"; +import { getDummyConnection } from "../../__test__/helper"; +import { ModuleManager } from "modules/ModuleManager"; +import { em, entity, text } from "data/prototype"; + +async function createApp(config: CreateAppConfig = {}) { + const app = internalCreateApp({ + connection: getDummyConnection().dummyConnection, + ...config, + options: { + ...config.options, + mode: "code", + }, + }); + await app.build(); + return app; +} + +describe("code-only", () => { + test("should create app with correct manager", async () => { + const app = await createApp(); + await app.build(); + + expect(app.version()).toBeDefined(); + expect(app.modules).toBeInstanceOf(ModuleManager); + }); + + test("should not perform database syncs", async () => { + const app = await createApp({ + config: { + data: em({ + test: entity("test", { + name: text(), + }), + }).toJSON(), + }, + }); + expect(app.em.entities.map((e) => e.name)).toEqual(["test"]); + expect( + await app.em.connection.kysely + .selectFrom("sqlite_master") + .where("type", "=", "table") + .selectAll() + .execute(), + ).toEqual([]); + + // only perform when explicitly forced + await app.em.schema().sync({ force: true }); + expect( + await app.em.connection.kysely + .selectFrom("sqlite_master") + .where("type", "=", "table") + .selectAll() + .execute() + .then((r) => r.map((r) => r.name)), + ).toEqual(["test", "sqlite_sequence"]); + }); + + test("should not perform seeding", async () => { + const called = mock(() => null); + const app = await createApp({ + config: { + data: em({ + test: entity("test", { + name: text(), + }), + }).toJSON(), + }, + options: { + seed: async (ctx) => { + called(); + await ctx.em.mutator("test").insertOne({ name: "test" }); + }, + }, + }); + await app.em.schema().sync({ force: true }); + expect(called).not.toHaveBeenCalled(); + expect( + await app.em + .repo("test") + .findMany({}) + .then((r) => r.data), + ).toEqual([]); + }); + + test("should sync and perform seeding", async () => { + const called = mock(() => null); + const app = await createApp({ + config: { + data: em({ + test: entity("test", { + name: text(), + }), + }).toJSON(), + }, + options: { + seed: async (ctx) => { + called(); + await ctx.em.mutator("test").insertOne({ name: "test" }); + }, + }, + }); + + await app.em.schema().sync({ force: true }); + await app.options?.seed?.({ + ...app.modules.ctx(), + app: app, + }); + expect(called).toHaveBeenCalled(); + expect( + await app.em + .repo("test") + .findMany({}) + .then((r) => r.data), + ).toEqual([{ id: 1, name: "test" }]); + }); + + test("should not allow to modify config", async () => { + const app = await createApp(); + // biome-ignore lint/suspicious/noPrototypeBuiltins: + expect(app.modules.hasOwnProperty("mutateConfigSafe")).toBe(false); + expect(() => { + app.modules.configs().auth.enabled = true; + }).toThrow(); + }); +}); diff --git a/app/__test__/app/mcp/mcp.auth.test.ts b/app/__test__/app/mcp/mcp.auth.test.ts index ea02274..e7658b4 100644 --- a/app/__test__/app/mcp/mcp.auth.test.ts +++ b/app/__test__/app/mcp/mcp.auth.test.ts @@ -29,7 +29,7 @@ describe("mcp auth", async () => { let server: McpServer; beforeEach(async () => { app = createApp({ - initialConfig: { + config: { auth: { enabled: true, jwt: { @@ -44,6 +44,7 @@ describe("mcp auth", async () => { }, }); await app.build(); + await app.getMcpClient().ping(); server = app.mcp!; server.setLogLevel("error"); server.onNotification((message) => { diff --git a/app/__test__/app/mcp/mcp.base.test.ts b/app/__test__/app/mcp/mcp.base.test.ts index df8bdeb..48acd2e 100644 --- a/app/__test__/app/mcp/mcp.base.test.ts +++ b/app/__test__/app/mcp/mcp.base.test.ts @@ -1,14 +1,18 @@ -import { describe, it, expect } from "bun:test"; +import { describe, it, expect, beforeAll, afterAll } from "bun:test"; import { createApp } from "core/test/utils"; import { registries } from "index"; import { StorageLocalAdapter } from "adapter/node/storage/StorageLocalAdapter"; +import { disableConsoleLog, enableConsoleLog } from "core/utils/test"; + +beforeAll(() => disableConsoleLog()); +afterAll(enableConsoleLog); describe("mcp", () => { it("should have tools", async () => { registries.media.register("local", StorageLocalAdapter); const app = createApp({ - initialConfig: { + config: { auth: { enabled: true, }, @@ -30,6 +34,11 @@ describe("mcp", () => { }); await app.build(); + // expect mcp to not be loaded yet + expect(app.mcp).toBeNull(); + + // after first request, mcp should be loaded + await app.getMcpClient().listTools(); expect(app.mcp?.tools.length).toBeGreaterThan(0); }); }); diff --git a/app/__test__/app/mcp/mcp.data.test.ts b/app/__test__/app/mcp/mcp.data.test.ts index 69b5106..43aecf5 100644 --- a/app/__test__/app/mcp/mcp.data.test.ts +++ b/app/__test__/app/mcp/mcp.data.test.ts @@ -41,7 +41,7 @@ describe("mcp data", async () => { beforeEach(async () => { const time = performance.now(); app = createApp({ - initialConfig: { + config: { server: { mcp: { enabled: true, @@ -50,6 +50,7 @@ describe("mcp data", async () => { }, }); await app.build(); + await app.getMcpClient().ping(); server = app.mcp!; server.setLogLevel("error"); server.onNotification((message) => { diff --git a/app/__test__/app/mcp/mcp.media.test.ts b/app/__test__/app/mcp/mcp.media.test.ts index c10e8bb..ad5383e 100644 --- a/app/__test__/app/mcp/mcp.media.test.ts +++ b/app/__test__/app/mcp/mcp.media.test.ts @@ -21,7 +21,7 @@ describe("mcp media", async () => { beforeEach(async () => { registries.media.register("local", StorageLocalAdapter); app = createApp({ - initialConfig: { + config: { media: { enabled: true, adapter: { @@ -39,6 +39,7 @@ describe("mcp media", async () => { }, }); await app.build(); + await app.getMcpClient().ping(); server = app.mcp!; server.setLogLevel("error"); server.onNotification((message) => { diff --git a/app/__test__/app/mcp/mcp.server.test.ts b/app/__test__/app/mcp/mcp.server.test.ts index 3ada557..ce072b6 100644 --- a/app/__test__/app/mcp/mcp.server.test.ts +++ b/app/__test__/app/mcp/mcp.server.test.ts @@ -1,6 +1,10 @@ -import { describe, test, expect, beforeAll, mock, beforeEach, afterAll } from "bun:test"; +import { describe, test, expect, beforeAll, afterAll } from "bun:test"; import { type App, createApp, createMcpToolCaller } from "core/test/utils"; import type { McpServer } from "bknd/utils"; +import { disableConsoleLog, enableConsoleLog } from "core/utils/test"; + +beforeAll(() => disableConsoleLog()); +afterAll(enableConsoleLog); /** * - [x] config_server_get @@ -11,7 +15,7 @@ describe("mcp system", async () => { let server: McpServer; beforeAll(async () => { app = createApp({ - initialConfig: { + config: { server: { mcp: { enabled: true, @@ -20,6 +24,7 @@ describe("mcp system", async () => { }, }); await app.build(); + await app.getMcpClient().ping(); server = app.mcp!; }); diff --git a/app/__test__/app/mcp/mcp.system.test.ts b/app/__test__/app/mcp/mcp.system.test.ts index 6b08628..de52198 100644 --- a/app/__test__/app/mcp/mcp.system.test.ts +++ b/app/__test__/app/mcp/mcp.system.test.ts @@ -14,7 +14,7 @@ describe("mcp system", async () => { let server: McpServer; beforeAll(async () => { app = createApp({ - initialConfig: { + config: { server: { mcp: { enabled: true, @@ -23,6 +23,7 @@ describe("mcp system", async () => { }, }); await app.build(); + await app.getMcpClient().ping(); server = app.mcp!; }); diff --git a/app/__test__/app/plugins/sync-config.test.ts b/app/__test__/app/plugins/sync-config.test.ts new file mode 100644 index 0000000..ba299c9 --- /dev/null +++ b/app/__test__/app/plugins/sync-config.test.ts @@ -0,0 +1,80 @@ +import { describe, it, expect, mock, beforeAll, afterAll } from "bun:test"; +import { createApp } from "core/test/utils"; +import { syncConfig } from "plugins/dev/sync-config.plugin"; +import { disableConsoleLog, enableConsoleLog } from "core/utils/test"; + +beforeAll(() => disableConsoleLog()); +afterAll(enableConsoleLog); + +describe("syncConfig", () => { + it("should only sync if enabled", async () => { + const called = mock(() => null); + const app = createApp(); + await app.build(); + + await syncConfig({ + write: () => { + called(); + }, + enabled: false, + includeFirstBoot: false, + })(app).onBuilt?.(); + expect(called).not.toHaveBeenCalled(); + + await syncConfig({ + write: () => { + called(); + }, + enabled: false, + includeFirstBoot: true, + })(app).onBuilt?.(); + expect(called).not.toHaveBeenCalled(); + + await syncConfig({ + write: () => { + called(); + }, + enabled: true, + includeFirstBoot: true, + })(app).onBuilt?.(); + expect(called).toHaveBeenCalledTimes(1); + }); + + it("should respect secrets", async () => { + const called = mock(() => null); + const app = createApp({ + config: { + auth: { + enabled: true, + jwt: { + secret: "test", + }, + }, + }, + }); + await app.build(); + + await syncConfig({ + write: async (config) => { + called(); + expect(config.auth.jwt.secret).toBe("test"); + }, + enabled: true, + includeSecrets: true, + includeFirstBoot: true, + })(app).onBuilt?.(); + + await syncConfig({ + write: async (config) => { + called(); + // it's an important test, because the `jwt` part is omitted if secrets=false in general app.toJSON() + // but it's required to get the app running + expect(config.auth.jwt.secret).toBe(""); + }, + enabled: true, + includeSecrets: false, + includeFirstBoot: true, + })(app).onBuilt?.(); + expect(called).toHaveBeenCalledTimes(2); + }); +}); diff --git a/app/__test__/app/repro.spec.ts b/app/__test__/app/repro.spec.ts index be2da38..54ff7e2 100644 --- a/app/__test__/app/repro.spec.ts +++ b/app/__test__/app/repro.spec.ts @@ -1,8 +1,12 @@ -import { describe, expect, test } from "bun:test"; +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; import { registries } from "../../src"; import { createApp } from "core/test/utils"; import * as proto from "../../src/data/prototype"; import { StorageLocalAdapter } from "adapter/node/storage/StorageLocalAdapter"; +import { disableConsoleLog, enableConsoleLog } from "core/utils/test"; + +beforeAll(() => disableConsoleLog()); +afterAll(enableConsoleLog); describe("repros", async () => { /** @@ -88,7 +92,7 @@ describe("repros", async () => { fns.relation(schema.product_likes).manyToOne(schema.users); }, ); - const app = createApp({ initialConfig: { data: schema.toJSON() } }); + const app = createApp({ config: { data: schema.toJSON() } }); await app.build(); const info = (await (await app.server.request("/api/data/info/products")).json()) as any; diff --git a/app/__test__/core/EventManager.spec.ts b/app/__test__/core/EventManager.spec.ts index 3d8b981..65d4433 100644 --- a/app/__test__/core/EventManager.spec.ts +++ b/app/__test__/core/EventManager.spec.ts @@ -1,6 +1,6 @@ import { afterAll, beforeAll, describe, expect, mock, test } from "bun:test"; import { Event, EventManager, InvalidEventReturn, NoParamEvent } from "../../src/core/events"; -import { disableConsoleLog, enableConsoleLog } from "../helper"; +import { disableConsoleLog, enableConsoleLog } from "core/utils/test"; beforeAll(disableConsoleLog); afterAll(enableConsoleLog); diff --git a/app/__test__/core/utils.spec.ts b/app/__test__/core/utils.spec.ts index 15428bf..36b4969 100644 --- a/app/__test__/core/utils.spec.ts +++ b/app/__test__/core/utils.spec.ts @@ -248,7 +248,7 @@ describe("Core Utils", async () => { expect(utils.getContentName(request)).toBe(name); }); - test.only("detectImageDimensions", async () => { + test("detectImageDimensions", async () => { // wrong // @ts-expect-error expect(utils.detectImageDimensions(new ArrayBuffer(), "text/plain")).rejects.toThrow(); @@ -267,12 +267,12 @@ describe("Core Utils", async () => { }); describe("dates", () => { - test.only("formats local time", () => { + test("formats local time", () => { expect(utils.datetimeStringUTC("2025-02-21T16:48:25.841Z")).toBe("2025-02-21 16:48:25"); - console.log(utils.datetimeStringUTC(new Date())); + /*console.log(utils.datetimeStringUTC(new Date())); console.log(utils.datetimeStringUTC()); console.log(new Date()); - console.log("timezone", Intl.DateTimeFormat().resolvedOptions().timeZone); + console.log("timezone", Intl.DateTimeFormat().resolvedOptions().timeZone); */ }); }); }); diff --git a/app/__test__/data/DataController.spec.ts b/app/__test__/data/DataController.spec.ts index ca4905d..9ca90d8 100644 --- a/app/__test__/data/DataController.spec.ts +++ b/app/__test__/data/DataController.spec.ts @@ -5,7 +5,8 @@ import { parse } from "core/utils/schema"; import { DataController } from "../../src/data/api/DataController"; import { dataConfigSchema } from "../../src/data/data-schema"; -import { disableConsoleLog, enableConsoleLog, getDummyConnection } from "../helper"; +import { getDummyConnection } from "../helper"; +import { disableConsoleLog, enableConsoleLog } from "core/utils/test"; import type { RepositoryResultJSON } from "data/entities/query/RepositoryResult"; import type { MutatorResultJSON } from "data/entities/mutation/MutatorResult"; import { Entity, EntityManager, type EntityData } from "data/entities"; @@ -13,7 +14,7 @@ import { TextField } from "data/fields"; import { ManyToOneRelation } from "data/relations"; const { dummyConnection, afterAllCleanup } = getDummyConnection(); -beforeAll(() => disableConsoleLog(["log", "warn"])); +beforeAll(() => disableConsoleLog()); afterAll(async () => (await afterAllCleanup()) && enableConsoleLog()); const dataConfig = parse(dataConfigSchema, {}); diff --git a/app/__test__/data/specs/JoinBuilder.spec.ts b/app/__test__/data/specs/JoinBuilder.spec.ts index 16f8d30..4ed1255 100644 --- a/app/__test__/data/specs/JoinBuilder.spec.ts +++ b/app/__test__/data/specs/JoinBuilder.spec.ts @@ -1,12 +1,15 @@ -import { afterAll, describe, expect, test } from "bun:test"; +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; import { Entity, EntityManager } from "data/entities"; import { ManyToOneRelation } from "data/relations"; import { TextField } from "data/fields"; import { JoinBuilder } from "data/entities/query/JoinBuilder"; import { getDummyConnection } from "../helper"; +import { disableConsoleLog, enableConsoleLog } from "core/utils/test"; + +beforeAll(() => disableConsoleLog()); const { dummyConnection, afterAllCleanup } = getDummyConnection(); -afterAll(afterAllCleanup); +afterAll(async () => (await afterAllCleanup()) && enableConsoleLog()); describe("[data] JoinBuilder", async () => { test("missing relation", async () => { diff --git a/app/__test__/data/specs/Mutator.spec.ts b/app/__test__/data/specs/Mutator.spec.ts index 45bbb28..c4014c8 100644 --- a/app/__test__/data/specs/Mutator.spec.ts +++ b/app/__test__/data/specs/Mutator.spec.ts @@ -9,13 +9,14 @@ import { } from "data/relations"; import { NumberField, TextField } from "data/fields"; import * as proto from "data/prototype"; -import { getDummyConnection, disableConsoleLog, enableConsoleLog } from "../../helper"; +import { getDummyConnection } from "../../helper"; +import { disableConsoleLog, enableConsoleLog } from "core/utils/test"; import { MutatorEvents } from "data/events"; const { dummyConnection, afterAllCleanup } = getDummyConnection(); afterAll(afterAllCleanup); -beforeAll(() => disableConsoleLog(["log", "warn"])); +beforeAll(() => disableConsoleLog()); afterAll(async () => (await afterAllCleanup()) && enableConsoleLog()); describe("[data] Mutator (base)", async () => { diff --git a/app/__test__/data/specs/Repository.spec.ts b/app/__test__/data/specs/Repository.spec.ts index 35c4ec5..d9b2dc2 100644 --- a/app/__test__/data/specs/Repository.spec.ts +++ b/app/__test__/data/specs/Repository.spec.ts @@ -1,4 +1,4 @@ -import { afterAll, describe, expect, test } from "bun:test"; +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; import type { Kysely, Transaction } from "kysely"; import { TextField } from "data/fields"; import { em as $em, entity as $entity, text as $text } from "data/prototype"; @@ -6,11 +6,13 @@ import { Entity, EntityManager } from "data/entities"; import { ManyToOneRelation } from "data/relations"; import { RepositoryEvents } from "data/events"; import { getDummyConnection } from "../helper"; +import { disableConsoleLog, enableConsoleLog } from "core/utils/test"; type E = Kysely | Transaction; const { dummyConnection, afterAllCleanup } = getDummyConnection(); -afterAll(afterAllCleanup); +beforeAll(() => disableConsoleLog()); +afterAll(async () => (await afterAllCleanup()) && enableConsoleLog()); async function sleep(ms: number) { return new Promise((resolve) => { diff --git a/app/__test__/data/specs/WithBuilder.spec.ts b/app/__test__/data/specs/WithBuilder.spec.ts index 31cfd96..b778b0a 100644 --- a/app/__test__/data/specs/WithBuilder.spec.ts +++ b/app/__test__/data/specs/WithBuilder.spec.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "bun:test"; +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; import { Entity, EntityManager } from "data/entities"; import { ManyToManyRelation, ManyToOneRelation, PolymorphicRelation } from "data/relations"; import { TextField } from "data/fields"; @@ -6,6 +6,10 @@ import * as proto from "data/prototype"; import { WithBuilder } from "data/entities/query/WithBuilder"; import { schemaToEm } from "../../helper"; import { getDummyConnection } from "../helper"; +import { disableConsoleLog, enableConsoleLog } from "core/utils/test"; + +beforeAll(() => disableConsoleLog()); +afterAll(enableConsoleLog); const { dummyConnection } = getDummyConnection(); diff --git a/app/__test__/data/specs/fields/FieldIndex.spec.ts b/app/__test__/data/specs/fields/FieldIndex.spec.ts index ab94214..fbcfabf 100644 --- a/app/__test__/data/specs/fields/FieldIndex.spec.ts +++ b/app/__test__/data/specs/fields/FieldIndex.spec.ts @@ -23,11 +23,4 @@ describe("FieldIndex", async () => { expect(index.name).toEqual("idx_test_name"); expect(index.unique).toEqual(false); }); - - test("it fails on non-unique", async () => { - const field = new TestField("name", { required: false }); - - expect(() => new EntityIndex(entity, [field], true)).toThrowError(); - expect(() => new EntityIndex(entity, [field])).toBeDefined(); - }); }); diff --git a/app/__test__/data/specs/relations/EntityRelation.spec.ts b/app/__test__/data/specs/relations/EntityRelation.spec.ts index 489a5be..12a8325 100644 --- a/app/__test__/data/specs/relations/EntityRelation.spec.ts +++ b/app/__test__/data/specs/relations/EntityRelation.spec.ts @@ -4,8 +4,10 @@ import { type BaseRelationConfig, EntityRelation, EntityRelationAnchor, + ManyToManyRelation, RelationTypes, } from "data/relations"; +import * as proto from "data/prototype"; class TestEntityRelation extends EntityRelation { constructor(config?: BaseRelationConfig) { @@ -75,4 +77,15 @@ describe("[data] EntityRelation", async () => { const relation2 = new TestEntityRelation({ required: true }); expect(relation2.required).toBe(true); }); + + it("correctly produces the relation name", async () => { + const relation = new ManyToManyRelation(new Entity("apps"), new Entity("organizations")); + expect(relation.getName()).not.toContain(","); + expect(relation.getName()).toBe("mn_apps_organizations"); + + const relation2 = new ManyToManyRelation(new Entity("apps"), new Entity("organizations"), { + connectionTableMappedName: "appOrganizations", + }); + expect(relation2.getName()).toBe("mn_apps_organizations_appOrganizations"); + }); }); diff --git a/app/__test__/helper.ts b/app/__test__/helper.ts index 2579a88..aaf88d2 100644 --- a/app/__test__/helper.ts +++ b/app/__test__/helper.ts @@ -39,26 +39,6 @@ export function getLocalLibsqlConnection() { return { url: "http://127.0.0.1:8080" }; } -type ConsoleSeverity = "debug" | "log" | "warn" | "error"; -const _oldConsoles = { - debug: console.debug, - log: console.log, - warn: console.warn, - error: console.error, -}; - -export function disableConsoleLog(severities: ConsoleSeverity[] = ["debug", "log", "warn"]) { - severities.forEach((severity) => { - console[severity] = () => null; - }); -} - -export function enableConsoleLog() { - Object.entries(_oldConsoles).forEach(([severity, fn]) => { - console[severity as ConsoleSeverity] = fn; - }); -} - export function compileQb(qb: SelectQueryBuilder) { const { sql, parameters } = qb.compile(); return { sql, parameters }; @@ -66,7 +46,7 @@ export function compileQb(qb: SelectQueryBuilder) { export function prettyPrintQb(qb: SelectQueryBuilder) { const { sql, parameters } = qb.compile(); - console.log("$", sqlFormat(sql), "\n[params]", parameters); + console.info("$", sqlFormat(sql), "\n[params]", parameters); } export function schemaToEm(s: ReturnType, conn?: Connection): EntityManager { diff --git a/app/__test__/integration/auth.integration.test.ts b/app/__test__/integration/auth.integration.test.ts index 298ad31..340ccaf 100644 --- a/app/__test__/integration/auth.integration.test.ts +++ b/app/__test__/integration/auth.integration.test.ts @@ -1,12 +1,9 @@ -import { afterAll, afterEach, beforeAll, describe, expect, it } from "bun:test"; -import { App, createApp } from "../../src"; -import type { AuthResponse } from "../../src/auth"; +import { afterAll, beforeAll, describe, expect, it } from "bun:test"; +import { App, createApp, type AuthResponse } from "../../src"; import { auth } from "../../src/auth/middlewares"; import { randomString, secureRandomString, withDisabledConsole } from "../../src/core/utils"; -import { disableConsoleLog, enableConsoleLog, getDummyConnection } from "../helper"; - -const { dummyConnection, afterAllCleanup } = getDummyConnection(); -afterEach(afterAllCleanup); +import { disableConsoleLog, enableConsoleLog } from "core/utils/test"; +import { getDummyConnection } from "../helper"; beforeAll(disableConsoleLog); afterAll(enableConsoleLog); @@ -66,9 +63,10 @@ const configs = { }; function createAuthApp() { + const { dummyConnection } = getDummyConnection(); const app = createApp({ connection: dummyConnection, - initialConfig: { + config: { auth: configs.auth, }, }); @@ -151,8 +149,8 @@ describe("integration auth", () => { const { data: users } = await app.em.repository("users").findMany(); expect(users.length).toBe(2); - expect(users[0].email).toBe(configs.users.normal.email); - expect(users[1].email).toBe(configs.users.admin.email); + expect(users[0]?.email).toBe(configs.users.normal.email); + expect(users[1]?.email).toBe(configs.users.admin.email); }); it("should log you in with API", async () => { @@ -223,7 +221,7 @@ describe("integration auth", () => { app.server.get("/get", auth(), async (c) => { return c.json({ - user: c.get("auth").user ?? null, + user: c.get("auth")?.user ?? null, }); }); app.server.get("/wait", auth(), async (c) => { @@ -242,7 +240,7 @@ describe("integration auth", () => { { await new Promise((r) => setTimeout(r, 10)); const res = await app.server.request("/get"); - const data = await res.json(); + const data = (await res.json()) as any; expect(data.user).toBe(null); expect(await $fns.me()).toEqual({ user: null as any }); } diff --git a/app/__test__/integration/config.integration.test.ts b/app/__test__/integration/config.integration.test.ts index 52c7df2..8e8ba4e 100644 --- a/app/__test__/integration/config.integration.test.ts +++ b/app/__test__/integration/config.integration.test.ts @@ -1,6 +1,10 @@ -import { describe, expect, it } from "bun:test"; +import { afterAll, beforeAll, describe, expect, it } from "bun:test"; import { createApp } from "core/test/utils"; import { Api } from "../../src/Api"; +import { disableConsoleLog, enableConsoleLog } from "core/utils/test"; + +beforeAll(disableConsoleLog); +afterAll(enableConsoleLog); describe("integration config", () => { it("should create an entity", async () => { diff --git a/app/__test__/media/MediaController.spec.ts b/app/__test__/media/MediaController.spec.ts index 7c9ae9f..bf62599 100644 --- a/app/__test__/media/MediaController.spec.ts +++ b/app/__test__/media/MediaController.spec.ts @@ -6,17 +6,20 @@ import { createApp } from "core/test/utils"; import { mergeObject, randomString } from "../../src/core/utils"; import type { TAppMediaConfig } from "../../src/media/media-schema"; import { StorageLocalAdapter } from "adapter/node/storage/StorageLocalAdapter"; -import { assetsPath, assetsTmpPath, disableConsoleLog, enableConsoleLog } from "../helper"; +import { assetsPath, assetsTmpPath } from "../helper"; +import { disableConsoleLog, enableConsoleLog } from "core/utils/test"; beforeAll(() => { + //disableConsoleLog(); registries.media.register("local", StorageLocalAdapter); }); +afterAll(enableConsoleLog); const path = `${assetsPath}/image.png`; async function makeApp(mediaOverride: Partial = {}) { const app = createApp({ - initialConfig: { + config: { media: mergeObject( { enabled: true, @@ -40,9 +43,6 @@ function makeName(ext: string) { return randomString(10) + "." + ext; } -beforeAll(disableConsoleLog); -afterAll(enableConsoleLog); - describe("MediaController", () => { test("accepts direct", async () => { const app = await makeApp(); @@ -94,4 +94,38 @@ describe("MediaController", () => { expect(res.status).toBe(413); expect(await Bun.file(assetsTmpPath + "/" + name).exists()).toBe(false); }); + + test("audio files", async () => { + const app = await makeApp(); + const file = Bun.file(`${assetsPath}/test.mp3`); + const name = makeName("mp3"); + const res = await app.server.request("/api/media/upload/" + name, { + method: "POST", + body: file, + }); + const result = (await res.json()) as any; + expect(result.data.mime_type).toStartWith("audio/mpeg"); + expect(result.name).toBe(name); + + const destFile = Bun.file(assetsTmpPath + "/" + name); + expect(destFile.exists()).resolves.toBe(true); + await destFile.delete(); + }); + + test("text files", async () => { + const app = await makeApp(); + const file = Bun.file(`${assetsPath}/test.txt`); + const name = makeName("txt"); + const res = await app.server.request("/api/media/upload/" + name, { + method: "POST", + body: file, + }); + const result = (await res.json()) as any; + expect(result.data.mime_type).toStartWith("text/plain"); + expect(result.name).toBe(name); + + const destFile = Bun.file(assetsTmpPath + "/" + name); + expect(destFile.exists()).resolves.toBe(true); + await destFile.delete(); + }); }); diff --git a/app/__test__/media/mime-types.spec.ts b/app/__test__/media/mime-types.spec.ts index dd13f7c..1435dd6 100644 --- a/app/__test__/media/mime-types.spec.ts +++ b/app/__test__/media/mime-types.spec.ts @@ -71,6 +71,8 @@ describe("media/mime-types", () => { ["application/zip", "zip"], ["text/tab-separated-values", "tsv"], ["application/zip", "zip"], + ["application/pdf", "pdf"], + ["audio/mpeg", "mp3"], ] as const; for (const [mime, ext] of tests) { @@ -88,6 +90,9 @@ describe("media/mime-types", () => { ["image.jpeg", "jpeg"], ["-473Wx593H-466453554-black-MODEL.jpg", "jpg"], ["-473Wx593H-466453554-black-MODEL.avif", "avif"], + ["file.pdf", "pdf"], + ["file.mp3", "mp3"], + ["robots.txt", "txt"], ] as const; for (const [filename, ext] of tests) { @@ -102,4 +107,36 @@ describe("media/mime-types", () => { const [, ext] = getRandomizedFilename(file).split("."); expect(ext).toBe("jpg"); }); + + test("getRandomizedFilename with body", async () => { + // should keep "pdf" + const [, ext] = getRandomizedFilename( + new File([""], "file.pdf", { type: "application/pdf" }), + ).split("."); + expect(ext).toBe("pdf"); + + { + // no ext, should use "pdf" only for known formats + const [, ext] = getRandomizedFilename( + new File([""], "file", { type: "application/pdf" }), + ).split("."); + expect(ext).toBe("pdf"); + } + + { + // wrong ext, should keep the wrong one + const [, ext] = getRandomizedFilename( + new File([""], "file.what", { type: "application/pdf" }), + ).split("."); + expect(ext).toBe("what"); + } + + { + // txt + const [, ext] = getRandomizedFilename( + new File([""], "file.txt", { type: "text/plain" }), + ).split("."); + expect(ext).toBe("txt"); + } + }); }); diff --git a/app/__test__/modules/AppAuth.spec.ts b/app/__test__/modules/AppAuth.spec.ts index e523fbc..89872de 100644 --- a/app/__test__/modules/AppAuth.spec.ts +++ b/app/__test__/modules/AppAuth.spec.ts @@ -3,11 +3,14 @@ import { createApp } from "core/test/utils"; import { AuthController } from "../../src/auth/api/AuthController"; import { em, entity, make, text } from "data/prototype"; import { AppAuth, type ModuleBuildContext } from "modules"; -import { disableConsoleLog, enableConsoleLog } from "../helper"; import { makeCtx, moduleTestSuite } from "./module-test-suite"; +import { disableConsoleLog, enableConsoleLog } from "core/utils/test"; + +beforeAll(disableConsoleLog); +afterAll(enableConsoleLog); describe("AppAuth", () => { - test.only("...", () => { + test.skip("...", () => { const auth = new AppAuth({}); console.log(auth.toJSON()); console.log(auth.config); @@ -147,7 +150,7 @@ describe("AppAuth", () => { test("registers auth middleware for bknd routes only", async () => { const app = createApp({ - initialConfig: { + config: { auth: { enabled: true, jwt: { @@ -177,7 +180,7 @@ describe("AppAuth", () => { test("should allow additional user fields", async () => { const app = createApp({ - initialConfig: { + config: { auth: { entity_name: "users", enabled: true, @@ -201,7 +204,7 @@ describe("AppAuth", () => { test("ensure user field configs is always correct", async () => { const app = createApp({ - initialConfig: { + config: { auth: { enabled: true, }, diff --git a/app/__test__/modules/AppMedia.spec.ts b/app/__test__/modules/AppMedia.spec.ts index d09041b..fb5464a 100644 --- a/app/__test__/modules/AppMedia.spec.ts +++ b/app/__test__/modules/AppMedia.spec.ts @@ -7,7 +7,7 @@ import { AppMedia } from "../../src/media/AppMedia"; import { moduleTestSuite } from "./module-test-suite"; describe("AppMedia", () => { - test.only("...", () => { + test.skip("...", () => { const media = new AppMedia(); console.log(media.toJSON()); }); @@ -18,7 +18,7 @@ describe("AppMedia", () => { registries.media.register("local", StorageLocalAdapter); const app = createApp({ - initialConfig: { + config: { media: { entity_name: "media", enabled: true, diff --git a/app/__test__/modules/DbModuleManager.spec.ts b/app/__test__/modules/DbModuleManager.spec.ts new file mode 100644 index 0000000..b85ebbf --- /dev/null +++ b/app/__test__/modules/DbModuleManager.spec.ts @@ -0,0 +1,76 @@ +import { it, expect, describe } from "bun:test"; +import { DbModuleManager } from "modules/db/DbModuleManager"; +import { getDummyConnection } from "../helper"; +import { TABLE_NAME } from "modules/db/migrations"; + +describe("DbModuleManager", () => { + it("should extract secrets", async () => { + const { dummyConnection } = getDummyConnection(); + const m = new DbModuleManager(dummyConnection, { + initial: { + auth: { + enabled: true, + jwt: { + secret: "test", + }, + }, + }, + }); + await m.build(); + expect(m.toJSON(true).auth.jwt.secret).toBe("test"); + await m.save(); + }); + + it("should work with initial secrets", async () => { + const { dummyConnection } = getDummyConnection(); + const db = dummyConnection.kysely; + const m = new DbModuleManager(dummyConnection, { + initial: { + auth: { + enabled: true, + jwt: { + secret: "", + }, + }, + }, + secrets: { + "auth.jwt.secret": "test", + }, + }); + await m.build(); + expect(m.toJSON(true).auth.jwt.secret).toBe("test"); + + const getSecrets = () => + db + .selectFrom(TABLE_NAME) + .selectAll() + .where("type", "=", "secrets") + .executeTakeFirst() + .then((r) => r?.json); + + expect(await getSecrets()).toEqual({ "auth.jwt.secret": "test" }); + + // also after rebuild + await m.build(); + await m.save(); + expect(await getSecrets()).toEqual({ "auth.jwt.secret": "test" }); + + // and ignore if already present + const m2 = new DbModuleManager(dummyConnection, { + initial: { + auth: { + enabled: true, + jwt: { + secret: "", + }, + }, + }, + secrets: { + "auth.jwt.secret": "something completely different", + }, + }); + await m2.build(); + await m2.save(); + expect(await getSecrets()).toEqual({ "auth.jwt.secret": "test" }); + }); +}); diff --git a/app/__test__/modules/ModuleManager.spec.ts b/app/__test__/modules/ModuleManager.spec.ts index 9c24de0..de5c889 100644 --- a/app/__test__/modules/ModuleManager.spec.ts +++ b/app/__test__/modules/ModuleManager.spec.ts @@ -1,14 +1,19 @@ -import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test"; -import { disableConsoleLog, enableConsoleLog } from "core/utils"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import { disableConsoleLog, enableConsoleLog } from "core/utils/test"; import { Module } from "modules/Module"; -import { type ConfigTable, getDefaultConfig, ModuleManager } from "modules/ModuleManager"; -import { CURRENT_VERSION, TABLE_NAME } from "modules/migrations"; +import { getDefaultConfig } from "modules/ModuleManager"; +import { type ConfigTable, DbModuleManager as ModuleManager } from "modules/db/DbModuleManager"; + +import { CURRENT_VERSION, TABLE_NAME } from "modules/db/migrations"; import { getDummyConnection } from "../helper"; import { s, stripMark } from "core/utils/schema"; import { Connection } from "data/connection/Connection"; import { entity, text } from "data/prototype"; +beforeAll(disableConsoleLog); +afterAll(enableConsoleLog); + describe("ModuleManager", async () => { test("s1: no config, no build", async () => { const { dummyConnection } = getDummyConnection(); @@ -133,7 +138,7 @@ describe("ModuleManager", async () => { const db = c2.dummyConnection.kysely; const mm2 = new ModuleManager(c2.dummyConnection, { - initial: { version: version - 1, ...json }, + initial: { version: version - 1, ...json } as any, }); await mm2.syncConfigTable(); await db diff --git a/app/__test__/modules/migrations/migrations.spec.ts b/app/__test__/modules/migrations/migrations.spec.ts index 4bd83ab..1266746 100644 --- a/app/__test__/modules/migrations/migrations.spec.ts +++ b/app/__test__/modules/migrations/migrations.spec.ts @@ -1,14 +1,22 @@ -import { describe, expect, test } from "bun:test"; -import { type InitialModuleConfigs, createApp } from "../../../src"; +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { App, type InitialModuleConfigs, createApp } from "/"; import { type Kysely, sql } from "kysely"; import { getDummyConnection } from "../../helper"; import v7 from "./samples/v7.json"; import v8 from "./samples/v8.json"; import v8_2 from "./samples/v8-2.json"; +import v9 from "./samples/v9.json"; +import { disableConsoleLog, enableConsoleLog } from "core/utils/test"; + +beforeAll(() => disableConsoleLog()); +afterAll(enableConsoleLog); // app expects migratable config to be present in database -async function createVersionedApp(config: InitialModuleConfigs | any) { +async function createVersionedApp( + config: InitialModuleConfigs | any, + opts?: { beforeCreateApp?: (db: Kysely) => Promise }, +) { const { dummyConnection } = getDummyConnection(); if (!("version" in config)) throw new Error("config must have a version"); @@ -34,6 +42,10 @@ async function createVersionedApp(config: InitialModuleConfigs | any) { }) .execute(); + if (opts?.beforeCreateApp) { + await opts.beforeCreateApp(db); + } + const app = createApp({ connection: dummyConnection, }); @@ -41,6 +53,19 @@ async function createVersionedApp(config: InitialModuleConfigs | any) { return app; } +async function getRawConfig( + app: App, + opts?: { version?: number; types?: ("config" | "diff" | "backup" | "secrets")[] }, +) { + const db = app.em.connection.kysely; + return await db + .selectFrom("__bknd") + .selectAll() + .$if(!!opts?.version, (qb) => qb.where("version", "=", opts?.version)) + .$if((opts?.types?.length ?? 0) > 0, (qb) => qb.where("type", "in", opts?.types)) + .execute(); +} + describe("Migrations", () => { /** * updated auth strategies to have "enabled" prop @@ -78,4 +103,30 @@ describe("Migrations", () => { // @ts-expect-error expect(app.toJSON(true).server.admin).toBeUndefined(); }); + + test("migration from 9 to 10", async () => { + expect(v9.version).toBe(9); + + const app = await createVersionedApp(v9); + + expect(app.version()).toBeGreaterThan(9); + // @ts-expect-error + expect(app.toJSON(true).media.adapter.config.secret_access_key).toBe( + "^^s3.secret_access_key^^", + ); + const [config, secrets] = (await getRawConfig(app, { + version: 10, + types: ["config", "secrets"], + })) as any; + + expect(config.json.auth.jwt.secret).toBe(""); + expect(config.json.media.adapter.config.access_key).toBe(""); + expect(config.json.media.adapter.config.secret_access_key).toBe(""); + + expect(secrets.json["auth.jwt.secret"]).toBe("^^jwt.secret^^"); + expect(secrets.json["media.adapter.config.access_key"]).toBe("^^s3.access_key^^"); + expect(secrets.json["media.adapter.config.secret_access_key"]).toBe( + "^^s3.secret_access_key^^", + ); + }); }); diff --git a/app/__test__/modules/migrations/samples/v9.json b/app/__test__/modules/migrations/samples/v9.json new file mode 100644 index 0000000..b6fc2a2 --- /dev/null +++ b/app/__test__/modules/migrations/samples/v9.json @@ -0,0 +1,612 @@ +{ + "version": 9, + "server": { + "cors": { + "origin": "*", + "allow_methods": ["GET", "POST", "PATCH", "PUT", "DELETE"], + "allow_headers": [ + "Content-Type", + "Content-Length", + "Authorization", + "Accept" + ], + "allow_credentials": true + }, + "mcp": { "enabled": false, "path": "/api/system/mcp" } + }, + "data": { + "basepath": "/api/data", + "default_primary_format": "integer", + "entities": { + "media": { + "type": "system", + "fields": { + "id": { + "type": "primary", + "config": { + "format": "integer", + "fillable": false, + "required": false + } + }, + "path": { "type": "text", "config": { "required": true } }, + "folder": { + "type": "boolean", + "config": { + "default_value": false, + "hidden": true, + "fillable": ["create"], + "required": false + } + }, + "mime_type": { "type": "text", "config": { "required": false } }, + "size": { "type": "number", "config": { "required": false } }, + "scope": { + "type": "text", + "config": { + "hidden": true, + "fillable": ["create"], + "required": false + } + }, + "etag": { "type": "text", "config": { "required": false } }, + "modified_at": { + "type": "date", + "config": { "type": "datetime", "required": false } + }, + "reference": { "type": "text", "config": { "required": false } }, + "entity_id": { "type": "number", "config": { "required": false } }, + "metadata": { "type": "json", "config": { "required": false } } + }, + "config": { "sort_field": "id", "sort_dir": "asc" } + }, + "users": { + "type": "system", + "fields": { + "id": { + "type": "primary", + "config": { + "format": "integer", + "fillable": false, + "required": false + } + }, + "email": { "type": "text", "config": { "required": true } }, + "strategy": { + "type": "enum", + "config": { + "options": { "type": "strings", "values": ["password"] }, + "required": true, + "hidden": ["update", "form"], + "fillable": ["create"] + } + }, + "strategy_value": { + "type": "text", + "config": { + "fillable": ["create"], + "hidden": ["read", "table", "update", "form"], + "required": true + } + }, + "role": { + "type": "enum", + "config": { + "options": { "type": "strings", "values": ["admin", "guest"] }, + "required": false + } + }, + "age": { + "type": "enum", + "config": { + "options": { + "type": "strings", + "values": ["18-24", "25-34", "35-44", "45-64", "65+"] + }, + "required": false + } + }, + "height": { "type": "number", "config": { "required": false } }, + "gender": { + "type": "enum", + "config": { + "options": { "type": "strings", "values": ["male", "female"] }, + "required": false + } + } + }, + "config": { "sort_field": "id", "sort_dir": "asc" } + }, + "avatars": { + "type": "regular", + "fields": { + "id": { + "type": "primary", + "config": { + "format": "integer", + "fillable": false, + "required": false + } + }, + "identifier": { "type": "text", "config": { "required": false } }, + "payload": { + "type": "json", + "config": { "required": false, "hidden": ["table"] } + }, + "created_at": { + "type": "date", + "config": { "type": "datetime", "required": false } + }, + "started_at": { + "type": "date", + "config": { "type": "datetime", "required": false } + }, + "completed_at": { + "type": "date", + "config": { "type": "datetime", "required": false } + }, + "input": { + "type": "media", + "config": { + "required": false, + "fillable": ["update"], + "hidden": false, + "mime_types": [], + "virtual": true, + "entity": "avatars" + } + }, + "output": { + "type": "media", + "config": { + "required": false, + "fillable": ["update"], + "hidden": false, + "mime_types": [], + "virtual": true, + "entity": "avatars" + } + }, + "users_id": { + "type": "relation", + "config": { + "label": "Users", + "required": false, + "reference": "users", + "target": "users", + "target_field": "id", + "target_field_type": "integer", + "on_delete": "set null" + } + } + }, + "config": { "sort_field": "id", "sort_dir": "desc" } + }, + "tryons": { + "type": "regular", + "fields": { + "id": { + "type": "primary", + "config": { + "format": "integer", + "fillable": false, + "required": false + } + }, + "created_at": { + "type": "date", + "config": { "type": "datetime", "required": false } + }, + "completed_at": { + "type": "date", + "config": { "type": "datetime", "required": false } + }, + "avatars_id": { + "type": "relation", + "config": { + "label": "Avatars", + "required": false, + "reference": "avatars", + "target": "avatars", + "target_field": "id", + "target_field_type": "integer", + "on_delete": "set null" + } + }, + "users_id": { + "type": "relation", + "config": { + "label": "Users", + "required": false, + "reference": "users", + "target": "users", + "target_field": "id", + "target_field_type": "integer", + "on_delete": "set null" + } + }, + "output": { + "type": "media", + "config": { + "required": false, + "fillable": ["update"], + "hidden": false, + "mime_types": [], + "virtual": true, + "entity": "tryons", + "max_items": 1 + } + }, + "products_id": { + "type": "relation", + "config": { + "label": "Products", + "required": false, + "reference": "products", + "target": "products", + "target_field": "id", + "target_field_type": "integer", + "on_delete": "set null" + } + }, + "payload": { + "type": "json", + "config": { "required": false, "hidden": ["table"] } + } + }, + "config": { "sort_field": "id", "sort_dir": "desc" } + }, + "products": { + "type": "regular", + "fields": { + "id": { + "type": "primary", + "config": { + "format": "integer", + "fillable": false, + "required": false + } + }, + "enabled": { "type": "boolean", "config": { "required": false } }, + "title": { "type": "text", "config": { "required": false } }, + "url": { "type": "text", "config": { "required": false } }, + "image": { + "type": "media", + "config": { + "required": false, + "fillable": ["update"], + "hidden": false, + "mime_types": [], + "virtual": true, + "entity": "products", + "max_items": 1 + } + }, + "created_at": { + "type": "date", + "config": { "type": "datetime", "required": false } + }, + "sites_id": { + "type": "relation", + "config": { + "label": "Sites", + "required": false, + "reference": "sites", + "target": "sites", + "target_field": "id", + "target_field_type": "integer", + "on_delete": "set null" + } + }, + "garment_type": { + "type": "enum", + "config": { + "options": { + "type": "strings", + "values": ["auto", "tops", "bottoms", "one-pieces"] + }, + "required": false + } + } + }, + "config": { "sort_field": "id", "sort_dir": "desc" } + }, + "sites": { + "type": "regular", + "fields": { + "id": { + "type": "primary", + "config": { + "format": "integer", + "fillable": false, + "required": false + } + }, + "origin": { + "type": "text", + "config": { + "pattern": "^(https?):\\/\\/([a-zA-Z0-9.-]+)(:\\d+)?$", + "required": true + } + }, + "name": { "type": "text", "config": { "required": false } }, + "active": { "type": "boolean", "config": { "required": false } }, + "logo": { + "type": "media", + "config": { + "required": false, + "fillable": ["update"], + "hidden": false, + "mime_types": [], + "virtual": true, + "entity": "sites", + "max_items": 1 + } + }, + "instructions": { + "type": "text", + "config": { + "html_config": { + "element": "textarea", + "props": { "rows": "2" } + }, + "required": false, + "hidden": ["table"] + } + } + }, + "config": { "sort_field": "id", "sort_dir": "desc" } + }, + "sessions": { + "type": "regular", + "fields": { + "id": { + "type": "primary", + "config": { "format": "uuid", "fillable": false, "required": false } + }, + "created_at": { + "type": "date", + "config": { "type": "datetime", "required": true } + }, + "claimed_at": { + "type": "date", + "config": { "type": "datetime", "required": false } + }, + "url": { "type": "text", "config": { "required": false } }, + "sites_id": { + "type": "relation", + "config": { + "label": "Sites", + "required": false, + "reference": "sites", + "target": "sites", + "target_field": "id", + "target_field_type": "integer", + "on_delete": "set null" + } + }, + "users_id": { + "type": "relation", + "config": { + "label": "Users", + "required": false, + "reference": "users", + "target": "users", + "target_field": "id", + "target_field_type": "integer", + "on_delete": "set null" + } + } + }, + "config": { "sort_field": "id", "sort_dir": "desc" } + } + }, + "relations": { + "poly_avatars_media_input": { + "type": "poly", + "source": "avatars", + "target": "media", + "config": { "mappedBy": "input" } + }, + "poly_avatars_media_output": { + "type": "poly", + "source": "avatars", + "target": "media", + "config": { "mappedBy": "output" } + }, + "n1_avatars_users": { + "type": "n:1", + "source": "avatars", + "target": "users", + "config": { + "mappedBy": "", + "inversedBy": "", + "required": false, + "with_limit": 5 + } + }, + "n1_tryons_avatars": { + "type": "n:1", + "source": "tryons", + "target": "avatars", + "config": { + "mappedBy": "", + "inversedBy": "", + "required": false, + "with_limit": 5 + } + }, + "n1_tryons_users": { + "type": "n:1", + "source": "tryons", + "target": "users", + "config": { + "mappedBy": "", + "inversedBy": "", + "required": false, + "with_limit": 5 + } + }, + "poly_tryons_media_output": { + "type": "poly", + "source": "tryons", + "target": "media", + "config": { "mappedBy": "output", "targetCardinality": 1 } + }, + "poly_products_media_image": { + "type": "poly", + "source": "products", + "target": "media", + "config": { "mappedBy": "image", "targetCardinality": 1 } + }, + "n1_tryons_products": { + "type": "n:1", + "source": "tryons", + "target": "products", + "config": { + "mappedBy": "", + "inversedBy": "", + "required": false, + "with_limit": 5 + } + }, + "poly_sites_media_logo": { + "type": "poly", + "source": "sites", + "target": "media", + "config": { "mappedBy": "logo", "targetCardinality": 1 } + }, + "n1_sessions_sites": { + "type": "n:1", + "source": "sessions", + "target": "sites", + "config": { + "mappedBy": "", + "inversedBy": "", + "required": false, + "with_limit": 5 + } + }, + "n1_sessions_users": { + "type": "n:1", + "source": "sessions", + "target": "users", + "config": { + "mappedBy": "", + "inversedBy": "", + "required": false, + "with_limit": 5 + } + }, + "n1_products_sites": { + "type": "n:1", + "source": "products", + "target": "sites", + "config": { + "mappedBy": "", + "inversedBy": "", + "required": false, + "with_limit": 5 + } + } + }, + "indices": { + "idx_unique_media_path": { + "entity": "media", + "fields": ["path"], + "unique": true + }, + "idx_media_reference": { + "entity": "media", + "fields": ["reference"], + "unique": false + }, + "idx_media_entity_id": { + "entity": "media", + "fields": ["entity_id"], + "unique": false + }, + "idx_unique_users_email": { + "entity": "users", + "fields": ["email"], + "unique": true + }, + "idx_users_strategy": { + "entity": "users", + "fields": ["strategy"], + "unique": false + }, + "idx_users_strategy_value": { + "entity": "users", + "fields": ["strategy_value"], + "unique": false + }, + "idx_sites_origin_active": { + "entity": "sites", + "fields": ["origin", "active"], + "unique": false + }, + "idx_sites_active": { + "entity": "sites", + "fields": ["active"], + "unique": false + }, + "idx_products_url": { + "entity": "products", + "fields": ["url"], + "unique": false + } + } + }, + "auth": { + "enabled": true, + "basepath": "/api/auth", + "entity_name": "users", + "allow_register": true, + "jwt": { + "secret": "^^jwt.secret^^", + "alg": "HS256", + "expires": 999999999, + "issuer": "issuer", + "fields": ["id", "email", "role"] + }, + "cookie": { + "path": "/", + "sameSite": "none", + "secure": true, + "httpOnly": true, + "expires": 604800, + "partitioned": false, + "renew": true, + "pathSuccess": "/admin", + "pathLoggedOut": "/" + }, + "strategies": { + "password": { + "enabled": true, + "type": "password", + "config": { "hashing": "sha256" } + } + }, + "guard": { "enabled": false }, + "roles": { + "admin": { "implicit_allow": true }, + "guest": { "is_default": true } + } + }, + "media": { + "enabled": true, + "basepath": "/api/media", + "entity_name": "media", + "storage": { "body_max_size": 0 }, + "adapter": { + "type": "s3", + "config": { + "access_key": "^^s3.access_key^^", + "secret_access_key": "^^s3.secret_access_key^^", + "url": "https://1234.r2.cloudflarestorage.com/bucket-name" + } + } + }, + "flows": { "basepath": "/api/flows", "flows": {} } +} diff --git a/app/build.cli.ts b/app/build.cli.ts index 999a6a1..fee5373 100644 --- a/app/build.cli.ts +++ b/app/build.cli.ts @@ -3,20 +3,25 @@ import c from "picocolors"; import { formatNumber } from "bknd/utils"; import * as esbuild from "esbuild"; +const deps = Object.keys(pkg.dependencies); +const external = ["jsonv-ts/*", "wrangler", "bknd", "bknd/*", ...deps]; + if (process.env.DEBUG) { - await esbuild.build({ + const result = await esbuild.build({ entryPoints: ["./src/cli/index.ts"], outdir: "./dist/cli", platform: "node", - minify: false, + minify: true, format: "esm", + metafile: true, bundle: true, - external: ["jsonv-ts", "jsonv-ts/*"], + external, define: { __isDev: "0", __version: JSON.stringify(pkg.version), }, }); + await Bun.write("./dist/cli/metafile-esm.json", JSON.stringify(result.metafile, null, 2)); process.exit(0); } @@ -26,7 +31,7 @@ const result = await Bun.build({ outdir: "./dist/cli", env: "PUBLIC_*", minify: true, - external: ["jsonv-ts", "jsonv-ts/*"], + external, define: { __isDev: "0", __version: JSON.stringify(pkg.version), diff --git a/app/build.ts b/app/build.ts index 66db700..b8729bd 100644 --- a/app/build.ts +++ b/app/build.ts @@ -252,6 +252,8 @@ async function buildAdapters() { platform: "neutral", entry: ["src/adapter/index.ts"], outDir: "dist/adapter", + // only way to keep @vite-ignore comments + minify: false, }), // specific adatpers @@ -270,6 +272,7 @@ async function buildAdapters() { ), tsup.build( baseConfig("cloudflare/proxy", { + target: "esnext", entry: ["src/adapter/cloudflare/proxy.ts"], outDir: "dist/adapter/cloudflare", metafile: false, diff --git a/app/internal/docs.build-assets.ts b/app/internal/docs.build-assets.ts index 730367f..4a4db73 100644 --- a/app/internal/docs.build-assets.ts +++ b/app/internal/docs.build-assets.ts @@ -3,7 +3,7 @@ import { createApp } from "bknd/adapter/bun"; async function generate() { console.info("Generating MCP documentation..."); const app = await createApp({ - initialConfig: { + config: { server: { mcp: { enabled: true, diff --git a/app/package.json b/app/package.json index 98d9426..9486c50 100644 --- a/app/package.json +++ b/app/package.json @@ -3,7 +3,7 @@ "type": "module", "sideEffects": false, "bin": "./dist/cli/index.js", - "version": "0.17.2", + "version": "0.18.0", "description": "Lightweight Firebase/Supabase alternative built to run anywhere — incl. Next.js, React Router, Astro, Cloudflare, Bun, Node, AWS Lambda & more.", "homepage": "https://bknd.io", "repository": { @@ -13,7 +13,7 @@ "bugs": { "url": "https://github.com/bknd-io/bknd/issues" }, - "packageManager": "bun@1.2.19", + "packageManager": "bun@1.2.22", "engines": { "node": ">=22.13" }, @@ -30,7 +30,7 @@ "build:types": "tsc -p tsconfig.build.json --emitDeclarationOnly && tsc-alias", "updater": "bun x npm-check-updates -ui", "cli": "LOCAL=1 bun src/cli/index.ts", - "prepublishOnly": "bun run types && bun run test && bun run test:node && VITE_DB_URL=:memory: bun run test:e2e && bun run build:all && cp ../README.md ./", + "prepublishOnly": "bun run types && bun run test && bun run test:node && NODE_NO_WARNINGS=1 VITE_DB_URL=:memory: bun run test:e2e && bun run build:all && cp ../README.md ./", "postpublish": "rm -f README.md", "test": "ALL_TESTS=1 bun test --bail", "test:all": "bun run test && bun run test:node", @@ -40,7 +40,7 @@ "test:coverage": "ALL_TESTS=1 bun test --bail --coverage", "test:vitest:coverage": "vitest run --coverage", "test:e2e": "playwright test", - "test:e2e:adapters": "bun run e2e/adapters.ts", + "test:e2e:adapters": "NODE_NO_WARNINGS=1 bun run e2e/adapters.ts", "test:e2e:ui": "VITE_DB_URL=:memory: playwright test --ui", "test:e2e:debug": "VITE_DB_URL=:memory: playwright test --debug", "test:e2e:report": "VITE_DB_URL=:memory: playwright show-report", @@ -65,7 +65,7 @@ "hono": "4.8.3", "json-schema-library": "10.0.0-rc7", "json-schema-to-ts": "^3.1.1", - "jsonv-ts": "0.8.2", + "jsonv-ts": "0.8.4", "kysely": "0.27.6", "lodash-es": "^4.17.21", "oauth4webapi": "^2.11.1", @@ -78,10 +78,10 @@ "@aws-sdk/client-s3": "^3.758.0", "@bluwy/giget-core": "^0.1.2", "@clack/prompts": "^0.11.0", - "@cloudflare/vitest-pool-workers": "^0.8.38", + "@cloudflare/vitest-pool-workers": "^0.9.3", "@cloudflare/workers-types": "^4.20250606.0", "@dagrejs/dagre": "^1.1.4", - "@hono/vite-dev-server": "^0.19.1", + "@hono/vite-dev-server": "^0.21.0", "@hookform/resolvers": "^4.1.3", "@libsql/client": "^0.15.9", "@mantine/modals": "^7.17.1", @@ -130,7 +130,9 @@ "vite-plugin-circular-dependency": "^0.5.0", "vite-tsconfig-paths": "^5.1.4", "vitest": "^3.0.9", - "wouter": "^3.6.0" + "wouter": "^3.6.0", + "wrangler": "^4.37.1", + "miniflare": "^4.20250913.0" }, "optionalDependencies": { "@hono/node-server": "^1.14.3" diff --git a/app/src/App.ts b/app/src/App.ts index bd62092..0f535f8 100644 --- a/app/src/App.ts +++ b/app/src/App.ts @@ -5,17 +5,18 @@ import type { em as prototypeEm } from "data/prototype"; import { Connection } from "data/connection/Connection"; import type { Hono } from "hono"; import { - ModuleManager, type InitialModuleConfigs, - type ModuleBuildContext, type ModuleConfigs, - type ModuleManagerOptions, type Modules, + ModuleManager, + type ModuleBuildContext, + type ModuleManagerOptions, } from "modules/ModuleManager"; +import { DbModuleManager } from "modules/db/DbModuleManager"; import * as SystemPermissions from "modules/permissions"; import { AdminController, type AdminControllerOptions } from "modules/server/AdminController"; import { SystemController } from "modules/server/SystemController"; -import type { MaybePromise } from "core/types"; +import type { MaybePromise, PartialRec } from "core/types"; import type { ServerEnv } from "modules/Controller"; import type { IEmailDriver, ICacheDriver } from "core/drivers"; @@ -93,20 +94,23 @@ export type AppOptions = { email?: IEmailDriver; cache?: ICacheDriver; }; + mode?: "db" | "code"; + readonly?: boolean; }; export type CreateAppConfig = { - /** - * bla - */ connection?: Connection | { url: string }; - initialConfig?: InitialModuleConfigs; + config?: PartialRec; options?: AppOptions; }; -export type AppConfig = InitialModuleConfigs; +export type AppConfig = { version: number } & ModuleConfigs; export type LocalApiOptions = Request | ApiOptions; -export class App { +export class App< + C extends Connection = Connection, + Config extends PartialRec = PartialRec, + Options extends AppOptions = AppOptions, +> { static readonly Events = AppEvents; modules: ModuleManager; @@ -121,8 +125,8 @@ export class App(module: Module) { - return this.modules.mutateConfigSafe(module); - } - get server() { return this.modules.server; } @@ -232,6 +244,10 @@ export class App Promise; - makeHandler?: ( - config?: Config, - args?: Args, - opts?: RuntimeOptions | FrameworkOptions, - ) => (request: Request) => Promise; + makeApp: (config: Config, args?: Args) => Promise; + makeHandler?: (config?: Config, args?: Args) => (request: Request) => Promise; label?: string; overrides?: { dbUrl?: string; }; }, ) { - const { test, expect, mock } = testRunner; - const id = crypto.randomUUID(); + const { test, expect, mock, beforeAll, afterAll } = testRunner; + beforeAll(() => disableConsoleLog()); + afterAll(() => enableConsoleLog()); test(`creates ${label}`, async () => { const beforeBuild = mock(async () => null) as any; @@ -39,7 +33,7 @@ export function adapterTestSuite< const config = { app: (env) => ({ connection: { url: env.url }, - initialConfig: { + config: { server: { cors: { origin: env.origin } }, }, }), @@ -53,11 +47,10 @@ export function adapterTestSuite< url: overrides.dbUrl ?? ":memory:", origin: "localhost", } as any, - { force: false, id }, ); expect(app).toBeDefined(); expect(app.toJSON().server.cors.origin).toEqual("localhost"); - expect(beforeBuild).toHaveBeenCalledTimes(1); + expect(beforeBuild).toHaveBeenCalledTimes(2); expect(onBuilt).toHaveBeenCalledTimes(1); }); @@ -68,8 +61,8 @@ export function adapterTestSuite< return { res, data }; }; - test("responds with the same app id", async () => { - const fetcher = makeHandler(undefined, undefined, { force: false, id }); + /* test.skip("responds with the same app id", async () => { + const fetcher = makeHandler(undefined, undefined, { id }); const { res, data } = await getConfig(fetcher); expect(res.ok).toBe(true); @@ -77,14 +70,14 @@ export function adapterTestSuite< expect(data.server.cors.origin).toEqual("localhost"); }); - test("creates fresh & responds to api config", async () => { + test.skip("creates fresh & responds to api config", async () => { // set the same id, but force recreate - const fetcher = makeHandler(undefined, undefined, { id, force: true }); + const fetcher = makeHandler(undefined, undefined, { id }); const { res, data } = await getConfig(fetcher); expect(res.ok).toBe(true); expect(res.status).toBe(200); expect(data.server.cors.origin).toEqual("*"); - }); + }); */ } } diff --git a/app/src/adapter/astro/astro.adapter.spec.ts b/app/src/adapter/astro/astro.adapter.spec.ts index 3f3d1e8..54f3eb2 100644 --- a/app/src/adapter/astro/astro.adapter.spec.ts +++ b/app/src/adapter/astro/astro.adapter.spec.ts @@ -10,6 +10,6 @@ afterAll(enableConsoleLog); describe("astro adapter", () => { adapterTestSuite(bunTestRunner, { makeApp: astro.getApp, - makeHandler: (c, a, o) => (request: Request) => astro.serve(c, a, o)({ request }), + makeHandler: (c, a) => (request: Request) => astro.serve(c, a)({ request }), }); }); diff --git a/app/src/adapter/astro/astro.adapter.ts b/app/src/adapter/astro/astro.adapter.ts index a684f73..7f24923 100644 --- a/app/src/adapter/astro/astro.adapter.ts +++ b/app/src/adapter/astro/astro.adapter.ts @@ -1,4 +1,4 @@ -import { type FrameworkBkndConfig, createFrameworkApp, type FrameworkOptions } from "bknd/adapter"; +import { type FrameworkBkndConfig, createFrameworkApp } from "bknd/adapter"; type AstroEnv = NodeJS.ProcessEnv; type TAstro = { @@ -9,17 +9,12 @@ export type AstroBkndConfig = FrameworkBkndConfig; export async function getApp( config: AstroBkndConfig = {}, args: Env = {} as Env, - opts: FrameworkOptions = {}, ) { - return await createFrameworkApp(config, args ?? import.meta.env, opts); + return await createFrameworkApp(config, args ?? import.meta.env); } -export function serve( - config: AstroBkndConfig = {}, - args: Env = {} as Env, - opts?: FrameworkOptions, -) { +export function serve(config: AstroBkndConfig = {}, args: Env = {} as Env) { return async (fnArgs: TAstro) => { - return (await getApp(config, args, opts)).fetch(fnArgs.request); + return (await getApp(config, args)).fetch(fnArgs.request); }; } diff --git a/app/src/adapter/aws/aws-lambda.adapter.ts b/app/src/adapter/aws/aws-lambda.adapter.ts index ad19047..8c43b3d 100644 --- a/app/src/adapter/aws/aws-lambda.adapter.ts +++ b/app/src/adapter/aws/aws-lambda.adapter.ts @@ -1,7 +1,7 @@ import type { App } from "bknd"; import { handle } from "hono/aws-lambda"; import { serveStatic } from "@hono/node-server/serve-static"; -import { type RuntimeBkndConfig, createRuntimeApp, type RuntimeOptions } from "bknd/adapter"; +import { type RuntimeBkndConfig, createRuntimeApp } from "bknd/adapter"; type AwsLambdaEnv = object; export type AwsLambdaBkndConfig = @@ -20,7 +20,6 @@ export type AwsLambdaBkndConfig = export async function createApp( { adminOptions = false, assets, ...config }: AwsLambdaBkndConfig = {}, args: Env = {} as Env, - opts?: RuntimeOptions, ): Promise { let additional: Partial = { adminOptions, @@ -57,17 +56,15 @@ export async function createApp( ...additional, }, args ?? process.env, - opts, ); } export function serve( config: AwsLambdaBkndConfig = {}, args: Env = {} as Env, - opts?: RuntimeOptions, ) { return async (event) => { - const app = await createApp(config, args, opts); + const app = await createApp(config, args); return await handle(app.server)(event); }; } diff --git a/app/src/adapter/aws/aws.adapter.spec.ts b/app/src/adapter/aws/aws.adapter.spec.ts index e6873d8..e3007e4 100644 --- a/app/src/adapter/aws/aws.adapter.spec.ts +++ b/app/src/adapter/aws/aws.adapter.spec.ts @@ -11,8 +11,8 @@ describe("aws adapter", () => { adapterTestSuite(bunTestRunner, { makeApp: awsLambda.createApp, // @todo: add a request to lambda event translator? - makeHandler: (c, a, o) => async (request: Request) => { - const app = await awsLambda.createApp(c, a, o); + makeHandler: (c, a) => async (request: Request) => { + const app = await awsLambda.createApp(c, a); return app.fetch(request); }, }); diff --git a/app/src/adapter/bun/bun.adapter.ts b/app/src/adapter/bun/bun.adapter.ts index 03689d5..00b61b5 100644 --- a/app/src/adapter/bun/bun.adapter.ts +++ b/app/src/adapter/bun/bun.adapter.ts @@ -1,7 +1,7 @@ /// import path from "node:path"; -import { type RuntimeBkndConfig, createRuntimeApp, type RuntimeOptions } from "bknd/adapter"; +import { type RuntimeBkndConfig, createRuntimeApp } from "bknd/adapter"; import { registerLocalMediaAdapter } from "."; import { config, type App } from "bknd"; import type { ServeOptions } from "bun"; @@ -13,7 +13,6 @@ export type BunBkndConfig = RuntimeBkndConfig & Omit( { distPath, serveStatic: _serveStatic, ...config }: BunBkndConfig = {}, args: Env = {} as Env, - opts?: RuntimeOptions, ) { const root = path.resolve(distPath ?? "./node_modules/bknd/dist", "static"); registerLocalMediaAdapter(); @@ -28,19 +27,17 @@ export async function createApp( ...config, }, args ?? (process.env as Env), - opts, ); } export function createHandler( config: BunBkndConfig = {}, args: Env = {} as Env, - opts?: RuntimeOptions, ) { let app: App | undefined; return async (req: Request) => { if (!app) { - app = await createApp(config, args ?? (process.env as Env), opts); + app = await createApp(config, args ?? (process.env as Env)); } return app.fetch(req); }; @@ -50,7 +47,7 @@ export function serve( { distPath, connection, - initialConfig, + config: _config, options, port = config.server.default_port, onBuilt, @@ -60,7 +57,6 @@ export function serve( ...serveOptions }: BunBkndConfig = {}, args: Env = {} as Env, - opts?: RuntimeOptions, ) { Bun.serve({ ...serveOptions, @@ -68,7 +64,7 @@ export function serve( fetch: createHandler( { connection, - initialConfig, + config: _config, options, onBuilt, buildConfig, @@ -77,7 +73,6 @@ export function serve( serveStatic, }, args, - opts, ), }); diff --git a/app/src/adapter/bun/connection/BunSqliteConnection.spec.ts b/app/src/adapter/bun/connection/BunSqliteConnection.spec.ts index 2b242a1..b18fe82 100644 --- a/app/src/adapter/bun/connection/BunSqliteConnection.spec.ts +++ b/app/src/adapter/bun/connection/BunSqliteConnection.spec.ts @@ -1,8 +1,9 @@ import { connectionTestSuite } from "data/connection/connection-test-suite"; import { bunSqlite } from "./BunSqliteConnection"; import { bunTestRunner } from "adapter/bun/test"; -import { describe } from "bun:test"; +import { describe, test, mock, expect } from "bun:test"; import { Database } from "bun:sqlite"; +import { GenericSqliteConnection } from "data/connection/sqlite/GenericSqliteConnection"; describe("BunSqliteConnection", () => { connectionTestSuite(bunTestRunner, { @@ -12,4 +13,20 @@ describe("BunSqliteConnection", () => { }), rawDialectDetails: [], }); + + test("onCreateConnection", async () => { + const called = mock(() => null); + + const conn = bunSqlite({ + onCreateConnection: (db) => { + expect(db).toBeInstanceOf(Database); + called(); + }, + }); + await conn.ping(); + + expect(conn).toBeInstanceOf(GenericSqliteConnection); + expect(conn.db).toBeInstanceOf(Database); + expect(called).toHaveBeenCalledTimes(1); + }); }); diff --git a/app/src/adapter/bun/connection/BunSqliteConnection.ts b/app/src/adapter/bun/connection/BunSqliteConnection.ts index 08444c5..39fff00 100644 --- a/app/src/adapter/bun/connection/BunSqliteConnection.ts +++ b/app/src/adapter/bun/connection/BunSqliteConnection.ts @@ -1,40 +1,53 @@ import { Database } from "bun:sqlite"; -import { genericSqlite, type GenericSqliteConnection } from "bknd"; +import { + genericSqlite, + type GenericSqliteConnection, + type GenericSqliteConnectionConfig, +} from "bknd"; +import { omitKeys } from "bknd/utils"; export type BunSqliteConnection = GenericSqliteConnection; -export type BunSqliteConnectionConfig = { - database: Database; -}; +export type BunSqliteConnectionConfig = Omit< + GenericSqliteConnectionConfig, + "name" | "supports" +> & + ({ database?: Database; url?: never } | { url?: string; database?: never }); -export function bunSqlite(config?: BunSqliteConnectionConfig | { url: string }) { - let db: Database; +export function bunSqlite(config?: BunSqliteConnectionConfig) { + let db: Database | undefined; if (config) { - if ("database" in config) { + if ("database" in config && config.database) { db = config.database; - } else { + } else if (config.url) { db = new Database(config.url); } - } else { + } + + if (!db) { db = new Database(":memory:"); } - return genericSqlite("bun-sqlite", db, (utils) => { - //const fn = cache ? "query" : "prepare"; - const getStmt = (sql: string) => db.prepare(sql); + return genericSqlite( + "bun-sqlite", + db, + (utils) => { + const getStmt = (sql: string) => db.prepare(sql); - return { - db, - query: utils.buildQueryFn({ - all: (sql, parameters) => getStmt(sql).all(...(parameters || [])), - run: (sql, parameters) => { - const { changes, lastInsertRowid } = getStmt(sql).run(...(parameters || [])); - return { - insertId: utils.parseBigInt(lastInsertRowid), - numAffectedRows: utils.parseBigInt(changes), - }; - }, - }), - close: () => db.close(), - }; - }); + return { + db, + query: utils.buildQueryFn({ + all: (sql, parameters) => getStmt(sql).all(...(parameters || [])), + run: (sql, parameters) => { + const { changes, lastInsertRowid } = getStmt(sql).run(...(parameters || [])); + return { + insertId: utils.parseBigInt(lastInsertRowid), + numAffectedRows: utils.parseBigInt(changes), + }; + }, + }), + close: () => db.close(), + }; + }, + omitKeys(config ?? ({} as any), ["database", "url", "name", "supports"]), + ); } diff --git a/app/src/adapter/bun/test.ts b/app/src/adapter/bun/test.ts index 4d453d7..b185776 100644 --- a/app/src/adapter/bun/test.ts +++ b/app/src/adapter/bun/test.ts @@ -1,4 +1,4 @@ -import { expect, test, mock, describe, beforeEach, afterEach, afterAll } from "bun:test"; +import { expect, test, mock, describe, beforeEach, afterEach, afterAll, beforeAll } from "bun:test"; export const bunTestRunner = { describe, @@ -8,4 +8,5 @@ export const bunTestRunner = { beforeEach, afterEach, afterAll, + beforeAll, }; diff --git a/app/src/adapter/cloudflare/cloudflare-workers.adapter.spec.ts b/app/src/adapter/cloudflare/cloudflare-workers.adapter.spec.ts index 64ba65b..65477b6 100644 --- a/app/src/adapter/cloudflare/cloudflare-workers.adapter.spec.ts +++ b/app/src/adapter/cloudflare/cloudflare-workers.adapter.spec.ts @@ -5,8 +5,8 @@ import { adapterTestSuite } from "adapter/adapter-test-suite"; import { bunTestRunner } from "adapter/bun/test"; import { type CloudflareBkndConfig, createApp } from "./cloudflare-workers.adapter"; -beforeAll(disableConsoleLog); -afterAll(enableConsoleLog); +/* beforeAll(disableConsoleLog); +afterAll(enableConsoleLog); */ describe("cf adapter", () => { const DB_URL = ":memory:"; @@ -20,31 +20,31 @@ describe("cf adapter", () => { const staticConfig = await makeConfig( { connection: { url: DB_URL }, - initialConfig: { data: { basepath: DB_URL } }, + config: { data: { basepath: DB_URL } }, }, $ctx({ DB_URL }), ); - expect(staticConfig.initialConfig).toEqual({ data: { basepath: DB_URL } }); + expect(staticConfig.config).toEqual({ data: { basepath: DB_URL } }); expect(staticConfig.connection).toBeDefined(); const dynamicConfig = await makeConfig( { app: (env) => ({ - initialConfig: { data: { basepath: env.DB_URL } }, + config: { data: { basepath: env.DB_URL } }, connection: { url: env.DB_URL }, }), }, $ctx({ DB_URL }), ); - expect(dynamicConfig.initialConfig).toEqual({ data: { basepath: DB_URL } }); + expect(dynamicConfig.config).toEqual({ data: { basepath: DB_URL } }); expect(dynamicConfig.connection).toBeDefined(); }); adapterTestSuite>(bunTestRunner, { - makeApp: async (c, a, o) => { - return await createApp(c, { env: a } as any, o); + makeApp: async (c, a) => { + return await createApp(c, { env: a } as any); }, - makeHandler: (c, a, o) => { + makeHandler: (c, a) => { console.log("args", a); return async (request: any) => { const app = await createApp( @@ -53,7 +53,6 @@ describe("cf adapter", () => { connection: { url: DB_URL }, }, a as any, - o, ); return app.fetch(request); }; diff --git a/app/src/adapter/cloudflare/cloudflare-workers.adapter.ts b/app/src/adapter/cloudflare/cloudflare-workers.adapter.ts index 091dc30..fe278c4 100644 --- a/app/src/adapter/cloudflare/cloudflare-workers.adapter.ts +++ b/app/src/adapter/cloudflare/cloudflare-workers.adapter.ts @@ -5,7 +5,7 @@ import { Hono } from "hono"; import { serveStatic } from "hono/cloudflare-workers"; import type { MaybePromise } from "bknd"; import { $console } from "bknd/utils"; -import { createRuntimeApp, type RuntimeOptions } from "bknd/adapter"; +import { createRuntimeApp } from "bknd/adapter"; import { registerAsyncsExecutionContext, makeConfig, type CloudflareContext } from "./config"; declare global { @@ -34,12 +34,8 @@ export type CloudflareBkndConfig = RuntimeBkndConfig & }; export async function createApp( - config: CloudflareBkndConfig, + config: CloudflareBkndConfig = {}, ctx: Partial> = {}, - opts: RuntimeOptions = { - // by default, require the app to be rebuilt every time - force: true, - }, ) { const appConfig = await makeConfig( { @@ -53,7 +49,7 @@ export async function createApp( }, ctx, ); - return await createRuntimeApp(appConfig, ctx?.env, opts); + return await createRuntimeApp(appConfig, ctx?.env); } // compatiblity diff --git a/app/src/adapter/cloudflare/connection/DoConnection.vitest.ts b/app/src/adapter/cloudflare/connection/DoConnection.vitest.ts index 695046e..cb209e8 100644 --- a/app/src/adapter/cloudflare/connection/DoConnection.vitest.ts +++ b/app/src/adapter/cloudflare/connection/DoConnection.vitest.ts @@ -1,11 +1,12 @@ /// -import { describe, test, expect } from "vitest"; +import { describe, beforeAll, afterAll } from "vitest"; import { viTestRunner } from "adapter/node/vitest"; import { connectionTestSuite } from "data/connection/connection-test-suite"; import { Miniflare } from "miniflare"; import { doSqlite } from "./DoConnection"; +import { disableConsoleLog, enableConsoleLog } from "core/utils/test"; const script = ` import { DurableObject } from "cloudflare:workers"; @@ -40,6 +41,9 @@ export default { } `; +beforeAll(() => disableConsoleLog()); +afterAll(() => enableConsoleLog()); + describe("doSqlite", async () => { connectionTestSuite(viTestRunner, { makeConnection: async () => { diff --git a/app/src/adapter/cloudflare/drivers/cache.vitest.ts b/app/src/adapter/cloudflare/drivers/cache.vitest.ts index d9a856d..65c6608 100644 --- a/app/src/adapter/cloudflare/drivers/cache.vitest.ts +++ b/app/src/adapter/cloudflare/drivers/cache.vitest.ts @@ -3,6 +3,10 @@ import { cacheWorkersKV } from "./cache"; import { viTestRunner } from "adapter/node/vitest"; import { cacheDriverTestSuite } from "core/drivers/cache/cache-driver-test-suite"; import { Miniflare } from "miniflare"; +import { disableConsoleLog, enableConsoleLog } from "core/utils/test"; + +beforeAll(() => disableConsoleLog()); +afterAll(() => enableConsoleLog()); describe("cacheWorkersKV", async () => { beforeAll(() => { diff --git a/app/src/adapter/cloudflare/index.ts b/app/src/adapter/cloudflare/index.ts index a09da7b..abc0719 100644 --- a/app/src/adapter/cloudflare/index.ts +++ b/app/src/adapter/cloudflare/index.ts @@ -16,7 +16,7 @@ export { type GetBindingType, type BindingMap, } from "./bindings"; -export { constants, type CloudflareContext } from "./config"; +export { constants, makeConfig, type CloudflareContext } from "./config"; export { StorageR2Adapter, registerMedia } from "./storage/StorageR2Adapter"; export { registries } from "bknd"; export { devFsVitePlugin, devFsWrite } from "./vite"; diff --git a/app/src/adapter/cloudflare/proxy.ts b/app/src/adapter/cloudflare/proxy.ts index ddbd4b3..9efd5c4 100644 --- a/app/src/adapter/cloudflare/proxy.ts +++ b/app/src/adapter/cloudflare/proxy.ts @@ -5,8 +5,9 @@ import { type CloudflareBkndConfig, type CloudflareEnv, } from "bknd/adapter/cloudflare"; -import type { PlatformProxy } from "wrangler"; +import type { GetPlatformProxyOptions, PlatformProxy } from "wrangler"; import process from "node:process"; +import { $console } from "bknd/utils"; export type WithPlatformProxyOptions = { /** @@ -14,22 +15,49 @@ export type WithPlatformProxyOptions = { * You can override/force this by setting this option. */ useProxy?: boolean; + proxyOptions?: GetPlatformProxyOptions; }; +async function getPlatformProxy(opts?: GetPlatformProxyOptions) { + try { + const { version } = await import("wrangler/package.json", { with: { type: "json" } }).then( + (pkg) => pkg.default, + ); + $console.log("Using wrangler version", version); + const { getPlatformProxy } = await import("wrangler"); + return getPlatformProxy(opts); + } catch (e) { + $console.error("Failed to import wrangler", String(e)); + const resolved = import.meta.resolve("wrangler"); + $console.log("Wrangler resolved to", resolved); + const file = resolved?.split("/").pop(); + if (file?.endsWith(".json")) { + $console.error( + "You have a `wrangler.json` in your current directory. Please change to .jsonc or .toml", + ); + } + } + + process.exit(1); +} + export function withPlatformProxy( - config?: CloudflareBkndConfig, + config: CloudflareBkndConfig = {}, opts?: WithPlatformProxyOptions, ) { const use_proxy = typeof opts?.useProxy === "boolean" ? opts.useProxy : process.env.PROXY === "1"; let proxy: PlatformProxy | undefined; + $console.log("Using cloudflare platform proxy"); + async function getEnv(env?: Env): Promise { if (use_proxy) { if (!proxy) { - const getPlatformProxy = await import("wrangler").then((mod) => mod.getPlatformProxy); - proxy = await getPlatformProxy(); - setTimeout(proxy?.dispose, 1000); + proxy = await getPlatformProxy(opts?.proxyOptions); + process.on("exit", () => { + proxy?.dispose(); + }); } return proxy.env as unknown as Env; } @@ -50,16 +78,22 @@ export function withPlatformProxy( // @ts-ignore app: async (_env) => { const env = await getEnv(_env); + const binding = use_proxy ? getBinding(env, "D1Database") : undefined; - if (config?.app === undefined && use_proxy) { - const binding = getBinding(env, "D1Database"); + if (config?.app === undefined && use_proxy && binding) { return { connection: d1Sqlite({ binding: binding.value, }), }; } else if (typeof config?.app === "function") { - return config?.app(env); + const appConfig = await config?.app(env); + if (binding) { + appConfig.connection = d1Sqlite({ + binding: binding.value, + }) as any; + } + return appConfig; } return config?.app || {}; }, diff --git a/app/src/adapter/cloudflare/storage/StorageR2Adapter.vitest.ts b/app/src/adapter/cloudflare/storage/StorageR2Adapter.vitest.ts index 437ec9b..6ecd8df 100644 --- a/app/src/adapter/cloudflare/storage/StorageR2Adapter.vitest.ts +++ b/app/src/adapter/cloudflare/storage/StorageR2Adapter.vitest.ts @@ -3,8 +3,12 @@ import { Miniflare } from "miniflare"; import { StorageR2Adapter } from "./StorageR2Adapter"; import { adapterTestSuite } from "media/storage/adapters/adapter-test-suite"; import path from "node:path"; -import { describe, afterAll, test, expect } from "vitest"; +import { describe, afterAll, test, expect, beforeAll } from "vitest"; import { viTestRunner } from "adapter/node/vitest"; +import { disableConsoleLog, enableConsoleLog } from "core/utils/test"; + +beforeAll(() => disableConsoleLog()); +afterAll(() => enableConsoleLog()); let mf: Miniflare | undefined; describe("StorageR2Adapter", async () => { @@ -24,7 +28,8 @@ describe("StorageR2Adapter", async () => { const buffer = readFileSync(path.join(basePath, "image.png")); const file = new File([buffer], "image.png", { type: "image/png" }); - await adapterTestSuite(viTestRunner, adapter, file); + // miniflare doesn't support range requests + await adapterTestSuite(viTestRunner, adapter, file, { testRange: false }); }); afterAll(async () => { diff --git a/app/src/adapter/cloudflare/vite.ts b/app/src/adapter/cloudflare/vite.ts index 22862b1..1b7640b 100644 --- a/app/src/adapter/cloudflare/vite.ts +++ b/app/src/adapter/cloudflare/vite.ts @@ -24,45 +24,157 @@ export function devFsVitePlugin({ projectRoot = config.root; }, configureServer(server) { - if (!isDev) return; + if (!isDev) { + verbose && console.debug("[dev-fs-plugin] Not in dev mode, skipping"); + return; + } + + // Track active chunked requests + const activeRequests = new Map< + string, + { + totalChunks: number; + filename: string; + chunks: string[]; + receivedChunks: number; + } + >(); // Intercept stdout to watch for our write requests const originalStdoutWrite = process.stdout.write; process.stdout.write = function (chunk: any, encoding?: any, callback?: any) { const output = chunk.toString(); - // Check if this output contains our special write request - if (output.includes("{{DEV_FS_WRITE_REQUEST}}")) { - try { - // Extract the JSON from the log line - const match = output.match(/{{DEV_FS_WRITE_REQUEST}} ({.*})/); - if (match) { - const writeRequest = JSON.parse(match[1]); - if (writeRequest.type === "DEV_FS_WRITE_REQUEST") { - if (verbose) { - console.debug("[dev-fs-plugin] Intercepted write request via stdout"); - } + // Skip our own debug output + if (output.includes("[dev-fs-plugin]") || output.includes("[dev-fs-polyfill]")) { + // @ts-ignore + // biome-ignore lint/style/noArguments: + return originalStdoutWrite.apply(process.stdout, arguments); + } - // Process the write request immediately - (async () => { - try { - const fullPath = resolve(projectRoot, writeRequest.filename); - await nodeWriteFile(fullPath, writeRequest.data); - if (verbose) { - console.debug("[dev-fs-plugin] File written successfully!"); - } - } catch (error) { - console.error("[dev-fs-plugin] Error writing file:", error); - } - })(); + // Track if we process any protocol messages (to suppress output) + let processedProtocolMessage = false; - // Don't output the raw write request to console - return true; + // Process all start markers in this output + if (output.includes("{{DEV_FS_START}}")) { + const startMatches = [ + ...output.matchAll(/{{DEV_FS_START}} ([a-z0-9]+) (\d+) (.+)/g), + ]; + for (const startMatch of startMatches) { + const requestId = startMatch[1]; + const totalChunks = Number.parseInt(startMatch[2]); + const filename = startMatch[3]; + + activeRequests.set(requestId, { + totalChunks, + filename, + chunks: new Array(totalChunks), + receivedChunks: 0, + }); + + verbose && + console.debug( + `[dev-fs-plugin] Started request ${requestId} for ${filename} (${totalChunks} chunks)`, + ); + } + processedProtocolMessage = true; + } + + // Process all chunk data in this output + if (output.includes("{{DEV_FS_CHUNK}}")) { + const chunkMatches = [ + ...output.matchAll(/{{DEV_FS_CHUNK}} ([a-z0-9]+) (\d+) ([A-Za-z0-9+/=]+)/g), + ]; + for (const chunkMatch of chunkMatches) { + const requestId = chunkMatch[1]; + const chunkIndex = Number.parseInt(chunkMatch[2]); + const chunkData = chunkMatch[3]; + + const request = activeRequests.get(requestId); + if (request) { + request.chunks[chunkIndex] = chunkData; + request.receivedChunks++; + verbose && + console.debug( + `[dev-fs-plugin] Received chunk ${chunkIndex}/${request.totalChunks - 1} for ${request.filename} (length: ${chunkData.length})`, + ); + + // Validate base64 chunk + if (chunkData.length < 1000 && chunkIndex < request.totalChunks - 1) { + verbose && + console.warn( + `[dev-fs-plugin] WARNING: Chunk ${chunkIndex} seems truncated (length: ${chunkData.length})`, + ); } } - } catch (error) { - // Not a valid write request, continue with normal output } + processedProtocolMessage = true; + } + + // Process all end markers in this output + if (output.includes("{{DEV_FS_END}}")) { + const endMatches = [...output.matchAll(/{{DEV_FS_END}} ([a-z0-9]+)/g)]; + for (const endMatch of endMatches) { + const requestId = endMatch[1]; + const request = activeRequests.get(requestId); + + if (request && request.receivedChunks === request.totalChunks) { + try { + // Reconstruct the base64 string + const fullBase64 = request.chunks.join(""); + verbose && + console.debug( + `[dev-fs-plugin] Reconstructed ${request.filename} - base64 length: ${fullBase64.length}`, + ); + + // Decode and parse + const decodedJson = atob(fullBase64); + const writeRequest = JSON.parse(decodedJson); + + if (writeRequest.type === "DEV_FS_WRITE_REQUEST") { + verbose && + console.debug( + `[dev-fs-plugin] Processing write request for ${writeRequest.filename}`, + ); + + // Process the write request + (async () => { + try { + const fullPath = resolve(projectRoot, writeRequest.filename); + verbose && + console.debug(`[dev-fs-plugin] Writing to: ${fullPath}`); + await nodeWriteFile(fullPath, writeRequest.data); + verbose && + console.debug("[dev-fs-plugin] File written successfully!"); + } catch (error) { + console.error("[dev-fs-plugin] Error writing file:", error); + } + })(); + + // Clean up + activeRequests.delete(requestId); + return true; + } + } catch (error) { + console.error( + "[dev-fs-plugin] Error processing chunked request:", + String(error), + ); + activeRequests.delete(requestId); + } + } else if (request) { + verbose && + console.debug( + `[dev-fs-plugin] Request ${requestId} incomplete: ${request.receivedChunks}/${request.totalChunks} chunks`, + ); + } + } + processedProtocolMessage = true; + } + + // If we processed any protocol messages, suppress output + if (processedProtocolMessage) { + return callback ? callback() : true; } // @ts-ignore @@ -78,7 +190,10 @@ export function devFsVitePlugin({ // @ts-ignore transform(code, id, options) { // Only transform in SSR mode during development - if (!isDev || !options?.ssr) return; + //if (!isDev || !options?.ssr) return; + if (!isDev) { + return; + } // Check if this is the bknd config file if (id.includes(configFile)) { @@ -92,7 +207,7 @@ export function devFsVitePlugin({ if (typeof globalThis !== 'undefined') { globalThis.__devFsPolyfill = { writeFile: async (filename, data) => { - ${verbose ? "console.debug('dev-fs polyfill: Intercepting write request for', filename);" : ""} + ${verbose ? "console.debug('[dev-fs-polyfill] Intercepting write request for', filename);" : ""} // Use console logging as a communication channel // The main process will watch for this specific log pattern @@ -103,16 +218,38 @@ if (typeof globalThis !== 'undefined') { timestamp: Date.now() }; - // Output as a specially formatted console message - console.log('{{DEV_FS_WRITE_REQUEST}}', JSON.stringify(writeRequest)); - ${verbose ? "console.debug('dev-fs polyfill: Write request sent via console');" : ""} + // Output as a specially formatted console message with end delimiter + // Base64 encode the JSON to avoid any control character issues + const jsonString = JSON.stringify(writeRequest); + const encodedJson = btoa(jsonString); + + // Split into reasonable chunks that balance performance vs reliability + const chunkSize = 2000; // 2KB chunks - safe for most environments + const chunks = []; + for (let i = 0; i < encodedJson.length; i += chunkSize) { + chunks.push(encodedJson.slice(i, i + chunkSize)); + } + + const requestId = Date.now().toString(36) + Math.random().toString(36).substr(2, 5); + + // Send start marker (use stdout.write to avoid console display) + process.stdout.write('{{DEV_FS_START}} ' + requestId + ' ' + chunks.length + ' ' + filename + '\\n'); + + // Send each chunk + chunks.forEach((chunk, index) => { + process.stdout.write('{{DEV_FS_CHUNK}} ' + requestId + ' ' + index + ' ' + chunk + '\\n'); + }); + + // Send end marker + process.stdout.write('{{DEV_FS_END}} ' + requestId + '\\n'); return Promise.resolve(); } }; -} -`; +}`; return polyfill + code; + } else { + verbose && console.debug("[dev-fs-plugin] Not transforming", id); } }, } satisfies Plugin; diff --git a/app/src/adapter/index.ts b/app/src/adapter/index.ts index 17827f7..2548efa 100644 --- a/app/src/adapter/index.ts +++ b/app/src/adapter/index.ts @@ -13,21 +13,14 @@ import type { AdminControllerOptions } from "modules/server/AdminController"; import type { Manifest } from "vite"; export type BkndConfig = CreateAppConfig & { - app?: CreateAppConfig | ((args: Args) => MaybePromise); - onBuilt?: (app: App) => Promise; - beforeBuild?: (app: App, registries?: typeof $registries) => Promise; + app?: Omit | ((args: Args) => MaybePromise, "app">>); + onBuilt?: (app: App) => MaybePromise; + beforeBuild?: (app?: App, registries?: typeof $registries) => MaybePromise; buildConfig?: Parameters[0]; }; export type FrameworkBkndConfig = BkndConfig; -export type CreateAdapterAppOptions = { - force?: boolean; - id?: string; -}; -export type FrameworkOptions = CreateAdapterAppOptions; -export type RuntimeOptions = CreateAdapterAppOptions; - export type RuntimeBkndConfig = BkndConfig & { distPath?: string; serveStatic?: MiddlewareHandler | [string, MiddlewareHandler]; @@ -41,7 +34,7 @@ export type DefaultArgs = { export async function makeConfig( config: BkndConfig, args?: Args, -): Promise { +): Promise, "app">> { let additionalConfig: CreateAppConfig = {}; const { app, ...rest } = config; if (app) { @@ -59,45 +52,34 @@ export async function makeConfig( } // a map that contains all apps by id -const apps = new Map(); export async function createAdapterApp( config: Config = {} as Config, args?: Args, - opts?: CreateAdapterAppOptions, ): Promise { - const id = opts?.id ?? "app"; - let app = apps.get(id); - if (!app || opts?.force) { - const appConfig = await makeConfig(config, args); - if (!appConfig.connection || !Connection.isConnection(appConfig.connection)) { - let connection: Connection | undefined; - if (Connection.isConnection(config.connection)) { - connection = config.connection; - } else { - const sqlite = (await import("bknd/adapter/sqlite")).sqlite; - const conf = appConfig.connection ?? { url: ":memory:" }; - connection = sqlite(conf) as any; - $console.info(`Using ${connection!.name} connection`, conf.url); - } - appConfig.connection = connection; - } + await config.beforeBuild?.(undefined, $registries); - app = App.create(appConfig); - - if (!opts?.force) { - apps.set(id, app); + const appConfig = await makeConfig(config, args); + if (!appConfig.connection || !Connection.isConnection(appConfig.connection)) { + let connection: Connection | undefined; + if (Connection.isConnection(config.connection)) { + connection = config.connection; + } else { + const sqlite = (await import("bknd/adapter/sqlite")).sqlite; + const conf = appConfig.connection ?? { url: ":memory:" }; + connection = sqlite(conf) as any; + $console.info(`Using ${connection!.name} connection`, conf.url); } + appConfig.connection = connection; } - return app; + return App.create(appConfig); } export async function createFrameworkApp( config: FrameworkBkndConfig = {}, args?: Args, - opts?: FrameworkOptions, ): Promise { - const app = await createAdapterApp(config, args, opts); + const app = await createAdapterApp(config, args); if (!app.isBuilt()) { if (config.onBuilt) { @@ -120,9 +102,8 @@ export async function createFrameworkApp( export async function createRuntimeApp( { serveStatic, adminOptions, ...config }: RuntimeBkndConfig = {}, args?: Args, - opts?: RuntimeOptions, ): Promise { - const app = await createAdapterApp(config, args, opts); + const app = await createAdapterApp(config, args); if (!app.isBuilt()) { app.emgr.onEvent( diff --git a/app/src/adapter/nextjs/nextjs.adapter.ts b/app/src/adapter/nextjs/nextjs.adapter.ts index bce7009..ba0953b 100644 --- a/app/src/adapter/nextjs/nextjs.adapter.ts +++ b/app/src/adapter/nextjs/nextjs.adapter.ts @@ -1,4 +1,4 @@ -import { createFrameworkApp, type FrameworkBkndConfig, type FrameworkOptions } from "bknd/adapter"; +import { createFrameworkApp, type FrameworkBkndConfig } from "bknd/adapter"; import { isNode } from "bknd/utils"; import type { NextApiRequest } from "next"; @@ -10,9 +10,8 @@ export type NextjsBkndConfig = FrameworkBkndConfig & { export async function getApp( config: NextjsBkndConfig, args: Env = {} as Env, - opts?: FrameworkOptions, ) { - return await createFrameworkApp(config, args ?? (process.env as Env), opts); + return await createFrameworkApp(config, args ?? (process.env as Env)); } function getCleanRequest(req: Request, cleanRequest: NextjsBkndConfig["cleanRequest"]) { @@ -41,10 +40,9 @@ function getCleanRequest(req: Request, cleanRequest: NextjsBkndConfig["cleanRequ export function serve( { cleanRequest, ...config }: NextjsBkndConfig = {}, args: Env = {} as Env, - opts?: FrameworkOptions, ) { return async (req: Request) => { - const app = await getApp(config, args, opts); + const app = await getApp(config, args); const request = getCleanRequest(req, cleanRequest); return app.fetch(request); }; diff --git a/app/src/adapter/node/connection/NodeSqliteConnection.ts b/app/src/adapter/node/connection/NodeSqliteConnection.ts index e215aad..915ab3d 100644 --- a/app/src/adapter/node/connection/NodeSqliteConnection.ts +++ b/app/src/adapter/node/connection/NodeSqliteConnection.ts @@ -1,19 +1,29 @@ -import { genericSqlite } from "bknd"; +import { + genericSqlite, + type GenericSqliteConnection, + type GenericSqliteConnectionConfig, +} from "bknd"; import { DatabaseSync } from "node:sqlite"; +import { omitKeys } from "bknd/utils"; -export type NodeSqliteConnectionConfig = { - database: DatabaseSync; -}; +export type NodeSqliteConnection = GenericSqliteConnection; +export type NodeSqliteConnectionConfig = Omit< + GenericSqliteConnectionConfig, + "name" | "supports" +> & + ({ database?: DatabaseSync; url?: never } | { url?: string; database?: never }); -export function nodeSqlite(config?: NodeSqliteConnectionConfig | { url: string }) { - let db: DatabaseSync; +export function nodeSqlite(config?: NodeSqliteConnectionConfig) { + let db: DatabaseSync | undefined; if (config) { - if ("database" in config) { + if ("database" in config && config.database) { db = config.database; - } else { + } else if (config.url) { db = new DatabaseSync(config.url); } - } else { + } + + if (!db) { db = new DatabaseSync(":memory:"); } @@ -21,11 +31,7 @@ export function nodeSqlite(config?: NodeSqliteConnectionConfig | { url: string } "node-sqlite", db, (utils) => { - const getStmt = (sql: string) => { - const stmt = db.prepare(sql); - //stmt.setReadBigInts(true); - return stmt; - }; + const getStmt = (sql: string) => db.prepare(sql); return { db, @@ -49,6 +55,7 @@ export function nodeSqlite(config?: NodeSqliteConnectionConfig | { url: string } }; }, { + ...omitKeys(config ?? ({} as any), ["database", "url", "name", "supports"]), supports: { batching: false, }, diff --git a/app/src/adapter/node/connection/NodeSqliteConnection.vi-test.ts b/app/src/adapter/node/connection/NodeSqliteConnection.vi-test.ts index 2cb9149..bbf85f5 100644 --- a/app/src/adapter/node/connection/NodeSqliteConnection.vi-test.ts +++ b/app/src/adapter/node/connection/NodeSqliteConnection.vi-test.ts @@ -1,8 +1,13 @@ import { nodeSqlite } from "./NodeSqliteConnection"; import { DatabaseSync } from "node:sqlite"; import { connectionTestSuite } from "data/connection/connection-test-suite"; -import { describe } from "vitest"; +import { describe, beforeAll, afterAll, test, expect, vi } from "vitest"; import { viTestRunner } from "../vitest"; +import { disableConsoleLog, enableConsoleLog } from "core/utils/test"; +import { GenericSqliteConnection } from "data/connection/sqlite/GenericSqliteConnection"; + +beforeAll(() => disableConsoleLog()); +afterAll(() => enableConsoleLog()); describe("NodeSqliteConnection", () => { connectionTestSuite(viTestRunner, { @@ -12,4 +17,20 @@ describe("NodeSqliteConnection", () => { }), rawDialectDetails: [], }); + + test("onCreateConnection", async () => { + const called = vi.fn(() => null); + + const conn = nodeSqlite({ + onCreateConnection: (db) => { + expect(db).toBeInstanceOf(DatabaseSync); + called(); + }, + }); + await conn.ping(); + + expect(conn).toBeInstanceOf(GenericSqliteConnection); + expect(conn.db).toBeInstanceOf(DatabaseSync); + expect(called).toHaveBeenCalledOnce(); + }); }); diff --git a/app/src/adapter/node/node.adapter.ts b/app/src/adapter/node/node.adapter.ts index 5a2c058..fd96086 100644 --- a/app/src/adapter/node/node.adapter.ts +++ b/app/src/adapter/node/node.adapter.ts @@ -2,7 +2,7 @@ import path from "node:path"; import { serve as honoServe } from "@hono/node-server"; import { serveStatic } from "@hono/node-server/serve-static"; import { registerLocalMediaAdapter } from "adapter/node/storage"; -import { type RuntimeBkndConfig, createRuntimeApp, type RuntimeOptions } from "bknd/adapter"; +import { type RuntimeBkndConfig, createRuntimeApp } from "bknd/adapter"; import { config as $config, type App } from "bknd"; import { $console } from "bknd/utils"; @@ -18,7 +18,6 @@ export type NodeBkndConfig = RuntimeBkndConfig & { export async function createApp( { distPath, relativeDistPath, ...config }: NodeBkndConfig = {}, args: Env = {} as Env, - opts?: RuntimeOptions, ) { const root = path.relative( process.cwd(), @@ -36,19 +35,17 @@ export async function createApp( }, // @ts-ignore args ?? { env: process.env }, - opts, ); } export function createHandler( config: NodeBkndConfig = {}, args: Env = {} as Env, - opts?: RuntimeOptions, ) { let app: App | undefined; return async (req: Request) => { if (!app) { - app = await createApp(config, args ?? (process.env as Env), opts); + app = await createApp(config, args ?? (process.env as Env)); } return app.fetch(req); }; @@ -57,13 +54,12 @@ export function createHandler( export function serve( { port = $config.server.default_port, hostname, listener, ...config }: NodeBkndConfig = {}, args: Env = {} as Env, - opts?: RuntimeOptions, ) { honoServe( { port, hostname, - fetch: createHandler(config, args, opts), + fetch: createHandler(config, args), }, (connInfo) => { $console.log(`Server is running on http://localhost:${connInfo.port}`); diff --git a/app/src/adapter/node/node.adapter.vi-test.ts b/app/src/adapter/node/node.adapter.vi-test.ts index 31cdb31..11bb0f7 100644 --- a/app/src/adapter/node/node.adapter.vi-test.ts +++ b/app/src/adapter/node/node.adapter.vi-test.ts @@ -2,10 +2,6 @@ import { describe, beforeAll, afterAll } from "vitest"; import * as node from "./node.adapter"; import { adapterTestSuite } from "adapter/adapter-test-suite"; import { viTestRunner } from "adapter/node/vitest"; -import { disableConsoleLog, enableConsoleLog } from "core/utils"; - -beforeAll(() => disableConsoleLog()); -afterAll(enableConsoleLog); describe("node adapter", () => { adapterTestSuite(viTestRunner, { diff --git a/app/src/adapter/node/storage/StorageLocalAdapter.ts b/app/src/adapter/node/storage/StorageLocalAdapter.ts index fa3e336..46db8fd 100644 --- a/app/src/adapter/node/storage/StorageLocalAdapter.ts +++ b/app/src/adapter/node/storage/StorageLocalAdapter.ts @@ -80,18 +80,79 @@ export class StorageLocalAdapter extends StorageAdapter { } } + private parseRangeHeader( + rangeHeader: string, + fileSize: number, + ): { start: number; end: number } | null { + // Parse "bytes=start-end" format + const match = rangeHeader.match(/^bytes=(\d*)-(\d*)$/); + if (!match) return null; + + const [, startStr, endStr] = match; + let start = startStr ? Number.parseInt(startStr, 10) : 0; + let end = endStr ? Number.parseInt(endStr, 10) : fileSize - 1; + + // Handle suffix-byte-range-spec (e.g., "bytes=-500") + if (!startStr && endStr) { + start = Math.max(0, fileSize - Number.parseInt(endStr, 10)); + end = fileSize - 1; + } + + // Validate range + if (start < 0 || end >= fileSize || start > end) { + return null; + } + + return { start, end }; + } + async getObject(key: string, headers: Headers): Promise { try { - const content = await readFile(`${this.config.path}/${key}`); + const filePath = `${this.config.path}/${key}`; + const stats = await stat(filePath); + const fileSize = stats.size; const mimeType = guessMimeType(key); - return new Response(content, { - status: 200, - headers: { - "Content-Type": mimeType || "application/octet-stream", - "Content-Length": content.length.toString(), - }, + const responseHeaders = new Headers({ + "Accept-Ranges": "bytes", + "Content-Type": mimeType || "application/octet-stream", }); + + const rangeHeader = headers.get("range"); + + if (rangeHeader) { + const range = this.parseRangeHeader(rangeHeader, fileSize); + + if (!range) { + // Invalid range - return 416 Range Not Satisfiable + responseHeaders.set("Content-Range", `bytes */${fileSize}`); + return new Response("", { + status: 416, + headers: responseHeaders, + }); + } + + const { start, end } = range; + const content = await readFile(filePath, { encoding: null }); + const chunk = content.slice(start, end + 1); + + responseHeaders.set("Content-Range", `bytes ${start}-${end}/${fileSize}`); + responseHeaders.set("Content-Length", chunk.length.toString()); + + return new Response(chunk, { + status: 206, // Partial Content + headers: responseHeaders, + }); + } else { + // Normal request - return entire file + const content = await readFile(filePath); + responseHeaders.set("Content-Length", content.length.toString()); + + return new Response(content, { + status: 200, + headers: responseHeaders, + }); + } } catch (error) { // Handle file reading errors return new Response("", { status: 404 }); diff --git a/app/src/adapter/node/storage/StorageLocalAdapter.vitest.ts b/app/src/adapter/node/storage/StorageLocalAdapter.vitest.ts index 3d13bd5..8b3acf1 100644 --- a/app/src/adapter/node/storage/StorageLocalAdapter.vitest.ts +++ b/app/src/adapter/node/storage/StorageLocalAdapter.vitest.ts @@ -1,9 +1,13 @@ -import { describe } from "vitest"; +import { describe, beforeAll, afterAll } from "vitest"; import { viTestRunner } from "adapter/node/vitest"; import { StorageLocalAdapter } from "adapter/node"; import { adapterTestSuite } from "media/storage/adapters/adapter-test-suite"; import { readFileSync } from "node:fs"; import path from "node:path"; +import { disableConsoleLog, enableConsoleLog } from "core/utils/test"; + +beforeAll(() => disableConsoleLog()); +afterAll(() => enableConsoleLog()); describe("StorageLocalAdapter (node)", async () => { const basePath = path.resolve(import.meta.dirname, "../../../../__test__/_assets"); diff --git a/app/src/adapter/node/test.ts b/app/src/adapter/node/test.ts index 3c78f25..bd1d8a5 100644 --- a/app/src/adapter/node/test.ts +++ b/app/src/adapter/node/test.ts @@ -1,5 +1,5 @@ import nodeAssert from "node:assert/strict"; -import { test, describe, beforeEach, afterEach } from "node:test"; +import { test, describe, beforeEach, afterEach, after, before } from "node:test"; import type { Matcher, Test, TestFn, TestRunner } from "core/test"; // Track mock function calls @@ -99,5 +99,6 @@ export const nodeTestRunner: TestRunner = { }), beforeEach: beforeEach, afterEach: afterEach, - afterAll: () => {}, + afterAll: after, + beforeAll: before, }; diff --git a/app/src/adapter/node/vitest.ts b/app/src/adapter/node/vitest.ts index 8f6988e..ad9afcf 100644 --- a/app/src/adapter/node/vitest.ts +++ b/app/src/adapter/node/vitest.ts @@ -1,5 +1,5 @@ import type { TestFn, TestRunner, Test } from "core/test"; -import { describe, test, expect, vi, beforeEach, afterEach, afterAll } from "vitest"; +import { describe, test, expect, vi, beforeEach, afterEach, afterAll, beforeAll } from "vitest"; function vitestTest(label: string, fn: TestFn, options?: any) { return test(label, fn as any); @@ -50,4 +50,5 @@ export const viTestRunner: TestRunner = { beforeEach: beforeEach, afterEach: afterEach, afterAll: afterAll, + beforeAll: beforeAll, }; diff --git a/app/src/adapter/react-router/react-router.adapter.spec.ts b/app/src/adapter/react-router/react-router.adapter.spec.ts index 25ef895..bb525c7 100644 --- a/app/src/adapter/react-router/react-router.adapter.spec.ts +++ b/app/src/adapter/react-router/react-router.adapter.spec.ts @@ -10,6 +10,6 @@ afterAll(enableConsoleLog); describe("react-router adapter", () => { adapterTestSuite(bunTestRunner, { makeApp: rr.getApp, - makeHandler: (c, a, o) => (request: Request) => rr.serve(c, a?.env, o)({ request }), + makeHandler: (c, a) => (request: Request) => rr.serve(c, a?.env)({ request }), }); }); diff --git a/app/src/adapter/react-router/react-router.adapter.ts b/app/src/adapter/react-router/react-router.adapter.ts index 4474509..f37260d 100644 --- a/app/src/adapter/react-router/react-router.adapter.ts +++ b/app/src/adapter/react-router/react-router.adapter.ts @@ -1,5 +1,4 @@ import { type FrameworkBkndConfig, createFrameworkApp } from "bknd/adapter"; -import type { FrameworkOptions } from "adapter"; type ReactRouterEnv = NodeJS.ProcessEnv; type ReactRouterFunctionArgs = { @@ -10,17 +9,15 @@ export type ReactRouterBkndConfig = FrameworkBkndConfig( config: ReactRouterBkndConfig, args: Env = {} as Env, - opts?: FrameworkOptions, ) { - return await createFrameworkApp(config, args ?? process.env, opts); + return await createFrameworkApp(config, args ?? process.env); } export function serve( config: ReactRouterBkndConfig = {}, args: Env = {} as Env, - opts?: FrameworkOptions, ) { return async (fnArgs: ReactRouterFunctionArgs) => { - return (await getApp(config, args, opts)).fetch(fnArgs.request); + return (await getApp(config, args)).fetch(fnArgs.request); }; } diff --git a/app/src/adapter/vite/vite.adapter.ts b/app/src/adapter/vite/vite.adapter.ts index c69bc1e..a4cb346 100644 --- a/app/src/adapter/vite/vite.adapter.ts +++ b/app/src/adapter/vite/vite.adapter.ts @@ -1,7 +1,7 @@ import { serveStatic } from "@hono/node-server/serve-static"; import { type DevServerOptions, default as honoViteDevServer } from "@hono/vite-dev-server"; import type { App } from "bknd"; -import { type RuntimeBkndConfig, createRuntimeApp, type FrameworkOptions } from "bknd/adapter"; +import { type RuntimeBkndConfig, createRuntimeApp } from "bknd/adapter"; import { registerLocalMediaAdapter } from "bknd/adapter/node"; import { devServerConfig } from "./dev-server-config"; import type { MiddlewareHandler } from "hono"; @@ -30,7 +30,6 @@ ${addBkndContext ? "" : ""} async function createApp( config: ViteBkndConfig = {}, env: ViteEnv = {} as ViteEnv, - opts: FrameworkOptions = {}, ): Promise { registerLocalMediaAdapter(); return await createRuntimeApp( @@ -47,18 +46,13 @@ async function createApp( ], }, env, - opts, ); } -export function serve( - config: ViteBkndConfig = {}, - args?: ViteEnv, - opts?: FrameworkOptions, -) { +export function serve(config: ViteBkndConfig = {}, args?: ViteEnv) { return { async fetch(request: Request, env: any, ctx: ExecutionContext) { - const app = await createApp(config, env, opts); + const app = await createApp(config, env); return app.fetch(request, env, ctx); }, }; diff --git a/app/src/auth/authenticate/Authenticator.ts b/app/src/auth/authenticate/Authenticator.ts index 0350aba..52a2e42 100644 --- a/app/src/auth/authenticate/Authenticator.ts +++ b/app/src/auth/authenticate/Authenticator.ts @@ -278,7 +278,9 @@ export class Authenticator< } return payload as any; - } catch (e) {} + } catch (e) { + $console.debug("Authenticator jwt verify error", String(e)); + } return; } @@ -396,8 +398,9 @@ export class Authenticator< if (headers.has("Authorization")) { const bearerHeader = String(headers.get("Authorization")); token = bearerHeader.replace("Bearer ", ""); - } else if (is_context) { - token = await this.getAuthCookie(c as Context); + } else { + const context = is_context ? (c as Context) : ({ req: { raw: { headers } } } as Context); + token = await this.getAuthCookie(context); } if (token) { diff --git a/app/src/cli/commands/config.ts b/app/src/cli/commands/config.ts index 154453f..ad9f428 100644 --- a/app/src/cli/commands/config.ts +++ b/app/src/cli/commands/config.ts @@ -4,6 +4,7 @@ import { makeAppFromEnv } from "cli/commands/run"; import { writeFile } from "node:fs/promises"; import c from "picocolors"; import { withConfigOptions } from "cli/utils/options"; +import { $console } from "bknd/utils"; export const config: CliCommand = (program) => { withConfigOptions(program.command("config")) @@ -19,7 +20,14 @@ export const config: CliCommand = (program) => { config = getDefaultConfig(); } else { const app = await makeAppFromEnv(options); - config = app.toJSON(options.secrets); + const manager = app.modules; + + if (options.secrets) { + $console.warn("Including secrets in output"); + config = manager.toJSON(true); + } else { + config = manager.extractSecrets().configs; + } } config = options.pretty ? JSON.stringify(config, null, 2) : JSON.stringify(config); @@ -31,5 +39,7 @@ export const config: CliCommand = (program) => { } else { console.info(JSON.parse(config)); } + + process.exit(0); }); }; diff --git a/app/src/cli/commands/copy-assets.ts b/app/src/cli/commands/copy-assets.ts index d9d1d12..0d9b27d 100644 --- a/app/src/cli/commands/copy-assets.ts +++ b/app/src/cli/commands/copy-assets.ts @@ -34,4 +34,5 @@ async function action(options: { out?: string; clean?: boolean }) { // biome-ignore lint/suspicious/noConsoleLog: console.log(c.green(`Assets copied to: ${c.bold(out)}`)); + process.exit(0); } diff --git a/app/src/cli/commands/debug.ts b/app/src/cli/commands/debug.ts index a9a1e6f..412b485 100644 --- a/app/src/cli/commands/debug.ts +++ b/app/src/cli/commands/debug.ts @@ -40,7 +40,9 @@ const subjects = { async function action(subject: string) { if (subject in subjects) { await subjects[subject](); + process.exit(0); } else { console.error("Invalid subject: ", subject); + process.exit(1); } } diff --git a/app/src/cli/commands/index.ts b/app/src/cli/commands/index.ts index ad014fb..87b6fcf 100644 --- a/app/src/cli/commands/index.ts +++ b/app/src/cli/commands/index.ts @@ -8,3 +8,4 @@ export { copyAssets } from "./copy-assets"; export { types } from "./types"; export { mcp } from "./mcp/mcp"; export { sync } from "./sync"; +export { secrets } from "./secrets"; diff --git a/app/src/cli/commands/run/platform.ts b/app/src/cli/commands/run/platform.ts index 061d44c..ed2e1aa 100644 --- a/app/src/cli/commands/run/platform.ts +++ b/app/src/cli/commands/run/platform.ts @@ -9,18 +9,28 @@ export const PLATFORMS = ["node", "bun"] as const; export type Platform = (typeof PLATFORMS)[number]; export async function serveStatic(server: Platform): Promise { + const onNotFound = (path: string) => { + $console.debug("Couldn't resolve static file at", path); + }; + switch (server) { case "node": { const m = await import("@hono/node-server/serve-static"); + const root = getRelativeDistPath() + "/static"; + $console.debug("Serving static files from", root); return m.serveStatic({ // somehow different for node - root: getRelativeDistPath() + "/static", + root, + onNotFound, }); } case "bun": { const m = await import("hono/bun"); + const root = path.resolve(getRelativeDistPath(), "static"); + $console.debug("Serving static files from", root); return m.serveStatic({ - root: path.resolve(getRelativeDistPath(), "static"), + root, + onNotFound, }); } } @@ -66,6 +76,9 @@ export async function getConfigPath(filePath?: string) { const config_path = path.resolve(process.cwd(), filePath); if (await fileExists(config_path)) { return config_path; + } else { + $console.error(`Config file could not be resolved: ${config_path}`); + process.exit(1); } } diff --git a/app/src/cli/commands/run/run.ts b/app/src/cli/commands/run/run.ts index 0e4efb0..154d50c 100644 --- a/app/src/cli/commands/run/run.ts +++ b/app/src/cli/commands/run/run.ts @@ -2,9 +2,8 @@ import type { Config } from "@libsql/client/node"; import { StorageLocalAdapter } from "adapter/node/storage"; import type { CliBkndConfig, CliCommand } from "cli/types"; import { Option } from "commander"; -import { config, type App, type CreateAppConfig } from "bknd"; +import { config, type App, type CreateAppConfig, type MaybePromise, registries } from "bknd"; import dotenv from "dotenv"; -import { registries } from "modules/registries"; import c from "picocolors"; import path from "node:path"; import { @@ -60,7 +59,7 @@ type MakeAppConfig = { connection?: CreateAppConfig["connection"]; server?: { platform?: Platform }; setAdminHtml?: boolean; - onBuilt?: (app: App) => Promise; + onBuilt?: (app: App) => MaybePromise; }; async function makeApp(config: MakeAppConfig) { diff --git a/app/src/cli/commands/schema.ts b/app/src/cli/commands/schema.ts index 5dceee9..f9fd510 100644 --- a/app/src/cli/commands/schema.ts +++ b/app/src/cli/commands/schema.ts @@ -8,7 +8,7 @@ export const schema: CliCommand = (program) => { .option("--pretty", "pretty print") .action((options) => { const schema = getDefaultSchema(); - // biome-ignore lint/suspicious/noConsoleLog: - console.log(options.pretty ? JSON.stringify(schema, null, 2) : JSON.stringify(schema)); + console.info(options.pretty ? JSON.stringify(schema, null, 2) : JSON.stringify(schema)); + process.exit(0); }); }; diff --git a/app/src/cli/commands/secrets.ts b/app/src/cli/commands/secrets.ts new file mode 100644 index 0000000..ceae59f --- /dev/null +++ b/app/src/cli/commands/secrets.ts @@ -0,0 +1,59 @@ +import type { CliCommand } from "../types"; +import { makeAppFromEnv } from "cli/commands/run"; +import { writeFile } from "node:fs/promises"; +import c from "picocolors"; +import { withConfigOptions, type WithConfigOptions } from "cli/utils/options"; +import { transformObject } from "bknd/utils"; +import { Option } from "commander"; + +export const secrets: CliCommand = (program) => { + withConfigOptions(program.command("secrets")) + .description("get app secrets") + .option("--template", "template output without the actual secrets") + .addOption( + new Option("--format ", "format output").choices(["json", "env"]).default("json"), + ) + .option("--out ", "output file") + .action( + async ( + options: WithConfigOptions<{ template: string; format: "json" | "env"; out: string }>, + ) => { + const app = await makeAppFromEnv(options); + const manager = app.modules; + + let secrets = manager.extractSecrets().secrets; + if (options.template) { + secrets = transformObject(secrets, () => ""); + } + + console.info(""); + if (options.out) { + if (options.format === "env") { + await writeFile( + options.out, + Object.entries(secrets) + .map(([key, value]) => `${key}=${value}`) + .join("\n"), + ); + } else { + await writeFile(options.out, JSON.stringify(secrets, null, 2)); + } + console.info(`Secrets written to ${c.cyan(options.out)}`); + } else { + if (options.format === "env") { + console.info( + c.cyan( + Object.entries(secrets) + .map(([key, value]) => `${key}=${value}`) + .join("\n"), + ), + ); + } else { + console.info(secrets); + } + } + console.info(""); + process.exit(0); + }, + ); +}; diff --git a/app/src/cli/commands/sync.ts b/app/src/cli/commands/sync.ts index d9b3ed5..8b8c5c4 100644 --- a/app/src/cli/commands/sync.ts +++ b/app/src/cli/commands/sync.ts @@ -7,13 +7,15 @@ import { withConfigOptions } from "cli/utils/options"; export const sync: CliCommand = (program) => { withConfigOptions(program.command("sync")) .description("sync database") - .option("--dump", "dump operations to console instead of executing them") + .option("--force", "perform database syncing operations") + .option("--seed", "perform seeding operations") .option("--drop", "include destructive DDL operations") .option("--out ", "output file") .option("--sql", "use sql output") .action(async (options) => { const app = await makeAppFromEnv(options); const schema = app.em.schema(); + console.info(c.dim("Checking database state...")); const stmts = await schema.sync({ drop: options.drop }); console.info(""); @@ -24,22 +26,41 @@ export const sync: CliCommand = (program) => { // @todo: currently assuming parameters aren't used const sql = stmts.map((d) => d.sql).join(";\n") + ";"; - if (options.dump) { + if (options.force) { + console.info(c.dim("Executing:") + "\n" + c.cyan(sql)); + await schema.sync({ force: true, drop: options.drop }); + + console.info(`\n${c.dim(`Executed ${c.cyan(stmts.length)} statement(s)`)}`); + console.info(`${c.green("Database synced")}`); + + if (options.seed) { + console.info(c.dim("\nExecuting seed...")); + const seed = app.options?.seed; + if (seed) { + await app.options?.seed?.({ + ...app.modules.ctx(), + app: app, + }); + console.info(c.green("Seed executed")); + } else { + console.info(c.yellow("No seed function provided")); + } + } + } else { if (options.out) { const output = options.sql ? sql : JSON.stringify(stmts, null, 2); await writeFile(options.out, output); console.info(`SQL written to ${c.cyan(options.out)}`); } else { - console.info(options.sql ? c.cyan(sql) : stmts); + console.info(c.dim("DDL to execute:") + "\n" + c.cyan(sql)); + + console.info( + c.yellow( + "\nNo statements have been executed. Use --force to perform database syncing operations", + ), + ); } - - process.exit(0); } - - await schema.sync({ force: true, drop: options.drop }); - console.info(c.cyan(sql)); - - console.info(`${c.gray(`Executed ${c.cyan(stmts.length)} statement(s)`)}`); - console.info(`${c.green("Database synced")}`); + process.exit(0); }); }; diff --git a/app/src/cli/commands/types/types.ts b/app/src/cli/commands/types/types.ts index b545d61..ebadf35 100644 --- a/app/src/cli/commands/types/types.ts +++ b/app/src/cli/commands/types/types.ts @@ -35,4 +35,6 @@ async function action({ await writeFile(outfile, et.toString()); console.info(`\nTypes written to ${c.cyan(outfile)}`); } + + process.exit(0); } diff --git a/app/src/cli/commands/user.ts b/app/src/cli/commands/user.ts index fb4bd4a..3721a2b 100644 --- a/app/src/cli/commands/user.ts +++ b/app/src/cli/commands/user.ts @@ -78,9 +78,11 @@ async function create(app: App, options: any) { password: await strategy.hash(password as string), }); $log.success(`Created user: ${c.cyan(created.email)}`); + process.exit(0); } catch (e) { $log.error("Error creating user"); $console.error(e); + process.exit(1); } } @@ -121,8 +123,10 @@ async function update(app: App, options: any) { if (await app.module.auth.changePassword(user.id, password)) { $log.success(`Updated user: ${c.cyan(user.email)}`); + process.exit(0); } else { $log.error("Error updating user"); + process.exit(1); } } @@ -158,4 +162,5 @@ async function token(app: App, options: any) { console.log( `\n${c.dim("Token:")}\n${c.yellow(await app.module.auth.authenticator.jwt(user))}\n`, ); + process.exit(0); } diff --git a/app/src/cli/index.ts b/app/src/cli/index.ts index 352cdd4..3385713 100644 --- a/app/src/cli/index.ts +++ b/app/src/cli/index.ts @@ -7,7 +7,7 @@ import { getVersion } from "./utils/sys"; import { capture, flush, init } from "cli/utils/telemetry"; const program = new Command(); -export async function main() { +async function main() { await init(); capture("start"); diff --git a/app/src/core/config.ts b/app/src/core/config.ts index 99a9013..581bea1 100644 --- a/app/src/core/config.ts +++ b/app/src/core/config.ts @@ -11,10 +11,12 @@ export interface AppEntity { export interface DB { // make sure to make unknown as "any" - [key: string]: { + /* [key: string]: { id: PrimaryFieldType; [key: string]: any; - }; + }; */ + // @todo: that's not good, but required for admin options + [key: string]: any; } export const config = { diff --git a/app/src/core/test/index.ts b/app/src/core/test/index.ts index 4e9bfef..0afecb2 100644 --- a/app/src/core/test/index.ts +++ b/app/src/core/test/index.ts @@ -31,6 +31,7 @@ export type TestRunner = { beforeEach: (fn: () => MaybePromise) => void; afterEach: (fn: () => MaybePromise) => void; afterAll: (fn: () => MaybePromise) => void; + beforeAll: (fn: () => MaybePromise) => void; }; export async function retry( diff --git a/app/src/core/types.ts b/app/src/core/types.ts index 1751766..03beae5 100644 --- a/app/src/core/types.ts +++ b/app/src/core/types.ts @@ -4,3 +4,5 @@ export interface Serializable { } export type MaybePromise = T | Promise; + +export type PartialRec = { [P in keyof T]?: PartialRec }; diff --git a/app/src/core/utils/objects.ts b/app/src/core/utils/objects.ts index 6f6a985..41902a9 100644 --- a/app/src/core/utils/objects.ts +++ b/app/src/core/utils/objects.ts @@ -396,6 +396,38 @@ export function getPath( } } +export function setPath(object: object, _path: string | (string | number)[], value: any) { + let path = _path; + // Optional string-path support. + // You can remove this `if` block if you don't need it. + if (typeof path === "string") { + const isQuoted = (str) => str[0] === '"' && str.at(-1) === '"'; + path = path + .split(/[.\[\]]+/) + .filter((x) => x) + .map((x) => (!Number.isNaN(Number(x)) ? Number(x) : x)) + .map((x) => (typeof x === "string" && isQuoted(x) ? x.slice(1, -1) : x)); + } + + if (path.length === 0) { + throw new Error("The path must have at least one entry in it"); + } + + const [head, ...tail] = path as any; + + if (tail.length === 0) { + object[head] = value; + return object; + } + + if (!(head in object)) { + object[head] = typeof tail[0] === "number" ? [] : {}; + } + + setPath(object[head], tail, value); + return object; +} + export function objectToJsLiteral(value: object, indent: number = 0, _level: number = 0): string { const nl = indent ? "\n" : ""; const pad = (lvl: number) => (indent ? " ".repeat(indent * lvl) : ""); diff --git a/app/src/core/utils/schema/index.ts b/app/src/core/utils/schema/index.ts index 5f10092..3d3692c 100644 --- a/app/src/core/utils/schema/index.ts +++ b/app/src/core/utils/schema/index.ts @@ -12,6 +12,7 @@ export { getMcpServer, stdioTransport, McpClient, + logLevels as mcpLogLevels, type McpClientConfig, type ToolAnnotation, type ToolHandlerCtx, @@ -21,8 +22,35 @@ export { secret, SecretSchema } from "./secret"; export { s }; -export const stripMark = (o: O): O => o; -export const mark = (o: O): O => o; +const symbol = Symbol("bknd-validation-mark"); + +export function stripMark(obj: O) { + const newObj = structuredClone(obj); + mark(newObj, false); + return newObj as O; +} + +export function mark(obj: any, validated = true) { + try { + if (typeof obj === "object" && obj !== null && !Array.isArray(obj)) { + if (validated) { + obj[symbol] = true; + } else { + delete obj[symbol]; + } + for (const key in obj) { + if (typeof obj[key] === "object" && obj[key] !== null) { + mark(obj[key], validated); + } + } + } + } catch (e) {} +} + +export function isMarked(obj: any) { + if (typeof obj !== "object" || obj === null) return false; + return obj[symbol] === true; +} export const stringIdentifier = s.string({ pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", @@ -38,7 +66,8 @@ export class InvalidSchemaError extends Error { ) { super( `Invalid schema given for ${JSON.stringify(value, null, 2)}\n\n` + - `Error: ${JSON.stringify(errors[0], null, 2)}`, + `Error: ${JSON.stringify(errors[0], null, 2)}\n\n` + + `Schema: ${JSON.stringify(schema.toJSON(), null, 2)}`, ); } @@ -73,6 +102,10 @@ export function parse : s.Static { + if (!opts?.forceParse && !opts?.coerce && isMarked(v)) { + return v as any; + } + const schema = (opts?.clone ? cloneSchema(_schema as any) : _schema) as s.Schema; let value = opts?.coerce !== false diff --git a/app/src/core/utils/test.ts b/app/src/core/utils/test.ts index 44d38d9..5107c4b 100644 --- a/app/src/core/utils/test.ts +++ b/app/src/core/utils/test.ts @@ -6,6 +6,8 @@ const _oldConsoles = { warn: console.warn, error: console.error, }; +let _oldStderr: any; +let _oldStdout: any; export async function withDisabledConsole( fn: () => Promise, @@ -36,10 +38,17 @@ export function disableConsoleLog(severities: ConsoleSeverity[] = ["log", "warn" severities.forEach((severity) => { console[severity] = () => null; }); + // Disable stderr + _oldStderr = process.stderr.write; + _oldStdout = process.stdout.write; + process.stderr.write = () => true; + process.stdout.write = () => true; $console?.setLevel("critical"); } export function enableConsoleLog() { + process.stderr.write = _oldStderr; + process.stdout.write = _oldStdout; Object.entries(_oldConsoles).forEach(([severity, fn]) => { console[severity as ConsoleSeverity] = fn; }); diff --git a/app/src/data/api/DataApi.ts b/app/src/data/api/DataApi.ts index b4deb5d..de88812 100644 --- a/app/src/data/api/DataApi.ts +++ b/app/src/data/api/DataApi.ts @@ -42,6 +42,9 @@ export class DataApi extends ModuleApi { ) { type Data = E extends keyof DB ? Selectable : EntityData; type T = RepositoryResultJSON; + + // @todo: if none found, still returns meta... + return this.readMany(entity, { ...query, limit: 1, diff --git a/app/src/data/api/DataController.ts b/app/src/data/api/DataController.ts index 055033c..163f0af 100644 --- a/app/src/data/api/DataController.ts +++ b/app/src/data/api/DataController.ts @@ -60,6 +60,7 @@ export class DataController extends Controller { "/sync", permission(DataPermissions.databaseSync), mcpTool("data_sync", { + // @todo: should be removed if readonly annotations: { destructiveHint: true, }, diff --git a/app/src/data/connection/Connection.ts b/app/src/data/connection/Connection.ts index 7db9227..1cc8b52 100644 --- a/app/src/data/connection/Connection.ts +++ b/app/src/data/connection/Connection.ts @@ -230,3 +230,15 @@ export function customIntrospector>( }, }; } + +export class DummyConnection extends Connection { + override name = "dummy"; + + constructor() { + super(undefined as any); + } + + override getFieldSchema(): SchemaResponse { + throw new Error("Method not implemented."); + } +} diff --git a/app/src/data/connection/connection-test-suite.ts b/app/src/data/connection/connection-test-suite.ts index aed28aa..270ccb0 100644 --- a/app/src/data/connection/connection-test-suite.ts +++ b/app/src/data/connection/connection-test-suite.ts @@ -4,6 +4,7 @@ import { getPath } from "bknd/utils"; import * as proto from "data/prototype"; import { createApp } from "App"; import type { MaybePromise } from "core/types"; +import { disableConsoleLog, enableConsoleLog } from "core/utils/test"; // @todo: add various datatypes: string, number, boolean, object, array, null, undefined, date, etc. // @todo: add toDriver/fromDriver tests on all types and fields @@ -21,7 +22,9 @@ export function connectionTestSuite( rawDialectDetails: string[]; }, ) { - const { test, expect, describe, beforeEach, afterEach, afterAll } = testRunner; + const { test, expect, describe, beforeEach, afterEach, afterAll, beforeAll } = testRunner; + beforeAll(() => disableConsoleLog()); + afterAll(() => enableConsoleLog()); describe("base", () => { let ctx: Awaited>; @@ -247,7 +250,7 @@ export function connectionTestSuite( const app = createApp({ connection: ctx.connection, - initialConfig: { + config: { data: schema.toJSON(), }, }); @@ -333,7 +336,7 @@ export function connectionTestSuite( const app = createApp({ connection: ctx.connection, - initialConfig: { + config: { data: schema.toJSON(), }, }); diff --git a/app/src/data/connection/sqlite/GenericSqliteConnection.ts b/app/src/data/connection/sqlite/GenericSqliteConnection.ts index 98a584b..0ec5212 100644 --- a/app/src/data/connection/sqlite/GenericSqliteConnection.ts +++ b/app/src/data/connection/sqlite/GenericSqliteConnection.ts @@ -1,7 +1,6 @@ import type { KyselyPlugin, QueryResult } from "kysely"; import { type IGenericSqlite, - type OnCreateConnection, type Promisable, parseBigInt, buildQueryFn, @@ -9,6 +8,7 @@ import { } from "kysely-generic-sqlite"; import { SqliteConnection } from "./SqliteConnection"; import type { ConnQuery, ConnQueryResults, Features } from "../Connection"; +import type { MaybePromise } from "bknd"; export type { IGenericSqlite }; export type TStatement = { sql: string; parameters?: any[] | readonly any[] }; @@ -16,11 +16,11 @@ export interface IGenericCustomSqlite extends IGenericSqlite { batch?: (stmts: TStatement[]) => Promisable[]>; } -export type GenericSqliteConnectionConfig = { +export type GenericSqliteConnectionConfig = { name?: string; additionalPlugins?: KyselyPlugin[]; excludeTables?: string[]; - onCreateConnection?: OnCreateConnection; + onCreateConnection?: (db: Database) => MaybePromise; supports?: Partial; }; @@ -35,7 +35,12 @@ export class GenericSqliteConnection extends SqliteConnection ) { super({ dialect: GenericSqliteDialect, - dialectArgs: [executor, config?.onCreateConnection], + dialectArgs: [ + executor, + config?.onCreateConnection && typeof config.onCreateConnection === "function" + ? (c: any) => config.onCreateConnection?.(c.db.db as any) + : undefined, + ], additionalPlugins: config?.additionalPlugins, excludeTables: config?.excludeTables, }); @@ -61,7 +66,6 @@ export class GenericSqliteConnection extends SqliteConnection override async executeQueries(...qbs: O): Promise> { const executor = await this.getExecutor(); if (!executor.batch) { - //$console.debug("Batching is not supported by this database"); return super.executeQueries(...qbs); } diff --git a/app/src/data/connection/sqlite/SqliteIntrospector.ts b/app/src/data/connection/sqlite/SqliteIntrospector.ts index 70c3ff6..8ff2688 100644 --- a/app/src/data/connection/sqlite/SqliteIntrospector.ts +++ b/app/src/data/connection/sqlite/SqliteIntrospector.ts @@ -55,7 +55,8 @@ export class SqliteIntrospector extends BaseIntrospector { )) FROM pragma_index_info(i.name) ii) )) FROM pragma_index_list(m.name) i LEFT JOIN sqlite_master im ON im.name = i.name - AND im.type = 'index' + AND im.type = 'index' + WHERE i.name not like 'sqlite_%' ) AS indices FROM sqlite_master m WHERE m.type IN ('table', 'view') diff --git a/app/src/data/entities/EntityTypescript.spec.ts b/app/src/data/entities/EntityTypescript.spec.ts new file mode 100644 index 0000000..7241659 --- /dev/null +++ b/app/src/data/entities/EntityTypescript.spec.ts @@ -0,0 +1,54 @@ +import { describe, it, expect } from "bun:test"; +import { EntityTypescript } from "./EntityTypescript"; +import * as proto from "../prototype"; +import { DummyConnection } from "../connection/Connection"; + +describe("EntityTypescript", () => { + it("should generate correct typescript for system entities", () => { + const schema = proto.em( + { + test: proto.entity("test", { + name: proto.text(), + }), + users: proto.systemEntity("users", { + name: proto.text(), + }), + }, + ({ relation }, { test, users }) => { + relation(test).manyToOne(users); + }, + ); + const et = new EntityTypescript(schema.proto.withConnection(new DummyConnection())); + expect(et.toString()).toContain('users?: DB["users"];'); + }); + + it("should generate correct typescript for system entities with uuid primary field", () => { + const schema = proto.em( + { + test: proto.entity( + "test", + { + name: proto.text(), + }, + { + primary_format: "uuid", + }, + ), + users: proto.systemEntity( + "users", + { + name: proto.text(), + }, + { + primary_format: "uuid", + }, + ), + }, + ({ relation }, { test, users }) => { + relation(test).manyToOne(users); + }, + ); + const et = new EntityTypescript(schema.proto.withConnection(new DummyConnection())); + expect(et.toString()).toContain("users_id?: string;"); + }); +}); diff --git a/app/src/data/entities/EntityTypescript.ts b/app/src/data/entities/EntityTypescript.ts index b7e2b2c..36d571b 100644 --- a/app/src/data/entities/EntityTypescript.ts +++ b/app/src/data/entities/EntityTypescript.ts @@ -40,7 +40,7 @@ const systemEntities = { export class EntityTypescript { constructor( - protected em: EntityManager, + protected em: EntityManager, protected _options: EntityTypescriptOptions = {}, ) {} @@ -50,7 +50,7 @@ export class EntityTypescript { indentWidth: 2, indentChar: " ", entityCommentMultiline: true, - fieldCommentMultiline: false, + fieldCommentMultiline: true, }; } @@ -82,7 +82,7 @@ export class EntityTypescript { } typeName(name: string) { - return autoFormatString(name); + return autoFormatString(name).replace(/ /g, ""); } fieldTypesToString(type: TEntityTSType, opts?: { ignore_fields?: string[]; indent?: number }) { diff --git a/app/src/data/entities/Result.ts b/app/src/data/entities/Result.ts index 2816efd..b637e1a 100644 --- a/app/src/data/entities/Result.ts +++ b/app/src/data/entities/Result.ts @@ -72,12 +72,12 @@ export class Result { return this.first().parameters; } - get data() { + get data(): T { if (this.options.single) { return this.first().data?.[0]; } - return this.first().data ?? []; + return this.first().data ?? ([] as T); } async execute(qb: Compilable | Compilable[]) { diff --git a/app/src/data/entities/index.ts b/app/src/data/entities/index.ts index 5beca37..59b16b9 100644 --- a/app/src/data/entities/index.ts +++ b/app/src/data/entities/index.ts @@ -4,5 +4,6 @@ export * from "./mutation/Mutator"; export * from "./query/Repository"; export * from "./query/WhereBuilder"; export * from "./query/WithBuilder"; +export * from "./Result"; export * from "./query/RepositoryResult"; export * from "./mutation/MutatorResult"; diff --git a/app/src/data/fields/indices/EntityIndex.ts b/app/src/data/fields/indices/EntityIndex.ts index e8af2e6..ec69ecc 100644 --- a/app/src/data/fields/indices/EntityIndex.ts +++ b/app/src/data/fields/indices/EntityIndex.ts @@ -15,17 +15,6 @@ export class EntityIndex { throw new Error("All fields must be instances of Field"); } - if (unique) { - const firstRequired = fields[0]?.isRequired(); - if (!firstRequired) { - throw new Error( - `Unique indices must have first field as required: ${fields - .map((f) => f.name) - .join(", ")}`, - ); - } - } - if (!name) { this.name = [ unique ? "idx_unique" : "idx", diff --git a/app/src/data/prototype/index.ts b/app/src/data/prototype/index.ts index 06483f5..43df6a1 100644 --- a/app/src/data/prototype/index.ts +++ b/app/src/data/prototype/index.ts @@ -210,8 +210,8 @@ type SystemEntities = { export function systemEntity< E extends keyof SystemEntities, Fields extends Record>, ->(name: E, fields: Fields) { - return entity(name, fields as any); +>(name: E, fields: Fields, config?: EntityConfig) { + return entity(name, fields as any, config, "system"); } export function relation(local: Local) { diff --git a/app/src/data/relations/ManyToManyRelation.ts b/app/src/data/relations/ManyToManyRelation.ts index 99b5740..e7a1233 100644 --- a/app/src/data/relations/ManyToManyRelation.ts +++ b/app/src/data/relations/ManyToManyRelation.ts @@ -180,8 +180,15 @@ export class ManyToManyRelation extends EntityRelation; @@ -70,7 +71,7 @@ export class PolymorphicRelation extends EntityRelation { + override getReferenceQuery(entity: Entity, id: PrimaryFieldType): Partial { const info = this.queryInfo(entity); return { @@ -101,8 +102,8 @@ export class PolymorphicRelation extends EntityRelation) { diff --git a/app/src/data/relations/RelationField.ts b/app/src/data/relations/RelationField.ts index 3a2ab07..f6e4c0d 100644 --- a/app/src/data/relations/RelationField.ts +++ b/app/src/data/relations/RelationField.ts @@ -84,9 +84,10 @@ export class RelationField extends Field { } override toType(): TFieldTSType { + const type = this.config.target_field_type === "integer" ? "number" : "string"; return { ...super.toType(), - type: "number", + type, }; } } diff --git a/app/src/data/schema/SchemaManager.ts b/app/src/data/schema/SchemaManager.ts index ab2d15f..78708d6 100644 --- a/app/src/data/schema/SchemaManager.ts +++ b/app/src/data/schema/SchemaManager.ts @@ -65,6 +65,7 @@ export class SchemaManager { if (SchemaManager.EXCLUDE_TABLES.includes(table.name)) { continue; } + if (!table.name) continue; cleanTables.push({ ...table, diff --git a/app/src/index.ts b/app/src/index.ts index 46902cd..ae01151 100644 --- a/app/src/index.ts +++ b/app/src/index.ts @@ -29,6 +29,7 @@ export { type InitialModuleConfigs, ModuleManagerEvents, } from "./modules/ModuleManager"; +export type * from "modules/ModuleApi"; export type { ServerEnv } from "modules/Controller"; export type { BkndConfig } from "bknd/adapter"; @@ -115,6 +116,7 @@ export { genericSqlite, genericSqliteUtils, type GenericSqliteConnection, + type GenericSqliteConnectionConfig, } from "data/connection/sqlite/GenericSqliteConnection"; export { EntityTypescript, @@ -128,6 +130,7 @@ export type { EntityRelation } from "data/relations"; export type * from "data/entities/Entity"; export type { EntityManager } from "data/entities/EntityManager"; export type { SchemaManager } from "data/schema/SchemaManager"; +export type * from "data/entities"; export { BaseIntrospector, Connection, diff --git a/app/src/media/AppMedia.ts b/app/src/media/AppMedia.ts index 0971187..2c1b6b2 100644 --- a/app/src/media/AppMedia.ts +++ b/app/src/media/AppMedia.ts @@ -8,6 +8,7 @@ import { MediaController } from "./api/MediaController"; import { buildMediaSchema, registry, type TAppMediaConfig } from "./media-schema"; import { mediaFields } from "./media-entities"; import * as MediaPermissions from "media/media-permissions"; +import * as DatabaseEvents from "data/events"; export type MediaFields = typeof AppMedia.mediaFields; export type MediaFieldSchema = FieldSchema; @@ -139,6 +140,30 @@ export class AppMedia extends Module> { }, { mode: "sync", id: "delete-data-media" }, ); + + emgr.onEvent( + DatabaseEvents.MutatorDeleteAfter, + async (e) => { + const { entity, data } = e.params; + const fields = entity.fields.filter((f) => f.type === "media"); + if (fields.length > 0) { + const references = fields.map((f) => `${entity.name}.${f.name}`); + $console.log("App:storage:file cleaning up", { + reference: { $in: references }, + entity_id: String(data.id), + }); + const { data: deleted } = await em.mutator(media).deleteWhere({ + reference: { $in: references }, + entity_id: String(data.id), + }); + for (const file of deleted) { + await this.storage.deleteFile(file.path); + } + $console.log("App:storage:file cleaned up files:", deleted.length); + } + }, + { mode: "async", id: "delete-data-media-after" }, + ); } override getOverwritePaths() { diff --git a/app/src/media/MediaField.ts b/app/src/media/MediaField.ts index 5f005bc..6ff8cd3 100644 --- a/app/src/media/MediaField.ts +++ b/app/src/media/MediaField.ts @@ -43,6 +43,10 @@ export class MediaField< return this.config.max_items; } + getAllowedMimeTypes(): string[] | undefined { + return this.config.mime_types; + } + getMinItems(): number | undefined { return this.config.min_items; } diff --git a/app/src/media/api/MediaController.ts b/app/src/media/api/MediaController.ts index 5be44fd..6a72048 100644 --- a/app/src/media/api/MediaController.ts +++ b/app/src/media/api/MediaController.ts @@ -181,16 +181,14 @@ export class MediaController extends Controller { "param", s.object({ entity: entitiesEnum, - id: s.number(), + id: s.anyOf([s.number(), s.string()]), field: s.string(), }), ), jsc("query", s.object({ overwrite: s.boolean().optional() })), permission([DataPermissions.entityCreate, MediaPermissions.uploadFile]), async (c) => { - const entity_name = c.req.param("entity"); - const field_name = c.req.param("field"); - const entity_id = Number.parseInt(c.req.param("id")); + const { entity: entity_name, id: entity_id, field: field_name } = c.req.valid("param"); // check if entity exists const entity = this.media.em.entity(entity_name); diff --git a/app/src/media/media-entities.ts b/app/src/media/media-entities.ts index a074b18..20bbf8b 100644 --- a/app/src/media/media-entities.ts +++ b/app/src/media/media-entities.ts @@ -9,6 +9,6 @@ export const mediaFields = { etag: text(), modified_at: datetime(), reference: text(), - entity_id: number(), + entity_id: text(), metadata: json(), }; diff --git a/app/src/media/storage/Storage.ts b/app/src/media/storage/Storage.ts index 893e25f..1d11b1d 100644 --- a/app/src/media/storage/Storage.ts +++ b/app/src/media/storage/Storage.ts @@ -71,22 +71,29 @@ export class Storage implements EmitsEvents { let info: FileUploadPayload = { name, - meta: { - size: 0, - type: "application/octet-stream", - }, + meta: isFile(file) + ? { + size: file.size, + type: file.type, + } + : { + size: 0, + type: "application/octet-stream", + }, etag: typeof result === "string" ? result : "", }; + // normally only etag is returned if (typeof result === "object") { info = result; - } else if (isFile(file)) { - info.meta.size = file.size; - info.meta.type = file.type; } // try to get better meta info - if (!isMimeType(info.meta.type, ["application/octet-stream", "application/json"])) { + if ( + !info.meta.type || + ["application/octet-stream", "application/json"].includes(info.meta.type) || + !info.meta.size + ) { const meta = await this.#adapter.getObjectMeta(name); if (!meta) { throw new Error("Failed to get object meta"); @@ -103,7 +110,7 @@ export class Storage implements EmitsEvents { ...dim, }; } catch (e) { - $console.warn("Failed to get image dimensions", e); + $console.warn("Failed to get image dimensions", e, file); } } diff --git a/app/src/media/storage/adapters/adapter-test-suite.ts b/app/src/media/storage/adapters/adapter-test-suite.ts index 1a92d34..a6d1917 100644 --- a/app/src/media/storage/adapters/adapter-test-suite.ts +++ b/app/src/media/storage/adapters/adapter-test-suite.ts @@ -11,12 +11,14 @@ export async function adapterTestSuite( retries?: number; retryTimeout?: number; skipExistsAfterDelete?: boolean; + testRange?: boolean; }, ) { const { test, expect } = testRunner; const options = { retries: opts?.retries ?? 1, retryTimeout: opts?.retryTimeout ?? 1000, + testRange: opts?.testRange ?? true, }; let objects = 0; @@ -53,9 +55,34 @@ export async function adapterTestSuite( await test("gets an object", async () => { const res = await adapter.getObject(filename, new Headers()); expect(res.ok).toBe(true); + expect(res.headers.get("Accept-Ranges")).toBe("bytes"); // @todo: check the content }); + if (options.testRange) { + await test("handles range request - partial content", async () => { + const headers = new Headers({ Range: "bytes=0-99" }); + const res = await adapter.getObject(filename, headers); + expect(res.status).toBe(206); // Partial Content + expect(/^bytes 0-99\/\d+$/.test(res.headers.get("Content-Range")!)).toBe(true); + expect(res.headers.get("Accept-Ranges")).toBe("bytes"); + }); + + await test("handles range request - suffix range", async () => { + const headers = new Headers({ Range: "bytes=-100" }); + const res = await adapter.getObject(filename, headers); + expect(res.status).toBe(206); // Partial Content + expect(/^bytes \d+-\d+\/\d+$/.test(res.headers.get("Content-Range")!)).toBe(true); + }); + + await test("handles invalid range request", async () => { + const headers = new Headers({ Range: "bytes=invalid" }); + const res = await adapter.getObject(filename, headers); + expect(res.status).toBe(416); // Range Not Satisfiable + expect(/^bytes \*\/\d+$/.test(res.headers.get("Content-Range")!)).toBe(true); + }); + } + await test("gets object meta", async () => { expect(await adapter.getObjectMeta(filename)).toEqual({ type: file.type, // image/png diff --git a/app/src/media/storage/adapters/cloudinary/StorageCloudinaryAdapter.ts b/app/src/media/storage/adapters/cloudinary/StorageCloudinaryAdapter.ts index 105dfef..490047b 100644 --- a/app/src/media/storage/adapters/cloudinary/StorageCloudinaryAdapter.ts +++ b/app/src/media/storage/adapters/cloudinary/StorageCloudinaryAdapter.ts @@ -1,12 +1,12 @@ -import { hash, pickHeaders, s, parse } from "bknd/utils"; +import { hash, pickHeaders, s, parse, secret } from "bknd/utils"; import type { FileBody, FileListObject, FileMeta } from "../../Storage"; import { StorageAdapter } from "../../StorageAdapter"; export const cloudinaryAdapterConfig = s.object( { cloud_name: s.string(), - api_key: s.string(), - api_secret: s.string(), + api_key: secret(), + api_secret: secret(), upload_preset: s.string().optional(), }, { title: "Cloudinary", description: "Cloudinary media storage" }, diff --git a/app/src/media/storage/adapters/s3/StorageS3Adapter.spec.ts b/app/src/media/storage/adapters/s3/StorageS3Adapter.spec.ts index 5a2fe89..62c86b6 100644 --- a/app/src/media/storage/adapters/s3/StorageS3Adapter.spec.ts +++ b/app/src/media/storage/adapters/s3/StorageS3Adapter.spec.ts @@ -22,6 +22,19 @@ afterAll(() => { cleanup(); }); */ +describe("StorageS3Adapter", async () => { + test("verify client's service is set to s3", async () => { + const adapter = new StorageS3Adapter({ + access_key: "", + secret_access_key: "", + url: "https://localhost", + }); + // this is important for minio to produce the correct headers + // and it won't harm s3 or r2 + expect(adapter.client.service).toBe("s3"); + }); +}); + describe.skipIf(ALL_TESTS)("StorageS3Adapter", async () => { if (ALL_TESTS) return; diff --git a/app/src/media/storage/adapters/s3/StorageS3Adapter.ts b/app/src/media/storage/adapters/s3/StorageS3Adapter.ts index bb89265..c36e0e3 100644 --- a/app/src/media/storage/adapters/s3/StorageS3Adapter.ts +++ b/app/src/media/storage/adapters/s3/StorageS3Adapter.ts @@ -8,15 +8,15 @@ import type { } from "@aws-sdk/client-s3"; import { AwsClient } from "core/clients/aws/AwsClient"; import { isDebug } from "core/env"; -import { isFile, pickHeaders2, parse, s } from "bknd/utils"; +import { isFile, pickHeaders2, parse, s, secret } from "bknd/utils"; import { transform } from "lodash-es"; import type { FileBody, FileListObject } from "../../Storage"; import { StorageAdapter } from "../../StorageAdapter"; export const s3AdapterConfig = s.object( { - access_key: s.string(), - secret_access_key: s.string(), + access_key: secret(), + secret_access_key: secret(), url: s.string({ pattern: "^https?://(?:.*)?[^/.]+$", description: "URL to S3 compatible endpoint without trailing slash", @@ -45,6 +45,7 @@ export class StorageS3Adapter extends StorageAdapter { accessKeyId: config.access_key, secretAccessKey: config.secret_access_key, retries: isDebug() ? 0 : 10, + service: "s3", }, { convertParams: "pascalToKebab", diff --git a/app/src/media/storage/mime-types-tiny.ts b/app/src/media/storage/mime-types-tiny.ts index e7c42bb..bbc8d9d 100644 --- a/app/src/media/storage/mime-types-tiny.ts +++ b/app/src/media/storage/mime-types-tiny.ts @@ -3,7 +3,7 @@ export const Q = { audio: ["ogg"], image: ["jpeg", "png", "gif", "webp", "bmp", "tiff", "avif", "heic", "heif"], text: ["html", "css", "mdx", "yaml", "vcard", "csv", "vtt"], - application: ["zip", "xml", "toml", "json", "json5"], + application: ["zip", "toml", "json", "json5", "pdf", "xml"], font: ["woff", "woff2", "ttf", "otf"], } as const; @@ -15,11 +15,13 @@ const c = { a: (w = "octet-stream") => `application/${w}`, i: (w) => `image/${w}`, v: (w) => `video/${w}`, + au: (w) => `audio/${w}`, } as const; export const M = new Map([ ["7z", c.z], ["7zip", c.z], - ["ai", c.a("pdf")], + ["txt", c.t()], + ["ai", c.a("postscript")], ["apk", c.a("vnd.android.package-archive")], ["doc", c.a("msword")], ["docx", `${c.vnd}.wordprocessingml.document`], @@ -32,12 +34,12 @@ export const M = new Map([ ["jpg", c.i("jpeg")], ["js", c.t("javascript")], ["log", c.t()], - ["m3u", c.t()], + ["m3u", c.au("x-mpegurl")], ["m3u8", c.a("vnd.apple.mpegurl")], ["manifest", c.t("cache-manifest")], ["md", c.t("markdown")], ["mkv", c.v("x-matroska")], - ["mp3", c.a("mpeg")], + ["mp3", c.au("mpeg")], ["mobi", c.a("x-mobipocket-ebook")], ["ppt", c.a("powerpoint")], ["pptx", `${c.vnd}.presentationml.presentation`], @@ -46,11 +48,10 @@ export const M = new Map([ ["tif", c.i("tiff")], ["tsv", c.t("tab-separated-values")], ["tgz", c.a("x-tar")], - ["txt", c.t()], ["text", c.t()], ["vcd", c.a("x-cdlink")], ["vcs", c.t("x-vcalendar")], - ["wav", c.a("x-wav")], + ["wav", c.au("vnd.wav")], ["webmanifest", c.a("manifest+json")], ["xls", c.a("vnd.ms-excel")], ["xlsx", `${c.vnd}.spreadsheetml.sheet`], diff --git a/app/src/modules/ModuleManager.ts b/app/src/modules/ModuleManager.ts index 2948b5c..8406eaa 100644 --- a/app/src/modules/ModuleManager.ts +++ b/app/src/modules/ModuleManager.ts @@ -1,26 +1,30 @@ -import { mark, stripMark, $console, s, objectEach, transformObject, McpServer } from "bknd/utils"; +import { + objectEach, + transformObject, + McpServer, + type s, + SecretSchema, + setPath, + mark, + $console, +} from "bknd/utils"; import { DebugLogger } from "core/utils/DebugLogger"; import { Guard } from "auth/authorize/Guard"; import { env } from "core/env"; -import { BkndError } from "core/errors"; import { EventManager, Event } from "core/events"; -import * as $diff from "core/object/diff"; import type { Connection } from "data/connection"; import { EntityManager } from "data/entities/EntityManager"; -import * as proto from "data/prototype"; -import { TransformPersistFailedException } from "data/errors"; import { Hono } from "hono"; -import type { Kysely } from "kysely"; -import { mergeWith } from "lodash-es"; -import { CURRENT_VERSION, TABLE_NAME, migrate } from "modules/migrations"; -import { AppServer } from "modules/server/AppServer"; -import { AppAuth } from "../auth/AppAuth"; -import { AppData } from "../data/AppData"; -import { AppFlows } from "../flows/AppFlows"; -import { AppMedia } from "../media/AppMedia"; import type { ServerEnv } from "./Controller"; import { Module, type ModuleBuildContext } from "./Module"; import { ModuleHelper } from "./ModuleHelper"; +import { AppServer } from "modules/server/AppServer"; +import { AppAuth } from "auth/AppAuth"; +import { AppData } from "data/AppData"; +import { AppFlows } from "flows/AppFlows"; +import { AppMedia } from "media/AppMedia"; +import type { PartialRec } from "core/types"; +import { mergeWith, pick } from "lodash-es"; export type { ModuleBuildContext }; @@ -47,13 +51,8 @@ export type ModuleSchemas = { export type ModuleConfigs = { [K in keyof ModuleSchemas]: s.Static; }; -type PartialRec = { [P in keyof T]?: PartialRec }; -export type InitialModuleConfigs = - | ({ - version: number; - } & ModuleConfigs) - | PartialRec; +export type InitialModuleConfigs = { version?: number } & PartialRec; enum Verbosity { silent = 0, @@ -75,47 +74,19 @@ export type ModuleManagerOptions = { // callback after server was created onServerInit?: (server: Hono) => void; // doesn't perform validity checks for given/fetched config - trustFetched?: boolean; + skipValidation?: boolean; // runs when initial config provided on a fresh database seed?: (ctx: ModuleBuildContext) => Promise; // called right after modules are built, before finish onModulesBuilt?: (ctx: ModuleBuildContext) => Promise; + // whether to store secrets in the database + storeSecrets?: boolean; + // provided secrets + secrets?: Record; /** @deprecated */ verbosity?: Verbosity; }; -export type ConfigTable = { - id?: number; - version: number; - type: "config" | "diff" | "backup"; - json: Json; - created_at?: Date; - updated_at?: Date; -}; - -const configJsonSchema = s.anyOf([ - getDefaultSchema(), - s.array( - s.strictObject({ - t: s.string({ enum: ["a", "r", "e"] }), - p: s.array(s.anyOf([s.string(), s.number()])), - o: s.any().optional(), - n: s.any().optional(), - }), - ), -]); -export const __bknd = proto.entity(TABLE_NAME, { - version: proto.number().required(), - type: proto.enumm({ enum: ["config", "diff", "backup"] }).required(), - json: proto.jsonSchema({ schema: configJsonSchema.toJSON() }).required(), - created_at: proto.datetime(), - updated_at: proto.datetime(), -}); -type ConfigTable2 = proto.Schema; -interface T_INTERNAL_EM { - __bknd: ConfigTable2; -} - const debug_modules = env("modules_debug"); abstract class ModuleManagerEvent extends Event<{ ctx: ModuleBuildContext } & A> {} @@ -127,8 +98,14 @@ export class ModuleManagerConfigUpdateEvent< }> { static override slug = "mm-config-update"; } +export class ModuleManagerSecretsExtractedEvent extends ModuleManagerEvent<{ + secrets: Record; +}> { + static override slug = "mm-secrets-extracted"; +} export const ModuleManagerEvents = { ModuleManagerConfigUpdateEvent, + ModuleManagerSecretsExtractedEvent, }; // @todo: cleanup old diffs on upgrade @@ -137,8 +114,6 @@ export class ModuleManager { static Events = ModuleManagerEvents; protected modules: Modules; - // internal em for __bknd config table - __em!: EntityManager; // ctx for modules em!: EntityManager; server!: Hono; @@ -146,42 +121,29 @@ export class ModuleManager { guard!: Guard; mcp!: ModuleBuildContext["mcp"]; - private _version: number = 0; - private _built = false; - private readonly _booted_with?: "provided" | "partial"; - private _stable_configs: ModuleConfigs | undefined; + protected _built = false; - private logger: DebugLogger; + protected logger: DebugLogger; constructor( - private readonly connection: Connection, - private options?: Partial, + protected readonly connection: Connection, + public options?: Partial, ) { - this.__em = new EntityManager([__bknd], this.connection); this.modules = {} as Modules; this.emgr = new EventManager({ ...ModuleManagerEvents }); this.logger = new DebugLogger(debug_modules); - let initial = {} as Partial; - if (options?.initial) { - if ("version" in options.initial) { - const { version, ...initialConfig } = options.initial; - this._version = version; - initial = stripMark(initialConfig); - - this._booted_with = "provided"; - } else { - initial = mergeWith(getDefaultConfig(), options.initial); - this._booted_with = "partial"; - } + const config = options?.initial ?? {}; + if (options?.skipValidation) { + mark(config, true); } - this.logger.log("booted with", this._booted_with); - - this.createModules(initial); + this.createModules(config); } - private createModules(initial: Partial) { + protected onModuleConfigUpdated(key: string, config: any) {} + + private createModules(initial: PartialRec) { this.logger.context("createModules").log("creating modules"); try { const context = this.ctx(true); @@ -211,46 +173,7 @@ export class ModuleManager { return this._built; } - /** - * This is set through module's setListener - * It's called everytime a module's config is updated in SchemaObject - * Needs to rebuild modules and save to database - */ - private async onModuleConfigUpdated(key: string, config: any) { - if (this.options?.onUpdated) { - await this.options.onUpdated(key as any, config); - } else { - await this.buildModules(); - } - } - - private repo() { - return this.__em.repo(__bknd, { - // to prevent exceptions when table doesn't exist - silent: true, - // disable counts for performance and compatibility - includeCounts: false, - }); - } - - private mutator() { - return this.__em.mutator(__bknd); - } - - private get db() { - // @todo: check why this is neccessary - return this.connection.kysely as unknown as Kysely<{ table: ConfigTable }>; - } - - // @todo: add indices for: version, type - async syncConfigTable() { - this.logger.context("sync").log("start"); - const result = await this.__em.schema().sync({ force: true }); - this.logger.log("done").clear(); - return result; - } - - private rebuildServer() { + protected rebuildServer() { this.server = new Hono(); if (this.options?.basePath) { this.server = this.server.basePath(this.options.basePath); @@ -299,252 +222,81 @@ export class ModuleManager { }; } - private async fetch(): Promise { - this.logger.context("fetch").log("fetching"); - const startTime = performance.now(); + extractSecrets() { + const moduleConfigs = structuredClone(this.configs()); + const secrets = { ...this.options?.secrets }; + const extractedKeys: string[] = []; - // disabling console log, because the table might not exist yet - const { data: result } = await this.repo().findOne( - { type: "config" }, - { - sort: { by: "version", dir: "desc" }, - }, - ); + for (const [key, module] of Object.entries(this.modules)) { + const config = moduleConfigs[key]; + const schema = module.getSchema(); - if (!result) { - this.logger.log("error fetching").clear(); - return undefined; - } + const extracted = [...schema.walk({ data: config })].filter( + (n) => n.schema instanceof SecretSchema, + ); - this.logger - .log("took", performance.now() - startTime, "ms", { - version: result.version, - id: result.id, - }) - .clear(); + for (const n of extracted) { + const path = [key, ...n.instancePath].join("."); - return result as unknown as ConfigTable; - } - - async save() { - this.logger.context("save").log("saving version", this.version()); - const configs = this.configs(); - const version = this.version(); - - try { - const state = await this.fetch(); - if (!state) throw new BkndError("no config found"); - this.logger.log("fetched version", state.version); - - if (state.version !== version) { - // @todo: mark all others as "backup" - this.logger.log("version conflict, storing new version", state.version, version); - await this.mutator().insertOne({ - version: state.version, - type: "backup", - json: configs, - }); - await this.mutator().insertOne({ - version: version, - type: "config", - json: configs, - }); - } else { - this.logger.log("version matches", state.version); - - // clean configs because of Diff() function - const diffs = $diff.diff(state.json, $diff.clone(configs)); - this.logger.log("checking diff", [diffs.length]); - - if (diffs.length > 0) { - // validate diffs, it'll throw on invalid - this.validateDiffs(diffs); - - const date = new Date(); - // store diff - await this.mutator().insertOne({ - version, - type: "diff", - json: $diff.clone(diffs), - created_at: date, - updated_at: date, - }); - - // store new version - await this.mutator().updateWhere( - { - version, - json: configs, - updated_at: date, - } as any, - { - type: "config", - version, - }, - ); - } else { - this.logger.log("no diff, not saving"); + if (typeof n.data === "string") { + extractedKeys.push(path); + secrets[path] = n.data; + setPath(moduleConfigs, path, ""); } } - } catch (e) { - if (e instanceof BkndError && e.message === "no config found") { - this.logger.log("no config, just save fresh"); - // no config, just save - await this.mutator().insertOne({ - type: "config", - version, - json: configs, - created_at: new Date(), - updated_at: new Date(), - }); - } else if (e instanceof TransformPersistFailedException) { - $console.error("ModuleManager: Cannot save invalid config"); - this.revertModules(); - throw e; - } else { - $console.error("ModuleManager: Aborting"); - this.revertModules(); - throw e; - } } - // re-apply configs to all modules (important for system entities) - this.setConfigs(configs); - - // @todo: cleanup old versions? - - this.logger.clear(); - return this; + return { + configs: moduleConfigs, + secrets: pick(secrets, extractedKeys), + extractedKeys, + }; } - private revertModules() { - if (this._stable_configs) { - $console.warn("ModuleManager: Reverting modules"); - this.setConfigs(this._stable_configs as any); - } else { - $console.error("ModuleManager: No stable configs to revert to"); - } - } - - /** - * Validates received diffs for an additional security control. - * Checks: - * - check if module is registered - * - run modules onBeforeUpdate() for added protection - * - * **Important**: Throw `Error` so it won't get catched. - * - * @param diffs - * @private - */ - private validateDiffs(diffs: $diff.DiffEntry[]): void { - // check top level paths, and only allow a single module to be modified in a single transaction - const modules = [...new Set(diffs.map((d) => d.p[0]))]; - if (modules.length === 0) { - return; - } - - for (const moduleName of modules) { - const name = moduleName as ModuleKey; - const module = this.get(name) as Module; - if (!module) { - const msg = "validateDiffs: module not registered"; - // biome-ignore format: ... - $console.error(msg, JSON.stringify({ module: name, diffs }, null, 2)); - throw new Error(msg); - } - - // pass diffs to the module to allow it to throw - if (this._stable_configs?.[name]) { - const current = $diff.clone(this._stable_configs?.[name]); - const modified = $diff.apply({ [name]: current }, diffs)[name]; - module.onBeforeUpdate(current, modified); - } - } - } - - private setConfigs(configs: ModuleConfigs): void { + protected async setConfigs(configs: ModuleConfigs): Promise { this.logger.log("setting configs"); - objectEach(configs, (config, key) => { + for await (const [key, config] of Object.entries(configs)) { + if (!(key in this.modules)) continue; + try { // setting "noEmit" to true, to not force listeners to update - this.modules[key].schema().set(config as any, true); + const result = await this.modules[key].schema().set(config as any, true); } catch (e) { console.error(e); throw new Error( `Failed to set config for module ${key}: ${JSON.stringify(config, null, 2)}`, ); } - }); + } } - async build(opts?: { fetch?: boolean }) { - this.logger.context("build").log("version", this.version()); - await this.ctx().connection.init(); + async build(opts?: any) { + this.createModules(this.options?.initial ?? {}); + await this.buildModules(); - // if no config provided, try fetch from db - if (this.version() === 0 || opts?.fetch === true) { - if (opts?.fetch) { - this.logger.log("force fetch"); - } + // if secrets were provided, extract, merge and build again + const provided_secrets = this.options?.secrets ?? {}; + if (Object.keys(provided_secrets).length > 0) { + const { configs, extractedKeys } = this.extractSecrets(); - const result = await this.fetch(); - - // if no version, and nothing found, go with initial - if (!result) { - this.logger.log("nothing in database, go initial"); - await this.setupInitial(); - } else { - this.logger.log("db has", result.version); - // set version and config from fetched - this._version = result.version; - - if (this.options?.trustFetched === true) { - this.logger.log("trusting fetched config (mark)"); - mark(result.json); - } - - // if version doesn't match, migrate before building - if (this.version() !== CURRENT_VERSION) { - this.logger.log("now migrating"); - - await this.syncConfigTable(); - - const version_before = this.version(); - const [_version, _configs] = await migrate(version_before, result.json, { - db: this.db, - }); - - this._version = _version; - this.ctx().flags.sync_required = true; - - this.logger.log("migrated to", _version); - $console.log("Migrated config from", version_before, "to", this.version()); - - this.createModules(_configs); - await this.buildModules(); - } else { - this.logger.log("version is current", this.version()); - this.createModules(result.json); - await this.buildModules(); + for (const key of extractedKeys) { + if (key in provided_secrets) { + setPath(configs, key, provided_secrets[key]); } } - } else { - if (this.version() !== CURRENT_VERSION) { - throw new Error( - `Given version (${this.version()}) and current version (${CURRENT_VERSION}) do not match.`, - ); - } - this.logger.log("current version is up to date", this.version()); + + await this.setConfigs(configs); await this.buildModules(); } - this.logger.log("done"); - this.logger.clear(); return this; } - private async buildModules(options?: { graceful?: boolean; ignoreFlags?: boolean }) { + protected async buildModules(options?: { + graceful?: boolean; + ignoreFlags?: boolean; + drop?: boolean; + }) { const state = { built: false, modules: [] as ModuleKey[], @@ -579,13 +331,8 @@ export class ModuleManager { ctx.flags.sync_required = false; this.logger.log("db sync requested"); - // sync db - await ctx.em.schema().sync({ force: true }); - state.synced = true; - - // save - await this.save(); - state.saved = true; + // ignore sync request on code mode since system tables + // are probably never fully in provided config } if (ctx.flags.ctx_reload_required) { @@ -601,92 +348,12 @@ export class ModuleManager { ctx.flags = Module.ctx_flags; // storing last stable config version - this._stable_configs = $diff.clone(this.configs()); + //this._stable_configs = $diff.clone(this.configs()); this.logger.clear(); return state; } - protected async setupInitial() { - this.logger.context("initial").log("start"); - this._version = CURRENT_VERSION; - await this.syncConfigTable(); - const state = await this.buildModules(); - if (!state.saved) { - await this.save(); - } - - const ctx = { - ...this.ctx(), - // disable events for initial setup - em: this.ctx().em.fork(), - }; - - // perform a sync - await ctx.em.schema().sync({ force: true }); - await this.options?.seed?.(ctx); - - // run first boot event - await this.options?.onFirstBoot?.(); - this.logger.clear(); - } - - mutateConfigSafe( - name: Module, - ): Pick, "set" | "patch" | "overwrite" | "remove"> { - const module = this.modules[name]; - - return new Proxy(module.schema(), { - get: (target, prop: string) => { - if (!["set", "patch", "overwrite", "remove"].includes(prop)) { - throw new Error(`Method ${prop} is not allowed`); - } - - return async (...args) => { - $console.log("[Safe Mutate]", name); - try { - // overwrite listener to run build inside this try/catch - module.setListener(async () => { - await this.emgr.emit( - new ModuleManagerConfigUpdateEvent({ - ctx: this.ctx(), - module: name, - config: module.config as any, - }), - ); - await this.buildModules(); - }); - - const result = await target[prop](...args); - - // revert to original listener - module.setListener(async (c) => { - await this.onModuleConfigUpdated(name, c); - }); - - // if there was an onUpdated listener, call it after success - // e.g. App uses it to register module routes - if (this.options?.onUpdated) { - await this.options.onUpdated(name, module.config as any); - } - - return result; - } catch (e) { - $console.error(`[Safe Mutate] failed "${name}":`, e); - - // revert to previous config & rebuild using original listener - this.revertModules(); - await this.onModuleConfigUpdated(name, module.config as any); - $console.warn(`[Safe Mutate] reverted "${name}":`); - - // make sure to throw the error - throw e; - } - }; - }, - }); - } - get(key: K): Modules[K] { if (!(key in this.modules)) { throw new Error(`Module "${key}" doesn't exist, cannot get`); @@ -695,7 +362,7 @@ export class ModuleManager { } version() { - return this._version; + return 0; } built() { diff --git a/app/src/modules/db/DbModuleManager.ts b/app/src/modules/db/DbModuleManager.ts new file mode 100644 index 0000000..a7bc903 --- /dev/null +++ b/app/src/modules/db/DbModuleManager.ts @@ -0,0 +1,592 @@ +import { mark, stripMark, $console, s, SecretSchema, setPath } from "bknd/utils"; +import { BkndError } from "core/errors"; +import * as $diff from "core/object/diff"; +import type { Connection } from "data/connection"; +import type { EntityManager } from "data/entities/EntityManager"; +import * as proto from "data/prototype"; +import { TransformPersistFailedException } from "data/errors"; +import type { Kysely } from "kysely"; +import { mergeWith } from "lodash-es"; +import { CURRENT_VERSION, TABLE_NAME, migrate } from "./migrations"; +import { Module, type ModuleBuildContext } from "../Module"; +import { + type InitialModuleConfigs, + type ModuleConfigs, + type Modules, + type ModuleKey, + getDefaultSchema, + getDefaultConfig, + ModuleManager, + ModuleManagerConfigUpdateEvent, + type ModuleManagerOptions, + ModuleManagerSecretsExtractedEvent, +} from "../ModuleManager"; + +export type { ModuleBuildContext }; + +export type ConfigTable = { + id?: number; + version: number; + type: "config" | "diff" | "backup"; + json: Json; + created_at?: Date; + updated_at?: Date; +}; + +const configJsonSchema = s.anyOf([ + getDefaultSchema(), + s.array( + s.strictObject({ + t: s.string({ enum: ["a", "r", "e"] }), + p: s.array(s.anyOf([s.string(), s.number()])), + o: s.any().optional(), + n: s.any().optional(), + }), + ), +]); +export const __bknd = proto.entity(TABLE_NAME, { + version: proto.number().required(), + type: proto.enumm({ enum: ["config", "diff", "backup", "secrets"] }).required(), + json: proto.jsonSchema({ schema: configJsonSchema.toJSON() }).required(), + created_at: proto.datetime(), + updated_at: proto.datetime(), +}); +const __schema = proto.em({ __bknd }, ({ index }, { __bknd }) => { + index(__bknd).on(["version", "type"]); +}); + +type ConfigTable2 = proto.Schema; +interface T_INTERNAL_EM { + __bknd: ConfigTable2; +} + +// @todo: cleanup old diffs on upgrade +// @todo: cleanup multiple backups on upgrade +export class DbModuleManager extends ModuleManager { + // internal em for __bknd config table + __em!: EntityManager; + + private _version: number = 0; + private readonly _booted_with?: "provided" | "partial"; + private _stable_configs: ModuleConfigs | undefined; + + constructor(connection: Connection, options?: Partial) { + let initial = {} as InitialModuleConfigs; + let booted_with = "partial" as any; + let version = 0; + + if (options?.initial) { + if ("version" in options.initial && options.initial.version) { + const { version: _v, ...config } = options.initial; + version = _v as number; + initial = stripMark(config) as any; + + booted_with = "provided"; + } else { + initial = mergeWith(getDefaultConfig(), options.initial); + booted_with = "partial"; + } + } + + super(connection, { ...options, initial }); + + this.__em = __schema.proto.withConnection(this.connection) as any; + //this.__em = new EntityManager(__schema.entities, this.connection); + this._version = version; + this._booted_with = booted_with; + + this.logger.log("booted with", this._booted_with); + } + + /** + * This is set through module's setListener + * It's called everytime a module's config is updated in SchemaObject + * Needs to rebuild modules and save to database + */ + protected override async onModuleConfigUpdated(key: string, config: any) { + if (this.options?.onUpdated) { + await this.options.onUpdated(key as any, config); + } else { + await this.buildModules(); + } + } + + private repo() { + return this.__em.repo(__bknd, { + // to prevent exceptions when table doesn't exist + silent: true, + // disable counts for performance and compatibility + includeCounts: false, + }); + } + + private mutator() { + return this.__em.mutator(__bknd); + } + + private get db() { + // @todo: check why this is neccessary + return this.connection.kysely as unknown as Kysely<{ table: ConfigTable }>; + } + + // @todo: add indices for: version, type + async syncConfigTable() { + this.logger.context("sync").log("start"); + const result = await this.__em.schema().sync({ force: true }); + this.logger.log("done").clear(); + return result; + } + + private async fetch(): Promise<{ configs?: ConfigTable; secrets?: ConfigTable } | undefined> { + this.logger.context("fetch").log("fetching"); + const startTime = performance.now(); + + // disabling console log, because the table might not exist yet + const { data: result } = await this.repo().findMany({ + where: { type: { $in: ["config", "secrets"] } }, + sort: { by: "version", dir: "desc" }, + }); + + if (!result.length) { + this.logger.log("error fetching").clear(); + return undefined; + } + + const configs = result.filter((r) => r.type === "config")[0]; + const secrets = result.filter((r) => r.type === "secrets")[0]; + + this.logger + .log("took", performance.now() - startTime, "ms", { + version: configs?.version, + id: configs?.id, + }) + .clear(); + + return { configs, secrets } as any; + } + + async save() { + this.logger.context("save").log("saving version", this.version()); + const { configs, secrets } = this.extractSecrets(); + const version = this.version(); + const store_secrets = this.options?.storeSecrets !== false; + + await this.emgr.emit( + new ModuleManagerSecretsExtractedEvent({ + ctx: this.ctx(), + secrets, + }), + ); + + try { + const state = await this.fetch(); + if (!state || !state.configs) throw new BkndError("no config found"); + this.logger.log("fetched version", state.configs.version); + + if (state.configs.version !== version) { + // @todo: mark all others as "backup" + this.logger.log( + "version conflict, storing new version", + state.configs.version, + version, + ); + const updates = [ + { + version: state.configs.version, + type: "backup", + json: state.configs.json, + }, + { + version: version, + type: "config", + json: configs, + }, + ]; + + if (store_secrets) { + updates.push({ + version: version, + type: "secrets", + json: secrets as any, + }); + } + await this.mutator().insertMany(updates); + } else { + this.logger.log("version matches", state.configs.version); + + // clean configs because of Diff() function + const diffs = $diff.diff(state.configs.json, $diff.clone(configs)); + this.logger.log("checking diff", [diffs.length]); + const date = new Date(); + + if (diffs.length > 0) { + // validate diffs, it'll throw on invalid + this.validateDiffs(diffs); + + // store diff + await this.mutator().insertOne({ + version, + type: "diff", + json: $diff.clone(diffs), + created_at: date, + updated_at: date, + }); + + // store new version + await this.mutator().updateWhere( + { + version, + json: configs, + updated_at: date, + } as any, + { + type: "config", + version, + }, + ); + } else { + this.logger.log("no diff, not saving"); + } + + // store secrets + if (store_secrets) { + if (!state.secrets || state.secrets?.version !== version) { + await this.mutator().insertOne({ + version, + type: "secrets", + json: secrets, + created_at: date, + updated_at: date, + }); + } else { + await this.mutator().updateOne(state.secrets.id!, { + version, + json: secrets, + updated_at: date, + } as any); + } + } + } + } catch (e) { + if (e instanceof BkndError && e.message === "no config found") { + this.logger.log("no config, just save fresh"); + // no config, just save + await this.mutator().insertOne({ + type: "config", + version, + json: configs, + created_at: new Date(), + updated_at: new Date(), + }); + if (store_secrets) { + await this.mutator().insertOne({ + type: "secrets", + version, + json: secrets, + created_at: new Date(), + updated_at: new Date(), + }); + } + } else if (e instanceof TransformPersistFailedException) { + $console.error("ModuleManager: Cannot save invalid config"); + this.revertModules(); + throw e; + } else { + $console.error("ModuleManager: Aborting"); + this.revertModules(); + throw e; + } + } + + // re-apply configs to all modules (important for system entities) + await this.setConfigs(this.configs()); + + // @todo: cleanup old versions? + + this.logger.clear(); + return this; + } + + private async revertModules() { + if (this._stable_configs) { + $console.warn("ModuleManager: Reverting modules"); + await this.setConfigs(this._stable_configs as any); + } else { + $console.error("ModuleManager: No stable configs to revert to"); + } + } + + /** + * Validates received diffs for an additional security control. + * Checks: + * - check if module is registered + * - run modules onBeforeUpdate() for added protection + * + * **Important**: Throw `Error` so it won't get catched. + * + * @param diffs + * @private + */ + private validateDiffs(diffs: $diff.DiffEntry[]): void { + // check top level paths, and only allow a single module to be modified in a single transaction + const modules = [...new Set(diffs.map((d) => d.p[0]))]; + if (modules.length === 0) { + return; + } + + for (const moduleName of modules) { + const name = moduleName as ModuleKey; + const module = this.get(name) as Module; + if (!module) { + const msg = "validateDiffs: module not registered"; + // biome-ignore format: ... + $console.error(msg, JSON.stringify({ module: name, diffs }, null, 2)); + throw new Error(msg); + } + + // pass diffs to the module to allow it to throw + if (this._stable_configs?.[name]) { + const current = $diff.clone(this._stable_configs?.[name]); + const modified = $diff.apply({ [name]: current }, diffs)[name]; + module.onBeforeUpdate(current, modified); + } + } + } + + override async build(opts?: { fetch?: boolean }) { + this.logger.context("build").log("version", this.version()); + await this.ctx().connection.init(); + + // if no config provided, try fetch from db + if (this.version() === 0 || opts?.fetch === true) { + if (opts?.fetch) { + this.logger.log("force fetch"); + } + + const result = await this.fetch(); + + // if no version, and nothing found, go with initial + if (!result?.configs) { + this.logger.log("nothing in database, go initial"); + await this.setupInitial(); + } else { + this.logger.log("db has", result.configs.version); + // set version and config from fetched + this._version = result.configs.version; + + if (result?.configs && result?.secrets) { + for (const [key, value] of Object.entries(result.secrets.json)) { + setPath(result.configs.json, key, value); + } + } + + if (this.options?.skipValidation === true) { + this.logger.log("skipping validation (mark)"); + mark(result.configs.json); + } + + // if version doesn't match, migrate before building + if (this.version() !== CURRENT_VERSION) { + this.logger.log("now migrating"); + + await this.syncConfigTable(); + + const version_before = this.version(); + const [_version, _configs] = await migrate(version_before, result.configs.json, { + db: this.db + }); + + this._version = _version; + this.ctx().flags.sync_required = true; + + this.logger.log("migrated to", _version); + $console.log("Migrated config from", version_before, "to", this.version()); + + // @ts-expect-error + await this.setConfigs(_configs); + await this.buildModules(); + } else { + this.logger.log("version is current", this.version()); + + await this.setConfigs(result.configs.json); + await this.buildModules(); + } + } + } else { + if (this.version() !== CURRENT_VERSION) { + throw new Error( + `Given version (${this.version()}) and current version (${CURRENT_VERSION}) do not match.`, + ); + } + this.logger.log("current version is up to date", this.version()); + await this.buildModules(); + } + + this.logger.log("done"); + this.logger.clear(); + return this; + } + + protected override async buildModules(options?: { graceful?: boolean; ignoreFlags?: boolean }) { + const state = { + built: false, + modules: [] as ModuleKey[], + synced: false, + saved: false, + reloaded: false, + }; + + this.logger.context("buildModules").log("triggered", options, this._built); + if (options?.graceful && this._built) { + this.logger.log("skipping build (graceful)"); + return state; + } + + this.logger.log("building"); + const ctx = this.ctx(true); + for (const key in this.modules) { + await this.modules[key].setContext(ctx).build(); + this.logger.log("built", key); + state.modules.push(key as ModuleKey); + } + + this._built = state.built = true; + this.logger.log("modules built", ctx.flags); + + if (this.options?.onModulesBuilt) { + await this.options.onModulesBuilt(ctx); + } + + if (options?.ignoreFlags !== true) { + if (ctx.flags.sync_required) { + ctx.flags.sync_required = false; + this.logger.log("db sync requested"); + + // sync db + await ctx.em.schema().sync({ force: true }); + state.synced = true; + + // save + await this.save(); + state.saved = true; + } + + if (ctx.flags.ctx_reload_required) { + ctx.flags.ctx_reload_required = false; + this.logger.log("ctx reload requested"); + this.ctx(true); + state.reloaded = true; + } + } + + // reset all falgs + this.logger.log("resetting flags"); + ctx.flags = Module.ctx_flags; + + // storing last stable config version + this._stable_configs = $diff.clone(this.configs()); + + this.logger.clear(); + return state; + } + + protected async setupInitial() { + this.logger.context("initial").log("start"); + this._version = CURRENT_VERSION; + await this.syncConfigTable(); + + const state = await this.buildModules(); + + // in case there are secrets provided, we need to extract the keys and merge them with the configs. Then another build is required. + if (this.options?.secrets) { + const { configs, extractedKeys } = this.extractSecrets(); + for (const key of extractedKeys) { + if (key in this.options.secrets) { + setPath(configs, key, this.options.secrets[key]); + } + } + await this.setConfigs(configs); + await this.buildModules(); + } + + // generally only save if not already done + // unless secrets are provided, then we need to save again + if (!state.saved || this.options?.secrets) { + await this.save(); + } + + const ctx = { + ...this.ctx(), + // disable events for initial setup + em: this.ctx().em.fork(), + }; + + // perform a sync + await ctx.em.schema().sync({ force: true }); + await this.options?.seed?.(ctx); + + // run first boot event + await this.options?.onFirstBoot?.(); + this.logger.clear(); + } + + mutateConfigSafe( + name: Module, + ): Pick, "set" | "patch" | "overwrite" | "remove"> { + const module = this.modules[name]; + + return new Proxy(module.schema(), { + get: (target, prop: string) => { + if (!["set", "patch", "overwrite", "remove"].includes(prop)) { + throw new Error(`Method ${prop} is not allowed`); + } + + return async (...args) => { + $console.log("[Safe Mutate]", name); + try { + // overwrite listener to run build inside this try/catch + module.setListener(async () => { + await this.emgr.emit( + new ModuleManagerConfigUpdateEvent({ + ctx: this.ctx(), + module: name, + config: module.config as any, + }), + ); + await this.buildModules(); + }); + + const result = await target[prop](...args); + + // revert to original listener + module.setListener(async (c) => { + await this.onModuleConfigUpdated(name, c); + }); + + // if there was an onUpdated listener, call it after success + // e.g. App uses it to register module routes + if (this.options?.onUpdated) { + await this.options.onUpdated(name, module.config as any); + } + + return result; + } catch (e) { + $console.error(`[Safe Mutate] failed "${name}":`, e); + + // revert to previous config & rebuild using original listener + this.revertModules(); + await this.onModuleConfigUpdated(name, module.config as any); + $console.warn(`[Safe Mutate] reverted "${name}":`); + + // make sure to throw the error + throw e; + } + }; + }, + }); + } + + override version() { + return this._version; + } +} diff --git a/app/src/modules/migrations.ts b/app/src/modules/db/migrations.ts similarity index 94% rename from app/src/modules/migrations.ts rename to app/src/modules/db/migrations.ts index e2834eb..13f39ee 100644 --- a/app/src/modules/migrations.ts +++ b/app/src/modules/db/migrations.ts @@ -99,6 +99,14 @@ export const migrations: Migration[] = [ }; }, }, + { + // remove secrets, automatic + // change media table `entity_id` from integer to text + version: 10, + up: async (config) => { + return config; + }, + }, ]; export const CURRENT_VERSION = migrations[migrations.length - 1]?.version ?? 0; diff --git a/app/src/modules/mcp/$object.ts b/app/src/modules/mcp/$object.ts index a57257b..f5fa6b4 100644 --- a/app/src/modules/mcp/$object.ts +++ b/app/src/modules/mcp/$object.ts @@ -6,7 +6,6 @@ import { type McpSchema, type SchemaWithMcpOptions, } from "./McpSchemaHelper"; -import type { Module } from "modules/Module"; export interface ObjectToolSchemaOptions extends s.IObjectOptions, SchemaWithMcpOptions {} @@ -79,6 +78,7 @@ export class ObjectToolSchema< private toolUpdate(node: s.Node) { const schema = this.mcp.cleanSchema; + return new Tool( [this.mcp.name, "update"].join("_"), { @@ -97,11 +97,12 @@ export class ObjectToolSchema< async (params, ctx: AppToolHandlerCtx) => { const { full, value, return_config } = params; const [module_name] = node.instancePath; + const manager = this.mcp.getManager(ctx); if (full) { - await ctx.context.app.mutateConfig(module_name as any).set(value); + await manager.mutateConfigSafe(module_name as any).set(value); } else { - await ctx.context.app.mutateConfig(module_name as any).patch("", value); + await manager.mutateConfigSafe(module_name as any).patch("", value); } let config: any = undefined; diff --git a/app/src/modules/mcp/$record.ts b/app/src/modules/mcp/$record.ts index a10054a..fc6dfaa 100644 --- a/app/src/modules/mcp/$record.ts +++ b/app/src/modules/mcp/$record.ts @@ -129,13 +129,14 @@ export class RecordToolSchema< const configs = ctx.context.app.toJSON(true); const config = getPath(configs, node.instancePath); const [module_name, ...rest] = node.instancePath; + const manager = this.mcp.getManager(ctx); if (params.key in config) { throw new Error(`Key "${params.key}" already exists in config`); } - await ctx.context.app - .mutateConfig(module_name as any) + await manager + .mutateConfigSafe(module_name as any) .patch([...rest, params.key], params.value); const newConfig = getPath(ctx.context.app.toJSON(), node.instancePath); @@ -175,13 +176,14 @@ export class RecordToolSchema< const configs = ctx.context.app.toJSON(true); const config = getPath(configs, node.instancePath); const [module_name, ...rest] = node.instancePath; + const manager = this.mcp.getManager(ctx); if (!(params.key in config)) { throw new Error(`Key "${params.key}" not found in config`); } - await ctx.context.app - .mutateConfig(module_name as any) + await manager + .mutateConfigSafe(module_name as any) .patch([...rest, params.key], params.value); const newConfig = getPath(ctx.context.app.toJSON(), node.instancePath); @@ -220,13 +222,14 @@ export class RecordToolSchema< const configs = ctx.context.app.toJSON(true); const config = getPath(configs, node.instancePath); const [module_name, ...rest] = node.instancePath; + const manager = this.mcp.getManager(ctx); if (!(params.key in config)) { throw new Error(`Key "${params.key}" not found in config`); } - await ctx.context.app - .mutateConfig(module_name as any) + await manager + .mutateConfigSafe(module_name as any) .remove([...rest, params.key].join(".")); const newConfig = getPath(ctx.context.app.toJSON(), node.instancePath); diff --git a/app/src/modules/mcp/$schema.ts b/app/src/modules/mcp/$schema.ts index 9c86d4a..c71424b 100644 --- a/app/src/modules/mcp/$schema.ts +++ b/app/src/modules/mcp/$schema.ts @@ -58,8 +58,9 @@ export const $schema = < async (params, ctx: AppToolHandlerCtx) => { const { value, return_config, secrets } = params; const [module_name, ...rest] = node.instancePath; + const manager = mcp.getManager(ctx); - await ctx.context.app.mutateConfig(module_name as any).overwrite(rest, value); + await manager.mutateConfigSafe(module_name as any).overwrite(rest, value); let config: any = undefined; if (return_config) { diff --git a/app/src/modules/mcp/McpSchemaHelper.ts b/app/src/modules/mcp/McpSchemaHelper.ts index 686e7ff..ecb16e4 100644 --- a/app/src/modules/mcp/McpSchemaHelper.ts +++ b/app/src/modules/mcp/McpSchemaHelper.ts @@ -10,6 +10,7 @@ import { } from "bknd/utils"; import type { ModuleBuildContext } from "modules"; import { excludePropertyTypes, rescursiveClean } from "./utils"; +import type { DbModuleManager } from "modules/db/DbModuleManager"; export const mcpSchemaSymbol = Symbol.for("bknd-mcp-schema"); @@ -74,4 +75,13 @@ export class McpSchemaHelper { }, }; } + + getManager(ctx: AppToolHandlerCtx): DbModuleManager { + const manager = ctx.context.app.modules; + if ("mutateConfigSafe" in manager) { + return manager as DbModuleManager; + } + + throw new Error("Manager not found"); + } } diff --git a/app/src/modules/mcp/system-mcp.ts b/app/src/modules/mcp/system-mcp.ts index bd278db..deabb4d 100644 --- a/app/src/modules/mcp/system-mcp.ts +++ b/app/src/modules/mcp/system-mcp.ts @@ -6,7 +6,7 @@ import { getVersion } from "core/env"; export function getSystemMcp(app: App) { const middlewareServer = getMcpServer(app.server); - const appConfig = app.modules.configs(); + //const appConfig = app.modules.configs(); const { version, ...appSchema } = app.getSchema(); const schema = s.strictObject(appSchema); const result = [...schema.walk({ maxDepth: 3 })]; @@ -19,9 +19,11 @@ export function getSystemMcp(app: App) { ].sort((a, b) => a.name.localeCompare(b.name)); // tools from app schema - tools.push( - ...nodes.flatMap((n) => n.schema.getTools(n)).sort((a, b) => a.name.localeCompare(b.name)), - ); + if (!app.isReadOnly()) { + tools.push( + ...nodes.flatMap((n) => n.schema.getTools(n)).sort((a, b) => a.name.localeCompare(b.name)), + ); + } const resources = [...middlewareServer.resources, ...app.modules.ctx().mcp.resources]; diff --git a/app/src/modules/server/AdminController.tsx b/app/src/modules/server/AdminController.tsx index a68519d..2800781 100644 --- a/app/src/modules/server/AdminController.tsx +++ b/app/src/modules/server/AdminController.tsx @@ -192,7 +192,7 @@ export class AdminController extends Controller { const assets = { js: "main.js", - css: "styles.css", + css: ["styles.css"], }; if (isProd) { @@ -213,9 +213,12 @@ export class AdminController extends Controller { } try { - // @todo: load all marked as entry (incl. css) - assets.js = manifest["src/ui/main.tsx"]?.file!; - assets.css = manifest["src/ui/main.tsx"]?.css?.[0] as any; + const entry = Object.values(manifest).find((m) => m.isEntry); + if (!entry) { + throw new Error("No entry found in manifest"); + } + assets.js = entry?.file; + assets.css = entry?.css ?? []; } catch (e) { $console.warn("Couldn't find assets in manifest", e); } @@ -245,7 +248,9 @@ export class AdminController extends Controller { {isProd ? (