From e8f2c7027982966eb39d92881f1ea3cc936c760a Mon Sep 17 00:00:00 2001 From: dswbx Date: Thu, 4 Sep 2025 15:20:12 +0200 Subject: [PATCH] mm: added secrets extraction for db mode --- .../integration/auth.integration.test.ts | 4 +- app/__test__/modules/DbModuleManager.spec.ts | 22 +++ app/src/App.ts | 8 +- app/src/core/utils/objects.ts | 32 ++++ .../cloudinary/StorageCloudinaryAdapter.ts | 6 +- .../storage/adapters/s3/StorageS3Adapter.ts | 6 +- app/src/modules/ModuleManager.ts | 10 ++ app/src/modules/db/DbModuleManager.ts | 155 +++++++++++++----- app/src/modules/server/SystemController.ts | 4 +- app/src/plugins/dev/sync-secrets.plugin.ts | 37 +++++ app/src/plugins/index.ts | 1 + app/vite.config.ts | 1 + 12 files changed, 231 insertions(+), 55 deletions(-) create mode 100644 app/__test__/modules/DbModuleManager.spec.ts create mode 100644 app/src/plugins/dev/sync-secrets.plugin.ts diff --git a/app/__test__/integration/auth.integration.test.ts b/app/__test__/integration/auth.integration.test.ts index ef5edfa..cf71f05 100644 --- a/app/__test__/integration/auth.integration.test.ts +++ b/app/__test__/integration/auth.integration.test.ts @@ -4,9 +4,6 @@ import { auth } from "../../src/auth/middlewares"; import { randomString, secureRandomString, withDisabledConsole } from "../../src/core/utils"; import { disableConsoleLog, enableConsoleLog, getDummyConnection } from "../helper"; -const { dummyConnection, afterAllCleanup } = getDummyConnection(); -afterEach(afterAllCleanup); - beforeAll(disableConsoleLog); afterAll(enableConsoleLog); @@ -65,6 +62,7 @@ const configs = { }; function createAuthApp() { + const { dummyConnection } = getDummyConnection(); const app = createApp({ connection: dummyConnection, config: { diff --git a/app/__test__/modules/DbModuleManager.spec.ts b/app/__test__/modules/DbModuleManager.spec.ts new file mode 100644 index 0000000..a148192 --- /dev/null +++ b/app/__test__/modules/DbModuleManager.spec.ts @@ -0,0 +1,22 @@ +import { it, expect, describe } from "bun:test"; +import { DbModuleManager } from "modules/db/DbModuleManager"; +import { getDummyConnection } from "../helper"; + +describe("DbModuleManager", () => { + it("should extract secrets", async () => { + const { dummyConnection } = getDummyConnection(false); + const m = new DbModuleManager(dummyConnection, { + initial: { + auth: { + enabled: true, + jwt: { + secret: "test", + }, + }, + }, + }); + await m.build(); + expect(m.toJSON(true).auth.jwt.secret).toBe("test"); + await m.save(); + }); +}); diff --git a/app/src/App.ts b/app/src/App.ts index 52826f3..4712c55 100644 --- a/app/src/App.ts +++ b/app/src/App.ts @@ -96,13 +96,7 @@ export type AppOptions = { }; mode?: "db" | "code"; readonly?: boolean; -} & ( - | { - mode?: "db"; - secrets?: Record; - } - | { mode?: "code" } -); +}; export type CreateAppConfig = { /** * bla diff --git a/app/src/core/utils/objects.ts b/app/src/core/utils/objects.ts index 4a5e129..1c3cd82 100644 --- a/app/src/core/utils/objects.ts +++ b/app/src/core/utils/objects.ts @@ -396,6 +396,38 @@ export function getPath( } } +export function setPath(object: object, _path: string | (string | number)[], value: any) { + let path = _path; + // Optional string-path support. + // You can remove this `if` block if you don't need it. + if (typeof path === "string") { + const isQuoted = (str) => str[0] === '"' && str.at(-1) === '"'; + path = path + .split(/[.\[\]]+/) + .filter((x) => x) + .map((x) => (!Number.isNaN(Number(x)) ? Number(x) : x)) + .map((x) => (typeof x === "string" && isQuoted(x) ? x.slice(1, -1) : x)); + } + + if (path.length === 0) { + throw new Error("The path must have at least one entry in it"); + } + + const [head, ...tail] = path as any; + + if (tail.length === 0) { + object[head] = value; + return object; + } + + if (!(head in object)) { + object[head] = typeof tail[0] === "number" ? [] : {}; + } + + setPath(object[head], tail, value); + return object; +} + export function objectToJsLiteral(value: object, indent: number = 0, _level: number = 0): string { const nl = indent ? "\n" : ""; const pad = (lvl: number) => (indent ? " ".repeat(indent * lvl) : ""); diff --git a/app/src/media/storage/adapters/cloudinary/StorageCloudinaryAdapter.ts b/app/src/media/storage/adapters/cloudinary/StorageCloudinaryAdapter.ts index 105dfef..490047b 100644 --- a/app/src/media/storage/adapters/cloudinary/StorageCloudinaryAdapter.ts +++ b/app/src/media/storage/adapters/cloudinary/StorageCloudinaryAdapter.ts @@ -1,12 +1,12 @@ -import { hash, pickHeaders, s, parse } from "bknd/utils"; +import { hash, pickHeaders, s, parse, secret } from "bknd/utils"; import type { FileBody, FileListObject, FileMeta } from "../../Storage"; import { StorageAdapter } from "../../StorageAdapter"; export const cloudinaryAdapterConfig = s.object( { cloud_name: s.string(), - api_key: s.string(), - api_secret: s.string(), + api_key: secret(), + api_secret: secret(), upload_preset: s.string().optional(), }, { title: "Cloudinary", description: "Cloudinary media storage" }, diff --git a/app/src/media/storage/adapters/s3/StorageS3Adapter.ts b/app/src/media/storage/adapters/s3/StorageS3Adapter.ts index bb89265..5926ebe 100644 --- a/app/src/media/storage/adapters/s3/StorageS3Adapter.ts +++ b/app/src/media/storage/adapters/s3/StorageS3Adapter.ts @@ -8,15 +8,15 @@ import type { } from "@aws-sdk/client-s3"; import { AwsClient } from "core/clients/aws/AwsClient"; import { isDebug } from "core/env"; -import { isFile, pickHeaders2, parse, s } from "bknd/utils"; +import { isFile, pickHeaders2, parse, s, secret } from "bknd/utils"; import { transform } from "lodash-es"; import type { FileBody, FileListObject } from "../../Storage"; import { StorageAdapter } from "../../StorageAdapter"; export const s3AdapterConfig = s.object( { - access_key: s.string(), - secret_access_key: s.string(), + access_key: secret(), + secret_access_key: secret(), url: s.string({ pattern: "^https?://(?:.*)?[^/.]+$", description: "URL to S3 compatible endpoint without trailing slash", diff --git a/app/src/modules/ModuleManager.ts b/app/src/modules/ModuleManager.ts index e035ad1..7a49225 100644 --- a/app/src/modules/ModuleManager.ts +++ b/app/src/modules/ModuleManager.ts @@ -69,6 +69,10 @@ export type ModuleManagerOptions = { seed?: (ctx: ModuleBuildContext) => Promise; // called right after modules are built, before finish onModulesBuilt?: (ctx: ModuleBuildContext) => Promise; + // whether to store secrets in the database + storeSecrets?: boolean; + // provided secrets + secrets?: Record; /** @deprecated */ verbosity?: Verbosity; }; @@ -84,8 +88,14 @@ export class ModuleManagerConfigUpdateEvent< }> { static override slug = "mm-config-update"; } +export class ModuleManagerSecretsExtractedEvent extends ModuleManagerEvent<{ + secrets: Record; +}> { + static override slug = "mm-secrets-extracted"; +} export const ModuleManagerEvents = { ModuleManagerConfigUpdateEvent, + ModuleManagerSecretsExtractedEvent, }; // @todo: cleanup old diffs on upgrade diff --git a/app/src/modules/db/DbModuleManager.ts b/app/src/modules/db/DbModuleManager.ts index 995d81d..126a5ea 100644 --- a/app/src/modules/db/DbModuleManager.ts +++ b/app/src/modules/db/DbModuleManager.ts @@ -1,8 +1,8 @@ -import { mark, stripMark, $console, s } from "bknd/utils"; +import { mark, stripMark, $console, s, SecretSchema, setPath } from "bknd/utils"; import { BkndError } from "core/errors"; import * as $diff from "core/object/diff"; import type { Connection } from "data/connection"; -import { EntityManager } from "data/entities/EntityManager"; +import type { EntityManager } from "data/entities/EntityManager"; import * as proto from "data/prototype"; import { TransformPersistFailedException } from "data/errors"; import type { Kysely } from "kysely"; @@ -19,6 +19,7 @@ import { ModuleManager, ModuleManagerConfigUpdateEvent, type ModuleManagerOptions, + ModuleManagerSecretsExtractedEvent, } from "../ModuleManager"; export type { ModuleBuildContext }; @@ -50,6 +51,10 @@ export const __bknd = proto.entity(TABLE_NAME, { created_at: proto.datetime(), updated_at: proto.datetime(), }); +const __schema = proto.em({ __bknd }, ({ index }, { __bknd }) => { + index(__bknd).on(["version", "type"]); +}); + type ConfigTable2 = proto.Schema; interface T_INTERNAL_EM { __bknd: ConfigTable2; @@ -85,7 +90,8 @@ export class DbModuleManager extends ModuleManager { super(connection, { ...options, initial }); - this.__em = new EntityManager([__bknd], this.connection); + this.__em = __schema.proto.withConnection(this.connection) as any; + //this.__em = new EntityManager(__schema.entities, this.connection); this._version = version; this._booted_with = booted_with; @@ -131,23 +137,24 @@ export class DbModuleManager extends ModuleManager { return result; } - private async fetch(): Promise { + private async fetch(): Promise<{ configs?: ConfigTable; secrets?: ConfigTable } | undefined> { this.logger.context("fetch").log("fetching"); const startTime = performance.now(); // disabling console log, because the table might not exist yet - const { data: result } = await this.repo().findOne( - { type: "config" }, - { - sort: { by: "version", dir: "desc" }, - }, - ); + const { data: result } = await this.repo().findMany({ + where: { type: { $in: ["config", "secrets"] } }, + sort: { by: "version", dir: "desc" }, + }); - if (!result) { + if (!result.length) { this.logger.log("error fetching").clear(); return undefined; } + const configs = result.filter((r) => r.type === "config")[0]; + const secrets = result.filter((r) => r.type === "secrets")[0]; + this.logger .log("took", performance.now() - startTime, "ms", { version: result.version, @@ -155,44 +162,93 @@ export class DbModuleManager extends ModuleManager { }) .clear(); - return result as unknown as ConfigTable; + return { configs, secrets }; + } + + extractSecrets() { + const moduleConfigs = structuredClone(this.configs()); + const secrets = this.options?.secrets || ({} as any); + + for (const [key, module] of Object.entries(this.modules)) { + const config = moduleConfigs[key]; + const schema = module.getSchema(); + + const extracted = [...schema.walk({ data: config })].filter( + (n) => n.schema instanceof SecretSchema, + ); + + //console.log("extracted", key, extracted, config); + for (const n of extracted) { + const path = [key, ...n.instancePath].join("."); + if (typeof n.data === "string" && n.data.length > 0) { + secrets[path] = n.data; + setPath(moduleConfigs, path, ""); + } + } + } + + return { + configs: moduleConfigs, + secrets, + }; } async save() { this.logger.context("save").log("saving version", this.version()); - const configs = this.configs(); + const { configs, secrets } = this.extractSecrets(); const version = this.version(); + await this.emgr.emit( + new ModuleManagerSecretsExtractedEvent({ + ctx: this.ctx(), + secrets, + }), + ); + try { const state = await this.fetch(); - if (!state) throw new BkndError("no config found"); - this.logger.log("fetched version", state.version); + if (!state || !state.configs) throw new BkndError("no config found"); + this.logger.log("fetched version", state.configs.version); - if (state.version !== version) { + if (state.configs.version !== version) { // @todo: mark all others as "backup" - this.logger.log("version conflict, storing new version", state.version, version); - await this.mutator().insertOne({ - version: state.version, - type: "backup", - json: configs, - }); - await this.mutator().insertOne({ - version: version, - type: "config", - json: configs, - }); + this.logger.log( + "version conflict, storing new version", + state.configs.version, + version, + ); + const updates = [ + { + version: state.configs.version, + type: "backup", + json: state.configs.json, + }, + { + version: version, + type: "config", + json: configs, + }, + ]; + if (this.options?.storeSecrets) { + updates.push({ + version: state.configs.version, + type: "secrets", + json: secrets, + }); + } + await this.mutator().insertMany(updates); } else { - this.logger.log("version matches", state.version); + this.logger.log("version matches", state.configs.version); // clean configs because of Diff() function - const diffs = $diff.diff(state.json, $diff.clone(configs)); + const diffs = $diff.diff(state.configs.json, $diff.clone(configs)); this.logger.log("checking diff", [diffs.length]); + const date = new Date(); if (diffs.length > 0) { // validate diffs, it'll throw on invalid this.validateDiffs(diffs); - const date = new Date(); // store diff await this.mutator().insertOne({ version, @@ -217,6 +273,25 @@ export class DbModuleManager extends ModuleManager { } else { this.logger.log("no diff, not saving"); } + + // store secrets + if (this.options?.storeSecrets) { + if (!state.secrets || state.secrets?.version !== version) { + await this.mutator().insertOne({ + version: state.configs.version, + type: "secrets", + json: secrets, + created_at: date, + updated_at: date, + }); + } else { + await this.mutator().updateOne(state.secrets.id!, { + version, + json: secrets, + updated_at: date, + } as any); + } + } } } catch (e) { if (e instanceof BkndError && e.message === "no config found") { @@ -241,7 +316,7 @@ export class DbModuleManager extends ModuleManager { } // re-apply configs to all modules (important for system entities) - await this.setConfigs(configs); + await this.setConfigs(this.configs()); // @todo: cleanup old versions? @@ -308,17 +383,23 @@ export class DbModuleManager extends ModuleManager { const result = await this.fetch(); // if no version, and nothing found, go with initial - if (!result) { + if (!result?.configs) { this.logger.log("nothing in database, go initial"); await this.setupInitial(); } else { - this.logger.log("db has", result.version); + this.logger.log("db has", result.configs.version); // set version and config from fetched - this._version = result.version; + this._version = result.configs.version; + + if (result?.configs && result?.secrets) { + for (const [key, value] of Object.entries(result.secrets.json)) { + setPath(result.configs.json, key, value); + } + } if (this.options?.trustFetched === true) { this.logger.log("trusting fetched config (mark)"); - mark(result.json); + mark(result.configs.json); } // if version doesn't match, migrate before building @@ -328,7 +409,7 @@ export class DbModuleManager extends ModuleManager { await this.syncConfigTable(); const version_before = this.version(); - const [_version, _configs] = await migrate(version_before, result.json, { + const [_version, _configs] = await migrate(version_before, result.configs.json, { db: this.db, }); @@ -344,7 +425,7 @@ export class DbModuleManager extends ModuleManager { } else { this.logger.log("version is current", this.version()); - await this.setConfigs(result.json); + await this.setConfigs(result.configs.json); await this.buildModules(); } } diff --git a/app/src/modules/server/SystemController.ts b/app/src/modules/server/SystemController.ts index fde3d63..f25b191 100644 --- a/app/src/modules/server/SystemController.ts +++ b/app/src/modules/server/SystemController.ts @@ -31,7 +31,7 @@ import * as SystemPermissions from "modules/permissions"; import { getVersion } from "core/env"; import type { Module } from "modules/Module"; import { getSystemMcp } from "modules/mcp/system-mcp"; -import { DbModuleManager } from "modules/db/DbModuleManager"; +import type { DbModuleManager } from "modules/db/DbModuleManager"; export type ConfigUpdate = { success: true; @@ -125,7 +125,7 @@ export class SystemController extends Controller { permission([SystemPermissions.configReadSecrets]), async (c) => { // @ts-expect-error "fetch" is private - return c.json(await this.app.modules.fetch()); + return c.json(await this.app.modules.fetch().then((r) => r?.configs)); }, ); diff --git a/app/src/plugins/dev/sync-secrets.plugin.ts b/app/src/plugins/dev/sync-secrets.plugin.ts new file mode 100644 index 0000000..16e0357 --- /dev/null +++ b/app/src/plugins/dev/sync-secrets.plugin.ts @@ -0,0 +1,37 @@ +import { type App, ModuleManagerEvents, type AppPlugin } from "bknd"; +import type { DbModuleManager } from "modules/db/DbModuleManager"; + +export type SyncSecretsOptions = { + enabled?: boolean; + write: (secrets: Record) => Promise; +}; + +export function syncSecrets({ enabled = true, write }: SyncSecretsOptions): AppPlugin { + let firstBoot = true; + return (app: App) => ({ + name: "bknd-sync-secrets", + onBuilt: async () => { + if (!enabled) return; + const manager = app.modules as DbModuleManager; + + if (!("extractSecrets" in manager)) { + throw new Error("ModuleManager does not support secrets"); + } + + app.emgr.onEvent( + ModuleManagerEvents.ModuleManagerSecretsExtractedEvent, + async ({ params: { secrets } }) => { + await write?.(secrets); + }, + { + id: "sync-secrets", + }, + ); + + if (firstBoot) { + firstBoot = false; + await write?.(manager.extractSecrets().secrets); + } + }, + }); +} diff --git a/app/src/plugins/index.ts b/app/src/plugins/index.ts index eab3bf6..45db2d5 100644 --- a/app/src/plugins/index.ts +++ b/app/src/plugins/index.ts @@ -6,3 +6,4 @@ export { export { showRoutes, type ShowRoutesOptions } from "./dev/show-routes.plugin"; export { syncConfig, type SyncConfigOptions } from "./dev/sync-config.plugin"; export { syncTypes, type SyncTypesOptions } from "./dev/sync-types.plugin"; +export { syncSecrets, type SyncSecretsOptions } from "./dev/sync-secrets.plugin"; diff --git a/app/vite.config.ts b/app/vite.config.ts index 2fb1930..c6eb2fe 100644 --- a/app/vite.config.ts +++ b/app/vite.config.ts @@ -30,6 +30,7 @@ export default defineConfig({ devServer({ ...devServerConfig, entry: "./vite.dev.ts", + //entry: "./vite.dev.code.ts", }), tailwindcss(), ],