shishantbiswas/bknd
1
0
Fork 0
mirror of https://github.com/shishantbiswas/bknd.git synced 2026-03-15 20:17:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

mm: added secrets extraction for db mode

Browse Source
This commit is contained in:
dswbx
2025-09-04 15:20:12 +02:00
parent 2c5371610b
commit e8f2c70279
12 changed files with 231 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/__test__/integration/auth.integration.test.ts
View File

@@ -4,9 +4,6 @@ import { auth } from "../../src/auth/middlewares";
import { randomString, secureRandomString, withDisabledConsole } from "../../src/core/utils";
import { disableConsoleLog, enableConsoleLog, getDummyConnection } from "../helper";
const { dummyConnection, afterAllCleanup } = getDummyConnection();
afterEach(afterAllCleanup);
beforeAll(disableConsoleLog);
afterAll(enableConsoleLog);
@@ -65,6 +62,7 @@ const configs = {
};
function createAuthApp() {
const { dummyConnection } = getDummyConnection();
const app = createApp({
connection: dummyConnection,
config: {

22
app/__test__/modules/DbModuleManager.spec.ts Normal file
View File

@@ -0,0 +1,22 @@
import { it, expect, describe } from "bun:test";
import { DbModuleManager } from "modules/db/DbModuleManager";
import { getDummyConnection } from "../helper";
describe("DbModuleManager", () => {
it("should extract secrets", async () => {
const { dummyConnection } = getDummyConnection(false);
const m = new DbModuleManager(dummyConnection, {
initial: {
auth: {
enabled: true,
jwt: {
secret: "test",
},
},
},
});
await m.build();
expect(m.toJSON(true).auth.jwt.secret).toBe("test");
await m.save();
});
});

8
app/src/App.ts
View File

@@ -96,13 +96,7 @@ export type AppOptions = {
};
mode?: "db" | "code";
readonly?: boolean;
} & (
| {
mode?: "db";
secrets?: Record<string, any>;
}
| { mode?: "code" }
);
};
export type CreateAppConfig = {
/**
* bla

32
app/src/core/utils/objects.ts
View File

@@ -396,6 +396,38 @@ export function getPath(
}
}
export function setPath(object: object, _path: string | (string | number)[], value: any) {
let path = _path;
// Optional string-path support.
// You can remove this `if` block if you don't need it.
if (typeof path === "string") {
const isQuoted = (str) => str[0] === '"' && str.at(-1) === '"';
path = path
.split(/[.\[\]]+/)
.filter((x) => x)
.map((x) => (!Number.isNaN(Number(x)) ? Number(x) : x))
.map((x) => (typeof x === "string" && isQuoted(x) ? x.slice(1, -1) : x));
}
if (path.length === 0) {
throw new Error("The path must have at least one entry in it");
}
const [head, ...tail] = path as any;
if (tail.length === 0) {
object[head] = value;
return object;
}
if (!(head in object)) {
object[head] = typeof tail[0] === "number" ? [] : {};
}
setPath(object[head], tail, value);
return object;
}
export function objectToJsLiteral(value: object, indent: number = 0, _level: number = 0): string {
const nl = indent ? "\n" : "";
const pad = (lvl: number) => (indent ? " ".repeat(indent * lvl) : "");

6
app/src/media/storage/adapters/cloudinary/StorageCloudinaryAdapter.ts
View File

@@ -1,12 +1,12 @@
import { hash, pickHeaders, s, parse } from "bknd/utils";
import { hash, pickHeaders, s, parse, secret } from "bknd/utils";
import type { FileBody, FileListObject, FileMeta } from "../../Storage";
import { StorageAdapter } from "../../StorageAdapter";
export const cloudinaryAdapterConfig = s.object(
{
cloud_name: s.string(),
api_key: s.string(),
api_secret: s.string(),
api_key: secret(),
api_secret: secret(),
upload_preset: s.string().optional(),
},
{ title: "Cloudinary", description: "Cloudinary media storage" },

6
app/src/media/storage/adapters/s3/StorageS3Adapter.ts
View File

@@ -8,15 +8,15 @@ import type {
} from "@aws-sdk/client-s3";
import { AwsClient } from "core/clients/aws/AwsClient";
import { isDebug } from "core/env";
import { isFile, pickHeaders2, parse, s } from "bknd/utils";
import { isFile, pickHeaders2, parse, s, secret } from "bknd/utils";
import { transform } from "lodash-es";
import type { FileBody, FileListObject } from "../../Storage";
import { StorageAdapter } from "../../StorageAdapter";
export const s3AdapterConfig = s.object(
{
access_key: s.string(),
secret_access_key: s.string(),
access_key: secret(),
secret_access_key: secret(),
url: s.string({
pattern: "^https?://(?:.*)?[^/.]+$",
description: "URL to S3 compatible endpoint without trailing slash",

10
app/src/modules/ModuleManager.ts
View File

@@ -69,6 +69,10 @@ export type ModuleManagerOptions = {
seed?: (ctx: ModuleBuildContext) => Promise<void>;
// called right after modules are built, before finish
onModulesBuilt?: (ctx: ModuleBuildContext) => Promise<void>;
// whether to store secrets in the database
storeSecrets?: boolean;
// provided secrets
secrets?: Record<string, any>;
/** @deprecated */
verbosity?: Verbosity;
};
@@ -84,8 +88,14 @@ export class ModuleManagerConfigUpdateEvent<
}> {
static override slug = "mm-config-update";
}
export class ModuleManagerSecretsExtractedEvent extends ModuleManagerEvent<{
secrets: Record<string, any>;
}> {
static override slug = "mm-secrets-extracted";
}
export const ModuleManagerEvents = {
ModuleManagerConfigUpdateEvent,
ModuleManagerSecretsExtractedEvent,
};
// @todo: cleanup old diffs on upgrade

155
app/src/modules/db/DbModuleManager.ts
View File

@@ -1,8 +1,8 @@
import { mark, stripMark, $console, s } from "bknd/utils";
import { mark, stripMark, $console, s, SecretSchema, setPath } from "bknd/utils";
import { BkndError } from "core/errors";
import * as $diff from "core/object/diff";
import type { Connection } from "data/connection";
import { EntityManager } from "data/entities/EntityManager";
import type { EntityManager } from "data/entities/EntityManager";
import * as proto from "data/prototype";
import { TransformPersistFailedException } from "data/errors";
import type { Kysely } from "kysely";
@@ -19,6 +19,7 @@ import {
ModuleManager,
ModuleManagerConfigUpdateEvent,
type ModuleManagerOptions,
ModuleManagerSecretsExtractedEvent,
} from "../ModuleManager";
export type { ModuleBuildContext };
@@ -50,6 +51,10 @@ export const __bknd = proto.entity(TABLE_NAME, {
created_at: proto.datetime(),
updated_at: proto.datetime(),
});
const __schema = proto.em({ __bknd }, ({ index }, { __bknd }) => {
index(__bknd).on(["version", "type"]);
});
type ConfigTable2 = proto.Schema<typeof __bknd>;
interface T_INTERNAL_EM {
__bknd: ConfigTable2;
@@ -85,7 +90,8 @@ export class DbModuleManager extends ModuleManager {
super(connection, { ...options, initial });
this.__em = new EntityManager([__bknd], this.connection);
this.__em = __schema.proto.withConnection(this.connection) as any;
//this.__em = new EntityManager(__schema.entities, this.connection);
this._version = version;
this._booted_with = booted_with;
@@ -131,23 +137,24 @@ export class DbModuleManager extends ModuleManager {
return result;
}
private async fetch(): Promise<ConfigTable | undefined> {
private async fetch(): Promise<{ configs?: ConfigTable; secrets?: ConfigTable } | undefined> {
this.logger.context("fetch").log("fetching");
const startTime = performance.now();
// disabling console log, because the table might not exist yet
const { data: result } = await this.repo().findOne(
{ type: "config" },
{
sort: { by: "version", dir: "desc" },
},
);
const { data: result } = await this.repo().findMany({
where: { type: { $in: ["config", "secrets"] } },
sort: { by: "version", dir: "desc" },
});
if (!result) {
if (!result.length) {
this.logger.log("error fetching").clear();
return undefined;
}
const configs = result.filter((r) => r.type === "config")[0];
const secrets = result.filter((r) => r.type === "secrets")[0];
this.logger
.log("took", performance.now() - startTime, "ms", {
version: result.version,
@@ -155,44 +162,93 @@ export class DbModuleManager extends ModuleManager {
})
.clear();
return result as unknown as ConfigTable;
return { configs, secrets };
}
extractSecrets() {
const moduleConfigs = structuredClone(this.configs());
const secrets = this.options?.secrets || ({} as any);
for (const [key, module] of Object.entries(this.modules)) {
const config = moduleConfigs[key];
const schema = module.getSchema();
const extracted = [...schema.walk({ data: config })].filter(
(n) => n.schema instanceof SecretSchema,
);
//console.log("extracted", key, extracted, config);
for (const n of extracted) {
const path = [key, ...n.instancePath].join(".");
if (typeof n.data === "string" && n.data.length > 0) {
secrets[path] = n.data;
setPath(moduleConfigs, path, "");
}
}
}
return {
configs: moduleConfigs,
secrets,
};
}
async save() {
this.logger.context("save").log("saving version", this.version());
const configs = this.configs();
const { configs, secrets } = this.extractSecrets();
const version = this.version();
await this.emgr.emit(
new ModuleManagerSecretsExtractedEvent({
ctx: this.ctx(),
secrets,
}),
);
try {
const state = await this.fetch();
if (!state) throw new BkndError("no config found");
this.logger.log("fetched version", state.version);
if (!state || !state.configs) throw new BkndError("no config found");
this.logger.log("fetched version", state.configs.version);
if (state.version !== version) {
if (state.configs.version !== version) {
// @todo: mark all others as "backup"
this.logger.log("version conflict, storing new version", state.version, version);
await this.mutator().insertOne({
version: state.version,
type: "backup",
json: configs,
});
await this.mutator().insertOne({
version: version,
type: "config",
json: configs,
});
this.logger.log(
"version conflict, storing new version",
state.configs.version,
version,
);
const updates = [
{
version: state.configs.version,
type: "backup",
json: state.configs.json,
},
{
version: version,
type: "config",
json: configs,
},
];
if (this.options?.storeSecrets) {
updates.push({
version: state.configs.version,
type: "secrets",
json: secrets,
});
}
await this.mutator().insertMany(updates);
} else {
this.logger.log("version matches", state.version);
this.logger.log("version matches", state.configs.version);
// clean configs because of Diff() function
const diffs = $diff.diff(state.json, $diff.clone(configs));
const diffs = $diff.diff(state.configs.json, $diff.clone(configs));
this.logger.log("checking diff", [diffs.length]);
const date = new Date();
if (diffs.length > 0) {
// validate diffs, it'll throw on invalid
this.validateDiffs(diffs);
const date = new Date();
// store diff
await this.mutator().insertOne({
version,
@@ -217,6 +273,25 @@ export class DbModuleManager extends ModuleManager {
} else {
this.logger.log("no diff, not saving");
}
// store secrets
if (this.options?.storeSecrets) {
if (!state.secrets || state.secrets?.version !== version) {
await this.mutator().insertOne({
version: state.configs.version,
type: "secrets",
json: secrets,
created_at: date,
updated_at: date,
});
} else {
await this.mutator().updateOne(state.secrets.id!, {
version,
json: secrets,
updated_at: date,
} as any);
}
}
}
} catch (e) {
if (e instanceof BkndError && e.message === "no config found") {
@@ -241,7 +316,7 @@ export class DbModuleManager extends ModuleManager {
}
// re-apply configs to all modules (important for system entities)
await this.setConfigs(configs);
await this.setConfigs(this.configs());
// @todo: cleanup old versions?
@@ -308,17 +383,23 @@ export class DbModuleManager extends ModuleManager {
const result = await this.fetch();
// if no version, and nothing found, go with initial
if (!result) {
if (!result?.configs) {
this.logger.log("nothing in database, go initial");
await this.setupInitial();
} else {
this.logger.log("db has", result.version);
this.logger.log("db has", result.configs.version);
// set version and config from fetched
this._version = result.version;
this._version = result.configs.version;
if (result?.configs && result?.secrets) {
for (const [key, value] of Object.entries(result.secrets.json)) {
setPath(result.configs.json, key, value);
}
}
if (this.options?.trustFetched === true) {
this.logger.log("trusting fetched config (mark)");
mark(result.json);
mark(result.configs.json);
}
// if version doesn't match, migrate before building
@@ -328,7 +409,7 @@ export class DbModuleManager extends ModuleManager {
await this.syncConfigTable();
const version_before = this.version();
const [_version, _configs] = await migrate(version_before, result.json, {
const [_version, _configs] = await migrate(version_before, result.configs.json, {
db: this.db,
});
@@ -344,7 +425,7 @@ export class DbModuleManager extends ModuleManager {
} else {
this.logger.log("version is current", this.version());
await this.setConfigs(result.json);
await this.setConfigs(result.configs.json);
await this.buildModules();
}
}

4
app/src/modules/server/SystemController.ts
View File

@@ -31,7 +31,7 @@ import * as SystemPermissions from "modules/permissions";
import { getVersion } from "core/env";
import type { Module } from "modules/Module";
import { getSystemMcp } from "modules/mcp/system-mcp";
import { DbModuleManager } from "modules/db/DbModuleManager";
import type { DbModuleManager } from "modules/db/DbModuleManager";
export type ConfigUpdate<Key extends ModuleKey = ModuleKey> = {
success: true;
@@ -125,7 +125,7 @@ export class SystemController extends Controller {
permission([SystemPermissions.configReadSecrets]),
async (c) => {
// @ts-expect-error "fetch" is private
return c.json(await this.app.modules.fetch());
return c.json(await this.app.modules.fetch().then((r) => r?.configs));
},
);

37
app/src/plugins/dev/sync-secrets.plugin.ts Normal file
View File

@@ -0,0 +1,37 @@
import { type App, ModuleManagerEvents, type AppPlugin } from "bknd";
import type { DbModuleManager } from "modules/db/DbModuleManager";
export type SyncSecretsOptions = {
enabled?: boolean;
write: (secrets: Record<string, any>) => Promise<void>;
};
export function syncSecrets({ enabled = true, write }: SyncSecretsOptions): AppPlugin {
let firstBoot = true;
return (app: App) => ({
name: "bknd-sync-secrets",
onBuilt: async () => {
if (!enabled) return;
const manager = app.modules as DbModuleManager;
if (!("extractSecrets" in manager)) {
throw new Error("ModuleManager does not support secrets");
}
app.emgr.onEvent(
ModuleManagerEvents.ModuleManagerSecretsExtractedEvent,
async ({ params: { secrets } }) => {
await write?.(secrets);
},
{
id: "sync-secrets",
},
);
if (firstBoot) {
firstBoot = false;
await write?.(manager.extractSecrets().secrets);
}
},
});
}

1
app/src/plugins/index.ts
View File

@@ -6,3 +6,4 @@ export {
export { showRoutes, type ShowRoutesOptions } from "./dev/show-routes.plugin";
export { syncConfig, type SyncConfigOptions } from "./dev/sync-config.plugin";
export { syncTypes, type SyncTypesOptions } from "./dev/sync-types.plugin";
export { syncSecrets, type SyncSecretsOptions } from "./dev/sync-secrets.plugin";

1
app/vite.config.ts
View File

@@ -30,6 +30,7 @@ export default defineConfig({
devServer({
...devServerConfig,
entry: "./vite.dev.ts",
//entry: "./vite.dev.code.ts",
}),
tailwindcss(),
],