mm: added secrets extraction for db mode

app/__test__/integration/auth.integration.test.ts

@@ -4,9 +4,6 @@ import { auth } from "../../src/auth/middlewares";
 import { randomString, secureRandomString, withDisabledConsole } from "../../src/core/utils";
 import { disableConsoleLog, enableConsoleLog, getDummyConnection } from "../helper";
 
-const { dummyConnection, afterAllCleanup } = getDummyConnection();
-afterEach(afterAllCleanup);
-
 beforeAll(disableConsoleLog);
 afterAll(enableConsoleLog);
 
@@ -65,6 +62,7 @@ const configs = {
 };
 
 function createAuthApp() {
+   const { dummyConnection } = getDummyConnection();
    const app = createApp({
       connection: dummyConnection,
       config: {

app/__test__/modules/DbModuleManager.spec.ts

@@ -0,0 +1,22 @@
+import { it, expect, describe } from "bun:test";
+import { DbModuleManager } from "modules/db/DbModuleManager";
+import { getDummyConnection } from "../helper";
+
+describe("DbModuleManager", () => {
+   it("should extract secrets", async () => {
+      const { dummyConnection } = getDummyConnection(false);
+      const m = new DbModuleManager(dummyConnection, {
+         initial: {
+            auth: {
+               enabled: true,
+               jwt: {
+                  secret: "test",
+               },
+            },
+         },
+      });
+      await m.build();
+      expect(m.toJSON(true).auth.jwt.secret).toBe("test");
+      await m.save();
+   });
+});

app/src/App.ts

@@ -96,13 +96,7 @@ export type AppOptions = {
    };
    mode?: "db" | "code";
    readonly?: boolean;
-} & (
+};
-   | {
-        mode?: "db";
-        secrets?: Record<string, any>;
-     }
-   | { mode?: "code" }
-);
 export type CreateAppConfig = {
    /**
     * bla

app/src/core/utils/objects.ts

@@ -396,6 +396,38 @@ export function getPath(
    }
 }
 
+export function setPath(object: object, _path: string | (string | number)[], value: any) {
+   let path = _path;
+   // Optional string-path support.
+   // You can remove this `if` block if you don't need it.
+   if (typeof path === "string") {
+      const isQuoted = (str) => str[0] === '"' && str.at(-1) === '"';
+      path = path
+         .split(/[.\[\]]+/)
+         .filter((x) => x)
+         .map((x) => (!Number.isNaN(Number(x)) ? Number(x) : x))
+         .map((x) => (typeof x === "string" && isQuoted(x) ? x.slice(1, -1) : x));
+   }
+
+   if (path.length === 0) {
+      throw new Error("The path must have at least one entry in it");
+   }
+
+   const [head, ...tail] = path as any;
+
+   if (tail.length === 0) {
+      object[head] = value;
+      return object;
+   }
+
+   if (!(head in object)) {
+      object[head] = typeof tail[0] === "number" ? [] : {};
+   }
+
+   setPath(object[head], tail, value);
+   return object;
+}
+
 export function objectToJsLiteral(value: object, indent: number = 0, _level: number = 0): string {
    const nl = indent ? "\n" : "";
    const pad = (lvl: number) => (indent ? " ".repeat(indent * lvl) : "");

app/src/media/storage/adapters/cloudinary/StorageCloudinaryAdapter.ts

@@ -1,12 +1,12 @@
-import { hash, pickHeaders, s, parse } from "bknd/utils";
+import { hash, pickHeaders, s, parse, secret } from "bknd/utils";
 import type { FileBody, FileListObject, FileMeta } from "../../Storage";
 import { StorageAdapter } from "../../StorageAdapter";
 
 export const cloudinaryAdapterConfig = s.object(
    {
       cloud_name: s.string(),
-      api_key: s.string(),
+      api_key: secret(),
-      api_secret: s.string(),
+      api_secret: secret(),
       upload_preset: s.string().optional(),
    },
    { title: "Cloudinary", description: "Cloudinary media storage" },

app/src/media/storage/adapters/s3/StorageS3Adapter.ts

@@ -8,15 +8,15 @@ import type {
 } from "@aws-sdk/client-s3";
 import { AwsClient } from "core/clients/aws/AwsClient";
 import { isDebug } from "core/env";
-import { isFile, pickHeaders2, parse, s } from "bknd/utils";
+import { isFile, pickHeaders2, parse, s, secret } from "bknd/utils";
 import { transform } from "lodash-es";
 import type { FileBody, FileListObject } from "../../Storage";
 import { StorageAdapter } from "../../StorageAdapter";
 
 export const s3AdapterConfig = s.object(
    {
-      access_key: s.string(),
+      access_key: secret(),
-      secret_access_key: s.string(),
+      secret_access_key: secret(),
       url: s.string({
          pattern: "^https?://(?:.*)?[^/.]+$",
          description: "URL to S3 compatible endpoint without trailing slash",

app/src/modules/ModuleManager.ts

@@ -69,6 +69,10 @@ export type ModuleManagerOptions = {
    seed?: (ctx: ModuleBuildContext) => Promise<void>;
    // called right after modules are built, before finish
    onModulesBuilt?: (ctx: ModuleBuildContext) => Promise<void>;
+   // whether to store secrets in the database
+   storeSecrets?: boolean;
+   // provided secrets
+   secrets?: Record<string, any>;
    /** @deprecated */
    verbosity?: Verbosity;
 };
@@ -84,8 +88,14 @@ export class ModuleManagerConfigUpdateEvent<
 }> {
    static override slug = "mm-config-update";
 }
+export class ModuleManagerSecretsExtractedEvent extends ModuleManagerEvent<{
+   secrets: Record<string, any>;
+}> {
+   static override slug = "mm-secrets-extracted";
+}
 export const ModuleManagerEvents = {
    ModuleManagerConfigUpdateEvent,
+   ModuleManagerSecretsExtractedEvent,
 };
 
 // @todo: cleanup old diffs on upgrade

app/src/modules/db/DbModuleManager.ts

@@ -1,8 +1,8 @@
-import { mark, stripMark, $console, s } from "bknd/utils";
+import { mark, stripMark, $console, s, SecretSchema, setPath } from "bknd/utils";
 import { BkndError } from "core/errors";
 import * as $diff from "core/object/diff";
 import type { Connection } from "data/connection";
-import { EntityManager } from "data/entities/EntityManager";
+import type { EntityManager } from "data/entities/EntityManager";
 import * as proto from "data/prototype";
 import { TransformPersistFailedException } from "data/errors";
 import type { Kysely } from "kysely";
@@ -19,6 +19,7 @@ import {
    ModuleManager,
    ModuleManagerConfigUpdateEvent,
    type ModuleManagerOptions,
+   ModuleManagerSecretsExtractedEvent,
 } from "../ModuleManager";
 
 export type { ModuleBuildContext };
@@ -50,6 +51,10 @@ export const __bknd = proto.entity(TABLE_NAME, {
    created_at: proto.datetime(),
    updated_at: proto.datetime(),
 });
+const __schema = proto.em({ __bknd }, ({ index }, { __bknd }) => {
+   index(__bknd).on(["version", "type"]);
+});
+
 type ConfigTable2 = proto.Schema<typeof __bknd>;
 interface T_INTERNAL_EM {
    __bknd: ConfigTable2;
@@ -85,7 +90,8 @@ export class DbModuleManager extends ModuleManager {
 
       super(connection, { ...options, initial });
 
-      this.__em = new EntityManager([__bknd], this.connection);
+      this.__em = __schema.proto.withConnection(this.connection) as any;
+      //this.__em = new EntityManager(__schema.entities, this.connection);
       this._version = version;
       this._booted_with = booted_with;
 
@@ -131,23 +137,24 @@ export class DbModuleManager extends ModuleManager {
       return result;
    }
 
-   private async fetch(): Promise<ConfigTable | undefined> {
+   private async fetch(): Promise<{ configs?: ConfigTable; secrets?: ConfigTable } | undefined> {
       this.logger.context("fetch").log("fetching");
       const startTime = performance.now();
 
       // disabling console log, because the table might not exist yet
-      const { data: result } = await this.repo().findOne(
+      const { data: result } = await this.repo().findMany({
-         { type: "config" },
+         where: { type: { $in: ["config", "secrets"] } },
-         {
          sort: { by: "version", dir: "desc" },
-         },
+      });
-      );
 
-      if (!result) {
+      if (!result.length) {
          this.logger.log("error fetching").clear();
          return undefined;
       }
 
+      const configs = result.filter((r) => r.type === "config")[0];
+      const secrets = result.filter((r) => r.type === "secrets")[0];
+
       this.logger
          .log("took", performance.now() - startTime, "ms", {
             version: result.version,
@@ -155,44 +162,93 @@ export class DbModuleManager extends ModuleManager {
          })
          .clear();
 
-      return result as unknown as ConfigTable;
+      return { configs, secrets };
+   }
+
+   extractSecrets() {
+      const moduleConfigs = structuredClone(this.configs());
+      const secrets = this.options?.secrets || ({} as any);
+
+      for (const [key, module] of Object.entries(this.modules)) {
+         const config = moduleConfigs[key];
+         const schema = module.getSchema();
+
+         const extracted = [...schema.walk({ data: config })].filter(
+            (n) => n.schema instanceof SecretSchema,
+         );
+
+         //console.log("extracted", key, extracted, config);
+         for (const n of extracted) {
+            const path = [key, ...n.instancePath].join(".");
+            if (typeof n.data === "string" && n.data.length > 0) {
+               secrets[path] = n.data;
+               setPath(moduleConfigs, path, "");
+            }
+         }
+      }
+
+      return {
+         configs: moduleConfigs,
+         secrets,
+      };
    }
 
    async save() {
       this.logger.context("save").log("saving version", this.version());
-      const configs = this.configs();
+      const { configs, secrets } = this.extractSecrets();
       const version = this.version();
 
+      await this.emgr.emit(
+         new ModuleManagerSecretsExtractedEvent({
+            ctx: this.ctx(),
+            secrets,
+         }),
+      );
+
       try {
          const state = await this.fetch();
-         if (!state) throw new BkndError("no config found");
+         if (!state || !state.configs) throw new BkndError("no config found");
-         this.logger.log("fetched version", state.version);
+         this.logger.log("fetched version", state.configs.version);
 
-         if (state.version !== version) {
+         if (state.configs.version !== version) {
             // @todo: mark all others as "backup"
-            this.logger.log("version conflict, storing new version", state.version, version);
+            this.logger.log(
-            await this.mutator().insertOne({
+               "version conflict, storing new version",
-               version: state.version,
+               state.configs.version,
+               version,
+            );
+            const updates = [
+               {
+                  version: state.configs.version,
                   type: "backup",
-               json: configs,
+                  json: state.configs.json,
-            });
+               },
-            await this.mutator().insertOne({
+               {
                   version: version,
                   type: "config",
                   json: configs,
+               },
+            ];
+            if (this.options?.storeSecrets) {
+               updates.push({
+                  version: state.configs.version,
+                  type: "secrets",
+                  json: secrets,
                });
+            }
+            await this.mutator().insertMany(updates);
          } else {
-            this.logger.log("version matches", state.version);
+            this.logger.log("version matches", state.configs.version);
 
             // clean configs because of Diff() function
-            const diffs = $diff.diff(state.json, $diff.clone(configs));
+            const diffs = $diff.diff(state.configs.json, $diff.clone(configs));
             this.logger.log("checking diff", [diffs.length]);
+            const date = new Date();
 
             if (diffs.length > 0) {
                // validate diffs, it'll throw on invalid
                this.validateDiffs(diffs);
 
-               const date = new Date();
                // store diff
                await this.mutator().insertOne({
                   version,
@@ -217,6 +273,25 @@ export class DbModuleManager extends ModuleManager {
             } else {
                this.logger.log("no diff, not saving");
             }
+
+            // store secrets
+            if (this.options?.storeSecrets) {
+               if (!state.secrets || state.secrets?.version !== version) {
+                  await this.mutator().insertOne({
+                     version: state.configs.version,
+                     type: "secrets",
+                     json: secrets,
+                     created_at: date,
+                     updated_at: date,
+                  });
+               } else {
+                  await this.mutator().updateOne(state.secrets.id!, {
+                     version,
+                     json: secrets,
+                     updated_at: date,
+                  } as any);
+               }
+            }
          }
       } catch (e) {
          if (e instanceof BkndError && e.message === "no config found") {
@@ -241,7 +316,7 @@ export class DbModuleManager extends ModuleManager {
       }
 
       // re-apply configs to all modules (important for system entities)
-      await this.setConfigs(configs);
+      await this.setConfigs(this.configs());
 
       // @todo: cleanup old versions?
 
@@ -308,17 +383,23 @@ export class DbModuleManager extends ModuleManager {
          const result = await this.fetch();
 
          // if no version, and nothing found, go with initial
-         if (!result) {
+         if (!result?.configs) {
             this.logger.log("nothing in database, go initial");
             await this.setupInitial();
          } else {
-            this.logger.log("db has", result.version);
+            this.logger.log("db has", result.configs.version);
             // set version and config from fetched
-            this._version = result.version;
+            this._version = result.configs.version;
+
+            if (result?.configs && result?.secrets) {
+               for (const [key, value] of Object.entries(result.secrets.json)) {
+                  setPath(result.configs.json, key, value);
+               }
+            }
 
             if (this.options?.trustFetched === true) {
                this.logger.log("trusting fetched config (mark)");
-               mark(result.json);
+               mark(result.configs.json);
             }
 
             // if version doesn't match, migrate before building
@@ -328,7 +409,7 @@ export class DbModuleManager extends ModuleManager {
                await this.syncConfigTable();
 
                const version_before = this.version();
-               const [_version, _configs] = await migrate(version_before, result.json, {
+               const [_version, _configs] = await migrate(version_before, result.configs.json, {
                   db: this.db,
                });
 
@@ -344,7 +425,7 @@ export class DbModuleManager extends ModuleManager {
             } else {
                this.logger.log("version is current", this.version());
 
-               await this.setConfigs(result.json);
+               await this.setConfigs(result.configs.json);
                await this.buildModules();
             }
          }

app/src/modules/server/SystemController.ts

@@ -31,7 +31,7 @@ import * as SystemPermissions from "modules/permissions";
 import { getVersion } from "core/env";
 import type { Module } from "modules/Module";
 import { getSystemMcp } from "modules/mcp/system-mcp";
-import { DbModuleManager } from "modules/db/DbModuleManager";
+import type { DbModuleManager } from "modules/db/DbModuleManager";
 
 export type ConfigUpdate<Key extends ModuleKey = ModuleKey> = {
    success: true;
@@ -125,7 +125,7 @@ export class SystemController extends Controller {
             permission([SystemPermissions.configReadSecrets]),
             async (c) => {
                // @ts-expect-error "fetch" is private
-               return c.json(await this.app.modules.fetch());
+               return c.json(await this.app.modules.fetch().then((r) => r?.configs));
             },
          );
 

app/src/plugins/dev/sync-secrets.plugin.ts

@@ -0,0 +1,37 @@
+import { type App, ModuleManagerEvents, type AppPlugin } from "bknd";
+import type { DbModuleManager } from "modules/db/DbModuleManager";
+
+export type SyncSecretsOptions = {
+   enabled?: boolean;
+   write: (secrets: Record<string, any>) => Promise<void>;
+};
+
+export function syncSecrets({ enabled = true, write }: SyncSecretsOptions): AppPlugin {
+   let firstBoot = true;
+   return (app: App) => ({
+      name: "bknd-sync-secrets",
+      onBuilt: async () => {
+         if (!enabled) return;
+         const manager = app.modules as DbModuleManager;
+
+         if (!("extractSecrets" in manager)) {
+            throw new Error("ModuleManager does not support secrets");
+         }
+
+         app.emgr.onEvent(
+            ModuleManagerEvents.ModuleManagerSecretsExtractedEvent,
+            async ({ params: { secrets } }) => {
+               await write?.(secrets);
+            },
+            {
+               id: "sync-secrets",
+            },
+         );
+
+         if (firstBoot) {
+            firstBoot = false;
+            await write?.(manager.extractSecrets().secrets);
+         }
+      },
+   });
+}

app/src/plugins/index.ts

@@ -6,3 +6,4 @@ export {
 export { showRoutes, type ShowRoutesOptions } from "./dev/show-routes.plugin";
 export { syncConfig, type SyncConfigOptions } from "./dev/sync-config.plugin";
 export { syncTypes, type SyncTypesOptions } from "./dev/sync-types.plugin";
+export { syncSecrets, type SyncSecretsOptions } from "./dev/sync-secrets.plugin";

app/vite.config.ts

@@ -30,6 +30,7 @@ export default defineConfig({
       devServer({
          ...devServerConfig,
          entry: "./vite.dev.ts",
+         //entry: "./vite.dev.code.ts",
       }),
       tailwindcss(),
    ],