Merge pull request #186 from bknd-io/feat/unify-connections

feat/unify-connections
This commit is contained in:
dswbx
2025-06-13 17:20:05 +02:00
committed by GitHub
86 changed files with 1816 additions and 1779 deletions

View File

@@ -2,10 +2,11 @@
import path from "node:path";
import { type RuntimeBkndConfig, createRuntimeApp, type RuntimeOptions } from "bknd/adapter";
import { registerLocalMediaAdapter } from "bknd/adapter/node";
import { registerLocalMediaAdapter } from ".";
import { config } from "bknd/core";
import type { ServeOptions } from "bun";
import { serveStatic } from "hono/bun";
import type { App } from "App";
type BunEnv = Bun.Env;
export type BunBkndConfig<Env = BunEnv> = RuntimeBkndConfig<Env> & Omit<ServeOptions, "fetch">;
@@ -33,8 +34,11 @@ export function createHandler<Env = BunEnv>(
args: Env = {} as Env,
opts?: RuntimeOptions,
) {
let app: App | undefined;
return async (req: Request) => {
const app = await createApp(config, args ?? (process.env as Env), opts);
if (!app) {
app = await createApp(config, args ?? (process.env as Env), opts);
}
return app.fetch(req);
};
}
@@ -72,5 +76,5 @@ export function serve<Env = BunEnv>(
),
});
console.log(`Server is running on http://localhost:${port}`);
console.info(`Server is running on http://localhost:${port}`);
}

View File

@@ -0,0 +1,12 @@
import { connectionTestSuite } from "data/connection/connection-test-suite";
import { bunSqlite } from "./BunSqliteConnection";
import { bunTestRunner } from "adapter/bun/test";
import { describe } from "bun:test";
import { Database } from "bun:sqlite";
describe("BunSqliteConnection", () => {
connectionTestSuite(bunTestRunner, {
makeConnection: () => bunSqlite({ database: new Database(":memory:") }),
rawDialectDetails: [],
});
});

View File

@@ -0,0 +1,48 @@
import { Database } from "bun:sqlite";
import {
buildQueryFn,
GenericSqliteConnection,
parseBigInt,
type IGenericSqlite,
} from "data/connection/sqlite/GenericSqliteConnection";
export type BunSqliteConnectionConfig = {
database: Database;
};
function bunSqliteExecutor(db: Database, cache: boolean): IGenericSqlite<Database> {
const fn = cache ? "query" : "prepare";
const getStmt = (sql: string) => db[fn](sql);
return {
db,
query: buildQueryFn({
all: (sql, parameters) => getStmt(sql).all(...(parameters || [])),
run: (sql, parameters) => {
const { changes, lastInsertRowid } = getStmt(sql).run(...(parameters || []));
return {
insertId: parseBigInt(lastInsertRowid),
numAffectedRows: parseBigInt(changes),
};
},
}),
close: () => db.close(),
};
}
export function bunSqlite(config?: BunSqliteConnectionConfig | { url: string }) {
let database: Database;
if (config) {
if ("database" in config) {
database = config.database;
} else {
database = new Database(config.url);
}
} else {
database = new Database(":memory:");
}
return new GenericSqliteConnection(database, () => bunSqliteExecutor(database, false), {
name: "bun-sqlite",
});
}

View File

@@ -1 +1,3 @@
export * from "./bun.adapter";
export * from "../node/storage";
export * from "./connection/BunSqliteConnection";

View File

@@ -1,6 +1,7 @@
import { expect, test, mock } from "bun:test";
import { expect, test, mock, describe } from "bun:test";
export const bunTestRunner = {
describe,
expect,
test,
mock,

View File

@@ -13,30 +13,32 @@ describe("cf adapter", () => {
const DB_URL = ":memory:";
const $ctx = (env?: any, request?: Request, ctx?: ExecutionContext) => ({
request: request ?? (null as any),
env: env ?? { DB_URL },
env: env ?? { url: DB_URL },
ctx: ctx ?? (null as any),
});
it("makes config", async () => {
expect(
makeConfig(
{
connection: { url: DB_URL },
},
$ctx({ DB_URL }),
),
).toEqual({ connection: { url: DB_URL } });
const staticConfig = makeConfig(
{
connection: { url: DB_URL },
initialConfig: { data: { basepath: DB_URL } },
},
$ctx({ DB_URL }),
);
expect(staticConfig.initialConfig).toEqual({ data: { basepath: DB_URL } });
expect(staticConfig.connection).toBeDefined();
expect(
makeConfig(
{
app: (env) => ({
connection: { url: env.DB_URL },
}),
},
$ctx({ DB_URL }),
),
).toEqual({ connection: { url: DB_URL } });
const dynamicConfig = makeConfig(
{
app: (env) => ({
initialConfig: { data: { basepath: env.DB_URL } },
connection: { url: env.DB_URL },
}),
},
$ctx({ DB_URL }),
);
expect(dynamicConfig.initialConfig).toEqual({ data: { basepath: DB_URL } });
expect(dynamicConfig.connection).toBeDefined();
});
adapterTestSuite<CloudflareBkndConfig, CfMakeConfigArgs<any>>(bunTestRunner, {

View File

@@ -9,6 +9,7 @@ import { makeConfig as makeAdapterConfig } from "bknd/adapter";
import type { Context, ExecutionContext } from "hono";
import { $console } from "core";
import { setCookie } from "hono/cookie";
import { sqlite } from "bknd/adapter/sqlite";
export const constants = {
exec_async_event_id: "cf_register_waituntil",
@@ -98,54 +99,70 @@ export function makeConfig<Env extends CloudflareEnv = CloudflareEnv>(
const appConfig = makeAdapterConfig(config, args?.env);
if (args?.env) {
const bindings = config.bindings?.(args?.env);
// if connection instance is given, don't do anything
// other than checking if D1 session is defined
if (D1Connection.isConnection(appConfig.connection)) {
if (config.d1?.session) {
// we cannot guarantee that db was opened with session
throw new Error(
"D1 session don't work when D1 is directly given as connection. Define it in bindings instead.",
);
}
// if connection is given, try to open with unified sqlite adapter
} else if (appConfig.connection) {
appConfig.connection = sqlite(appConfig.connection);
// if connection is not given, but env is set
// try to make D1 from bindings
} else if (args?.env) {
const bindings = config.bindings?.(args?.env);
const sessionHelper = d1SessionHelper(config);
const sessionId = sessionHelper.get(args.request);
let session: D1DatabaseSession | undefined;
let db: D1Database | undefined;
if (!appConfig.connection) {
let db: D1Database | undefined;
if (bindings?.db) {
$console.log("Using database from bindings");
db = bindings.db;
} else if (Object.keys(args).length > 0) {
const binding = getBinding(args.env, "D1Database");
if (binding) {
$console.log(`Using database from env "${binding.key}"`);
db = binding.value;
}
}
// if db is given in bindings, use it
if (bindings?.db) {
$console.log("Using database from bindings");
db = bindings.db;
if (db) {
if (config.d1?.session) {
session = db.withSession(sessionId ?? config.d1?.first);
appConfig.connection = new D1Connection({ binding: session });
} else {
appConfig.connection = new D1Connection({ binding: db });
}
} else {
throw new Error("No database connection given");
// scan for D1Database in args
} else {
const binding = getBinding(args.env, "D1Database");
if (binding) {
$console.log(`Using database from env "${binding.key}"`);
db = binding.value;
}
}
if (config.d1?.session) {
appConfig.options = {
...appConfig.options,
manager: {
...appConfig.options?.manager,
onServerInit: (server) => {
server.use(async (c, next) => {
sessionHelper.set(c, session);
await next();
});
// if db is found, check if session is requested
if (db) {
if (config.d1?.session) {
session = db.withSession(sessionId ?? config.d1?.first);
appConfig.connection = new D1Connection({ binding: session });
appConfig.options = {
...appConfig.options,
manager: {
...appConfig.options?.manager,
onServerInit: (server) => {
server.use(async (c, next) => {
sessionHelper.set(c, session);
await next();
});
},
},
},
};
};
} else {
appConfig.connection = new D1Connection({ binding: db });
}
}
}
if (!D1Connection.isConnection(appConfig.connection)) {
throw new Error("Couldn't find database connection");
}
return appConfig;
}

View File

@@ -1,65 +1,42 @@
/// <reference types="@cloudflare/workers-types" />
import { KyselyPluginRunner, SqliteConnection, SqliteIntrospector } from "bknd/data";
import type { QB } from "data/connection/Connection";
import { type DatabaseIntrospector, Kysely, ParseJSONResultsPlugin } from "kysely";
import { SqliteConnection } from "bknd/data";
import type { ConnQuery, ConnQueryResults } from "data/connection/Connection";
import { D1Dialect } from "kysely-d1";
export type D1ConnectionConfig<DB extends D1Database | D1DatabaseSession = D1Database> = {
binding: DB;
};
class CustomD1Dialect extends D1Dialect {
override createIntrospector(db: Kysely<any>): DatabaseIntrospector {
return new SqliteIntrospector(db, {
excludeTables: ["_cf_KV", "_cf_METADATA"],
});
}
}
export class D1Connection<
DB extends D1Database | D1DatabaseSession = D1Database,
> extends SqliteConnection {
> extends SqliteConnection<DB> {
override name = "sqlite-d1";
protected override readonly supported = {
batching: true,
softscans: false,
};
constructor(private config: D1ConnectionConfig<DB>) {
const plugins = [new ParseJSONResultsPlugin()];
const kysely = new Kysely({
dialect: new CustomD1Dialect({ database: config.binding as D1Database }),
plugins,
super({
excludeTables: ["_cf_KV", "_cf_METADATA"],
dialect: D1Dialect,
dialectArgs: [{ database: config.binding as D1Database }],
});
super(kysely, {}, plugins);
}
get client(): DB {
return this.config.binding;
}
override async executeQueries<O extends ConnQuery[]>(...qbs: O): Promise<ConnQueryResults<O>> {
const compiled = this.getCompiled(...qbs);
protected override async batch<Queries extends QB[]>(
queries: [...Queries],
): Promise<{
[K in keyof Queries]: Awaited<ReturnType<Queries[K]["execute"]>>;
}> {
const db = this.config.binding;
const res = await db.batch(
queries.map((q) => {
const { sql, parameters } = q.compile();
compiled.map(({ sql, parameters }) => {
return db.prepare(sql).bind(...parameters);
}),
);
// let it run through plugins
const kyselyPlugins = new KyselyPluginRunner(this.plugins);
const data: any = [];
for (const r of res) {
const rows = await kyselyPlugins.transformResultRows(r.results);
data.push(rows);
}
return data;
return this.withTransformedRows(res, "results") as any;
}
}

View File

@@ -64,7 +64,7 @@ export class DurableBkndApp extends DurableObject {
"type" in config.connection &&
config.connection.type === "libsql"
) {
config.connection.config.protocol = "wss";
//config.connection.config.protocol = "wss";
}
this.app = await createRuntimeApp({

View File

@@ -1,32 +0,0 @@
import { createWriteStream, readFileSync } from "node:fs";
import { test } from "node:test";
import { Miniflare } from "miniflare";
import { StorageR2Adapter } from "./StorageR2Adapter";
import { adapterTestSuite } from "media";
import { nodeTestRunner } from "adapter/node/test";
import path from "node:path";
// https://github.com/nodejs/node/issues/44372#issuecomment-1736530480
console.log = async (message: any) => {
const tty = createWriteStream("/dev/tty");
const msg = typeof message === "string" ? message : JSON.stringify(message, null, 2);
return tty.write(`${msg}\n`);
};
test("StorageR2Adapter", async () => {
const mf = new Miniflare({
modules: true,
script: "export default { async fetch() { return new Response(null); } }",
r2Buckets: ["BUCKET"],
});
const bucket = (await mf.getR2Bucket("BUCKET")) as unknown as R2Bucket;
const adapter = new StorageR2Adapter(bucket);
const basePath = path.resolve(import.meta.dirname, "../../../../__test__/_assets");
const buffer = readFileSync(path.join(basePath, "image.png"));
const file = new File([buffer], "image.png", { type: "image/png" });
await adapterTestSuite(nodeTestRunner, adapter, file);
await mf.dispose();
});

View File

@@ -0,0 +1,32 @@
import { readFileSync } from "node:fs";
import { Miniflare } from "miniflare";
import { StorageR2Adapter } from "./StorageR2Adapter";
import { adapterTestSuite } from "media/storage/adapters/adapter-test-suite";
import path from "node:path";
import { describe, afterAll, test, expect } from "vitest";
import { viTestRunner } from "adapter/node/vitest";
let mf: Miniflare | undefined;
describe("StorageR2Adapter", async () => {
mf = new Miniflare({
modules: true,
script: "export default { async fetch() { return new Response(null); } }",
r2Buckets: ["BUCKET"],
});
const bucket = (await mf?.getR2Bucket("BUCKET")) as unknown as R2Bucket;
test("test", () => {
expect(bucket).toBeDefined();
});
const adapter = new StorageR2Adapter(bucket);
const basePath = path.resolve(import.meta.dirname, "../../../../__test__/_assets");
const buffer = readFileSync(path.join(basePath, "image.png"));
const file = new File([buffer], "image.png", { type: "image/png" });
await adapterTestSuite(viTestRunner, adapter, file);
});
afterAll(async () => {
await mf?.dispose();
});

View File

@@ -0,0 +1,14 @@
import { defineWorkersConfig } from "@cloudflare/vitest-pool-workers/config";
export default defineWorkersConfig({
test: {
poolOptions: {
workers: {
miniflare: {
compatibilityDate: "2025-06-04",
},
},
},
include: ["**/*.vi-test.ts", "**/*.vitest.ts"],
},
});

View File

@@ -1,7 +1,10 @@
import { App, type CreateAppConfig } from "bknd";
import { config as $config } from "bknd/core";
import { config as $config, $console } from "bknd/core";
import type { MiddlewareHandler } from "hono";
import type { AdminControllerOptions } from "modules/server/AdminController";
import { Connection } from "bknd/data";
export { Connection } from "bknd/data";
export type BkndConfig<Args = any> = CreateAppConfig & {
app?: CreateAppConfig | ((args: Args) => CreateAppConfig);
@@ -59,7 +62,20 @@ export async function createAdapterApp<Config extends BkndConfig = BkndConfig, A
const id = opts?.id ?? "app";
let app = apps.get(id);
if (!app || opts?.force) {
app = App.create(makeConfig(config, args));
const appConfig = makeConfig(config, args);
if (!appConfig.connection || !Connection.isConnection(appConfig.connection)) {
let connection: Connection | undefined;
if (Connection.isConnection(config.connection)) {
connection = config.connection;
} else {
const sqlite = (await import("bknd/adapter/sqlite")).sqlite;
connection = sqlite(config.connection ?? { url: ":memory:" });
$console.info(`Using ${connection.name} connection`);
}
appConfig.connection = connection;
}
app = App.create(appConfig);
apps.set(id, app);
}
return app;

View File

@@ -0,0 +1,57 @@
import {
buildQueryFn,
GenericSqliteConnection,
parseBigInt,
type IGenericSqlite,
} from "../../../data/connection/sqlite/GenericSqliteConnection";
import { DatabaseSync } from "node:sqlite";
export type NodeSqliteConnectionConfig = {
database: DatabaseSync;
};
function nodeSqliteExecutor(db: DatabaseSync): IGenericSqlite<DatabaseSync> {
const getStmt = (sql: string) => {
const stmt = db.prepare(sql);
//stmt.setReadBigInts(true);
return stmt;
};
return {
db,
query: buildQueryFn({
all: (sql, parameters = []) => getStmt(sql).all(...parameters),
run: (sql, parameters = []) => {
const { changes, lastInsertRowid } = getStmt(sql).run(...parameters);
return {
insertId: parseBigInt(lastInsertRowid),
numAffectedRows: parseBigInt(changes),
};
},
}),
close: () => db.close(),
iterator: (isSelect, sql, parameters = []) => {
if (!isSelect) {
throw new Error("Only support select in stream()");
}
return getStmt(sql).iterate(...parameters) as any;
},
};
}
export function nodeSqlite(config?: NodeSqliteConnectionConfig | { url: string }) {
let database: DatabaseSync;
if (config) {
if ("database" in config) {
database = config.database;
} else {
database = new DatabaseSync(config.url);
}
} else {
database = new DatabaseSync(":memory:");
}
return new GenericSqliteConnection(database, () => nodeSqliteExecutor(database), {
name: "node-sqlite",
});
}

View File

@@ -0,0 +1,11 @@
import { nodeSqlite } from "./NodeSqliteConnection";
import { DatabaseSync } from "node:sqlite";
import { connectionTestSuite } from "data/connection/connection-test-suite";
import { describe, test, expect } from "vitest";
describe("NodeSqliteConnection", () => {
connectionTestSuite({ describe, test, expect } as any, {
makeConnection: () => nodeSqlite({ database: new DatabaseSync(":memory:") }),
rawDialectDetails: [],
});
});

View File

@@ -1,18 +1,3 @@
import { registries } from "bknd";
import { type LocalAdapterConfig, StorageLocalAdapter } from "./storage/StorageLocalAdapter";
export * from "./node.adapter";
export { StorageLocalAdapter, type LocalAdapterConfig };
let registered = false;
export function registerLocalMediaAdapter() {
if (!registered) {
registries.media.register("local", StorageLocalAdapter);
registered = true;
}
return (config: Partial<LocalAdapterConfig> = {}) => {
const adapter = new StorageLocalAdapter(config);
return adapter.toJSON(true);
};
}
export * from "./storage";
export * from "./connection/NodeSqliteConnection";

View File

@@ -1,5 +1,5 @@
import { afterAll, beforeAll, describe } from "bun:test";
import * as node from "./node.adapter";
import { createApp, createHandler } from "./node.adapter";
import { adapterTestSuite } from "adapter/adapter-test-suite";
import { bunTestRunner } from "adapter/bun/test";
import { disableConsoleLog, enableConsoleLog } from "core/utils";
@@ -9,7 +9,7 @@ afterAll(enableConsoleLog);
describe("node adapter (bun)", () => {
adapterTestSuite(bunTestRunner, {
makeApp: node.createApp,
makeHandler: node.createHandler,
makeApp: createApp,
makeHandler: createHandler,
});
});

View File

@@ -1,10 +1,11 @@
import path from "node:path";
import { serve as honoServe } from "@hono/node-server";
import { serveStatic } from "@hono/node-server/serve-static";
import { registerLocalMediaAdapter } from "adapter/node/index";
import { registerLocalMediaAdapter } from "adapter/node/storage";
import { type RuntimeBkndConfig, createRuntimeApp, type RuntimeOptions } from "bknd/adapter";
import { config as $config } from "bknd/core";
import { $console } from "core";
import type { App } from "App";
type NodeEnv = NodeJS.ProcessEnv;
export type NodeBkndConfig<Env = NodeEnv> = RuntimeBkndConfig<Env> & {
@@ -45,8 +46,11 @@ export function createHandler<Env = NodeEnv>(
args: Env = {} as Env,
opts?: RuntimeOptions,
) {
let app: App | undefined;
return async (req: Request) => {
const app = await createApp(config, args ?? (process.env as Env), opts);
if (!app) {
app = await createApp(config, args ?? (process.env as Env), opts);
}
return app.fetch(req);
};
}

View File

@@ -1,14 +1,14 @@
import { describe, before, after } from "node:test";
import { describe, beforeAll, afterAll } from "vitest";
import * as node from "./node.adapter";
import { adapterTestSuite } from "adapter/adapter-test-suite";
import { nodeTestRunner } from "adapter/node/test";
import { viTestRunner } from "adapter/node/vitest";
import { disableConsoleLog, enableConsoleLog } from "core/utils";
before(() => disableConsoleLog());
after(enableConsoleLog);
beforeAll(() => disableConsoleLog());
afterAll(enableConsoleLog);
describe("node adapter", () => {
adapterTestSuite(nodeTestRunner, {
adapterTestSuite(viTestRunner, {
makeApp: node.createApp,
makeHandler: node.createHandler,
});

View File

@@ -1,5 +1,5 @@
import { describe } from "node:test";
import { nodeTestRunner } from "adapter/node/test";
import { describe } from "vitest";
import { viTestRunner } from "adapter/node/vitest";
import { StorageLocalAdapter } from "adapter/node";
import { adapterTestSuite } from "media/storage/adapters/adapter-test-suite";
import { readFileSync } from "node:fs";
@@ -14,5 +14,5 @@ describe("StorageLocalAdapter (node)", async () => {
path: path.join(basePath, "tmp"),
});
await adapterTestSuite(nodeTestRunner, adapter, file);
await adapterTestSuite(viTestRunner, adapter, file);
});

View File

@@ -0,0 +1,17 @@
import { registries } from "bknd";
import { type LocalAdapterConfig, StorageLocalAdapter } from "./StorageLocalAdapter";
export * from "./StorageLocalAdapter";
let registered = false;
export function registerLocalMediaAdapter() {
if (!registered) {
registries.media.register("local", StorageLocalAdapter);
registered = true;
}
return (config: Partial<LocalAdapterConfig> = {}) => {
const adapter = new StorageLocalAdapter(config);
return adapter.toJSON(true);
};
}

View File

@@ -1,5 +1,5 @@
import nodeAssert from "node:assert/strict";
import { test } from "node:test";
import { test, describe } from "node:test";
import type { Matcher, Test, TestFn, TestRunner } from "core/test";
// Track mock function calls
@@ -85,6 +85,7 @@ nodeTest.skipIf = (condition: boolean): Test => {
};
export const nodeTestRunner: TestRunner = {
describe,
test: nodeTest,
mock: createMockFunction,
expect: <T = unknown>(actual?: T, failMsg?: string) => ({

View File

@@ -0,0 +1,50 @@
import type { TestFn, TestRunner, Test } from "core/test";
import { describe, test, expect, vi } from "vitest";
function vitestTest(label: string, fn: TestFn, options?: any) {
return test(label, fn as any);
}
vitestTest.if = (condition: boolean): Test => {
if (condition) {
return vitestTest;
}
return (() => {}) as any;
};
vitestTest.skip = (label: string, fn: TestFn) => {
return test.skip(label, fn as any);
};
vitestTest.skipIf = (condition: boolean): Test => {
if (condition) {
return (() => {}) as any;
}
return vitestTest;
};
const vitestExpect = <T = unknown>(actual: T, parentFailMsg?: string) => {
return {
toEqual: (expected: T, failMsg = parentFailMsg) => {
expect(actual, failMsg).toEqual(expected);
},
toBe: (expected: T, failMsg = parentFailMsg) => {
expect(actual, failMsg).toBe(expected);
},
toBeString: () => expect(typeof actual, parentFailMsg).toBe("string"),
toBeUndefined: () => expect(actual, parentFailMsg).toBeUndefined(),
toBeDefined: () => expect(actual, parentFailMsg).toBeDefined(),
toBeOneOf: (expected: T | Array<T> | Iterable<T>, failMsg = parentFailMsg) => {
const e = Array.isArray(expected) ? expected : [expected];
expect(actual, failMsg).toBeOneOf(e);
},
toHaveBeenCalled: () => expect(actual, parentFailMsg).toHaveBeenCalled(),
toHaveBeenCalledTimes: (expected: number, failMsg = parentFailMsg) => {
expect(actual, failMsg).toHaveBeenCalledTimes(expected);
},
};
};
export const viTestRunner: TestRunner = {
describe,
test: vitestTest,
expect: vitestExpect as any,
mock: (fn) => vi.fn(fn),
};

View File

@@ -0,0 +1,6 @@
import type { Connection } from "bknd/data";
import { bunSqlite } from "../bun/connection/BunSqliteConnection";
export function sqlite(config: { url: string }): Connection {
return bunSqlite(config);
}

View File

@@ -0,0 +1,6 @@
import type { Connection } from "bknd/data";
import { libsql } from "../../data/connection/sqlite/LibsqlConnection";
export function sqlite(config: { url: string }): Connection {
return libsql(config);
}

View File

@@ -0,0 +1,6 @@
import type { Connection } from "bknd/data";
import { nodeSqlite } from "../node/connection/NodeSqliteConnection";
export function sqlite(config: { url: string }): Connection {
return nodeSqlite(config);
}

View File

@@ -0,0 +1,3 @@
import type { Connection } from "bknd/data";
export type SqliteConnection = (config: { url: string }) => Connection;