shishantbiswas/bknd
1
0
Fork 0
mirror of https://github.com/shishantbiswas/bknd.git synced 2026-03-16 20:37:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #186 from bknd-io/feat/unify-connections

Browse Source 
feat/unify-connections
This commit is contained in:
dswbx
2025-06-13 17:20:05 +02:00
committed by GitHub
parent 9c4aac8843 2239333f08
commit c09d1d53b3
86 changed files with 1816 additions and 1779 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
app/src/App.ts
View File

@@ -1,7 +1,7 @@
import type { CreateUserPayload } from "auth/AppAuth";
import { $console } from "core";
import { Event } from "core/events";
import { Connection, type LibSqlCredentials, LibsqlConnection } from "data";
import { Connection } from "data/connection/Connection";
import type { Hono } from "hono";
import {
ModuleManager,
@@ -14,10 +14,10 @@ import {
import * as SystemPermissions from "modules/permissions";
import { AdminController, type AdminControllerOptions } from "modules/server/AdminController";
import { SystemController } from "modules/server/SystemController";
import type { ServerEnv } from "modules/Controller";
// biome-ignore format: must be here
import { Api, type ApiOptions } from "Api";
import type { ServerEnv } from "modules/Controller";
export type AppPlugin = (app: App) => Promise<void> | void;
@@ -51,15 +51,8 @@ export type AppOptions = {
manager?: Omit<ModuleManagerOptions, "initial" | "onUpdated" | "seed">;
asyncEventsMode?: "sync" | "async" | "none";
};
export type CreateAppConfig = {
connection?:
| Connection
| {
// @deprecated
type: "libsql";
config: LibSqlCredentials;
}
| LibSqlCredentials;
export type CreateAppConfig<C extends Connection = Connection> = {
connection?: C | { url: string };
initialConfig?: InitialModuleConfigs;
options?: AppOptions;
};
@@ -67,7 +60,7 @@ export type CreateAppConfig = {
export type AppConfig = InitialModuleConfigs;
export type LocalApiOptions = Request | ApiOptions;
export class App {
export class App<C extends Connection = Connection> {
static readonly Events = AppEvents;
modules: ModuleManager;
@@ -79,7 +72,7 @@ export class App {
private _building: boolean = false;
constructor(
private connection: Connection,
public connection: C,
_initialConfig?: InitialModuleConfigs,
private options?: AppOptions,
) {
@@ -262,31 +255,9 @@ export class App {
}
export function createApp(config: CreateAppConfig = {}) {
let connection: Connection | undefined = undefined;
try {
if (Connection.isConnection(config.connection)) {
connection = config.connection;
} else if (typeof config.connection === "object") {
if ("type" in config.connection) {
$console.warn(
"Using deprecated connection type 'libsql', use the 'config' object directly.",
);
connection = new LibsqlConnection(config.connection.config);
} else {
connection = new LibsqlConnection(config.connection);
}
} else {
connection = new LibsqlConnection({ url: ":memory:" });
$console.warn("No connection provided, using in-memory database");
}
} catch (e) {
$console.error("Could not create connection", e);
}
if (!connection) {
if (!config.connection || !Connection.isConnection(config.connection)) {
throw new Error("Invalid connection");
}
return new App(connection, config.initialConfig, config.options);
return new App(config.connection, config.initialConfig, config.options);
}

10
app/src/adapter/bun/bun.adapter.ts
View File

@@ -2,10 +2,11 @@
import path from "node:path";
import { type RuntimeBkndConfig, createRuntimeApp, type RuntimeOptions } from "bknd/adapter";
import { registerLocalMediaAdapter } from "bknd/adapter/node";
import { registerLocalMediaAdapter } from ".";
import { config } from "bknd/core";
import type { ServeOptions } from "bun";
import { serveStatic } from "hono/bun";
import type { App } from "App";
type BunEnv = Bun.Env;
export type BunBkndConfig<Env = BunEnv> = RuntimeBkndConfig<Env> & Omit<ServeOptions, "fetch">;
@@ -33,8 +34,11 @@ export function createHandler<Env = BunEnv>(
args: Env = {} as Env,
opts?: RuntimeOptions,
) {
let app: App | undefined;
return async (req: Request) => {
const app = await createApp(config, args ?? (process.env as Env), opts);
if (!app) {
app = await createApp(config, args ?? (process.env as Env), opts);
}
return app.fetch(req);
};
}
@@ -72,5 +76,5 @@ export function serve<Env = BunEnv>(
),
});
console.log(`Server is running on http://localhost:${port}`);
console.info(`Server is running on http://localhost:${port}`);
}

12
app/src/adapter/bun/connection/BunSqliteConnection.spec.ts Normal file
View File

@@ -0,0 +1,12 @@
import { connectionTestSuite } from "data/connection/connection-test-suite";
import { bunSqlite } from "./BunSqliteConnection";
import { bunTestRunner } from "adapter/bun/test";
import { describe } from "bun:test";
import { Database } from "bun:sqlite";
describe("BunSqliteConnection", () => {
connectionTestSuite(bunTestRunner, {
makeConnection: () => bunSqlite({ database: new Database(":memory:") }),
rawDialectDetails: [],
});
});

48
app/src/adapter/bun/connection/BunSqliteConnection.ts Normal file
View File

@@ -0,0 +1,48 @@
import { Database } from "bun:sqlite";
import {
buildQueryFn,
GenericSqliteConnection,
parseBigInt,
type IGenericSqlite,
} from "data/connection/sqlite/GenericSqliteConnection";
export type BunSqliteConnectionConfig = {
database: Database;
};
function bunSqliteExecutor(db: Database, cache: boolean): IGenericSqlite<Database> {
const fn = cache ? "query" : "prepare";
const getStmt = (sql: string) => db[fn](sql);
return {
db,
query: buildQueryFn({
all: (sql, parameters) => getStmt(sql).all(...(parameters || [])),
run: (sql, parameters) => {
const { changes, lastInsertRowid } = getStmt(sql).run(...(parameters || []));
return {
insertId: parseBigInt(lastInsertRowid),
numAffectedRows: parseBigInt(changes),
};
},
}),
close: () => db.close(),
};
}
export function bunSqlite(config?: BunSqliteConnectionConfig | { url: string }) {
let database: Database;
if (config) {
if ("database" in config) {
database = config.database;
} else {
database = new Database(config.url);
}
} else {
database = new Database(":memory:");
}
return new GenericSqliteConnection(database, () => bunSqliteExecutor(database, false), {
name: "bun-sqlite",
});
}

2
app/src/adapter/bun/index.ts
View File

@@ -1 +1,3 @@
export * from "./bun.adapter";
export * from "../node/storage";
export * from "./connection/BunSqliteConnection";

3
app/src/adapter/bun/test.ts
View File

@@ -1,6 +1,7 @@
import { expect, test, mock } from "bun:test";
import { expect, test, mock, describe } from "bun:test";
export const bunTestRunner = {
describe,
expect,
test,
mock,

40
app/src/adapter/cloudflare/cloudflare-workers.adapter.spec.ts
View File

@@ -13,30 +13,32 @@ describe("cf adapter", () => {
const DB_URL = ":memory:";
const $ctx = (env?: any, request?: Request, ctx?: ExecutionContext) => ({
request: request ?? (null as any),
env: env ?? { DB_URL },
env: env ?? { url: DB_URL },
ctx: ctx ?? (null as any),
});
it("makes config", async () => {
expect(
makeConfig(
{
connection: { url: DB_URL },
},
$ctx({ DB_URL }),
),
).toEqual({ connection: { url: DB_URL } });
const staticConfig = makeConfig(
{
connection: { url: DB_URL },
initialConfig: { data: { basepath: DB_URL } },
},
$ctx({ DB_URL }),
);
expect(staticConfig.initialConfig).toEqual({ data: { basepath: DB_URL } });
expect(staticConfig.connection).toBeDefined();
expect(
makeConfig(
{
app: (env) => ({
connection: { url: env.DB_URL },
}),
},
$ctx({ DB_URL }),
),
).toEqual({ connection: { url: DB_URL } });
const dynamicConfig = makeConfig(
{
app: (env) => ({
initialConfig: { data: { basepath: env.DB_URL } },
connection: { url: env.DB_URL },
}),
},
$ctx({ DB_URL }),
);
expect(dynamicConfig.initialConfig).toEqual({ data: { basepath: DB_URL } });
expect(dynamicConfig.connection).toBeDefined();
});
adapterTestSuite<CloudflareBkndConfig, CfMakeConfigArgs<any>>(bunTestRunner, {

87
app/src/adapter/cloudflare/config.ts
View File

@@ -9,6 +9,7 @@ import { makeConfig as makeAdapterConfig } from "bknd/adapter";
import type { Context, ExecutionContext } from "hono";
import { $console } from "core";
import { setCookie } from "hono/cookie";
import { sqlite } from "bknd/adapter/sqlite";
export const constants = {
exec_async_event_id: "cf_register_waituntil",
@@ -98,54 +99,70 @@ export function makeConfig<Env extends CloudflareEnv = CloudflareEnv>(
const appConfig = makeAdapterConfig(config, args?.env);
if (args?.env) {
const bindings = config.bindings?.(args?.env);
// if connection instance is given, don't do anything
// other than checking if D1 session is defined
if (D1Connection.isConnection(appConfig.connection)) {
if (config.d1?.session) {
// we cannot guarantee that db was opened with session
throw new Error(
"D1 session don't work when D1 is directly given as connection. Define it in bindings instead.",
);
}
// if connection is given, try to open with unified sqlite adapter
} else if (appConfig.connection) {
appConfig.connection = sqlite(appConfig.connection);
// if connection is not given, but env is set
// try to make D1 from bindings
} else if (args?.env) {
const bindings = config.bindings?.(args?.env);
const sessionHelper = d1SessionHelper(config);
const sessionId = sessionHelper.get(args.request);
let session: D1DatabaseSession | undefined;
let db: D1Database | undefined;
if (!appConfig.connection) {
let db: D1Database | undefined;
if (bindings?.db) {
$console.log("Using database from bindings");
db = bindings.db;
} else if (Object.keys(args).length > 0) {
const binding = getBinding(args.env, "D1Database");
if (binding) {
$console.log(`Using database from env "${binding.key}"`);
db = binding.value;
}
}
// if db is given in bindings, use it
if (bindings?.db) {
$console.log("Using database from bindings");
db = bindings.db;
if (db) {
if (config.d1?.session) {
session = db.withSession(sessionId ?? config.d1?.first);
appConfig.connection = new D1Connection({ binding: session });
} else {
appConfig.connection = new D1Connection({ binding: db });
}
} else {
throw new Error("No database connection given");
// scan for D1Database in args
} else {
const binding = getBinding(args.env, "D1Database");
if (binding) {
$console.log(`Using database from env "${binding.key}"`);
db = binding.value;
}
}
if (config.d1?.session) {
appConfig.options = {
...appConfig.options,
manager: {
...appConfig.options?.manager,
onServerInit: (server) => {
server.use(async (c, next) => {
sessionHelper.set(c, session);
await next();
});
// if db is found, check if session is requested
if (db) {
if (config.d1?.session) {
session = db.withSession(sessionId ?? config.d1?.first);
appConfig.connection = new D1Connection({ binding: session });
appConfig.options = {
...appConfig.options,
manager: {
...appConfig.options?.manager,
onServerInit: (server) => {
server.use(async (c, next) => {
sessionHelper.set(c, session);
await next();
});
},
},
},
};
};
} else {
appConfig.connection = new D1Connection({ binding: db });
}
}
}
if (!D1Connection.isConnection(appConfig.connection)) {
throw new Error("Couldn't find database connection");
}
return appConfig;
}

51
app/src/adapter/cloudflare/connection/D1Connection.ts
View File

@@ -1,65 +1,42 @@
/// <reference types="@cloudflare/workers-types" />
import { KyselyPluginRunner, SqliteConnection, SqliteIntrospector } from "bknd/data";
import type { QB } from "data/connection/Connection";
import { type DatabaseIntrospector, Kysely, ParseJSONResultsPlugin } from "kysely";
import { SqliteConnection } from "bknd/data";
import type { ConnQuery, ConnQueryResults } from "data/connection/Connection";
import { D1Dialect } from "kysely-d1";
export type D1ConnectionConfig<DB extends D1Database | D1DatabaseSession = D1Database> = {
binding: DB;
};
class CustomD1Dialect extends D1Dialect {
override createIntrospector(db: Kysely<any>): DatabaseIntrospector {
return new SqliteIntrospector(db, {
excludeTables: ["_cf_KV", "_cf_METADATA"],
});
}
}
export class D1Connection<
DB extends D1Database | D1DatabaseSession = D1Database,
> extends SqliteConnection {
> extends SqliteConnection<DB> {
override name = "sqlite-d1";
protected override readonly supported = {
batching: true,
softscans: false,
};
constructor(private config: D1ConnectionConfig<DB>) {
const plugins = [new ParseJSONResultsPlugin()];
const kysely = new Kysely({
dialect: new CustomD1Dialect({ database: config.binding as D1Database }),
plugins,
super({
excludeTables: ["_cf_KV", "_cf_METADATA"],
dialect: D1Dialect,
dialectArgs: [{ database: config.binding as D1Database }],
});
super(kysely, {}, plugins);
}
get client(): DB {
return this.config.binding;
}
override async executeQueries<O extends ConnQuery[]>(...qbs: O): Promise<ConnQueryResults<O>> {
const compiled = this.getCompiled(...qbs);
protected override async batch<Queries extends QB[]>(
queries: [...Queries],
): Promise<{
[K in keyof Queries]: Awaited<ReturnType<Queries[K]["execute"]>>;
}> {
const db = this.config.binding;
const res = await db.batch(
queries.map((q) => {
const { sql, parameters } = q.compile();
compiled.map(({ sql, parameters }) => {
return db.prepare(sql).bind(...parameters);
}),
);
// let it run through plugins
const kyselyPlugins = new KyselyPluginRunner(this.plugins);
const data: any = [];
for (const r of res) {
const rows = await kyselyPlugins.transformResultRows(r.results);
data.push(rows);
}
return data;
return this.withTransformedRows(res, "results") as any;
}
}

2
app/src/adapter/cloudflare/modes/durable.ts
View File

@@ -64,7 +64,7 @@ export class DurableBkndApp extends DurableObject {
"type" in config.connection &&
config.connection.type === "libsql"
) {
config.connection.config.protocol = "wss";
//config.connection.config.protocol = "wss";
}
this.app = await createRuntimeApp({

32
app/src/adapter/cloudflare/storage/StorageR2Adapter.native-spec.ts
View File

@@ -1,32 +0,0 @@
import { createWriteStream, readFileSync } from "node:fs";
import { test } from "node:test";
import { Miniflare } from "miniflare";
import { StorageR2Adapter } from "./StorageR2Adapter";
import { adapterTestSuite } from "media";
import { nodeTestRunner } from "adapter/node/test";
import path from "node:path";
// https://github.com/nodejs/node/issues/44372#issuecomment-1736530480
console.log = async (message: any) => {
const tty = createWriteStream("/dev/tty");
const msg = typeof message === "string" ? message : JSON.stringify(message, null, 2);
return tty.write(`${msg}\n`);
};
test("StorageR2Adapter", async () => {
const mf = new Miniflare({
modules: true,
script: "export default { async fetch() { return new Response(null); } }",
r2Buckets: ["BUCKET"],
});
const bucket = (await mf.getR2Bucket("BUCKET")) as unknown as R2Bucket;
const adapter = new StorageR2Adapter(bucket);
const basePath = path.resolve(import.meta.dirname, "../../../../__test__/_assets");
const buffer = readFileSync(path.join(basePath, "image.png"));
const file = new File([buffer], "image.png", { type: "image/png" });
await adapterTestSuite(nodeTestRunner, adapter, file);
await mf.dispose();
});

32
app/src/adapter/cloudflare/storage/StorageR2Adapter.vitest.ts Normal file
View File

@@ -0,0 +1,32 @@
import { readFileSync } from "node:fs";
import { Miniflare } from "miniflare";
import { StorageR2Adapter } from "./StorageR2Adapter";
import { adapterTestSuite } from "media/storage/adapters/adapter-test-suite";
import path from "node:path";
import { describe, afterAll, test, expect } from "vitest";
import { viTestRunner } from "adapter/node/vitest";
let mf: Miniflare | undefined;
describe("StorageR2Adapter", async () => {
mf = new Miniflare({
modules: true,
script: "export default { async fetch() { return new Response(null); } }",
r2Buckets: ["BUCKET"],
});
const bucket = (await mf?.getR2Bucket("BUCKET")) as unknown as R2Bucket;
test("test", () => {
expect(bucket).toBeDefined();
});
const adapter = new StorageR2Adapter(bucket);
const basePath = path.resolve(import.meta.dirname, "../../../../__test__/_assets");
const buffer = readFileSync(path.join(basePath, "image.png"));
const file = new File([buffer], "image.png", { type: "image/png" });
await adapterTestSuite(viTestRunner, adapter, file);
});
afterAll(async () => {
await mf?.dispose();
});

14
app/src/adapter/cloudflare/vitest.config.ts Normal file
View File

@@ -0,0 +1,14 @@
import { defineWorkersConfig } from "@cloudflare/vitest-pool-workers/config";
export default defineWorkersConfig({
test: {
poolOptions: {
workers: {
miniflare: {
compatibilityDate: "2025-06-04",
},
},
},
include: ["**/*.vi-test.ts", "**/*.vitest.ts"],
},
});

20
app/src/adapter/index.ts
View File

@@ -1,7 +1,10 @@
import { App, type CreateAppConfig } from "bknd";
import { config as $config } from "bknd/core";
import { config as $config, $console } from "bknd/core";
import type { MiddlewareHandler } from "hono";
import type { AdminControllerOptions } from "modules/server/AdminController";
import { Connection } from "bknd/data";
export { Connection } from "bknd/data";
export type BkndConfig<Args = any> = CreateAppConfig & {
app?: CreateAppConfig | ((args: Args) => CreateAppConfig);
@@ -59,7 +62,20 @@ export async function createAdapterApp<Config extends BkndConfig = BkndConfig, A
const id = opts?.id ?? "app";
let app = apps.get(id);
if (!app || opts?.force) {
app = App.create(makeConfig(config, args));
const appConfig = makeConfig(config, args);
if (!appConfig.connection || !Connection.isConnection(appConfig.connection)) {
let connection: Connection | undefined;
if (Connection.isConnection(config.connection)) {
connection = config.connection;
} else {
const sqlite = (await import("bknd/adapter/sqlite")).sqlite;
connection = sqlite(config.connection ?? { url: ":memory:" });
$console.info(`Using ${connection.name} connection`);
}
appConfig.connection = connection;
}
app = App.create(appConfig);
apps.set(id, app);
}
return app;

57
app/src/adapter/node/connection/NodeSqliteConnection.ts Normal file
View File

@@ -0,0 +1,57 @@
import {
buildQueryFn,
GenericSqliteConnection,
parseBigInt,
type IGenericSqlite,
} from "../../../data/connection/sqlite/GenericSqliteConnection";
import { DatabaseSync } from "node:sqlite";
export type NodeSqliteConnectionConfig = {
database: DatabaseSync;
};
function nodeSqliteExecutor(db: DatabaseSync): IGenericSqlite<DatabaseSync> {
const getStmt = (sql: string) => {
const stmt = db.prepare(sql);
//stmt.setReadBigInts(true);
return stmt;
};
return {
db,
query: buildQueryFn({
all: (sql, parameters = []) => getStmt(sql).all(...parameters),
run: (sql, parameters = []) => {
const { changes, lastInsertRowid } = getStmt(sql).run(...parameters);
return {
insertId: parseBigInt(lastInsertRowid),
numAffectedRows: parseBigInt(changes),
};
},
}),
close: () => db.close(),
iterator: (isSelect, sql, parameters = []) => {
if (!isSelect) {
throw new Error("Only support select in stream()");
}
return getStmt(sql).iterate(...parameters) as any;
},
};
}
export function nodeSqlite(config?: NodeSqliteConnectionConfig | { url: string }) {
let database: DatabaseSync;
if (config) {
if ("database" in config) {
database = config.database;
} else {
database = new DatabaseSync(config.url);
}
} else {
database = new DatabaseSync(":memory:");
}
return new GenericSqliteConnection(database, () => nodeSqliteExecutor(database), {
name: "node-sqlite",
});
}

11
app/src/adapter/node/connection/NodeSqliteConnection.vi-test.ts Normal file
View File

@@ -0,0 +1,11 @@
import { nodeSqlite } from "./NodeSqliteConnection";
import { DatabaseSync } from "node:sqlite";
import { connectionTestSuite } from "data/connection/connection-test-suite";
import { describe, test, expect } from "vitest";
describe("NodeSqliteConnection", () => {
connectionTestSuite({ describe, test, expect } as any, {
makeConnection: () => nodeSqlite({ database: new DatabaseSync(":memory:") }),
rawDialectDetails: [],
});
});

19
app/src/adapter/node/index.ts
View File

@@ -1,18 +1,3 @@
import { registries } from "bknd";
import { type LocalAdapterConfig, StorageLocalAdapter } from "./storage/StorageLocalAdapter";
export * from "./node.adapter";
export { StorageLocalAdapter, type LocalAdapterConfig };
let registered = false;
export function registerLocalMediaAdapter() {
if (!registered) {
registries.media.register("local", StorageLocalAdapter);
registered = true;
}
return (config: Partial<LocalAdapterConfig> = {}) => {
const adapter = new StorageLocalAdapter(config);
return adapter.toJSON(true);
};
}
export * from "./storage";
export * from "./connection/NodeSqliteConnection";

6
app/src/adapter/node/node.adapter.spec.ts
View File

@@ -1,5 +1,5 @@
import { afterAll, beforeAll, describe } from "bun:test";
import * as node from "./node.adapter";
import { createApp, createHandler } from "./node.adapter";
import { adapterTestSuite } from "adapter/adapter-test-suite";
import { bunTestRunner } from "adapter/bun/test";
import { disableConsoleLog, enableConsoleLog } from "core/utils";
@@ -9,7 +9,7 @@ afterAll(enableConsoleLog);
describe("node adapter (bun)", () => {
adapterTestSuite(bunTestRunner, {
makeApp: node.createApp,
makeHandler: node.createHandler,
makeApp: createApp,
makeHandler: createHandler,
});
});

8
app/src/adapter/node/node.adapter.ts
View File

@@ -1,10 +1,11 @@
import path from "node:path";
import { serve as honoServe } from "@hono/node-server";
import { serveStatic } from "@hono/node-server/serve-static";
import { registerLocalMediaAdapter } from "adapter/node/index";
import { registerLocalMediaAdapter } from "adapter/node/storage";
import { type RuntimeBkndConfig, createRuntimeApp, type RuntimeOptions } from "bknd/adapter";
import { config as $config } from "bknd/core";
import { $console } from "core";
import type { App } from "App";
type NodeEnv = NodeJS.ProcessEnv;
export type NodeBkndConfig<Env = NodeEnv> = RuntimeBkndConfig<Env> & {
@@ -45,8 +46,11 @@ export function createHandler<Env = NodeEnv>(
args: Env = {} as Env,
opts?: RuntimeOptions,
) {
let app: App | undefined;
return async (req: Request) => {
const app = await createApp(config, args ?? (process.env as Env), opts);
if (!app) {
app = await createApp(config, args ?? (process.env as Env), opts);
}
return app.fetch(req);
};
}

10
app/src/adapter/node/node.adapter.native-spec.ts → app/src/adapter/node/node.adapter.vi-test.ts
View File

@@ -1,14 +1,14 @@
import { describe, before, after } from "node:test";
import { describe, beforeAll, afterAll } from "vitest";
import * as node from "./node.adapter";
import { adapterTestSuite } from "adapter/adapter-test-suite";
import { nodeTestRunner } from "adapter/node/test";
import { viTestRunner } from "adapter/node/vitest";
import { disableConsoleLog, enableConsoleLog } from "core/utils";
before(() => disableConsoleLog());
after(enableConsoleLog);
beforeAll(() => disableConsoleLog());
afterAll(enableConsoleLog);
describe("node adapter", () => {
adapterTestSuite(nodeTestRunner, {
adapterTestSuite(viTestRunner, {
makeApp: node.createApp,
makeHandler: node.createHandler,
});

6
app/src/adapter/node/storage/StorageLocalAdapter.native-spec.ts → app/src/adapter/node/storage/StorageLocalAdapter.vitest.ts
View File

@@ -1,5 +1,5 @@
import { describe } from "node:test";
import { nodeTestRunner } from "adapter/node/test";
import { describe } from "vitest";
import { viTestRunner } from "adapter/node/vitest";
import { StorageLocalAdapter } from "adapter/node";
import { adapterTestSuite } from "media/storage/adapters/adapter-test-suite";
import { readFileSync } from "node:fs";
@@ -14,5 +14,5 @@ describe("StorageLocalAdapter (node)", async () => {
path: path.join(basePath, "tmp"),
});
await adapterTestSuite(nodeTestRunner, adapter, file);
await adapterTestSuite(viTestRunner, adapter, file);
});

17
app/src/adapter/node/storage/index.ts Normal file
View File

@@ -0,0 +1,17 @@
import { registries } from "bknd";
import { type LocalAdapterConfig, StorageLocalAdapter } from "./StorageLocalAdapter";
export * from "./StorageLocalAdapter";
let registered = false;
export function registerLocalMediaAdapter() {
if (!registered) {
registries.media.register("local", StorageLocalAdapter);
registered = true;
}
return (config: Partial<LocalAdapterConfig> = {}) => {
const adapter = new StorageLocalAdapter(config);
return adapter.toJSON(true);
};
}

3
app/src/adapter/node/test.ts
View File

@@ -1,5 +1,5 @@
import nodeAssert from "node:assert/strict";
import { test } from "node:test";
import { test, describe } from "node:test";
import type { Matcher, Test, TestFn, TestRunner } from "core/test";
// Track mock function calls
@@ -85,6 +85,7 @@ nodeTest.skipIf = (condition: boolean): Test => {
};
export const nodeTestRunner: TestRunner = {
describe,
test: nodeTest,
mock: createMockFunction,
expect: <T = unknown>(actual?: T, failMsg?: string) => ({

50
app/src/adapter/node/vitest.ts Normal file
View File

@@ -0,0 +1,50 @@
import type { TestFn, TestRunner, Test } from "core/test";
import { describe, test, expect, vi } from "vitest";
function vitestTest(label: string, fn: TestFn, options?: any) {
return test(label, fn as any);
}
vitestTest.if = (condition: boolean): Test => {
if (condition) {
return vitestTest;
}
return (() => {}) as any;
};
vitestTest.skip = (label: string, fn: TestFn) => {
return test.skip(label, fn as any);
};
vitestTest.skipIf = (condition: boolean): Test => {
if (condition) {
return (() => {}) as any;
}
return vitestTest;
};
const vitestExpect = <T = unknown>(actual: T, parentFailMsg?: string) => {
return {
toEqual: (expected: T, failMsg = parentFailMsg) => {
expect(actual, failMsg).toEqual(expected);
},
toBe: (expected: T, failMsg = parentFailMsg) => {
expect(actual, failMsg).toBe(expected);
},
toBeString: () => expect(typeof actual, parentFailMsg).toBe("string"),
toBeUndefined: () => expect(actual, parentFailMsg).toBeUndefined(),
toBeDefined: () => expect(actual, parentFailMsg).toBeDefined(),
toBeOneOf: (expected: T | Array<T> | Iterable<T>, failMsg = parentFailMsg) => {
const e = Array.isArray(expected) ? expected : [expected];
expect(actual, failMsg).toBeOneOf(e);
},
toHaveBeenCalled: () => expect(actual, parentFailMsg).toHaveBeenCalled(),
toHaveBeenCalledTimes: (expected: number, failMsg = parentFailMsg) => {
expect(actual, failMsg).toHaveBeenCalledTimes(expected);
},
};
};
export const viTestRunner: TestRunner = {
describe,
test: vitestTest,
expect: vitestExpect as any,
mock: (fn) => vi.fn(fn),
};

6
app/src/adapter/sqlite/bun.ts Normal file
View File

@@ -0,0 +1,6 @@
import type { Connection } from "bknd/data";
import { bunSqlite } from "../bun/connection/BunSqliteConnection";
export function sqlite(config: { url: string }): Connection {
return bunSqlite(config);
}

6
app/src/adapter/sqlite/edge.ts Normal file
View File

@@ -0,0 +1,6 @@
import type { Connection } from "bknd/data";
import { libsql } from "../../data/connection/sqlite/LibsqlConnection";
export function sqlite(config: { url: string }): Connection {
return libsql(config);
}

6
app/src/adapter/sqlite/node.ts Normal file
View File

@@ -0,0 +1,6 @@
import type { Connection } from "bknd/data";
import { nodeSqlite } from "../node/connection/NodeSqliteConnection";
export function sqlite(config: { url: string }): Connection {
return nodeSqlite(config);
}

3
app/src/adapter/sqlite/types.ts Normal file
View File

@@ -0,0 +1,3 @@
import type { Connection } from "bknd/data";
export type SqliteConnection = (config: { url: string }) => Connection;

7
app/src/cli/commands/run/platform.ts
View File

@@ -1,6 +1,5 @@
import path from "node:path";
import type { Config } from "@libsql/client/node";
import { $console, config } from "core";
import { $console } from "core";
import type { MiddlewareHandler } from "hono";
import open from "open";
import { fileExists, getRelativeDistPath } from "../../utils/sys";
@@ -27,10 +26,6 @@ export async function serveStatic(server: Platform): Promise<MiddlewareHandler>
}
}
export async function attachServeStatic(app: any, platform: Platform) {
app.module.server.client.get(config.server.assets_path + "*", await serveStatic(platform));
}
export async function startServer(
server: Platform,
app: App,

42
app/src/cli/commands/run/run.ts
View File

@@ -1,6 +1,6 @@
import type { Config } from "@libsql/client/node";
import { App, type CreateAppConfig } from "App";
import { StorageLocalAdapter } from "adapter/node";
import type { App, CreateAppConfig } from "App";
import { StorageLocalAdapter } from "adapter/node/storage";
import type { CliBkndConfig, CliCommand } from "cli/types";
import { Option } from "commander";
import { colorizeConsole, config } from "core";
@@ -11,19 +11,19 @@ import path from "node:path";
import {
PLATFORMS,
type Platform,
attachServeStatic,
getConfigPath,
getConnectionCredentialsFromEnv,
serveStatic,
startServer,
} from "./platform";
import { makeConfig } from "adapter";
import { isBun as $isBun } from "cli/utils/sys";
import { createRuntimeApp, makeConfig } from "adapter";
import { isBun } from "core/utils";
const env_files = [".env", ".dev.vars"];
dotenv.config({
path: env_files.map((file) => path.resolve(process.cwd(), file)),
});
const isBun = $isBun();
const is_bun = isBun();
export const run: CliCommand = (program) => {
program
@@ -44,15 +44,14 @@ export const run: CliCommand = (program) => {
)
.addOption(new Option("-c, --config <config>", "config file"))
.addOption(
new Option("--db-url <db>", "database url, can be any valid libsql url").conflicts(
new Option("--db-url <db>", "database url, can be any valid sqlite url").conflicts(
"config",
),
)
.addOption(new Option("--db-token <db>", "database token").conflicts("config"))
.addOption(
new Option("--server <server>", "server type")
.choices(PLATFORMS)
.default(isBun ? "bun" : "node"),
.default(is_bun ? "bun" : "node"),
)
.addOption(new Option("--no-open", "don't open browser window on start"))
.action(action);
@@ -72,23 +71,9 @@ type MakeAppConfig = {
};
async function makeApp(config: MakeAppConfig) {
const app = App.create({ connection: config.connection });
app.emgr.onEvent(
App.Events.AppBuiltEvent,
async () => {
if (config.onBuilt) {
await config.onBuilt(app);
}
await attachServeStatic(app, config.server?.platform ?? "node");
app.registerAdminController();
},
"sync",
);
await app.build();
return app;
return await createRuntimeApp({
serveStatic: await serveStatic(config.server?.platform ?? "node"),
});
}
export async function makeConfigApp(_config: CliBkndConfig, platform?: Platform) {
@@ -104,7 +89,6 @@ type RunOptions = {
memory?: boolean;
config?: string;
dbUrl?: string;
dbToken?: string;
server: Platform;
open?: boolean;
};
@@ -116,9 +100,7 @@ export async function makeAppFromEnv(options: Partial<RunOptions> = {}) {
// first start from arguments if given
if (options.dbUrl) {
console.info("Using connection from", c.cyan("--db-url"));
const connection = options.dbUrl
? { url: options.dbUrl, authToken: options.dbToken }
: undefined;
const connection = options.dbUrl ? { url: options.dbUrl } : undefined;
app = await makeApp({ connection, server: { platform: options.server } });
// check configuration file to be present

2
app/src/cli/commands/user.ts
View File

@@ -11,7 +11,7 @@ import type { CliCommand } from "cli/types";
import { Argument } from "commander";
import { $console } from "core";
import c from "picocolors";
import { isBun } from "cli/utils/sys";
import { isBun } from "core/utils";
export const user: CliCommand = (program) => {
program

8
app/src/cli/utils/sys.ts
View File

@@ -4,14 +4,6 @@ import { readFile, writeFile as nodeWriteFile } from "node:fs/promises";
import path from "node:path";
import url from "node:url";
export function isBun(): boolean {
try {
return typeof Bun !== "undefined";
} catch (e) {
return false;
}
}
export function getRootPath() {
const _path = path.dirname(url.fileURLToPath(import.meta.url));
// because of "src", local needs one more level up

1
app/src/core/test/index.ts
View File

@@ -16,6 +16,7 @@ export interface Test {
skipIf: (condition: boolean) => (label: string, fn: TestFn) => void;
}
export type TestRunner = {
describe: (label: string, asyncFn: () => Promise<void>) => void;
test: Test;
mock: <T extends (...args: any[]) => any>(fn: T) => T | any;
expect: <T = unknown>(

12
app/src/core/test/utils.ts Normal file
View File

@@ -0,0 +1,12 @@
import { createApp as createAppInternal, type CreateAppConfig } from "App";
import { bunSqlite } from "adapter/bun/connection/BunSqliteConnection";
import { Connection } from "data/connection/Connection";
export { App } from "App";
export function createApp({ connection, ...config }: CreateAppConfig = {}) {
return createAppInternal({
...config,
connection: Connection.isConnection(connection) ? connection : bunSqlite(connection as any),
});
}

8
app/src/core/utils/runtime.ts
View File

@@ -48,6 +48,14 @@ export function isNode() {
}
}
export function isBun() {
try {
return typeof Bun !== "undefined";
} catch (e) {
return false;
}
}
export function invariant(condition: boolean | any, message: string) {
if (!condition) {
throw new Error(message);

29
app/src/data/api/DataApi.ts
View File

@@ -1,5 +1,5 @@
import type { DB } from "core";
import type { EntityData, RepoQueryIn, RepositoryResponse } from "data";
import type { EntityData, RepoQueryIn, RepositoryResultJSON } from "data";
import type { Insertable, Selectable, Updateable } from "kysely";
import { type BaseModuleApiOptions, ModuleApi, type PrimaryFieldType } from "modules";
import type { FetchPromise, ResponseObject } from "modules/ModuleApi";
@@ -32,10 +32,7 @@ export class DataApi extends ModuleApi<DataApiOptions> {
query: Omit<RepoQueryIn, "where" | "limit" | "offset"> = {},
) {
type Data = E extends keyof DB ? Selectable<DB[E]> : EntityData;
return this.get<Pick<RepositoryResponse<Data>, "meta" | "data">>(
["entity", entity as any, id],
query,
);
return this.get<RepositoryResultJSON<Data>>(["entity", entity as any, id], query);
}
readOneBy<E extends keyof DB | string>(
@@ -43,7 +40,7 @@ export class DataApi extends ModuleApi<DataApiOptions> {
query: Omit<RepoQueryIn, "limit" | "offset" | "sort"> = {},
) {
type Data = E extends keyof DB ? Selectable<DB[E]> : EntityData;
type T = Pick<RepositoryResponse<Data>, "meta" | "data">;
type T = RepositoryResultJSON<Data>;
return this.readMany(entity, {
...query,
limit: 1,
@@ -53,7 +50,7 @@ export class DataApi extends ModuleApi<DataApiOptions> {
readMany<E extends keyof DB | string>(entity: E, query: RepoQueryIn = {}) {
type Data = E extends keyof DB ? Selectable<DB[E]> : EntityData;
type T = Pick<RepositoryResponse<Data[]>, "meta" | "data">;
type T = RepositoryResultJSON<Data[]>;
const input = query ?? this.options.defaultQuery;
const req = this.get<T>(["entity", entity as any], input);
@@ -72,7 +69,7 @@ export class DataApi extends ModuleApi<DataApiOptions> {
query: RepoQueryIn = {},
) {
type Data = R extends keyof DB ? Selectable<DB[R]> : EntityData;
return this.get<Pick<RepositoryResponse<Data[]>, "meta" | "data">>(
return this.get<RepositoryResultJSON<Data[]>>(
["entity", entity as any, id, reference],
query ?? this.options.defaultQuery,
);
@@ -83,7 +80,7 @@ export class DataApi extends ModuleApi<DataApiOptions> {
input: Insertable<Input>,
) {
type Data = E extends keyof DB ? Selectable<DB[E]> : EntityData;
return this.post<RepositoryResponse<Data>>(["entity", entity as any], input);
return this.post<RepositoryResultJSON<Data>>(["entity", entity as any], input);
}
createMany<E extends keyof DB | string, Input = E extends keyof DB ? DB[E] : EntityData>(
@@ -94,7 +91,7 @@ export class DataApi extends ModuleApi<DataApiOptions> {
throw new Error("input is required");
}
type Data = E extends keyof DB ? Selectable<DB[E]> : EntityData;
return this.post<RepositoryResponse<Data[]>>(["entity", entity as any], input);
return this.post<RepositoryResultJSON<Data[]>>(["entity", entity as any], input);
}
updateOne<E extends keyof DB | string, Input = E extends keyof DB ? DB[E] : EntityData>(
@@ -104,7 +101,7 @@ export class DataApi extends ModuleApi<DataApiOptions> {
) {
if (!id) throw new Error("ID is required");
type Data = E extends keyof DB ? Selectable<DB[E]> : EntityData;
return this.patch<RepositoryResponse<Data>>(["entity", entity as any, id], input);
return this.patch<RepositoryResultJSON<Data>>(["entity", entity as any, id], input);
}
updateMany<E extends keyof DB | string, Input = E extends keyof DB ? DB[E] : EntityData>(
@@ -114,7 +111,7 @@ export class DataApi extends ModuleApi<DataApiOptions> {
) {
this.requireObjectSet(where);
type Data = E extends keyof DB ? Selectable<DB[E]> : EntityData;
return this.patch<RepositoryResponse<Data[]>>(["entity", entity as any], {
return this.patch<RepositoryResultJSON<Data[]>>(["entity", entity as any], {
update,
where,
});
@@ -123,24 +120,24 @@ export class DataApi extends ModuleApi<DataApiOptions> {
deleteOne<E extends keyof DB | string>(entity: E, id: PrimaryFieldType) {
if (!id) throw new Error("ID is required");
type Data = E extends keyof DB ? Selectable<DB[E]> : EntityData;
return this.delete<RepositoryResponse<Data>>(["entity", entity as any, id]);
return this.delete<RepositoryResultJSON<Data>>(["entity", entity as any, id]);
}
deleteMany<E extends keyof DB | string>(entity: E, where: RepoQueryIn["where"]) {
this.requireObjectSet(where);
type Data = E extends keyof DB ? Selectable<DB[E]> : EntityData;
return this.delete<RepositoryResponse<Data>>(["entity", entity as any], where);
return this.delete<RepositoryResultJSON<Data>>(["entity", entity as any], where);
}
count<E extends keyof DB | string>(entity: E, where: RepoQueryIn["where"] = {}) {
return this.post<RepositoryResponse<{ entity: E; count: number }>>(
return this.post<RepositoryResultJSON<{ entity: E; count: number }>>(
["entity", entity as any, "fn", "count"],
where,
);
}
exists<E extends keyof DB | string>(entity: E, where: RepoQueryIn["where"] = {}) {
return this.post<RepositoryResponse<{ entity: E; exists: boolean }>>(
return this.post<RepositoryResultJSON<{ entity: E; exists: boolean }>>(
["entity", entity as any, "fn", "exists"],
where,
);

53
app/src/data/api/DataController.ts
View File

@@ -3,9 +3,7 @@ import {
DataPermissions,
type EntityData,
type EntityManager,
type MutatorResponse,
type RepoQuery,
type RepositoryResponse,
repoQuery,
} from "data";
import type { Handler } from "hono/types";
@@ -32,33 +30,6 @@ export class DataController extends Controller {
return this.ctx.guard;
}
repoResult<T extends RepositoryResponse<any> = RepositoryResponse>(
res: T,
): Pick<T, "meta" | "data"> {
let meta: Partial<RepositoryResponse["meta"]> = {};
if ("meta" in res) {
const { query, ...rest } = res.meta;
meta = rest;
if (isDebug()) meta.query = query;
}
const template = { data: res.data, meta };
// @todo: this works but it breaks in FE (need to improve DataTable)
// filter empty
return Object.fromEntries(
Object.entries(template).filter(([_, v]) => typeof v !== "undefined" && v !== null),
) as any;
}
mutatorResult(res: MutatorResponse | MutatorResponse<EntityData>) {
const template = { data: res.data };
// filter empty
return Object.fromEntries(Object.entries(template).filter(([_, v]) => v !== undefined));
}
entityExists(entity: string) {
try {
return !!this.em.entity(entity);
@@ -257,7 +228,7 @@ export class DataController extends Controller {
const where = c.req.valid("json") as any;
const result = await this.em.repository(entity).count(where);
return c.json({ entity, count: result.count });
return c.json({ entity, ...result.data });
},
);
@@ -279,7 +250,7 @@ export class DataController extends Controller {
const where = c.req.valid("json") as any;
const result = await this.em.repository(entity).exists(where);
return c.json({ entity, exists: result.exists });
return c.json({ entity, ...result.data });
},
);
@@ -318,7 +289,7 @@ export class DataController extends Controller {
const options = c.req.valid("query") as RepoQuery;
const result = await this.em.repository(entity).findMany(options);
return c.json(this.repoResult(result), { status: result.data ? 200 : 404 });
return c.json(result, { status: result.data ? 200 : 404 });
},
);
@@ -347,7 +318,7 @@ export class DataController extends Controller {
const options = c.req.valid("query") as RepoQuery;
const result = await this.em.repository(entity).findId(id, options);
return c.json(this.repoResult(result), { status: result.data ? 200 : 404 });
return c.json(result, { status: result.data ? 200 : 404 });
},
);
@@ -380,7 +351,7 @@ export class DataController extends Controller {
.repository(entity)
.findManyByReference(id, reference, options);
return c.json(this.repoResult(result), { status: result.data ? 200 : 404 });
return c.json(result, { status: result.data ? 200 : 404 });
},
);
@@ -414,7 +385,7 @@ export class DataController extends Controller {
const options = (await c.req.json()) as RepoQuery;
const result = await this.em.repository(entity).findMany(options);
return c.json(this.repoResult(result), { status: result.data ? 200 : 404 });
return c.json(result, { status: result.data ? 200 : 404 });
},
);
@@ -440,11 +411,11 @@ export class DataController extends Controller {
if (Array.isArray(body)) {
const result = await this.em.mutator(entity).insertMany(body);
return c.json(this.mutatorResult(result), 201);
return c.json(result, 201);
}
const result = await this.em.mutator(entity).insertOne(body);
return c.json(this.mutatorResult(result), 201);
return c.json(result, 201);
},
);
@@ -475,7 +446,7 @@ export class DataController extends Controller {
};
const result = await this.em.mutator(entity).updateWhere(update, where);
return c.json(this.mutatorResult(result));
return c.json(result);
},
);
@@ -497,7 +468,7 @@ export class DataController extends Controller {
const body = (await c.req.json()) as EntityData;
const result = await this.em.mutator(entity).updateOne(id, body);
return c.json(this.mutatorResult(result));
return c.json(result);
},
);
@@ -517,7 +488,7 @@ export class DataController extends Controller {
}
const result = await this.em.mutator(entity).deleteOne(id);
return c.json(this.mutatorResult(result));
return c.json(result);
},
);
@@ -539,7 +510,7 @@ export class DataController extends Controller {
const where = (await c.req.json()) as RepoQuery["where"];
const result = await this.em.mutator(entity).deleteWhere(where);
return c.json(this.mutatorResult(result));
return c.json(result);
},
);

94
app/src/data/connection/Connection.ts
View File

@@ -2,12 +2,15 @@ import {
type AliasableExpression,
type ColumnBuilderCallback,
type ColumnDataType,
type Compilable,
type CompiledQuery,
type DatabaseIntrospector,
type Dialect,
type Expression,
type Kysely,
type KyselyPlugin,
type OnModifyForeignAction,
type QueryResult,
type RawBuilder,
type SelectQueryBuilder,
type SelectQueryNode,
@@ -15,7 +18,8 @@ import {
sql,
} from "kysely";
import type { BaseIntrospector, BaseIntrospectorConfig } from "./BaseIntrospector";
import type { Constructor } from "core";
import type { Constructor, DB } from "core";
import { KyselyPluginRunner } from "data/plugins/KyselyPluginRunner";
export type QB = SelectQueryBuilder<any, any, any>;
@@ -75,22 +79,44 @@ export type DbFunctions = {
>;
};
export type ConnQuery = CompiledQuery | Compilable;
export type ConnQueryResult<T extends ConnQuery> = T extends CompiledQuery<infer R>
? QueryResult<R>
: T extends Compilable<infer R>
? QueryResult<R>
: never;
export type ConnQueryResults<T extends ConnQuery[]> = {
[K in keyof T]: ConnQueryResult<T[K]>;
};
const CONN_SYMBOL = Symbol.for("bknd:connection");
export abstract class Connection<DB = any> {
export type Features = {
batching: boolean;
softscans: boolean;
};
export abstract class Connection<Client = unknown> {
abstract name: string;
protected initialized = false;
kysely: Kysely<DB>;
protected readonly supported = {
protected pluginRunner: KyselyPluginRunner;
protected readonly supported: Partial<Features> = {
batching: false,
softscans: true,
};
kysely: Kysely<DB>;
client!: Client;
constructor(
kysely: Kysely<DB>,
kysely: Kysely<any>,
public fn: Partial<DbFunctions> = {},
protected plugins: KyselyPlugin[] = [],
) {
this.kysely = kysely;
this[CONN_SYMBOL] = true;
this.pluginRunner = new KyselyPluginRunner(plugins);
}
// @todo: consider moving constructor logic here, required by sqlocal
@@ -121,30 +147,46 @@ export abstract class Connection<DB = any> {
return res.rows.length > 0;
}
protected async batch<Queries extends QB[]>(
queries: [...Queries],
): Promise<{
[K in keyof Queries]: Awaited<ReturnType<Queries[K]["execute"]>>;
}> {
throw new Error("Batching not supported");
protected async transformResultRows(result: any[]): Promise<any[]> {
return await this.pluginRunner.transformResultRows(result);
}
async batchQuery<Queries extends QB[]>(
queries: [...Queries],
): Promise<{
[K in keyof Queries]: Awaited<ReturnType<Queries[K]["execute"]>>;
}> {
// bypass if no client support
if (!this.supports("batching")) {
const data: any = [];
for (const q of queries) {
const result = await q.execute();
data.push(result);
}
return data;
}
/**
* Execute a query and return the result including all metadata
* returned from the dialect.
*/
async executeQueries<O extends ConnQuery[]>(...qbs: O): Promise<ConnQueryResults<O>> {
return Promise.all(qbs.map(async (qb) => await this.kysely.executeQuery(qb))) as any;
}
return await this.batch(queries);
async executeQuery<O extends ConnQuery>(qb: O): Promise<ConnQueryResult<O>> {
const res = await this.executeQueries(qb);
return res[0] as any;
}
protected getCompiled(...qbs: ConnQuery[]): CompiledQuery[] {
return qbs.map((qb) => {
if ("compile" in qb) {
return qb.compile();
}
return qb;
});
}
protected async withTransformedRows<
Key extends string = "rows",
O extends { [K in Key]: any[] }[] = [],
>(result: O, _key?: Key): Promise<O> {
return (await Promise.all(
result.map(async (row) => {
const key = _key ?? "rows";
const { [key]: rows, ...r } = row;
return {
...r,
rows: await this.transformResultRows(rows),
};
}),
)) as any;
}
protected validateFieldSpecType(type: string): type is FieldSpec["type"] {

2
app/src/data/connection/DummyConnection.ts
View File

@@ -1,6 +1,8 @@
import { Connection, type FieldSpec, type SchemaResponse } from "./Connection";
export class DummyConnection extends Connection {
override name: string = "dummy";
protected override readonly supported = {
batching: true,
};

187
app/src/data/connection/connection-test-suite.ts Normal file
View File

@@ -0,0 +1,187 @@
import type { TestRunner } from "core/test";
import { Connection, type FieldSpec } from "./Connection";
export function connectionTestSuite(
testRunner: TestRunner,
{
makeConnection,
rawDialectDetails,
}: {
makeConnection: () => Connection;
rawDialectDetails: string[];
},
) {
const { test, expect, describe } = testRunner;
test("pings", async () => {
const connection = makeConnection();
const res = await connection.ping();
expect(res).toBe(true);
});
test("initializes", async () => {
const connection = makeConnection();
await connection.init();
// @ts-expect-error
expect(connection.initialized).toBe(true);
expect(connection.client).toBeDefined();
});
test("isConnection", async () => {
const connection = makeConnection();
expect(Connection.isConnection(connection)).toBe(true);
});
test("getFieldSchema", async () => {
const c = makeConnection();
const specToNode = (spec: FieldSpec) => {
// @ts-expect-error
const schema = c.kysely.schema.createTable("test").addColumn(...c.getFieldSchema(spec));
return schema.toOperationNode();
};
{
// primary
const node = specToNode({
type: "integer",
name: "id",
primary: true,
});
const col = node.columns[0]!;
expect(col.primaryKey).toBe(true);
expect(col.notNull).toBe(true);
}
{
// normal
const node = specToNode({
type: "text",
name: "text",
});
const col = node.columns[0]!;
expect(!col.primaryKey).toBe(true);
expect(!col.notNull).toBe(true);
}
{
// nullable (expect to be same as normal)
const node = specToNode({
type: "text",
name: "text",
nullable: true,
});
const col = node.columns[0]!;
expect(!col.primaryKey).toBe(true);
expect(!col.notNull).toBe(true);
}
});
describe("schema", async () => {
const connection = makeConnection();
const fields = [
{
type: "integer",
name: "id",
primary: true,
},
{
type: "text",
name: "text",
},
{
type: "json",
name: "json",
},
] as const satisfies FieldSpec[];
let b = connection.kysely.schema.createTable("test");
for (const field of fields) {
// @ts-expect-error
b = b.addColumn(...connection.getFieldSchema(field));
}
await b.execute();
// add index
await connection.kysely.schema.createIndex("test_index").on("test").columns(["id"]).execute();
test("executes query", async () => {
await connection.kysely
.insertInto("test")
.values({ id: 1, text: "test", json: JSON.stringify({ a: 1 }) })
.execute();
const expected = { id: 1, text: "test", json: { a: 1 } };
const qb = connection.kysely.selectFrom("test").selectAll();
const res = await connection.executeQuery(qb);
expect(res.rows).toEqual([expected]);
expect(rawDialectDetails.every((detail) => detail in res)).toBe(true);
{
const res = await connection.executeQueries(qb, qb);
expect(res.length).toBe(2);
res.map((r) => {
expect(r.rows).toEqual([expected]);
expect(rawDialectDetails.every((detail) => detail in r)).toBe(true);
});
}
});
test("introspects", async () => {
const tables = await connection.getIntrospector().getTables({
withInternalKyselyTables: false,
});
const clean = tables.map((t) => ({
...t,
columns: t.columns.map((c) => ({
...c,
dataType: undefined,
})),
}));
expect(clean).toEqual([
{
name: "test",
isView: false,
columns: [
{
name: "id",
dataType: undefined,
isNullable: false,
isAutoIncrementing: true,
hasDefaultValue: false,
},
{
name: "text",
dataType: undefined,
isNullable: true,
isAutoIncrementing: false,
hasDefaultValue: false,
},
{
name: "json",
dataType: undefined,
isNullable: true,
isAutoIncrementing: false,
hasDefaultValue: false,
},
],
},
]);
});
expect(await connection.getIntrospector().getIndices()).toEqual([
{
name: "test_index",
table: "test",
isUnique: false,
columns: [
{
name: "id",
order: 0,
},
],
},
]);
});
}

4
app/src/data/connection/index.ts
View File

@@ -5,11 +5,13 @@ export {
type IndexSpec,
type DbFunctions,
type SchemaResponse,
type ConnQuery,
type ConnQueryResults,
customIntrospector,
} from "./Connection";
// sqlite
export { LibsqlConnection, type LibSqlCredentials } from "./sqlite/LibsqlConnection";
//export { libsql, LibsqlConnection, type LibSqlCredentials } from "./sqlite/LibsqlConnection";
export { SqliteConnection } from "./sqlite/SqliteConnection";
export { SqliteIntrospector } from "./sqlite/SqliteIntrospector";
export { SqliteLocalConnection } from "./sqlite/SqliteLocalConnection";

49
app/src/data/connection/sqlite/GenericSqliteConnection.ts Normal file
View File

@@ -0,0 +1,49 @@
import type { KyselyPlugin } from "kysely";
import {
type IGenericSqlite,
type OnCreateConnection,
type Promisable,
parseBigInt,
buildQueryFn,
GenericSqliteDialect,
} from "kysely-generic-sqlite";
import { SqliteConnection } from "./SqliteConnection";
import type { Features } from "../Connection";
export type GenericSqliteConnectionConfig = {
name: string;
additionalPlugins?: KyselyPlugin[];
excludeTables?: string[];
onCreateConnection?: OnCreateConnection;
supports?: Partial<Features>;
};
export { parseBigInt, buildQueryFn, GenericSqliteDialect, type IGenericSqlite };
export class GenericSqliteConnection<DB = unknown> extends SqliteConnection<DB> {
override name = "generic-sqlite";
constructor(
db: DB,
executor: () => Promisable<IGenericSqlite>,
config?: GenericSqliteConnectionConfig,
) {
super({
dialect: GenericSqliteDialect,
dialectArgs: [executor, config?.onCreateConnection],
additionalPlugins: config?.additionalPlugins,
excludeTables: config?.excludeTables,
});
this.client = db;
if (config?.name) {
this.name = config.name;
}
if (config?.supports) {
for (const [key, value] of Object.entries(config.supports)) {
if (value) {
this.supported[key] = value;
}
}
}
}
}

12
app/src/data/connection/sqlite/LibsqlConnection.spec.ts Normal file
View File

@@ -0,0 +1,12 @@
import { connectionTestSuite } from "../connection-test-suite";
import { LibsqlConnection } from "./LibsqlConnection";
import { bunTestRunner } from "adapter/bun/test";
import { describe } from "bun:test";
import { createClient } from "@libsql/client";
describe("LibsqlConnection", () => {
connectionTestSuite(bunTestRunner, {
makeConnection: () => new LibsqlConnection(createClient({ url: ":memory:" })),
rawDialectDetails: ["rowsAffected", "lastInsertRowid"],
});
});

112
app/src/data/connection/sqlite/LibsqlConnection.ts
View File

@@ -1,92 +1,62 @@
import { type Client, type Config, type InStatement, createClient } from "@libsql/client";
import type { Client, Config, InStatement } from "@libsql/client";
import { createClient } from "libsql-stateless-easy";
import { LibsqlDialect } from "@libsql/kysely-libsql";
import { FilterNumericKeysPlugin } from "data/plugins/FilterNumericKeysPlugin";
import { KyselyPluginRunner } from "data/plugins/KyselyPluginRunner";
import { type DatabaseIntrospector, Kysely, ParseJSONResultsPlugin } from "kysely";
import type { QB } from "../Connection";
import { SqliteConnection } from "./SqliteConnection";
import { SqliteIntrospector } from "./SqliteIntrospector";
import { $console } from "core";
import { type ConnQuery, type ConnQueryResults, SqliteConnection } from "bknd/data";
export const LIBSQL_PROTOCOLS = ["wss", "https", "libsql"] as const;
export type LibSqlCredentials = Config & {
protocol?: (typeof LIBSQL_PROTOCOLS)[number];
};
const plugins = [new FilterNumericKeysPlugin(), new ParseJSONResultsPlugin()];
class CustomLibsqlDialect extends LibsqlDialect {
override createIntrospector(db: Kysely<any>): DatabaseIntrospector {
return new SqliteIntrospector(db, {
excludeTables: ["libsql_wasm_func_table"],
plugins,
});
}
}
export class LibsqlConnection extends SqliteConnection {
private client: Client;
protected override readonly supported = {
batching: true,
};
constructor(client: Client);
constructor(credentials: LibSqlCredentials);
constructor(clientOrCredentials: Client | LibSqlCredentials) {
let client: Client;
let batching_enabled = true;
if (clientOrCredentials && "url" in clientOrCredentials) {
let { url, authToken, protocol } = clientOrCredentials;
if (protocol && LIBSQL_PROTOCOLS.includes(protocol)) {
$console.log("changing protocol to", protocol);
const [, rest] = url.split("://");
url = `${protocol}://${rest}`;
}
client = createClient({ url, authToken });
} else {
client = clientOrCredentials;
function getClient(clientOrCredentials: Client | LibSqlCredentials): Client {
if (clientOrCredentials && "url" in clientOrCredentials) {
let { url, authToken, protocol } = clientOrCredentials;
if (protocol && LIBSQL_PROTOCOLS.includes(protocol)) {
console.info("changing protocol to", protocol);
const [, rest] = url.split("://");
url = `${protocol}://${rest}`;
}
const kysely = new Kysely({
// @ts-expect-error libsql has type issues
dialect: new CustomLibsqlDialect({ client }),
plugins,
return createClient({ url, authToken });
}
return clientOrCredentials as Client;
}
export class LibsqlConnection extends SqliteConnection<Client> {
override name = "libsql";
protected override readonly supported = {
batching: true,
softscans: true,
};
constructor(clientOrCredentials: Client | LibSqlCredentials) {
const client = getClient(clientOrCredentials);
super({
excludeTables: ["libsql_wasm_func_table"],
dialect: LibsqlDialect,
dialectArgs: [{ client }],
additionalPlugins: [new FilterNumericKeysPlugin()],
});
super(kysely, {}, plugins);
this.client = client;
this.supported.batching = batching_enabled;
}
getClient(): Client {
return this.client;
}
protected override async batch<Queries extends QB[]>(
queries: [...Queries],
): Promise<{
[K in keyof Queries]: Awaited<ReturnType<Queries[K]["execute"]>>;
}> {
const stms: InStatement[] = queries.map((q) => {
const compiled = q.compile();
override async executeQueries<O extends ConnQuery[]>(...qbs: O): Promise<ConnQueryResults<O>> {
const compiled = this.getCompiled(...qbs);
const stms: InStatement[] = compiled.map((q) => {
return {
sql: compiled.sql,
args: compiled.parameters as any[],
sql: q.sql,
args: q.parameters as any[],
};
});
const res = await this.client.batch(stms);
// let it run through plugins
const kyselyPlugins = new KyselyPluginRunner(this.plugins);
const data: any = [];
for (const r of res) {
const rows = await kyselyPlugins.transformResultRows(r.rows);
data.push(rows);
}
return data;
return this.withTransformedRows(await this.client.batch(stms)) as any;
}
}
export function libsql(credentials: LibSqlCredentials): LibsqlConnection {
return new LibsqlConnection(credentials);
}

43
app/src/data/connection/sqlite/SqliteConnection.ts
View File

@@ -1,16 +1,49 @@
import type { ColumnDataType, ColumnDefinitionBuilder, Kysely, KyselyPlugin } from "kysely";
import {
ParseJSONResultsPlugin,
type ColumnDataType,
type ColumnDefinitionBuilder,
type Dialect,
Kysely,
type KyselyPlugin,
} from "kysely";
import { jsonArrayFrom, jsonBuildObject, jsonObjectFrom } from "kysely/helpers/sqlite";
import { Connection, type DbFunctions, type FieldSpec, type SchemaResponse } from "../Connection";
import type { Constructor } from "core";
import { customIntrospector } from "../Connection";
import { SqliteIntrospector } from "./SqliteIntrospector";
export type SqliteConnectionConfig<
CustomDialect extends Constructor<Dialect> = Constructor<Dialect>,
> = {
excludeTables?: string[];
dialect: CustomDialect;
dialectArgs?: ConstructorParameters<CustomDialect>;
additionalPlugins?: KyselyPlugin[];
customFn?: Partial<DbFunctions>;
};
export abstract class SqliteConnection<Client = unknown> extends Connection<Client> {
override name = "sqlite";
constructor(config: SqliteConnectionConfig) {
const { excludeTables, dialect, dialectArgs = [], additionalPlugins } = config;
const plugins = [new ParseJSONResultsPlugin(), ...(additionalPlugins ?? [])];
const kysely = new Kysely({
dialect: customIntrospector(dialect, SqliteIntrospector, {
excludeTables,
plugins,
}).create(...dialectArgs),
plugins,
});
export class SqliteConnection extends Connection {
constructor(kysely: Kysely<any>, fn: Partial<DbFunctions> = {}, plugins: KyselyPlugin[] = []) {
super(
kysely,
{
...fn,
jsonArrayFrom,
jsonObjectFrom,
jsonBuildObject,
...(config.customFn ?? {}),
},
plugins,
);
@@ -43,7 +76,7 @@ export class SqliteConnection extends Connection {
if (spec.onUpdate) relCol = relCol.onUpdate(spec.onUpdate);
return relCol;
}
return spec.nullable ? col : col.notNull();
return col;
},
] as const;
}

33
app/src/data/connection/sqlite/SqliteLocalConnection.ts
View File

@@ -1,31 +1,14 @@
import {
type DatabaseIntrospector,
Kysely,
ParseJSONResultsPlugin,
type SqliteDatabase,
SqliteDialect,
} from "kysely";
import { type SqliteDatabase, SqliteDialect } from "kysely";
import { SqliteConnection } from "./SqliteConnection";
import { SqliteIntrospector } from "./SqliteIntrospector";
const plugins = [new ParseJSONResultsPlugin()];
export class SqliteLocalConnection extends SqliteConnection<SqliteDatabase> {
override name = "sqlite-local";
class CustomSqliteDialect extends SqliteDialect {
override createIntrospector(db: Kysely<any>): DatabaseIntrospector {
return new SqliteIntrospector(db, {
excludeTables: ["test_table"],
plugins,
constructor(database: SqliteDatabase) {
super({
dialect: SqliteDialect,
dialectArgs: [{ database }],
});
}
}
export class SqliteLocalConnection extends SqliteConnection {
constructor(private database: SqliteDatabase) {
const kysely = new Kysely({
dialect: new CustomSqliteDialect({ database }),
plugins,
});
super(kysely, {}, plugins);
this.client = database;
}
}

3
app/src/data/entities/EntityManager.ts
View File

@@ -207,8 +207,9 @@ export class EntityManager<TBD extends object = DefaultDB> {
repository<E extends Entity | keyof TBD | string>(
entity: E,
opts: Omit<RepositoryOptions, "emgr"> = {},
): Repository<TBD, EntitySchema<TBD, E>> {
return this.repo(entity);
return this.repo(entity, opts);
}
repo<E extends Entity | keyof TBD | string>(

126
app/src/data/entities/Result.ts Normal file
View File

@@ -0,0 +1,126 @@
import { isDebug } from "core";
import { pick } from "core/utils";
import type { Connection } from "data/connection";
import type {
Compilable,
CompiledQuery,
QueryResult as KyselyQueryResult,
SelectQueryBuilder,
} from "kysely";
export type ResultHydrator<T = any> = (rows: T[]) => any;
export type ResultOptions<T = any> = {
hydrator?: ResultHydrator<T>;
beforeExecute?: (compiled: CompiledQuery) => void | Promise<void>;
onError?: (error: Error) => void | Promise<void>;
single?: boolean;
};
export type ResultJSON<T = any> = {
data: T;
meta: {
items: number;
time: number;
sql?: string;
parameters?: any[];
[key: string]: any;
};
};
export interface QueryResult<T = any> extends Omit<KyselyQueryResult<T>, "rows"> {
time: number;
items: number;
data: T;
rows: unknown[];
sql: string;
parameters: any[];
count?: number;
total?: number;
}
export class Result<T = unknown> {
results: QueryResult<T>[] = [];
time: number = 0;
constructor(
protected conn: Connection,
protected options: ResultOptions<T> = {},
) {}
get(): QueryResult<T> {
if (!this.results) {
throw new Error("Result not executed");
}
if (Array.isArray(this.results)) {
return (this.results ?? []) as any;
}
return this.results[0] as any;
}
first(): QueryResult<T> {
const res = this.get();
const first = Array.isArray(res) ? res[0] : res;
return first ?? ({} as any);
}
get sql() {
return this.first().sql;
}
get parameters() {
return this.first().parameters;
}
get data() {
if (this.options.single) {
return this.first().data?.[0];
}
return this.first().data ?? [];
}
async execute(qb: Compilable | Compilable[]) {
const qbs = Array.isArray(qb) ? qb : [qb];
for (const qb of qbs) {
const compiled = qb.compile();
await this.options.beforeExecute?.(compiled);
try {
const start = performance.now();
const res = await this.conn.executeQuery(compiled);
this.time = Number.parseFloat((performance.now() - start).toFixed(2));
this.results.push({
...res,
data: this.options.hydrator?.(res.rows as T[]),
items: res.rows.length,
time: this.time,
sql: compiled.sql,
parameters: [...compiled.parameters],
});
} catch (e) {
if (this.options.onError) {
await this.options.onError(e as Error);
} else {
throw e;
}
}
}
return this;
}
protected additionalMetaKeys(): string[] {
return [];
}
toJSON(): ResultJSON<T> {
const { rows, data, ...metaRaw } = this.first();
const keys = isDebug() ? ["items", "time", "sql", "parameters"] : ["items", "time"];
const meta = pick(metaRaw, [...keys, ...this.additionalMetaKeys()] as any);
return {
data: this.data,
meta,
};
}
}

4
app/src/data/entities/index.ts
View File

@@ -1,6 +1,8 @@
export * from "./Entity";
export * from "./EntityManager";
export * from "./Mutator";
export * from "./mutation/Mutator";
export * from "./query/Repository";
export * from "./query/WhereBuilder";
export * from "./query/WithBuilder";
export * from "./query/RepositoryResult";
export * from "./mutation/MutatorResult";

80
app/src/data/entities/Mutator.ts → app/src/data/entities/mutation/Mutator.ts
View File

@@ -1,12 +1,13 @@
import { $console, type DB as DefaultDB, type PrimaryFieldType } from "core";
import { type EmitsEvents, EventManager } from "core/events";
import type { DeleteQueryBuilder, InsertQueryBuilder, UpdateQueryBuilder } from "kysely";
import { type TActionContext, WhereBuilder } from "..";
import type { Entity, EntityData, EntityManager } from "../entities";
import { InvalidSearchParamsException } from "../errors";
import { MutatorEvents } from "../events";
import { RelationMutator } from "../relations";
import type { RepoQuery } from "../server/query";
import { type TActionContext, WhereBuilder } from "../..";
import type { Entity, EntityData, EntityManager } from "../../entities";
import { InvalidSearchParamsException } from "../../errors";
import { MutatorEvents } from "../../events";
import { RelationMutator } from "../../relations";
import type { RepoQuery } from "../../server/query";
import { MutatorResult, type MutatorResultOptions } from "./MutatorResult";
type MutatorQB =
| InsertQueryBuilder<any, any, any>
@@ -17,14 +18,6 @@ type MutatorUpdateOrDelete =
| UpdateQueryBuilder<any, any, any, any>
| DeleteQueryBuilder<any, any, any>;
export type MutatorResponse<T = EntityData[]> = {
entity: Entity;
sql: string;
parameters: any[];
result: EntityData[];
data: T;
};
export class Mutator<
TBD extends object = DefaultDB,
TB extends keyof TBD = any,
@@ -103,35 +96,18 @@ export class Mutator<
return validatedData as Given;
}
protected async many(qb: MutatorQB): Promise<MutatorResponse> {
const entity = this.entity;
const { sql, parameters } = qb.compile();
try {
const result = await qb.execute();
const data = this.em.hydrate(entity.name, result) as EntityData[];
return {
entity,
sql,
parameters: [...parameters],
result: result,
data,
};
} catch (e) {
// @todo: redact
$console.error("[Error in query]", sql);
throw e;
}
protected async performQuery<T = EntityData[]>(
qb: MutatorQB,
opts?: MutatorResultOptions,
): Promise<MutatorResult<T>> {
const result = new MutatorResult(this.em, this.entity, {
silent: false,
...opts,
});
return (await result.execute(qb)) as any;
}
protected async single(qb: MutatorQB): Promise<MutatorResponse<EntityData>> {
const { data, ...response } = await this.many(qb);
return { ...response, data: data[0]! };
}
async insertOne(data: Input): Promise<MutatorResponse<Output>> {
async insertOne(data: Input): Promise<MutatorResult<Output>> {
const entity = this.entity;
if (entity.type === "system" && this.__unstable_disable_system_entity_creation) {
throw new Error(`Creation of system entity "${entity.name}" is disabled`);
@@ -174,7 +150,7 @@ export class Mutator<
.values(validatedData)
.returning(entity.getSelect());
const res = await this.single(query);
const res = await this.performQuery(query, { single: true });
await this.emgr.emit(
new Mutator.Events.MutatorInsertAfter({ entity, data: res.data, changed: validatedData }),
@@ -183,7 +159,7 @@ export class Mutator<
return res as any;
}
async updateOne(id: PrimaryFieldType, data: Partial<Input>): Promise<MutatorResponse<Output>> {
async updateOne(id: PrimaryFieldType, data: Partial<Input>): Promise<MutatorResult<Output>> {
const entity = this.entity;
if (!id) {
throw new Error("ID must be provided for update");
@@ -206,7 +182,7 @@ export class Mutator<
.where(entity.id().name, "=", id)
.returning(entity.getSelect());
const res = await this.single(query);
const res = await this.performQuery(query, { single: true });
await this.emgr.emit(
new Mutator.Events.MutatorUpdateAfter({
@@ -220,7 +196,7 @@ export class Mutator<
return res as any;
}
async deleteOne(id: PrimaryFieldType): Promise<MutatorResponse<Output>> {
async deleteOne(id: PrimaryFieldType): Promise<MutatorResult<Output>> {
const entity = this.entity;
if (!id) {
throw new Error("ID must be provided for deletion");
@@ -233,7 +209,7 @@ export class Mutator<
.where(entity.id().name, "=", id)
.returning(entity.getSelect());
const res = await this.single(query);
const res = await this.performQuery(query, { single: true });
await this.emgr.emit(
new Mutator.Events.MutatorDeleteAfter({ entity, entityId: id, data: res.data }),
@@ -286,7 +262,7 @@ export class Mutator<
}
// @todo: decide whether entries should be deleted all at once or one by one (for events)
async deleteWhere(where: RepoQuery["where"]): Promise<MutatorResponse<Output[]>> {
async deleteWhere(where: RepoQuery["where"]): Promise<MutatorResult<Output[]>> {
const entity = this.entity;
// @todo: add a way to delete all by adding force?
@@ -298,13 +274,13 @@ export class Mutator<
entity.getSelect(),
);
return (await this.many(qb)) as any;
return await this.performQuery(qb);
}
async updateWhere(
data: Partial<Input>,
where: RepoQuery["where"],
): Promise<MutatorResponse<Output[]>> {
): Promise<MutatorResult<Output[]>> {
const entity = this.entity;
const validatedData = await this.getValidatedData(data, "update");
@@ -317,10 +293,10 @@ export class Mutator<
.set(validatedData as any)
.returning(entity.getSelect());
return (await this.many(query)) as any;
return await this.performQuery(query);
}
async insertMany(data: Input[]): Promise<MutatorResponse<Output[]>> {
async insertMany(data: Input[]): Promise<MutatorResult<Output[]>> {
const entity = this.entity;
if (entity.type === "system" && this.__unstable_disable_system_entity_creation) {
throw new Error(`Creation of system entity "${entity.name}" is disabled`);
@@ -352,6 +328,6 @@ export class Mutator<
.values(validated)
.returning(entity.getSelect());
return (await this.many(query)) as any;
return await this.performQuery(query);
}
}

33
app/src/data/entities/mutation/MutatorResult.ts Normal file
View File

@@ -0,0 +1,33 @@
import { $console } from "core/console";
import type { Entity, EntityData } from "../Entity";
import type { EntityManager } from "../EntityManager";
import { Result, type ResultJSON, type ResultOptions } from "../Result";
export type MutatorResultOptions = ResultOptions & {
silent?: boolean;
};
export type MutatorResultJSON<T = EntityData[]> = ResultJSON<T>;
export class MutatorResult<T = EntityData[]> extends Result<T> {
constructor(
protected em: EntityManager<any>,
public entity: Entity,
options?: MutatorResultOptions,
) {
super(em.connection, {
hydrator: (rows) => em.hydrate(entity.name, rows as any),
beforeExecute: (compiled) => {
if (!options?.silent) {
$console.debug(`[Mutation]\n${compiled.sql}\n`);
}
},
onError: (error) => {
if (!options?.silent) {
$console.error("[ERROR] Mutator:", error.message);
}
},
...options,
});
}
}

215
app/src/data/entities/query/Repository.ts
View File

@@ -13,37 +13,11 @@ import {
WithBuilder,
} from "../index";
import { JoinBuilder } from "./JoinBuilder";
import { ensureInt } from "core/utils";
import { RepositoryResult, type RepositoryResultOptions } from "./RepositoryResult";
import type { ResultOptions } from "../Result";
export type RepositoryQB = SelectQueryBuilder<any, any, any>;
export type RepositoryRawResponse = {
sql: string;
parameters: any[];
result: EntityData[];
};
export type RepositoryResponse<T = EntityData[]> = RepositoryRawResponse & {
entity: Entity;
data: T;
meta: {
items: number;
total?: number;
count?: number;
time?: number;
query?: {
sql: string;
parameters: readonly any[];
};
};
};
export type RepositoryCountResponse = RepositoryRawResponse & {
count: number;
};
export type RepositoryExistsResponse = RepositoryRawResponse & {
exists: boolean;
};
export type RepositoryOptions = {
silent?: boolean;
includeCounts?: boolean;
@@ -182,126 +156,18 @@ export class Repository<TBD extends object = DefaultDB, TB extends keyof TBD = a
return validated;
}
protected async executeQb(qb: RepositoryQB) {
const compiled = qb.compile();
if (this.options?.silent !== true) {
$console.debug(`Repository: query\n${compiled.sql}\n`, compiled.parameters);
}
let result: any;
try {
result = await qb.execute();
} catch (e) {
if (this.options?.silent !== true) {
if (e instanceof Error) {
$console.error("[ERROR] Repository.executeQb", e.message);
}
throw e;
}
}
return {
result,
sql: compiled.sql,
parameters: [...compiled.parameters],
};
}
protected async performQuery(qb: RepositoryQB): Promise<RepositoryResponse> {
const entity = this.entity;
const compiled = qb.compile();
const payload = {
entity,
sql: compiled.sql,
parameters: [...compiled.parameters],
result: [],
data: [],
meta: {
total: 0,
count: 0,
items: 0,
time: 0,
query: { sql: compiled.sql, parameters: compiled.parameters },
},
};
// don't batch (add counts) if `includeCounts` is set to false
// or when explicitly set to true and batching is not supported
if (
this.options?.includeCounts === false ||
(this.options?.includeCounts === true && !this.em.connection.supports("batching"))
) {
const start = performance.now();
const res = await this.executeQb(qb);
const time = Number.parseFloat((performance.now() - start).toFixed(2));
const result = res.result ?? [];
const data = this.em.hydrate(entity.name, result);
return {
...payload,
result,
data,
meta: {
...payload.meta,
total: undefined,
count: undefined,
items: data.length,
time,
},
};
}
if (this.options?.silent !== true) {
$console.debug(`Repository: query\n${compiled.sql}\n`, compiled.parameters);
}
const selector = (as = "count") => this.conn.fn.countAll<number>().as(as);
const countQuery = qb
.clearSelect()
.select(selector())
.clearLimit()
.clearOffset()
.clearGroupBy()
.clearOrderBy();
const totalQuery = this.conn.selectFrom(entity.name).select(selector());
try {
const start = performance.now();
const [_count, _total, result] = await this.em.connection.batchQuery([
countQuery,
totalQuery,
qb,
]);
const time = Number.parseFloat((performance.now() - start).toFixed(2));
const data = this.em.hydrate(entity.name, result);
return {
...payload,
result,
data,
meta: {
...payload.meta,
// parsing is important since pg returns string
total: ensureInt(_total[0]?.count),
count: ensureInt(_count[0]?.count),
items: result.length,
time,
},
};
} catch (e) {
if (this.options?.silent !== true) {
if (e instanceof Error) {
$console.error("[ERROR] Repository.performQuery", e.message);
}
throw e;
} else {
return payload;
}
}
protected async performQuery<T = EntityData[]>(
qb: RepositoryQB,
opts?: RepositoryResultOptions,
execOpts?: { includeCounts?: boolean },
): Promise<RepositoryResult<T>> {
const result = new RepositoryResult(this.em, this.entity, {
silent: this.options.silent,
...opts,
});
return (await result.execute(qb, {
includeCounts: execOpts?.includeCounts ?? this.options.includeCounts,
})) as any;
}
private async triggerFindBefore(entity: Entity, options: RepoQuery): Promise<void> {
@@ -319,7 +185,7 @@ export class Repository<TBD extends object = DefaultDB, TB extends keyof TBD = a
): Promise<void> {
if (options.limit === 1) {
await this.emgr.emit(
new Repository.Events.RepositoryFindOneAfter({ entity, options, data: data[0]! }),
new Repository.Events.RepositoryFindOneAfter({ entity, options, data }),
);
} else {
await this.emgr.emit(
@@ -331,12 +197,11 @@ export class Repository<TBD extends object = DefaultDB, TB extends keyof TBD = a
protected async single(
qb: RepositoryQB,
options: RepoQuery,
): Promise<RepositoryResponse<EntityData>> {
): Promise<RepositoryResult<TBD[TB] | undefined>> {
await this.triggerFindBefore(this.entity, options);
const { data, ...response } = await this.performQuery(qb);
await this.triggerFindAfter(this.entity, options, data);
return { ...response, data: data[0]! };
const result = await this.performQuery(qb, { single: true });
await this.triggerFindAfter(this.entity, options, result.data);
return result as any;
}
addOptionsToQueryBuilder(
@@ -413,7 +278,7 @@ export class Repository<TBD extends object = DefaultDB, TB extends keyof TBD = a
async findId(
id: PrimaryFieldType,
_options?: Partial<Omit<RepoQuery, "where" | "limit" | "offset">>,
): Promise<RepositoryResponse<TBD[TB] | undefined>> {
): Promise<RepositoryResult<TBD[TB] | undefined>> {
const { qb, options } = this.buildQuery(
{
..._options,
@@ -429,7 +294,7 @@ export class Repository<TBD extends object = DefaultDB, TB extends keyof TBD = a
async findOne(
where: RepoQuery["where"],
_options?: Partial<Omit<RepoQuery, "where" | "limit" | "offset">>,
): Promise<RepositoryResponse<TBD[TB] | undefined>> {
): Promise<RepositoryResult<TBD[TB] | undefined>> {
const { qb, options } = this.buildQuery({
..._options,
where,
@@ -439,7 +304,7 @@ export class Repository<TBD extends object = DefaultDB, TB extends keyof TBD = a
return (await this.single(qb, options)) as any;
}
async findMany(_options?: Partial<RepoQuery>): Promise<RepositoryResponse<TBD[TB][]>> {
async findMany(_options?: Partial<RepoQuery>): Promise<RepositoryResult<TBD[TB][]>> {
const { qb, options } = this.buildQuery(_options);
await this.triggerFindBefore(this.entity, options);
@@ -454,7 +319,7 @@ export class Repository<TBD extends object = DefaultDB, TB extends keyof TBD = a
id: PrimaryFieldType,
reference: string,
_options?: Partial<Omit<RepoQuery, "limit" | "offset">>,
): Promise<RepositoryResponse<EntityData>> {
): Promise<RepositoryResult<EntityData>> {
const entity = this.entity;
const listable_relations = this.em.relations.listableRelationsOf(entity);
const relation = listable_relations.find((r) => r.ref(reference).reference === reference);
@@ -482,10 +347,10 @@ export class Repository<TBD extends object = DefaultDB, TB extends keyof TBD = a
},
};
return this.cloneFor(newEntity).findMany(findManyOptions);
return this.cloneFor(newEntity).findMany(findManyOptions) as any;
}
async count(where?: RepoQuery["where"]): Promise<RepositoryCountResponse> {
async count(where?: RepoQuery["where"]): Promise<RepositoryResult<{ count: number }>> {
const entity = this.entity;
const options = this.getValidOptions({ where });
@@ -497,17 +362,18 @@ export class Repository<TBD extends object = DefaultDB, TB extends keyof TBD = a
qb = WhereBuilder.addClause(qb, options.where);
}
const { result, ...compiled } = await this.executeQb(qb);
return {
sql: compiled.sql,
parameters: [...compiled.parameters],
result,
count: result[0]?.count ?? 0,
};
return await this.performQuery(
qb,
{
hydrator: (rows) => ({ count: rows[0]?.count ?? 0 }),
},
{ includeCounts: false },
);
}
async exists(where: Required<RepoQuery>["where"]): Promise<RepositoryExistsResponse> {
async exists(
where: Required<RepoQuery>["where"],
): Promise<RepositoryResult<{ exists: boolean }>> {
const entity = this.entity;
const options = this.getValidOptions({ where });
@@ -517,13 +383,8 @@ export class Repository<TBD extends object = DefaultDB, TB extends keyof TBD = a
// add mandatory where
qb = WhereBuilder.addClause(qb, options.where!).limit(1);
const { result, ...compiled } = await this.executeQb(qb);
return {
sql: compiled.sql,
parameters: [...compiled.parameters],
result,
exists: result[0]!.count > 0,
};
return await this.performQuery(qb, {
hydrator: (rows) => ({ exists: rows[0]?.count > 0 }),
});
}
}

105
app/src/data/entities/query/RepositoryResult.ts Normal file
View File

@@ -0,0 +1,105 @@
import { $console } from "core/console";
import type { Entity, EntityData } from "../Entity";
import type { EntityManager } from "../EntityManager";
import { Result, type ResultJSON, type ResultOptions } from "../Result";
import type { Compilable, SelectQueryBuilder } from "kysely";
import { ensureInt } from "core/utils";
export type RepositoryResultOptions = ResultOptions & {
silent?: boolean;
};
export type RepositoryResultJSON<T = EntityData[]> = ResultJSON<T>;
export class RepositoryResult<T = EntityData[]> extends Result<T> {
constructor(
protected em: EntityManager<any>,
public entity: Entity,
options?: RepositoryResultOptions,
) {
super(em.connection, {
hydrator: (rows) => em.hydrate(entity.name, rows as any),
beforeExecute: (compiled) => {
if (!options?.silent) {
$console.debug(`Query:\n${compiled.sql}\n`, compiled.parameters);
}
},
onError: (error) => {
if (options?.silent !== true) {
$console.error("Repository:", String(error));
throw error;
}
},
...options,
});
}
private shouldIncludeCounts(intent?: boolean) {
if (intent === undefined) return this.conn.supports("softscans");
return intent;
}
override async execute(
qb: SelectQueryBuilder<any, any, any>,
opts?: { includeCounts?: boolean },
) {
const includeCounts = this.shouldIncludeCounts(opts?.includeCounts);
if (includeCounts) {
const selector = (as = "count") => this.conn.kysely.fn.countAll<number>().as(as);
const countQuery = qb
.clearSelect()
.select(selector())
.clearLimit()
.clearOffset()
.clearGroupBy()
.clearOrderBy();
const totalQuery = this.conn.kysely.selectFrom(this.entity.name).select(selector());
const compiled = qb.compile();
this.options.beforeExecute?.(compiled);
try {
const start = performance.now();
const [main, count, total] = await this.em.connection.executeQueries(
compiled,
countQuery,
totalQuery,
);
this.time = Number.parseFloat((performance.now() - start).toFixed(2));
this.results.push({
...main,
data: this.options.hydrator?.(main.rows as T[]),
items: main.rows.length,
count: ensureInt(count.rows[0]?.count ?? 0),
total: ensureInt(total.rows[0]?.count ?? 0),
time: this.time,
sql: compiled.sql,
parameters: [...compiled.parameters],
});
} catch (e) {
if (this.options.onError) {
await this.options.onError(e as Error);
} else {
throw e;
}
}
return this;
}
return await super.execute(qb);
}
get count() {
return this.first().count;
}
get total() {
return this.first().total;
}
protected override additionalMetaKeys(): string[] {
return ["count", "total"];
}
}

2
app/src/data/relations/ManyToOneRelation.ts
View File

@@ -208,7 +208,7 @@ export class ManyToOneRelation extends EntityRelation<typeof ManyToOneRelation.s
[field.targetField()]: primaryReference as any,
});
if (!query.exists) {
if (!query.data.exists) {
const idProp = field.targetField();
throw new Error(
`Cannot connect "${entity.name}.${key}" to ` +

2
app/src/data/relations/RelationMutator.ts
View File

@@ -70,7 +70,7 @@ export class RelationMutator {
[field.targetField()]: value,
});
if (!query.exists) {
if (!query.data.exists) {
const idProp = field.targetField();
throw new Error(
`Cannot connect "${this.entity.name}.${key}" to ` +

8
app/src/media/api/MediaController.ts
View File

@@ -216,7 +216,9 @@ export class MediaController extends Controller {
const paths_to_delete: string[] = [];
if (max_items) {
const { overwrite } = c.req.valid("query");
const { count } = await this.media.em.repository(media_entity).count(mediaRef);
const {
data: { count },
} = await this.media.em.repository(media_entity).count(mediaRef);
// if there are more than or equal to max items
if (count >= max_items) {
@@ -255,7 +257,9 @@ export class MediaController extends Controller {
}
// check if entity exists in database
const { exists } = await this.media.em.repository(entity).exists({ id: entity_id });
const {
data: { exists },
} = await this.media.em.repository(entity).exists({ id: entity_id });
if (!exists) {
return c.json(
{ error: `Entity "${entity_name}" with ID "${entity_id}" doesn't exist found` },

28
app/src/modules/ModuleManager.ts
View File

@@ -11,16 +11,9 @@ import {
stripMark,
transformObject,
} from "core/utils";
import {
type Connection,
EntityManager,
type Schema,
datetime,
entity,
enumm,
jsonSchema,
number,
} from "data";
import type { Connection, Schema } from "data";
import { EntityManager } from "data/entities/EntityManager";
import * as proto from "data/prototype";
import { TransformPersistFailedException } from "data/errors";
import { Hono } from "hono";
import type { Kysely } from "kysely";
@@ -116,12 +109,12 @@ const configJsonSchema = Type.Union([
}),
),
]);
export const __bknd = entity(TABLE_NAME, {
version: number().required(),
type: enumm({ enum: ["config", "diff", "backup"] }).required(),
json: jsonSchema({ schema: configJsonSchema }).required(),
created_at: datetime(),
updated_at: datetime(),
export const __bknd = proto.entity(TABLE_NAME, {
version: proto.number().required(),
type: proto.enumm({ enum: ["config", "diff", "backup"] }).required(),
json: proto.jsonSchema({ schema: configJsonSchema }).required(),
created_at: proto.datetime(),
updated_at: proto.datetime(),
});
type ConfigTable2 = Schema<typeof __bknd>;
interface T_INTERNAL_EM {
@@ -234,7 +227,8 @@ export class ModuleManager {
}
private get db() {
return this.connection.kysely as Kysely<{ table: ConfigTable }>;
// @todo: check why this is neccessary
return this.connection.kysely as unknown as Kysely<{ table: ConfigTable }>;
}
// @todo: add indices for: version, type

5
app/src/modules/server/SystemController.ts
View File

@@ -311,6 +311,11 @@ export class SystemController extends Controller {
c.json({
version: c.get("app")?.version(),
runtime: getRuntimeKey(),
connection: {
name: this.app.em.connection.name,
// @ts-expect-error
supports: this.app.em.connection.supported,
},
timezone: {
name: getTimezone(),
offset: getTimezoneOffset(),

8
app/src/ui/client/api/use-entity.ts
View File

@@ -1,7 +1,7 @@
import type { DB, PrimaryFieldType } from "core";
import { objectTransform } from "core/utils/objects";
import { encodeSearch } from "core/utils/reqres";
import type { EntityData, RepoQueryIn, RepositoryResponse } from "data";
import type { EntityData, RepoQueryIn, RepositoryResult } from "data";
import type { Insertable, Selectable, Updateable } from "kysely";
import type { FetchPromise, ModuleApi, ResponseObject } from "modules/ModuleApi";
import useSWR, { type SWRConfiguration, type SWRResponse, mutate } from "swr";
@@ -28,15 +28,13 @@ interface UseEntityReturn<
Entity extends keyof DB | string,
Id extends PrimaryFieldType | undefined,
Data = Entity extends keyof DB ? DB[Entity] : EntityData,
Response = ResponseObject<RepositoryResponse<Selectable<Data>>>,
Response = ResponseObject<RepositoryResult<Selectable<Data>>>,
> {
create: (input: Insertable<Data>) => Promise<Response>;
read: (
query?: RepoQueryIn,
) => Promise<
ResponseObject<
RepositoryResponse<Id extends undefined ? Selectable<Data>[] : Selectable<Data>>
>
ResponseObject<RepositoryResult<Id extends undefined ? Selectable<Data>[] : Selectable<Data>>>
>;
update: Id extends undefined
? (input: Updateable<Data>, id: Id) => Promise<Response>