From 6c2e5795962a21a2294776636bfb72ccadb16d8c Mon Sep 17 00:00:00 2001 From: dswbx Date: Thu, 12 Jun 2025 09:02:18 +0200 Subject: [PATCH] connection: rewrote query execution, batching, added generic sqlite, added node/bun sqlite, aligned repo/mutator results --- app/__test__/App.spec.ts | 2 +- app/__test__/api/DataApi.spec.ts | 2 +- app/__test__/data/DataController.spec.ts | 103 +++------ app/__test__/data/data.test.ts | 13 +- app/__test__/data/mutation.simple.test.ts | 8 +- app/__test__/data/specs/Mutator.spec.ts | 7 +- app/__test__/data/specs/Repository.spec.ts | 154 +++---------- app/__test__/helper.ts | 5 +- app/package.json | 5 +- .../connection/BunSqliteConnection.spec.ts | 12 + .../bun/connection/BunSqliteConnection.ts | 41 ++++ app/src/adapter/bun/test.ts | 3 +- .../cloudflare/connection/D1Connection.ts | 51 ++--- .../node/connection/NodeSqliteConnection.ts | 46 ++++ .../NodeSqliteConnection.vi-test.ts | 11 + ...native-spec.ts => node.adapter.vi-test.ts} | 10 +- app/src/adapter/node/test.ts | 3 +- app/src/adapter/node/vitest.ts | 50 ++++ app/src/core/test/index.ts | 1 + app/src/data/api/DataController.ts | 53 +---- app/src/data/connection/Connection.ts | 94 +++++--- .../data/connection/connection-test-suite.ts | 187 +++++++++++++++ .../sqlite/GenericSqliteConnection.ts | 37 +++ .../sqlite/LibsqlConnection.spec.ts | 11 + .../connection/sqlite/LibsqlConnection.ts | 70 ++---- .../connection/sqlite/SqliteConnection.ts | 43 +++- .../sqlite/SqliteLocalConnection.ts | 33 +-- app/src/data/entities/EntityManager.ts | 3 +- app/src/data/entities/Result.ts | 126 ++++++++++ app/src/data/entities/index.ts | 2 +- .../data/entities/{ => mutation}/Mutator.ts | 80 +++---- .../data/entities/mutation/MutatorResult.ts | 33 +++ app/src/data/entities/query/Repository.ts | 215 ++++-------------- .../data/entities/query/RepositoryResult.ts | 105 +++++++++ app/src/data/relations/ManyToOneRelation.ts | 2 +- app/src/data/relations/RelationMutator.ts | 2 +- app/src/modules/server/SystemController.ts | 5 + app/vite.dev.ts | 4 + app/vitest.config.ts | 2 +- bun.lock | 5 +- 40 files changed, 990 insertions(+), 649 deletions(-) create mode 100644 app/src/adapter/bun/connection/BunSqliteConnection.spec.ts create mode 100644 app/src/adapter/bun/connection/BunSqliteConnection.ts create mode 100644 app/src/adapter/node/connection/NodeSqliteConnection.ts create mode 100644 app/src/adapter/node/connection/NodeSqliteConnection.vi-test.ts rename app/src/adapter/node/{node.adapter.native-spec.ts => node.adapter.vi-test.ts} (57%) create mode 100644 app/src/adapter/node/vitest.ts create mode 100644 app/src/data/connection/connection-test-suite.ts create mode 100644 app/src/data/connection/sqlite/GenericSqliteConnection.ts create mode 100644 app/src/data/connection/sqlite/LibsqlConnection.spec.ts create mode 100644 app/src/data/entities/Result.ts rename app/src/data/entities/{ => mutation}/Mutator.ts (84%) create mode 100644 app/src/data/entities/mutation/MutatorResult.ts create mode 100644 app/src/data/entities/query/RepositoryResult.ts diff --git a/app/__test__/App.spec.ts b/app/__test__/App.spec.ts index 79fdc51..3b36c85 100644 --- a/app/__test__/App.spec.ts +++ b/app/__test__/App.spec.ts @@ -1,4 +1,4 @@ -import { afterAll, afterEach, describe, expect, test } from "bun:test"; +import { afterEach, describe, test } from "bun:test"; import { App } from "../src"; import { getDummyConnection } from "./helper"; diff --git a/app/__test__/api/DataApi.spec.ts b/app/__test__/api/DataApi.spec.ts index 51786ca..5be5e1e 100644 --- a/app/__test__/api/DataApi.spec.ts +++ b/app/__test__/api/DataApi.spec.ts @@ -153,7 +153,7 @@ describe("DataApi", () => { const oneBy = api.readOneBy("posts", { where: { title: "baz" }, select: ["title"] }); const oneByRes = await oneBy; expect(oneByRes.data).toEqual({ title: "baz" } as any); - expect(oneByRes.body.meta.count).toEqual(1); + expect(oneByRes.body.meta.items).toEqual(1); }); it("exists/count", async () => { diff --git a/app/__test__/data/DataController.spec.ts b/app/__test__/data/DataController.spec.ts index 21ae226..96c30c6 100644 --- a/app/__test__/data/DataController.spec.ts +++ b/app/__test__/data/DataController.spec.ts @@ -7,13 +7,13 @@ import { type EntityData, EntityManager, ManyToOneRelation, - type MutatorResponse, - type RepositoryResponse, TextField, } from "../../src/data"; import { DataController } from "../../src/data/api/DataController"; import { dataConfigSchema } from "../../src/data/data-schema"; import { disableConsoleLog, enableConsoleLog, getDummyConnection } from "../helper"; +import type { RepositoryResultJSON } from "data/entities/query/RepositoryResult"; +import type { MutatorResultJSON } from "data/entities/mutation/MutatorResult"; const { dummyConnection, afterAllCleanup } = getDummyConnection(); beforeAll(() => disableConsoleLog(["log", "warn"])); @@ -21,52 +21,6 @@ afterAll(async () => (await afterAllCleanup()) && enableConsoleLog()); const dataConfig = parse(dataConfigSchema, {}); describe("[data] DataController", async () => { - test("repoResult", async () => { - const em = new EntityManager([], dummyConnection); - const ctx: any = { em, guard: new Guard() }; - const controller = new DataController(ctx, dataConfig); - - const res = controller.repoResult({ - entity: null as any, - data: [] as any, - sql: "", - parameters: [] as any, - result: [] as any, - meta: { - total: 0, - count: 0, - items: 0, - }, - }); - - expect(res).toEqual({ - meta: { - total: 0, - count: 0, - items: 0, - }, - data: [], - }); - }); - - test("mutatorResult", async () => { - const em = new EntityManager([], dummyConnection); - const ctx: any = { em, guard: new Guard() }; - const controller = new DataController(ctx, dataConfig); - - const res = controller.mutatorResult({ - entity: null as any, - data: [] as any, - sql: "", - parameters: [] as any, - result: [] as any, - }); - - expect(res).toEqual({ - data: [], - }); - }); - describe("getController", async () => { const users = new Entity("users", [ new TextField("name", { required: true }), @@ -120,8 +74,7 @@ describe("[data] DataController", async () => { method: "POST", body: JSON.stringify(_user), }); - //console.log("res", { _user }, res); - const result = (await res.json()) as MutatorResponse; + const result = (await res.json()) as MutatorResultJSON; const { id, ...data } = result.data as any; expect(res.status).toBe(201); @@ -135,7 +88,7 @@ describe("[data] DataController", async () => { method: "POST", body: JSON.stringify(_post), }); - const result = (await res.json()) as MutatorResponse; + const result = (await res.json()) as MutatorResultJSON; const { id, ...data } = result.data as any; expect(res.status).toBe(201); @@ -146,13 +99,13 @@ describe("[data] DataController", async () => { test("/:entity (read many)", async () => { const res = await app.request("/entity/users"); - const data = (await res.json()) as RepositoryResponse; + const data = (await res.json()) as RepositoryResultJSON; - expect(data.meta.total).toBe(3); - expect(data.meta.count).toBe(3); + //expect(data.meta.total).toBe(3); + //expect(data.meta.count).toBe(3); expect(data.meta.items).toBe(3); expect(data.data.length).toBe(3); - expect(data.data[0].name).toBe("foo"); + expect(data.data[0]?.name).toBe("foo"); }); test("/:entity/query (func query)", async () => { @@ -165,33 +118,32 @@ describe("[data] DataController", async () => { where: { bio: { $isnull: 1 } }, }), }); - const data = (await res.json()) as RepositoryResponse; + const data = (await res.json()) as RepositoryResultJSON; - expect(data.meta.total).toBe(3); - expect(data.meta.count).toBe(1); + //expect(data.meta.total).toBe(3); + //expect(data.meta.count).toBe(1); expect(data.meta.items).toBe(1); expect(data.data.length).toBe(1); - expect(data.data[0].name).toBe("bar"); + expect(data.data[0]?.name).toBe("bar"); }); test("/:entity (read many, paginated)", async () => { const res = await app.request("/entity/users?limit=1&offset=2"); - const data = (await res.json()) as RepositoryResponse; + const data = (await res.json()) as RepositoryResultJSON; - expect(data.meta.total).toBe(3); - expect(data.meta.count).toBe(3); + //expect(data.meta.total).toBe(3); + //expect(data.meta.count).toBe(3); expect(data.meta.items).toBe(1); expect(data.data.length).toBe(1); - expect(data.data[0].name).toBe("baz"); + expect(data.data[0]?.name).toBe("baz"); }); test("/:entity/:id (read one)", async () => { const res = await app.request("/entity/users/3"); - const data = (await res.json()) as RepositoryResponse; - console.log("data", data); + const data = (await res.json()) as RepositoryResultJSON; - expect(data.meta.total).toBe(3); - expect(data.meta.count).toBe(1); + //expect(data.meta.total).toBe(3); + //expect(data.meta.count).toBe(1); expect(data.meta.items).toBe(1); expect(data.data).toEqual({ id: 3, ...fixtures.users[2] }); }); @@ -201,7 +153,7 @@ describe("[data] DataController", async () => { method: "PATCH", body: JSON.stringify({ name: "new name" }), }); - const { data } = (await res.json()) as MutatorResponse; + const { data } = (await res.json()) as MutatorResultJSON; expect(res.ok).toBe(true); expect(data as any).toEqual({ id: 3, ...fixtures.users[2], name: "new name" }); @@ -209,27 +161,26 @@ describe("[data] DataController", async () => { test("/:entity/:id/:reference (read references)", async () => { const res = await app.request("/entity/users/1/posts"); - const data = (await res.json()) as RepositoryResponse; - console.log("data", data); + const data = (await res.json()) as RepositoryResultJSON; - expect(data.meta.total).toBe(2); - expect(data.meta.count).toBe(1); + //expect(data.meta.total).toBe(2); + //expect(data.meta.count).toBe(1); expect(data.meta.items).toBe(1); expect(data.data.length).toBe(1); - expect(data.data[0].content).toBe("post 1"); + expect(data.data[0]?.content).toBe("post 1"); }); test("/:entity/:id (delete one)", async () => { const res = await app.request("/entity/posts/2", { method: "DELETE", }); - const { data } = (await res.json()) as RepositoryResponse; + const { data } = (await res.json()) as RepositoryResultJSON; expect(data).toEqual({ id: 2, ...fixtures.posts[1] }); // verify const res2 = await app.request("/entity/posts"); - const data2 = (await res2.json()) as RepositoryResponse; - expect(data2.meta.total).toBe(1); + const data2 = (await res2.json()) as RepositoryResultJSON; + //expect(data2.meta.total).toBe(1); }); }); }); diff --git a/app/__test__/data/data.test.ts b/app/__test__/data/data.test.ts index 79b4301..8032167 100644 --- a/app/__test__/data/data.test.ts +++ b/app/__test__/data/data.test.ts @@ -34,19 +34,12 @@ describe("some tests", async () => { test("findId", async () => { const query = await em.repository(users).findId(1); - /*const { result, total, count, time } = query; - console.log("query", query.result, { - result, - total, - count, - time, - });*/ expect(query.sql).toBe( 'select "users"."id" as "id", "users"."username" as "username", "users"."email" as "email" from "users" where "id" = ? limit ?', ); expect(query.parameters).toEqual([1, 1]); - expect(query.result).toEqual([]); + expect(query.data).toBeUndefined(); }); test("findMany", async () => { @@ -56,7 +49,7 @@ describe("some tests", async () => { 'select "users"."id" as "id", "users"."username" as "username", "users"."email" as "email" from "users" order by "users"."id" asc limit ? offset ?', ); expect(query.parameters).toEqual([10, 0]); - expect(query.result).toEqual([]); + expect(query.data).toEqual([]); }); test("findMany with number", async () => { @@ -66,7 +59,7 @@ describe("some tests", async () => { 'select "posts"."id" as "id", "posts"."title" as "title", "posts"."content" as "content", "posts"."created_at" as "created_at", "posts"."likes" as "likes" from "posts" order by "posts"."id" asc limit ? offset ?', ); expect(query.parameters).toEqual([10, 0]); - expect(query.result).toEqual([]); + expect(query.data).toEqual([]); }); test("try adding an existing field name", async () => { diff --git a/app/__test__/data/mutation.simple.test.ts b/app/__test__/data/mutation.simple.test.ts index f425733..93b17da 100644 --- a/app/__test__/data/mutation.simple.test.ts +++ b/app/__test__/data/mutation.simple.test.ts @@ -45,7 +45,7 @@ describe("Mutator simple", async () => { }, }); - expect(query.result).toEqual([{ id: 1, label: "test", count: 1 }]); + expect(query.data).toEqual([{ id: 1, label: "test", count: 1 }]); }); test("update inserted row", async () => { @@ -87,7 +87,7 @@ describe("Mutator simple", async () => { expect(mutation.data).toEqual({ id, label: "new label", count: 100 }); const query2 = await em.repository(items).findId(id); - expect(query2.result.length).toBe(0); + expect(query2.data).toBeUndefined(); }); test("validation: insert incomplete row", async () => { @@ -177,13 +177,13 @@ describe("Mutator simple", async () => { }); test("insertMany", async () => { - const oldCount = (await em.repo(items).count()).count; + const oldCount = (await em.repo(items).count()).data.count; const inserts = [{ label: "insert 1" }, { label: "insert 2" }]; const { data } = await em.mutator(items).insertMany(inserts); expect(data.length).toBe(2); expect(data.map((d) => ({ label: d.label }))).toEqual(inserts); - const newCount = (await em.repo(items).count()).count; + const newCount = (await em.repo(items).count()).data.count; expect(newCount).toBe(oldCount + inserts.length); const { data: data2 } = await em.repo(items).findMany({ offset: oldCount }); diff --git a/app/__test__/data/specs/Mutator.spec.ts b/app/__test__/data/specs/Mutator.spec.ts index 7110956..d7f09e0 100644 --- a/app/__test__/data/specs/Mutator.spec.ts +++ b/app/__test__/data/specs/Mutator.spec.ts @@ -1,4 +1,4 @@ -import { afterAll, describe, expect, test } from "bun:test"; +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; import type { EventManager } from "../../../src/core/events"; import { Entity, @@ -12,11 +12,14 @@ import { TextField, } from "../../../src/data"; import * as proto from "../../../src/data/prototype"; -import { getDummyConnection } from "../helper"; +import { getDummyConnection, disableConsoleLog, enableConsoleLog } from "../../helper"; const { dummyConnection, afterAllCleanup } = getDummyConnection(); afterAll(afterAllCleanup); +beforeAll(() => disableConsoleLog(["log", "warn"])); +afterAll(async () => (await afterAllCleanup()) && enableConsoleLog()); + describe("[data] Mutator (base)", async () => { const entity = new Entity("items", [ new TextField("label", { required: true }), diff --git a/app/__test__/data/specs/Repository.spec.ts b/app/__test__/data/specs/Repository.spec.ts index 54254d4..23f094c 100644 --- a/app/__test__/data/specs/Repository.spec.ts +++ b/app/__test__/data/specs/Repository.spec.ts @@ -26,120 +26,6 @@ async function sleep(ms: number) { } describe("[Repository]", async () => { - test.skip("bulk", async () => { - //const connection = dummyConnection; - //const connection = getLocalLibsqlConnection(); - const credentials = null as any; // @todo: determine what to do here - const connection = new LibsqlConnection(credentials); - - const em = new EntityManager([], connection); - /*const emLibsql = new EntityManager([], { - url: connection.url.replace("https", "libsql"), - authToken: connection.authToken, - });*/ - const table = "posts"; - - const client = connection.getClient(); - if (!client) { - console.log("Cannot perform test without libsql connection"); - return; - } - - const conn = em.connection.kysely; - const selectQ = (e: E) => e.selectFrom(table).selectAll().limit(2); - const countQ = (e: E) => e.selectFrom(table).select(e.fn.count("*").as("count")); - - async function executeTransaction(em: EntityManager) { - return await em.connection.kysely.transaction().execute(async (e) => { - const res = await selectQ(e).execute(); - const count = await countQ(e).execute(); - - return [res, count]; - }); - } - - async function executeBatch(em: EntityManager) { - const queries = [selectQ(conn), countQ(conn)]; - return await em.connection.batchQuery(queries); - } - - async function executeSingleKysely(em: EntityManager) { - const res = await selectQ(conn).execute(); - const count = await countQ(conn).execute(); - return [res, count]; - } - - async function executeSingleClient(em: EntityManager) { - const q1 = selectQ(conn).compile(); - const res = await client.execute({ - sql: q1.sql, - args: q1.parameters as any, - }); - - const q2 = countQ(conn).compile(); - const count = await client.execute({ - sql: q2.sql, - args: q2.parameters as any, - }); - return [res, count]; - } - - const transaction = await executeTransaction(em); - const batch = await executeBatch(em); - - expect(batch).toEqual(transaction as any); - - const testperf = false; - if (testperf) { - const times = 5; - - const exec = async ( - name: string, - fn: (em: EntityManager) => Promise, - em: EntityManager, - ) => { - const res = await Perf.execute(() => fn(em), times); - await sleep(1000); - const info = { - name, - total: res.total.toFixed(2), - avg: (res.total / times).toFixed(2), - first: res.marks[0].time.toFixed(2), - last: res.marks[res.marks.length - 1].time.toFixed(2), - }; - console.log(info.name, info, res.marks); - return info; - }; - - const data: any[] = []; - data.push(await exec("transaction.http", executeTransaction, em)); - data.push(await exec("bulk.http", executeBatch, em)); - data.push(await exec("singleKy.http", executeSingleKysely, em)); - data.push(await exec("singleCl.http", executeSingleClient, em)); - - /*data.push(await exec("transaction.libsql", executeTransaction, emLibsql)); - data.push(await exec("bulk.libsql", executeBatch, emLibsql)); - data.push(await exec("singleKy.libsql", executeSingleKysely, emLibsql)); - data.push(await exec("singleCl.libsql", executeSingleClient, emLibsql));*/ - - console.table(data); - /** - * ┌───┬────────────────────┬────────┬────────┬────────┬────────┐ - * │ │ name │ total │ avg │ first │ last │ - * ├───┼────────────────────┼────────┼────────┼────────┼────────┤ - * │ 0 │ transaction.http │ 681.29 │ 136.26 │ 136.46 │ 396.09 │ - * │ 1 │ bulk.http │ 164.82 │ 32.96 │ 32.95 │ 99.91 │ - * │ 2 │ singleKy.http │ 330.01 │ 66.00 │ 65.86 │ 195.41 │ - * │ 3 │ singleCl.http │ 326.17 │ 65.23 │ 61.32 │ 198.08 │ - * │ 4 │ transaction.libsql │ 856.79 │ 171.36 │ 132.31 │ 595.24 │ - * │ 5 │ bulk.libsql │ 180.63 │ 36.13 │ 35.39 │ 107.71 │ - * │ 6 │ singleKy.libsql │ 347.11 │ 69.42 │ 65.00 │ 207.14 │ - * │ 7 │ singleCl.libsql │ 328.60 │ 65.72 │ 62.19 │ 195.04 │ - * └───┴────────────────────┴────────┴────────┴────────┴────────┘ - */ - } - }); - test("count & exists", async () => { const items = new Entity("items", [new TextField("label")]); const em = new EntityManager([items], dummyConnection); @@ -160,25 +46,44 @@ describe("[Repository]", async () => { // count all const res = await em.repository(items).count(); expect(res.sql).toBe('select count(*) as "count" from "items"'); - expect(res.count).toBe(3); + expect(res.data.count).toBe(3); + + // + { + const res = await em.repository(items).findMany(); + expect(res.count).toBeUndefined(); + } + + { + const res = await em + .repository(items, { + includeCounts: true, + }) + .findMany(); + expect(res.count).toBe(3); + } // count filtered - const res2 = await em.repository(items).count({ label: { $in: ["a", "b"] } }); + const res2 = await em + .repository(items, { + includeCounts: true, + }) + .count({ label: { $in: ["a", "b"] } }); expect(res2.sql).toBe('select count(*) as "count" from "items" where "label" in (?, ?)'); expect(res2.parameters).toEqual(["a", "b"]); - expect(res2.count).toBe(2); + expect(res2.data.count).toBe(2); // check exists const res3 = await em.repository(items).exists({ label: "a" }); - expect(res3.exists).toBe(true); + expect(res3.data.exists).toBe(true); const res4 = await em.repository(items).exists({ label: "d" }); - expect(res4.exists).toBe(false); + expect(res4.data.exists).toBe(false); // for now, allow empty filter const res5 = await em.repository(items).exists({}); - expect(res5.exists).toBe(true); + expect(res5.data.exists).toBe(true); }); test("option: silent", async () => { @@ -191,6 +96,9 @@ describe("[Repository]", async () => { // should throw because table doesn't exist expect(em.repo("items").findMany({})).rejects.toThrow(/no such table/); // should silently return empty result + em.repo("items", { silent: true }) + .findMany({}) + .then((r) => r.data); expect( em .repo("items", { silent: true }) @@ -209,16 +117,16 @@ describe("[Repository]", async () => { expect( em - .repo("items") + .repo("items", { includeCounts: true }) .findMany({}) - .then((r) => [r.meta.count, r.meta.total]), + .then((r) => [r.count, r.total]), ).resolves.toEqual([0, 0]); expect( em .repo("items", { includeCounts: false }) .findMany({}) - .then((r) => [r.meta.count, r.meta.total]), + .then((r) => [r.count, r.total]), ).resolves.toEqual([undefined, undefined]); }); }); diff --git a/app/__test__/helper.ts b/app/__test__/helper.ts index 16b8b8e..ba09d4c 100644 --- a/app/__test__/helper.ts +++ b/app/__test__/helper.ts @@ -38,14 +38,15 @@ export function getLocalLibsqlConnection() { return { url: "http://127.0.0.1:8080" }; } -type ConsoleSeverity = "log" | "warn" | "error"; +type ConsoleSeverity = "debug" | "log" | "warn" | "error"; const _oldConsoles = { + debug: console.debug, log: console.log, warn: console.warn, error: console.error, }; -export function disableConsoleLog(severities: ConsoleSeverity[] = ["log", "warn"]) { +export function disableConsoleLog(severities: ConsoleSeverity[] = ["debug", "log", "warn"]) { severities.forEach((severity) => { console[severity] = () => null; }); diff --git a/app/package.json b/app/package.json index fc458c8..c8d326e 100644 --- a/app/package.json +++ b/app/package.json @@ -76,6 +76,7 @@ "devDependencies": { "@aws-sdk/client-s3": "^3.758.0", "@bluwy/giget-core": "^0.1.2", + "@cloudflare/workers-types": "^4.20250606.0", "@dagrejs/dagre": "^1.1.4", "@hono/typebox-validator": "^0.3.3", "@hono/vite-dev-server": "^0.19.1", @@ -102,6 +103,7 @@ "jsdom": "^26.0.0", "jsonv-ts": "^0.1.0", "kysely-d1": "^0.3.0", + "kysely-generic-sqlite": "^1.2.1", "open": "^10.1.0", "openapi-types": "^12.1.3", "picocolors": "^1.1.1", @@ -124,8 +126,7 @@ "vite": "^6.3.5", "vite-tsconfig-paths": "^5.1.4", "vitest": "^3.0.9", - "wouter": "^3.6.0", - "@cloudflare/workers-types": "^4.20250606.0" + "wouter": "^3.6.0" }, "optionalDependencies": { "@hono/node-server": "^1.14.3" diff --git a/app/src/adapter/bun/connection/BunSqliteConnection.spec.ts b/app/src/adapter/bun/connection/BunSqliteConnection.spec.ts new file mode 100644 index 0000000..5099fbc --- /dev/null +++ b/app/src/adapter/bun/connection/BunSqliteConnection.spec.ts @@ -0,0 +1,12 @@ +import { connectionTestSuite } from "data/connection/connection-test-suite"; +import { bunSqlite } from "./BunSqliteConnection"; +import { bunTestRunner } from "adapter/bun/test"; +import { describe } from "bun:test"; +import { Database } from "bun:sqlite"; + +describe("BunSqliteConnection", () => { + connectionTestSuite(bunTestRunner, { + makeConnection: () => bunSqlite({ database: new Database(":memory:") }), + rawDialectDetails: [], + }); +}); diff --git a/app/src/adapter/bun/connection/BunSqliteConnection.ts b/app/src/adapter/bun/connection/BunSqliteConnection.ts new file mode 100644 index 0000000..dc5638f --- /dev/null +++ b/app/src/adapter/bun/connection/BunSqliteConnection.ts @@ -0,0 +1,41 @@ +import type { Database } from "bun:sqlite"; +import { + buildQueryFn, + GenericSqliteConnection, + parseBigInt, + type IGenericSqlite, +} from "data/connection/sqlite/GenericSqliteConnection"; + +export type BunSqliteConnectionConfig = { + database: Database; +}; + +function bunSqliteExecutor(db: Database, cache: boolean): IGenericSqlite { + const fn = cache ? "query" : "prepare"; + const getStmt = (sql: string) => db[fn](sql); + + return { + db, + query: buildQueryFn({ + all: (sql, parameters) => getStmt(sql).all(...(parameters || [])), + run: (sql, parameters) => { + const { changes, lastInsertRowid } = getStmt(sql).run(...(parameters || [])); + return { + insertId: parseBigInt(lastInsertRowid), + numAffectedRows: parseBigInt(changes), + }; + }, + }), + close: () => db.close(), + }; +} + +export function bunSqlite(config: BunSqliteConnectionConfig) { + return new GenericSqliteConnection( + config.database, + () => bunSqliteExecutor(config.database, false), + { + name: "bun-sqlite", + }, + ); +} diff --git a/app/src/adapter/bun/test.ts b/app/src/adapter/bun/test.ts index 7bd314a..1cb6ca5 100644 --- a/app/src/adapter/bun/test.ts +++ b/app/src/adapter/bun/test.ts @@ -1,6 +1,7 @@ -import { expect, test, mock } from "bun:test"; +import { expect, test, mock, describe } from "bun:test"; export const bunTestRunner = { + describe, expect, test, mock, diff --git a/app/src/adapter/cloudflare/connection/D1Connection.ts b/app/src/adapter/cloudflare/connection/D1Connection.ts index ddf6be8..d7eb5a0 100644 --- a/app/src/adapter/cloudflare/connection/D1Connection.ts +++ b/app/src/adapter/cloudflare/connection/D1Connection.ts @@ -1,65 +1,42 @@ /// -import { KyselyPluginRunner, SqliteConnection, SqliteIntrospector } from "bknd/data"; -import type { QB } from "data/connection/Connection"; -import { type DatabaseIntrospector, Kysely, ParseJSONResultsPlugin } from "kysely"; +import { SqliteConnection } from "bknd/data"; +import type { ConnQuery, ConnQueryResults } from "data/connection/Connection"; import { D1Dialect } from "kysely-d1"; export type D1ConnectionConfig = { binding: DB; }; -class CustomD1Dialect extends D1Dialect { - override createIntrospector(db: Kysely): DatabaseIntrospector { - return new SqliteIntrospector(db, { - excludeTables: ["_cf_KV", "_cf_METADATA"], - }); - } -} - export class D1Connection< DB extends D1Database | D1DatabaseSession = D1Database, -> extends SqliteConnection { +> extends SqliteConnection { + override name = "sqlite-d1"; + protected override readonly supported = { batching: true, + softscans: false, }; constructor(private config: D1ConnectionConfig) { - const plugins = [new ParseJSONResultsPlugin()]; - - const kysely = new Kysely({ - dialect: new CustomD1Dialect({ database: config.binding as D1Database }), - plugins, + super({ + excludeTables: ["_cf_KV", "_cf_METADATA"], + dialect: D1Dialect, + dialectArgs: [{ database: config.binding as D1Database }], }); - super(kysely, {}, plugins); } - get client(): DB { - return this.config.binding; - } + override async executeQueries(...qbs: O): Promise> { + const compiled = this.getCompiled(...qbs); - protected override async batch( - queries: [...Queries], - ): Promise<{ - [K in keyof Queries]: Awaited>; - }> { const db = this.config.binding; const res = await db.batch( - queries.map((q) => { - const { sql, parameters } = q.compile(); + compiled.map(({ sql, parameters }) => { return db.prepare(sql).bind(...parameters); }), ); - // let it run through plugins - const kyselyPlugins = new KyselyPluginRunner(this.plugins); - const data: any = []; - for (const r of res) { - const rows = await kyselyPlugins.transformResultRows(r.results); - data.push(rows); - } - - return data; + return this.withTransformedRows(res, "results") as any; } } diff --git a/app/src/adapter/node/connection/NodeSqliteConnection.ts b/app/src/adapter/node/connection/NodeSqliteConnection.ts new file mode 100644 index 0000000..b6faad4 --- /dev/null +++ b/app/src/adapter/node/connection/NodeSqliteConnection.ts @@ -0,0 +1,46 @@ +import { + buildQueryFn, + GenericSqliteConnection, + parseBigInt, + type IGenericSqlite, +} from "../../../data/connection/sqlite/GenericSqliteConnection"; +import type { DatabaseSync } from "node:sqlite"; + +export type NodeSqliteConnectionConfig = { + database: DatabaseSync; +}; + +function nodeSqliteExecutor(db: DatabaseSync): IGenericSqlite { + const getStmt = (sql: string) => { + const stmt = db.prepare(sql); + //stmt.setReadBigInts(true); + return stmt; + }; + + return { + db, + query: buildQueryFn({ + all: (sql, parameters = []) => getStmt(sql).all(...parameters), + run: (sql, parameters = []) => { + const { changes, lastInsertRowid } = getStmt(sql).run(...parameters); + return { + insertId: parseBigInt(lastInsertRowid), + numAffectedRows: parseBigInt(changes), + }; + }, + }), + close: () => db.close(), + iterator: (isSelect, sql, parameters = []) => { + if (!isSelect) { + throw new Error("Only support select in stream()"); + } + return getStmt(sql).iterate(...parameters) as any; + }, + }; +} + +export function nodeSqlite(config: NodeSqliteConnectionConfig) { + return new GenericSqliteConnection(config.database, () => nodeSqliteExecutor(config.database), { + name: "node-sqlite", + }); +} diff --git a/app/src/adapter/node/connection/NodeSqliteConnection.vi-test.ts b/app/src/adapter/node/connection/NodeSqliteConnection.vi-test.ts new file mode 100644 index 0000000..62ee9cb --- /dev/null +++ b/app/src/adapter/node/connection/NodeSqliteConnection.vi-test.ts @@ -0,0 +1,11 @@ +import { nodeSqlite } from "./NodeSqliteConnection"; +import { DatabaseSync } from "node:sqlite"; +import { connectionTestSuite } from "data/connection/connection-test-suite"; +import { describe, test, expect } from "vitest"; + +describe("NodeSqliteConnection", () => { + connectionTestSuite({ describe, test, expect } as any, { + makeConnection: () => nodeSqlite({ database: new DatabaseSync(":memory:") }), + rawDialectDetails: [], + }); +}); diff --git a/app/src/adapter/node/node.adapter.native-spec.ts b/app/src/adapter/node/node.adapter.vi-test.ts similarity index 57% rename from app/src/adapter/node/node.adapter.native-spec.ts rename to app/src/adapter/node/node.adapter.vi-test.ts index 62dcc1b..31cdb31 100644 --- a/app/src/adapter/node/node.adapter.native-spec.ts +++ b/app/src/adapter/node/node.adapter.vi-test.ts @@ -1,14 +1,14 @@ -import { describe, before, after } from "node:test"; +import { describe, beforeAll, afterAll } from "vitest"; import * as node from "./node.adapter"; import { adapterTestSuite } from "adapter/adapter-test-suite"; -import { nodeTestRunner } from "adapter/node/test"; +import { viTestRunner } from "adapter/node/vitest"; import { disableConsoleLog, enableConsoleLog } from "core/utils"; -before(() => disableConsoleLog()); -after(enableConsoleLog); +beforeAll(() => disableConsoleLog()); +afterAll(enableConsoleLog); describe("node adapter", () => { - adapterTestSuite(nodeTestRunner, { + adapterTestSuite(viTestRunner, { makeApp: node.createApp, makeHandler: node.createHandler, }); diff --git a/app/src/adapter/node/test.ts b/app/src/adapter/node/test.ts index 992cbee..5a634ae 100644 --- a/app/src/adapter/node/test.ts +++ b/app/src/adapter/node/test.ts @@ -1,5 +1,5 @@ import nodeAssert from "node:assert/strict"; -import { test } from "node:test"; +import { test, describe } from "node:test"; import type { Matcher, Test, TestFn, TestRunner } from "core/test"; // Track mock function calls @@ -85,6 +85,7 @@ nodeTest.skipIf = (condition: boolean): Test => { }; export const nodeTestRunner: TestRunner = { + describe, test: nodeTest, mock: createMockFunction, expect: (actual?: T, failMsg?: string) => ({ diff --git a/app/src/adapter/node/vitest.ts b/app/src/adapter/node/vitest.ts new file mode 100644 index 0000000..569be7a --- /dev/null +++ b/app/src/adapter/node/vitest.ts @@ -0,0 +1,50 @@ +import type { TestFn, TestRunner, Test } from "core/test"; +import { describe, test, expect, vi } from "vitest"; + +function vitestTest(label: string, fn: TestFn, options?: any) { + return test(label, fn as any); +} +vitestTest.if = (condition: boolean): Test => { + if (condition) { + return vitestTest; + } + return (() => {}) as any; +}; +vitestTest.skip = (label: string, fn: TestFn) => { + return test.skip(label, fn as any); +}; +vitestTest.skipIf = (condition: boolean): Test => { + if (condition) { + return (() => {}) as any; + } + return vitestTest; +}; + +const vitestExpect = (actual: T, parentFailMsg?: string) => { + return { + toEqual: (expected: T, failMsg = parentFailMsg) => { + expect(actual, failMsg).toEqual(expected); + }, + toBe: (expected: T, failMsg = parentFailMsg) => { + expect(actual, failMsg).toBe(expected); + }, + toBeString: () => expect(typeof actual, parentFailMsg).toBe("string"), + toBeUndefined: () => expect(actual, parentFailMsg).toBeUndefined(), + toBeDefined: () => expect(actual, parentFailMsg).toBeDefined(), + toBeOneOf: (expected: T | Array | Iterable, failMsg = parentFailMsg) => { + const e = Array.isArray(expected) ? expected : [expected]; + expect(actual, failMsg).toBeOneOf(e); + }, + toHaveBeenCalled: () => expect(actual, parentFailMsg).toHaveBeenCalled(), + toHaveBeenCalledTimes: (expected: number, failMsg = parentFailMsg) => { + expect(actual, failMsg).toHaveBeenCalledTimes(expected); + }, + }; +}; + +export const viTestRunner: TestRunner = { + describe, + test: vitestTest, + expect: vitestExpect as any, + mock: (fn) => vi.fn(fn), +}; diff --git a/app/src/core/test/index.ts b/app/src/core/test/index.ts index ca1ffba..c731938 100644 --- a/app/src/core/test/index.ts +++ b/app/src/core/test/index.ts @@ -16,6 +16,7 @@ export interface Test { skipIf: (condition: boolean) => (label: string, fn: TestFn) => void; } export type TestRunner = { + describe: (label: string, asyncFn: () => Promise) => void; test: Test; mock: any>(fn: T) => T | any; expect: ( diff --git a/app/src/data/api/DataController.ts b/app/src/data/api/DataController.ts index bc86a8d..de3e715 100644 --- a/app/src/data/api/DataController.ts +++ b/app/src/data/api/DataController.ts @@ -3,9 +3,7 @@ import { DataPermissions, type EntityData, type EntityManager, - type MutatorResponse, type RepoQuery, - type RepositoryResponse, repoQuery, } from "data"; import type { Handler } from "hono/types"; @@ -32,33 +30,6 @@ export class DataController extends Controller { return this.ctx.guard; } - repoResult = RepositoryResponse>( - res: T, - ): Pick { - let meta: Partial = {}; - - if ("meta" in res) { - const { query, ...rest } = res.meta; - meta = rest; - if (isDebug()) meta.query = query; - } - - const template = { data: res.data, meta }; - - // @todo: this works but it breaks in FE (need to improve DataTable) - // filter empty - return Object.fromEntries( - Object.entries(template).filter(([_, v]) => typeof v !== "undefined" && v !== null), - ) as any; - } - - mutatorResult(res: MutatorResponse | MutatorResponse) { - const template = { data: res.data }; - - // filter empty - return Object.fromEntries(Object.entries(template).filter(([_, v]) => v !== undefined)); - } - entityExists(entity: string) { try { return !!this.em.entity(entity); @@ -257,7 +228,7 @@ export class DataController extends Controller { const where = c.req.valid("json") as any; const result = await this.em.repository(entity).count(where); - return c.json({ entity, count: result.count }); + return c.json({ entity, ...result.data }); }, ); @@ -279,7 +250,7 @@ export class DataController extends Controller { const where = c.req.valid("json") as any; const result = await this.em.repository(entity).exists(where); - return c.json({ entity, exists: result.exists }); + return c.json({ entity, ...result.data }); }, ); @@ -318,7 +289,7 @@ export class DataController extends Controller { const options = c.req.valid("query") as RepoQuery; const result = await this.em.repository(entity).findMany(options); - return c.json(this.repoResult(result), { status: result.data ? 200 : 404 }); + return c.json(result, { status: result.data ? 200 : 404 }); }, ); @@ -347,7 +318,7 @@ export class DataController extends Controller { const options = c.req.valid("query") as RepoQuery; const result = await this.em.repository(entity).findId(id, options); - return c.json(this.repoResult(result), { status: result.data ? 200 : 404 }); + return c.json(result, { status: result.data ? 200 : 404 }); }, ); @@ -380,7 +351,7 @@ export class DataController extends Controller { .repository(entity) .findManyByReference(id, reference, options); - return c.json(this.repoResult(result), { status: result.data ? 200 : 404 }); + return c.json(result, { status: result.data ? 200 : 404 }); }, ); @@ -414,7 +385,7 @@ export class DataController extends Controller { const options = (await c.req.json()) as RepoQuery; const result = await this.em.repository(entity).findMany(options); - return c.json(this.repoResult(result), { status: result.data ? 200 : 404 }); + return c.json(result, { status: result.data ? 200 : 404 }); }, ); @@ -440,11 +411,11 @@ export class DataController extends Controller { if (Array.isArray(body)) { const result = await this.em.mutator(entity).insertMany(body); - return c.json(this.mutatorResult(result), 201); + return c.json(result, 201); } const result = await this.em.mutator(entity).insertOne(body); - return c.json(this.mutatorResult(result), 201); + return c.json(result, 201); }, ); @@ -475,7 +446,7 @@ export class DataController extends Controller { }; const result = await this.em.mutator(entity).updateWhere(update, where); - return c.json(this.mutatorResult(result)); + return c.json(result); }, ); @@ -497,7 +468,7 @@ export class DataController extends Controller { const body = (await c.req.json()) as EntityData; const result = await this.em.mutator(entity).updateOne(id, body); - return c.json(this.mutatorResult(result)); + return c.json(result); }, ); @@ -517,7 +488,7 @@ export class DataController extends Controller { } const result = await this.em.mutator(entity).deleteOne(id); - return c.json(this.mutatorResult(result)); + return c.json(result); }, ); @@ -539,7 +510,7 @@ export class DataController extends Controller { const where = (await c.req.json()) as RepoQuery["where"]; const result = await this.em.mutator(entity).deleteWhere(where); - return c.json(this.mutatorResult(result)); + return c.json(result); }, ); diff --git a/app/src/data/connection/Connection.ts b/app/src/data/connection/Connection.ts index 814c03d..fab8e36 100644 --- a/app/src/data/connection/Connection.ts +++ b/app/src/data/connection/Connection.ts @@ -2,12 +2,15 @@ import { type AliasableExpression, type ColumnBuilderCallback, type ColumnDataType, + type Compilable, + type CompiledQuery, type DatabaseIntrospector, type Dialect, type Expression, type Kysely, type KyselyPlugin, type OnModifyForeignAction, + type QueryResult, type RawBuilder, type SelectQueryBuilder, type SelectQueryNode, @@ -15,7 +18,8 @@ import { sql, } from "kysely"; import type { BaseIntrospector, BaseIntrospectorConfig } from "./BaseIntrospector"; -import type { Constructor } from "core"; +import type { Constructor, DB } from "core"; +import { KyselyPluginRunner } from "data/plugins/KyselyPluginRunner"; export type QB = SelectQueryBuilder; @@ -75,22 +79,44 @@ export type DbFunctions = { >; }; +export type ConnQuery = CompiledQuery | Compilable; + +export type ConnQueryResult = T extends CompiledQuery + ? QueryResult + : T extends Compilable + ? QueryResult + : never; + +export type ConnQueryResults = { + [K in keyof T]: ConnQueryResult; +}; + const CONN_SYMBOL = Symbol.for("bknd:connection"); -export abstract class Connection { +export type Features = { + batching: boolean; + softscans: boolean; +}; + +export abstract class Connection { + abstract name: string; protected initialized = false; - kysely: Kysely; - protected readonly supported = { + protected pluginRunner: KyselyPluginRunner; + protected readonly supported: Partial = { batching: false, + softscans: true, }; + kysely: Kysely; + client!: Client; constructor( - kysely: Kysely, + kysely: Kysely, public fn: Partial = {}, protected plugins: KyselyPlugin[] = [], ) { this.kysely = kysely; this[CONN_SYMBOL] = true; + this.pluginRunner = new KyselyPluginRunner(plugins); } // @todo: consider moving constructor logic here, required by sqlocal @@ -121,30 +147,46 @@ export abstract class Connection { return res.rows.length > 0; } - protected async batch( - queries: [...Queries], - ): Promise<{ - [K in keyof Queries]: Awaited>; - }> { - throw new Error("Batching not supported"); + protected async transformResultRows(result: any[]): Promise { + return await this.pluginRunner.transformResultRows(result); } - async batchQuery( - queries: [...Queries], - ): Promise<{ - [K in keyof Queries]: Awaited>; - }> { - // bypass if no client support - if (!this.supports("batching")) { - const data: any = []; - for (const q of queries) { - const result = await q.execute(); - data.push(result); - } - return data; - } + /** + * Execute a query and return the result including all metadata + * returned from the dialect. + */ + async executeQueries(...qbs: O): Promise> { + return Promise.all(qbs.map(async (qb) => await this.kysely.executeQuery(qb))) as any; + } - return await this.batch(queries); + async executeQuery(qb: O): Promise> { + const res = await this.executeQueries(qb); + return res[0] as any; + } + + protected getCompiled(...qbs: ConnQuery[]): CompiledQuery[] { + return qbs.map((qb) => { + if ("compile" in qb) { + return qb.compile(); + } + return qb; + }); + } + + protected async withTransformedRows< + Key extends string = "rows", + O extends { [K in Key]: any[] }[] = [], + >(result: O, _key?: Key): Promise { + return (await Promise.all( + result.map(async (row) => { + const key = _key ?? "rows"; + const { [key]: rows, ...r } = row; + return { + ...r, + rows: await this.transformResultRows(rows), + }; + }), + )) as any; } protected validateFieldSpecType(type: string): type is FieldSpec["type"] { diff --git a/app/src/data/connection/connection-test-suite.ts b/app/src/data/connection/connection-test-suite.ts new file mode 100644 index 0000000..5305337 --- /dev/null +++ b/app/src/data/connection/connection-test-suite.ts @@ -0,0 +1,187 @@ +import type { TestRunner } from "core/test"; +import { Connection, type FieldSpec } from "./Connection"; + +export function connectionTestSuite( + testRunner: TestRunner, + { + makeConnection, + rawDialectDetails, + }: { + makeConnection: () => Connection; + rawDialectDetails: string[]; + }, +) { + const { test, expect, describe } = testRunner; + + test("pings", async () => { + const connection = makeConnection(); + const res = await connection.ping(); + expect(res).toBe(true); + }); + + test("initializes", async () => { + const connection = makeConnection(); + await connection.init(); + // @ts-expect-error + expect(connection.initialized).toBe(true); + expect(connection.client).toBeDefined(); + }); + + test("isConnection", async () => { + const connection = makeConnection(); + expect(Connection.isConnection(connection)).toBe(true); + }); + + test("getFieldSchema", async () => { + const c = makeConnection(); + const specToNode = (spec: FieldSpec) => { + // @ts-expect-error + const schema = c.kysely.schema.createTable("test").addColumn(...c.getFieldSchema(spec)); + return schema.toOperationNode(); + }; + + { + // primary + const node = specToNode({ + type: "integer", + name: "id", + primary: true, + }); + const col = node.columns[0]!; + expect(col.primaryKey).toBe(true); + expect(col.notNull).toBe(true); + } + + { + // normal + const node = specToNode({ + type: "text", + name: "text", + }); + const col = node.columns[0]!; + expect(!col.primaryKey).toBe(true); + expect(!col.notNull).toBe(true); + } + + { + // nullable (expect to be same as normal) + const node = specToNode({ + type: "text", + name: "text", + nullable: true, + }); + const col = node.columns[0]!; + expect(!col.primaryKey).toBe(true); + expect(!col.notNull).toBe(true); + } + }); + + describe("schema", async () => { + const connection = makeConnection(); + const fields = [ + { + type: "integer", + name: "id", + primary: true, + }, + { + type: "text", + name: "text", + }, + { + type: "json", + name: "json", + }, + ] as const satisfies FieldSpec[]; + + let b = connection.kysely.schema.createTable("test"); + for (const field of fields) { + // @ts-expect-error + b = b.addColumn(...connection.getFieldSchema(field)); + } + await b.execute(); + + // add index + await connection.kysely.schema.createIndex("test_index").on("test").columns(["id"]).execute(); + + test("executes query", async () => { + await connection.kysely + .insertInto("test") + .values({ id: 1, text: "test", json: JSON.stringify({ a: 1 }) }) + .execute(); + + const expected = { id: 1, text: "test", json: { a: 1 } }; + + const qb = connection.kysely.selectFrom("test").selectAll(); + const res = await connection.executeQuery(qb); + expect(res.rows).toEqual([expected]); + expect(rawDialectDetails.every((detail) => detail in res)).toBe(true); + + { + const res = await connection.executeQueries(qb, qb); + expect(res.length).toBe(2); + res.map((r) => { + expect(r.rows).toEqual([expected]); + expect(rawDialectDetails.every((detail) => detail in r)).toBe(true); + }); + } + }); + + test("introspects", async () => { + const tables = await connection.getIntrospector().getTables({ + withInternalKyselyTables: false, + }); + const clean = tables.map((t) => ({ + ...t, + columns: t.columns.map((c) => ({ + ...c, + dataType: undefined, + })), + })); + + expect(clean).toEqual([ + { + name: "test", + isView: false, + columns: [ + { + name: "id", + dataType: undefined, + isNullable: false, + isAutoIncrementing: true, + hasDefaultValue: false, + }, + { + name: "text", + dataType: undefined, + isNullable: true, + isAutoIncrementing: false, + hasDefaultValue: false, + }, + { + name: "json", + dataType: undefined, + isNullable: true, + isAutoIncrementing: false, + hasDefaultValue: false, + }, + ], + }, + ]); + }); + + expect(await connection.getIntrospector().getIndices()).toEqual([ + { + name: "test_index", + table: "test", + isUnique: false, + columns: [ + { + name: "id", + order: 0, + }, + ], + }, + ]); + }); +} diff --git a/app/src/data/connection/sqlite/GenericSqliteConnection.ts b/app/src/data/connection/sqlite/GenericSqliteConnection.ts new file mode 100644 index 0000000..3707181 --- /dev/null +++ b/app/src/data/connection/sqlite/GenericSqliteConnection.ts @@ -0,0 +1,37 @@ +import type { KyselyPlugin } from "kysely"; +import { + type IGenericSqlite, + type OnCreateConnection, + type Promisable, + parseBigInt, + buildQueryFn, + GenericSqliteDialect, +} from "kysely-generic-sqlite"; +import { SqliteConnection } from "./SqliteConnection"; + +export type GenericSqliteConnectionConfig = { + name: string; + additionalPlugins?: KyselyPlugin[]; + excludeTables?: string[]; + onCreateConnection?: OnCreateConnection; +}; + +export { parseBigInt, buildQueryFn, GenericSqliteDialect, type IGenericSqlite }; + +export class GenericSqliteConnection extends SqliteConnection { + override name = "generic-sqlite"; + + constructor( + db: DB, + executor: () => Promisable, + config?: GenericSqliteConnectionConfig, + ) { + super({ + dialect: GenericSqliteDialect, + dialectArgs: [executor, config?.onCreateConnection], + additionalPlugins: config?.additionalPlugins, + excludeTables: config?.excludeTables, + }); + this.client = db; + } +} diff --git a/app/src/data/connection/sqlite/LibsqlConnection.spec.ts b/app/src/data/connection/sqlite/LibsqlConnection.spec.ts new file mode 100644 index 0000000..dde5a84 --- /dev/null +++ b/app/src/data/connection/sqlite/LibsqlConnection.spec.ts @@ -0,0 +1,11 @@ +import { connectionTestSuite } from "../connection-test-suite"; +import { LibsqlConnection } from "./LibsqlConnection"; +import { bunTestRunner } from "adapter/bun/test"; +import { describe } from "bun:test"; + +describe("LibsqlConnection", () => { + connectionTestSuite(bunTestRunner, { + makeConnection: () => new LibsqlConnection({ url: ":memory:" }), + rawDialectDetails: ["rowsAffected", "lastInsertRowid"], + }); +}); diff --git a/app/src/data/connection/sqlite/LibsqlConnection.ts b/app/src/data/connection/sqlite/LibsqlConnection.ts index 210f8cf..cf0ff58 100644 --- a/app/src/data/connection/sqlite/LibsqlConnection.ts +++ b/app/src/data/connection/sqlite/LibsqlConnection.ts @@ -1,40 +1,26 @@ -import { type Client, type Config, type InStatement, createClient } from "@libsql/client"; +import { createClient, type Client, type Config, type InStatement } from "@libsql/client"; import { LibsqlDialect } from "@libsql/kysely-libsql"; -import { FilterNumericKeysPlugin } from "data/plugins/FilterNumericKeysPlugin"; -import { KyselyPluginRunner } from "data/plugins/KyselyPluginRunner"; -import { type DatabaseIntrospector, Kysely, ParseJSONResultsPlugin } from "kysely"; -import type { QB } from "../Connection"; -import { SqliteConnection } from "./SqliteConnection"; -import { SqliteIntrospector } from "./SqliteIntrospector"; import { $console } from "core"; +import { FilterNumericKeysPlugin } from "data/plugins/FilterNumericKeysPlugin"; +import type { ConnQuery, ConnQueryResults } from "../Connection"; +import { SqliteConnection } from "./SqliteConnection"; export const LIBSQL_PROTOCOLS = ["wss", "https", "libsql"] as const; export type LibSqlCredentials = Config & { protocol?: (typeof LIBSQL_PROTOCOLS)[number]; }; -const plugins = [new FilterNumericKeysPlugin(), new ParseJSONResultsPlugin()]; - -class CustomLibsqlDialect extends LibsqlDialect { - override createIntrospector(db: Kysely): DatabaseIntrospector { - return new SqliteIntrospector(db, { - excludeTables: ["libsql_wasm_func_table"], - plugins, - }); - } -} - -export class LibsqlConnection extends SqliteConnection { - private client: Client; +export class LibsqlConnection extends SqliteConnection { + override name = "libsql"; protected override readonly supported = { batching: true, + softscans: true, }; constructor(client: Client); constructor(credentials: LibSqlCredentials); constructor(clientOrCredentials: Client | LibSqlCredentials) { let client: Client; - let batching_enabled = true; if (clientOrCredentials && "url" in clientOrCredentials) { let { url, authToken, protocol } = clientOrCredentials; if (protocol && LIBSQL_PROTOCOLS.includes(protocol)) { @@ -48,45 +34,25 @@ export class LibsqlConnection extends SqliteConnection { client = clientOrCredentials; } - const kysely = new Kysely({ - // @ts-expect-error libsql has type issues - dialect: new CustomLibsqlDialect({ client }), - plugins, + super({ + excludeTables: ["libsql_wasm_func_table"], + dialect: LibsqlDialect, + dialectArgs: [{ client }], + additionalPlugins: [new FilterNumericKeysPlugin()], }); - super(kysely, {}, plugins); this.client = client; - this.supported.batching = batching_enabled; } - getClient(): Client { - return this.client; - } - - protected override async batch( - queries: [...Queries], - ): Promise<{ - [K in keyof Queries]: Awaited>; - }> { - const stms: InStatement[] = queries.map((q) => { - const compiled = q.compile(); + override async executeQueries(...qbs: O): Promise> { + const compiled = this.getCompiled(...qbs); + const stms: InStatement[] = compiled.map((q) => { return { - sql: compiled.sql, - args: compiled.parameters as any[], + sql: q.sql, + args: q.parameters as any[], }; }); - const res = await this.client.batch(stms); - - // let it run through plugins - const kyselyPlugins = new KyselyPluginRunner(this.plugins); - - const data: any = []; - for (const r of res) { - const rows = await kyselyPlugins.transformResultRows(r.rows); - data.push(rows); - } - - return data; + return this.withTransformedRows(await this.client.batch(stms)) as any; } } diff --git a/app/src/data/connection/sqlite/SqliteConnection.ts b/app/src/data/connection/sqlite/SqliteConnection.ts index 33a3fd8..705046d 100644 --- a/app/src/data/connection/sqlite/SqliteConnection.ts +++ b/app/src/data/connection/sqlite/SqliteConnection.ts @@ -1,16 +1,49 @@ -import type { ColumnDataType, ColumnDefinitionBuilder, Kysely, KyselyPlugin } from "kysely"; +import { + ParseJSONResultsPlugin, + type ColumnDataType, + type ColumnDefinitionBuilder, + type Dialect, + Kysely, + type KyselyPlugin, +} from "kysely"; import { jsonArrayFrom, jsonBuildObject, jsonObjectFrom } from "kysely/helpers/sqlite"; import { Connection, type DbFunctions, type FieldSpec, type SchemaResponse } from "../Connection"; +import type { Constructor } from "core"; +import { customIntrospector } from "../Connection"; +import { SqliteIntrospector } from "./SqliteIntrospector"; + +export type SqliteConnectionConfig< + CustomDialect extends Constructor = Constructor, +> = { + excludeTables?: string[]; + dialect: CustomDialect; + dialectArgs?: ConstructorParameters; + additionalPlugins?: KyselyPlugin[]; + customFn?: Partial; +}; + +export abstract class SqliteConnection extends Connection { + override name = "sqlite"; + + constructor(config: SqliteConnectionConfig) { + const { excludeTables, dialect, dialectArgs = [], additionalPlugins } = config; + const plugins = [new ParseJSONResultsPlugin(), ...(additionalPlugins ?? [])]; + + const kysely = new Kysely({ + dialect: customIntrospector(dialect, SqliteIntrospector, { + excludeTables, + plugins, + }).create(...dialectArgs), + plugins, + }); -export class SqliteConnection extends Connection { - constructor(kysely: Kysely, fn: Partial = {}, plugins: KyselyPlugin[] = []) { super( kysely, { - ...fn, jsonArrayFrom, jsonObjectFrom, jsonBuildObject, + ...(config.customFn ?? {}), }, plugins, ); @@ -43,7 +76,7 @@ export class SqliteConnection extends Connection { if (spec.onUpdate) relCol = relCol.onUpdate(spec.onUpdate); return relCol; } - return spec.nullable ? col : col.notNull(); + return col; }, ] as const; } diff --git a/app/src/data/connection/sqlite/SqliteLocalConnection.ts b/app/src/data/connection/sqlite/SqliteLocalConnection.ts index a92577b..34d9845 100644 --- a/app/src/data/connection/sqlite/SqliteLocalConnection.ts +++ b/app/src/data/connection/sqlite/SqliteLocalConnection.ts @@ -1,31 +1,14 @@ -import { - type DatabaseIntrospector, - Kysely, - ParseJSONResultsPlugin, - type SqliteDatabase, - SqliteDialect, -} from "kysely"; +import { type SqliteDatabase, SqliteDialect } from "kysely"; import { SqliteConnection } from "./SqliteConnection"; -import { SqliteIntrospector } from "./SqliteIntrospector"; -const plugins = [new ParseJSONResultsPlugin()]; +export class SqliteLocalConnection extends SqliteConnection { + override name = "sqlite-local"; -class CustomSqliteDialect extends SqliteDialect { - override createIntrospector(db: Kysely): DatabaseIntrospector { - return new SqliteIntrospector(db, { - excludeTables: ["test_table"], - plugins, + constructor(database: SqliteDatabase) { + super({ + dialect: SqliteDialect, + dialectArgs: [{ database }], }); - } -} - -export class SqliteLocalConnection extends SqliteConnection { - constructor(private database: SqliteDatabase) { - const kysely = new Kysely({ - dialect: new CustomSqliteDialect({ database }), - plugins, - }); - - super(kysely, {}, plugins); + this.client = database; } } diff --git a/app/src/data/entities/EntityManager.ts b/app/src/data/entities/EntityManager.ts index e4e71c2..0401d4d 100644 --- a/app/src/data/entities/EntityManager.ts +++ b/app/src/data/entities/EntityManager.ts @@ -207,8 +207,9 @@ export class EntityManager { repository( entity: E, + opts: Omit = {}, ): Repository> { - return this.repo(entity); + return this.repo(entity, opts); } repo( diff --git a/app/src/data/entities/Result.ts b/app/src/data/entities/Result.ts new file mode 100644 index 0000000..a37cbd7 --- /dev/null +++ b/app/src/data/entities/Result.ts @@ -0,0 +1,126 @@ +import { isDebug } from "core"; +import { pick } from "core/utils"; +import type { Connection } from "data/connection"; +import type { + Compilable, + CompiledQuery, + QueryResult as KyselyQueryResult, + SelectQueryBuilder, +} from "kysely"; + +export type ResultHydrator = (rows: T[]) => any; +export type ResultOptions = { + hydrator?: ResultHydrator; + beforeExecute?: (compiled: CompiledQuery) => void | Promise; + onError?: (error: Error) => void | Promise; + single?: boolean; +}; +export type ResultJSON = { + data: T; + meta: { + items: number; + time: number; + sql?: string; + parameters?: any[]; + [key: string]: any; + }; +}; + +export interface QueryResult extends Omit, "rows"> { + time: number; + items: number; + data: T; + rows: unknown[]; + sql: string; + parameters: any[]; + count?: number; + total?: number; +} + +export class Result { + results: QueryResult[] = []; + time: number = 0; + + constructor( + protected conn: Connection, + protected options: ResultOptions = {}, + ) {} + + get(): QueryResult { + if (!this.results) { + throw new Error("Result not executed"); + } + + if (Array.isArray(this.results)) { + return (this.results ?? []) as any; + } + + return this.results[0] as any; + } + + first(): QueryResult { + const res = this.get(); + const first = Array.isArray(res) ? res[0] : res; + return first ?? ({} as any); + } + + get sql() { + return this.first().sql; + } + + get parameters() { + return this.first().parameters; + } + + get data() { + if (this.options.single) { + return this.first().data?.[0]; + } + + return this.first().data ?? []; + } + + async execute(qb: Compilable | Compilable[]) { + const qbs = Array.isArray(qb) ? qb : [qb]; + + for (const qb of qbs) { + const compiled = qb.compile(); + await this.options.beforeExecute?.(compiled); + try { + const start = performance.now(); + const res = await this.conn.executeQuery(compiled); + this.time = Number.parseFloat((performance.now() - start).toFixed(2)); + this.results.push({ + ...res, + data: this.options.hydrator?.(res.rows as T[]), + items: res.rows.length, + time: this.time, + sql: compiled.sql, + parameters: [...compiled.parameters], + }); + } catch (e) { + if (this.options.onError) { + await this.options.onError(e as Error); + } else { + throw e; + } + } + } + + return this; + } + + protected additionalMetaKeys(): string[] { + return []; + } + + toJSON(): ResultJSON { + const { rows, data, ...metaRaw } = this.first(); + const keys = isDebug() ? ["items", "time", "sql", "parameters"] : ["items", "time"]; + const meta = pick(metaRaw, [...keys, ...this.additionalMetaKeys()] as any); + return { + data: this.data, + meta, + }; + } +} diff --git a/app/src/data/entities/index.ts b/app/src/data/entities/index.ts index efe6446..eb5f3b2 100644 --- a/app/src/data/entities/index.ts +++ b/app/src/data/entities/index.ts @@ -1,6 +1,6 @@ export * from "./Entity"; export * from "./EntityManager"; -export * from "./Mutator"; +export * from "./mutation/Mutator"; export * from "./query/Repository"; export * from "./query/WhereBuilder"; export * from "./query/WithBuilder"; diff --git a/app/src/data/entities/Mutator.ts b/app/src/data/entities/mutation/Mutator.ts similarity index 84% rename from app/src/data/entities/Mutator.ts rename to app/src/data/entities/mutation/Mutator.ts index 7dcb2dd..bf8cebe 100644 --- a/app/src/data/entities/Mutator.ts +++ b/app/src/data/entities/mutation/Mutator.ts @@ -1,12 +1,13 @@ import { $console, type DB as DefaultDB, type PrimaryFieldType } from "core"; import { type EmitsEvents, EventManager } from "core/events"; import type { DeleteQueryBuilder, InsertQueryBuilder, UpdateQueryBuilder } from "kysely"; -import { type TActionContext, WhereBuilder } from ".."; -import type { Entity, EntityData, EntityManager } from "../entities"; -import { InvalidSearchParamsException } from "../errors"; -import { MutatorEvents } from "../events"; -import { RelationMutator } from "../relations"; -import type { RepoQuery } from "../server/query"; +import { type TActionContext, WhereBuilder } from "../.."; +import type { Entity, EntityData, EntityManager } from "../../entities"; +import { InvalidSearchParamsException } from "../../errors"; +import { MutatorEvents } from "../../events"; +import { RelationMutator } from "../../relations"; +import type { RepoQuery } from "../../server/query"; +import { MutatorResult, type MutatorResultOptions } from "./MutatorResult"; type MutatorQB = | InsertQueryBuilder @@ -17,14 +18,6 @@ type MutatorUpdateOrDelete = | UpdateQueryBuilder | DeleteQueryBuilder; -export type MutatorResponse = { - entity: Entity; - sql: string; - parameters: any[]; - result: EntityData[]; - data: T; -}; - export class Mutator< TBD extends object = DefaultDB, TB extends keyof TBD = any, @@ -103,35 +96,18 @@ export class Mutator< return validatedData as Given; } - protected async many(qb: MutatorQB): Promise { - const entity = this.entity; - const { sql, parameters } = qb.compile(); - - try { - const result = await qb.execute(); - - const data = this.em.hydrate(entity.name, result) as EntityData[]; - - return { - entity, - sql, - parameters: [...parameters], - result: result, - data, - }; - } catch (e) { - // @todo: redact - $console.error("[Error in query]", sql); - throw e; - } + protected async performQuery( + qb: MutatorQB, + opts?: MutatorResultOptions, + ): Promise> { + const result = new MutatorResult(this.em, this.entity, { + silent: false, + ...opts, + }); + return (await result.execute(qb)) as any; } - protected async single(qb: MutatorQB): Promise> { - const { data, ...response } = await this.many(qb); - return { ...response, data: data[0]! }; - } - - async insertOne(data: Input): Promise> { + async insertOne(data: Input): Promise> { const entity = this.entity; if (entity.type === "system" && this.__unstable_disable_system_entity_creation) { throw new Error(`Creation of system entity "${entity.name}" is disabled`); @@ -174,7 +150,7 @@ export class Mutator< .values(validatedData) .returning(entity.getSelect()); - const res = await this.single(query); + const res = await this.performQuery(query, { single: true }); await this.emgr.emit( new Mutator.Events.MutatorInsertAfter({ entity, data: res.data, changed: validatedData }), @@ -183,7 +159,7 @@ export class Mutator< return res as any; } - async updateOne(id: PrimaryFieldType, data: Partial): Promise> { + async updateOne(id: PrimaryFieldType, data: Partial): Promise> { const entity = this.entity; if (!id) { throw new Error("ID must be provided for update"); @@ -206,7 +182,7 @@ export class Mutator< .where(entity.id().name, "=", id) .returning(entity.getSelect()); - const res = await this.single(query); + const res = await this.performQuery(query, { single: true }); await this.emgr.emit( new Mutator.Events.MutatorUpdateAfter({ @@ -220,7 +196,7 @@ export class Mutator< return res as any; } - async deleteOne(id: PrimaryFieldType): Promise> { + async deleteOne(id: PrimaryFieldType): Promise> { const entity = this.entity; if (!id) { throw new Error("ID must be provided for deletion"); @@ -233,7 +209,7 @@ export class Mutator< .where(entity.id().name, "=", id) .returning(entity.getSelect()); - const res = await this.single(query); + const res = await this.performQuery(query, { single: true }); await this.emgr.emit( new Mutator.Events.MutatorDeleteAfter({ entity, entityId: id, data: res.data }), @@ -286,7 +262,7 @@ export class Mutator< } // @todo: decide whether entries should be deleted all at once or one by one (for events) - async deleteWhere(where: RepoQuery["where"]): Promise> { + async deleteWhere(where: RepoQuery["where"]): Promise> { const entity = this.entity; // @todo: add a way to delete all by adding force? @@ -298,13 +274,13 @@ export class Mutator< entity.getSelect(), ); - return (await this.many(qb)) as any; + return await this.performQuery(qb); } async updateWhere( data: Partial, where: RepoQuery["where"], - ): Promise> { + ): Promise> { const entity = this.entity; const validatedData = await this.getValidatedData(data, "update"); @@ -317,10 +293,10 @@ export class Mutator< .set(validatedData as any) .returning(entity.getSelect()); - return (await this.many(query)) as any; + return await this.performQuery(query); } - async insertMany(data: Input[]): Promise> { + async insertMany(data: Input[]): Promise> { const entity = this.entity; if (entity.type === "system" && this.__unstable_disable_system_entity_creation) { throw new Error(`Creation of system entity "${entity.name}" is disabled`); @@ -352,6 +328,6 @@ export class Mutator< .values(validated) .returning(entity.getSelect()); - return (await this.many(query)) as any; + return await this.performQuery(query); } } diff --git a/app/src/data/entities/mutation/MutatorResult.ts b/app/src/data/entities/mutation/MutatorResult.ts new file mode 100644 index 0000000..88cf435 --- /dev/null +++ b/app/src/data/entities/mutation/MutatorResult.ts @@ -0,0 +1,33 @@ +import { $console } from "core/console"; +import type { Entity, EntityData } from "../Entity"; +import type { EntityManager } from "../EntityManager"; +import { Result, type ResultJSON, type ResultOptions } from "../Result"; + +export type MutatorResultOptions = ResultOptions & { + silent?: boolean; +}; + +export type MutatorResultJSON = ResultJSON; + +export class MutatorResult extends Result { + constructor( + protected em: EntityManager, + public entity: Entity, + options?: MutatorResultOptions, + ) { + super(em.connection, { + hydrator: (rows) => em.hydrate(entity.name, rows as any), + beforeExecute: (compiled) => { + if (!options?.silent) { + $console.debug(`[Mutation]\n${compiled.sql}\n`, compiled.parameters); + } + }, + onError: (error) => { + if (!options?.silent) { + $console.error("[ERROR] Mutator:", error.message); + } + }, + ...options, + }); + } +} diff --git a/app/src/data/entities/query/Repository.ts b/app/src/data/entities/query/Repository.ts index 625e701..ce39c6a 100644 --- a/app/src/data/entities/query/Repository.ts +++ b/app/src/data/entities/query/Repository.ts @@ -13,37 +13,11 @@ import { WithBuilder, } from "../index"; import { JoinBuilder } from "./JoinBuilder"; -import { ensureInt } from "core/utils"; +import { RepositoryResult, type RepositoryResultOptions } from "./RepositoryResult"; +import type { ResultOptions } from "../Result"; export type RepositoryQB = SelectQueryBuilder; -export type RepositoryRawResponse = { - sql: string; - parameters: any[]; - result: EntityData[]; -}; -export type RepositoryResponse = RepositoryRawResponse & { - entity: Entity; - data: T; - meta: { - items: number; - total?: number; - count?: number; - time?: number; - query?: { - sql: string; - parameters: readonly any[]; - }; - }; -}; - -export type RepositoryCountResponse = RepositoryRawResponse & { - count: number; -}; -export type RepositoryExistsResponse = RepositoryRawResponse & { - exists: boolean; -}; - export type RepositoryOptions = { silent?: boolean; includeCounts?: boolean; @@ -182,126 +156,18 @@ export class Repository { - const entity = this.entity; - const compiled = qb.compile(); - - const payload = { - entity, - sql: compiled.sql, - parameters: [...compiled.parameters], - result: [], - data: [], - meta: { - total: 0, - count: 0, - items: 0, - time: 0, - query: { sql: compiled.sql, parameters: compiled.parameters }, - }, - }; - - // don't batch (add counts) if `includeCounts` is set to false - // or when explicitly set to true and batching is not supported - if ( - this.options?.includeCounts === false || - (this.options?.includeCounts === true && !this.em.connection.supports("batching")) - ) { - const start = performance.now(); - const res = await this.executeQb(qb); - const time = Number.parseFloat((performance.now() - start).toFixed(2)); - const result = res.result ?? []; - const data = this.em.hydrate(entity.name, result); - - return { - ...payload, - result, - data, - meta: { - ...payload.meta, - total: undefined, - count: undefined, - items: data.length, - time, - }, - }; - } - - if (this.options?.silent !== true) { - $console.debug(`Repository: query\n${compiled.sql}\n`, compiled.parameters); - } - - const selector = (as = "count") => this.conn.fn.countAll().as(as); - const countQuery = qb - .clearSelect() - .select(selector()) - .clearLimit() - .clearOffset() - .clearGroupBy() - .clearOrderBy(); - const totalQuery = this.conn.selectFrom(entity.name).select(selector()); - - try { - const start = performance.now(); - const [_count, _total, result] = await this.em.connection.batchQuery([ - countQuery, - totalQuery, - qb, - ]); - - const time = Number.parseFloat((performance.now() - start).toFixed(2)); - const data = this.em.hydrate(entity.name, result); - - return { - ...payload, - result, - data, - meta: { - ...payload.meta, - // parsing is important since pg returns string - total: ensureInt(_total[0]?.count), - count: ensureInt(_count[0]?.count), - items: result.length, - time, - }, - }; - } catch (e) { - if (this.options?.silent !== true) { - if (e instanceof Error) { - $console.error("[ERROR] Repository.performQuery", e.message); - } - - throw e; - } else { - return payload; - } - } + protected async performQuery( + qb: RepositoryQB, + opts?: RepositoryResultOptions, + execOpts?: { includeCounts?: boolean }, + ): Promise> { + const result = new RepositoryResult(this.em, this.entity, { + silent: this.options.silent, + ...opts, + }); + return (await result.execute(qb, { + includeCounts: execOpts?.includeCounts ?? this.options.includeCounts, + })) as any; } private async triggerFindBefore(entity: Entity, options: RepoQuery): Promise { @@ -319,7 +185,7 @@ export class Repository { if (options.limit === 1) { await this.emgr.emit( - new Repository.Events.RepositoryFindOneAfter({ entity, options, data: data[0]! }), + new Repository.Events.RepositoryFindOneAfter({ entity, options, data }), ); } else { await this.emgr.emit( @@ -331,12 +197,11 @@ export class Repository> { + ): Promise> { await this.triggerFindBefore(this.entity, options); - const { data, ...response } = await this.performQuery(qb); - - await this.triggerFindAfter(this.entity, options, data); - return { ...response, data: data[0]! }; + const result = await this.performQuery(qb, { single: true }); + await this.triggerFindAfter(this.entity, options, result.data); + return result as any; } addOptionsToQueryBuilder( @@ -413,7 +278,7 @@ export class Repository>, - ): Promise> { + ): Promise> { const { qb, options } = this.buildQuery( { ..._options, @@ -429,7 +294,7 @@ export class Repository>, - ): Promise> { + ): Promise> { const { qb, options } = this.buildQuery({ ..._options, where, @@ -439,7 +304,7 @@ export class Repository): Promise> { + async findMany(_options?: Partial): Promise> { const { qb, options } = this.buildQuery(_options); await this.triggerFindBefore(this.entity, options); @@ -454,7 +319,7 @@ export class Repository>, - ): Promise> { + ): Promise> { const entity = this.entity; const listable_relations = this.em.relations.listableRelationsOf(entity); const relation = listable_relations.find((r) => r.ref(reference).reference === reference); @@ -482,10 +347,10 @@ export class Repository { + async count(where?: RepoQuery["where"]): Promise> { const entity = this.entity; const options = this.getValidOptions({ where }); @@ -497,17 +362,18 @@ export class Repository ({ count: rows[0]?.count ?? 0 }), + }, + { includeCounts: false }, + ); } - async exists(where: Required["where"]): Promise { + async exists( + where: Required["where"], + ): Promise> { const entity = this.entity; const options = this.getValidOptions({ where }); @@ -517,13 +383,8 @@ export class Repository 0, - }; + return await this.performQuery(qb, { + hydrator: (rows) => ({ exists: rows[0]?.count > 0 }), + }); } } diff --git a/app/src/data/entities/query/RepositoryResult.ts b/app/src/data/entities/query/RepositoryResult.ts new file mode 100644 index 0000000..fee0dff --- /dev/null +++ b/app/src/data/entities/query/RepositoryResult.ts @@ -0,0 +1,105 @@ +import { $console } from "core/console"; +import type { Entity, EntityData } from "../Entity"; +import type { EntityManager } from "../EntityManager"; +import { Result, type ResultJSON, type ResultOptions } from "../Result"; +import type { Compilable, SelectQueryBuilder } from "kysely"; +import { ensureInt } from "core/utils"; + +export type RepositoryResultOptions = ResultOptions & { + silent?: boolean; +}; + +export type RepositoryResultJSON = ResultJSON; + +export class RepositoryResult extends Result { + constructor( + protected em: EntityManager, + public entity: Entity, + options?: RepositoryResultOptions, + ) { + super(em.connection, { + hydrator: (rows) => em.hydrate(entity.name, rows as any), + beforeExecute: (compiled) => { + if (!options?.silent) { + $console.debug(`Query:\n${compiled.sql}\n`, compiled.parameters); + } + }, + onError: (error) => { + if (options?.silent !== true) { + $console.error("Repository:", String(error)); + throw error; + } + }, + ...options, + }); + } + + private shouldIncludeCounts(intent?: boolean) { + if (intent === undefined) return this.conn.supports("softscans"); + return intent; + } + + override async execute( + qb: SelectQueryBuilder, + opts?: { includeCounts?: boolean }, + ) { + const includeCounts = this.shouldIncludeCounts(opts?.includeCounts); + + if (includeCounts) { + const selector = (as = "count") => this.conn.kysely.fn.countAll().as(as); + const countQuery = qb + .clearSelect() + .select(selector()) + .clearLimit() + .clearOffset() + .clearGroupBy() + .clearOrderBy(); + const totalQuery = this.conn.kysely.selectFrom(this.entity.name).select(selector()); + + const compiled = qb.compile(); + this.options.beforeExecute?.(compiled); + + try { + const start = performance.now(); + const [main, count, total] = await this.em.connection.executeQueries( + compiled, + countQuery, + totalQuery, + ); + this.time = Number.parseFloat((performance.now() - start).toFixed(2)); + this.results.push({ + ...main, + data: this.options.hydrator?.(main.rows as T[]), + items: main.rows.length, + count: ensureInt(count.rows[0]?.count ?? 0), + total: ensureInt(total.rows[0]?.count ?? 0), + time: this.time, + sql: compiled.sql, + parameters: [...compiled.parameters], + }); + } catch (e) { + if (this.options.onError) { + await this.options.onError(e as Error); + } else { + throw e; + } + } + + return this; + } + + return await super.execute(qb); + } + + get count() { + return this.first().count; + } + + get total() { + return this.first().total; + } + + protected override additionalMetaKeys(): string[] { + return ["count", "total"]; + } +} diff --git a/app/src/data/relations/ManyToOneRelation.ts b/app/src/data/relations/ManyToOneRelation.ts index 7fde72a..c9a1e0b 100644 --- a/app/src/data/relations/ManyToOneRelation.ts +++ b/app/src/data/relations/ManyToOneRelation.ts @@ -208,7 +208,7 @@ export class ManyToOneRelation extends EntityRelation=0.26" } }, "sha512-/Bs3/Uktn04nQ9g/4oSphLMEtSHkS5+j5hbKjK5gMqXQfqr/v3V3FKtoN4pLTmo2W35hNdrIpQnBukGL1zZc6g=="], + "kysely-neon": ["kysely-neon@1.3.0", "", { "peerDependencies": { "@neondatabase/serverless": "^0.4.3", "kysely": "0.x.x", "ws": "^8.13.0" }, "optionalPeers": ["ws"] }, "sha512-CIIlbmqpIXVJDdBEYtEOwbmALag0jmqYrGfBeM4cHKb9AgBGs+X1SvXUZ8TqkDacQEqEZN2XtsDoUkcMIISjHw=="], "kysely-postgres-js": ["kysely-postgres-js@2.0.0", "", { "peerDependencies": { "kysely": ">= 0.24.0 < 1", "postgres": ">= 3.4.0 < 4" } }, "sha512-R1tWx6/x3tSatWvsmbHJxpBZYhNNxcnMw52QzZaHKg7ZOWtHib4iZyEaw4gb2hNKVctWQ3jfMxZT/ZaEMK6kBQ=="],