connection: rewrote query execution, batching, added generic sqlite, added node/bun sqlite, aligned repo/mutator results

app/src/data/connection/sqlite/GenericSqliteConnection.ts

@@ -0,0 +1,37 @@
+import type { KyselyPlugin } from "kysely";
+import {
+   type IGenericSqlite,
+   type OnCreateConnection,
+   type Promisable,
+   parseBigInt,
+   buildQueryFn,
+   GenericSqliteDialect,
+} from "kysely-generic-sqlite";
+import { SqliteConnection } from "./SqliteConnection";
+
+export type GenericSqliteConnectionConfig = {
+   name: string;
+   additionalPlugins?: KyselyPlugin[];
+   excludeTables?: string[];
+   onCreateConnection?: OnCreateConnection;
+};
+
+export { parseBigInt, buildQueryFn, GenericSqliteDialect, type IGenericSqlite };
+
+export class GenericSqliteConnection<DB = unknown> extends SqliteConnection<DB> {
+   override name = "generic-sqlite";
+
+   constructor(
+      db: DB,
+      executor: () => Promisable<IGenericSqlite>,
+      config?: GenericSqliteConnectionConfig,
+   ) {
+      super({
+         dialect: GenericSqliteDialect,
+         dialectArgs: [executor, config?.onCreateConnection],
+         additionalPlugins: config?.additionalPlugins,
+         excludeTables: config?.excludeTables,
+      });
+      this.client = db;
+   }
+}

app/src/data/connection/sqlite/LibsqlConnection.spec.ts

@@ -0,0 +1,11 @@
+import { connectionTestSuite } from "../connection-test-suite";
+import { LibsqlConnection } from "./LibsqlConnection";
+import { bunTestRunner } from "adapter/bun/test";
+import { describe } from "bun:test";
+
+describe("LibsqlConnection", () => {
+   connectionTestSuite(bunTestRunner, {
+      makeConnection: () => new LibsqlConnection({ url: ":memory:" }),
+      rawDialectDetails: ["rowsAffected", "lastInsertRowid"],
+   });
+});

app/src/data/connection/sqlite/LibsqlConnection.ts

@@ -1,40 +1,26 @@
-import { type Client, type Config, type InStatement, createClient } from "@libsql/client";
+import { createClient, type Client, type Config, type InStatement } from "@libsql/client";
 import { LibsqlDialect } from "@libsql/kysely-libsql";
-import { FilterNumericKeysPlugin } from "data/plugins/FilterNumericKeysPlugin";
-import { KyselyPluginRunner } from "data/plugins/KyselyPluginRunner";
-import { type DatabaseIntrospector, Kysely, ParseJSONResultsPlugin } from "kysely";
-import type { QB } from "../Connection";
-import { SqliteConnection } from "./SqliteConnection";
-import { SqliteIntrospector } from "./SqliteIntrospector";
 import { $console } from "core";
+import { FilterNumericKeysPlugin } from "data/plugins/FilterNumericKeysPlugin";
+import type { ConnQuery, ConnQueryResults } from "../Connection";
+import { SqliteConnection } from "./SqliteConnection";
 
 export const LIBSQL_PROTOCOLS = ["wss", "https", "libsql"] as const;
 export type LibSqlCredentials = Config & {
    protocol?: (typeof LIBSQL_PROTOCOLS)[number];
 };
 
-const plugins = [new FilterNumericKeysPlugin(), new ParseJSONResultsPlugin()];
-
-class CustomLibsqlDialect extends LibsqlDialect {
-   override createIntrospector(db: Kysely<any>): DatabaseIntrospector {
-      return new SqliteIntrospector(db, {
-         excludeTables: ["libsql_wasm_func_table"],
-         plugins,
-      });
-   }
-}
-
-export class LibsqlConnection extends SqliteConnection {
-   private client: Client;
+export class LibsqlConnection extends SqliteConnection<Client> {
+   override name = "libsql";
    protected override readonly supported = {
       batching: true,
+      softscans: true,
    };
 
    constructor(client: Client);
    constructor(credentials: LibSqlCredentials);
    constructor(clientOrCredentials: Client | LibSqlCredentials) {
       let client: Client;
-      let batching_enabled = true;
       if (clientOrCredentials && "url" in clientOrCredentials) {
          let { url, authToken, protocol } = clientOrCredentials;
          if (protocol && LIBSQL_PROTOCOLS.includes(protocol)) {
@@ -48,45 +34,25 @@ export class LibsqlConnection extends SqliteConnection {
          client = clientOrCredentials;
       }
 
-      const kysely = new Kysely({
-         // @ts-expect-error libsql has type issues
-         dialect: new CustomLibsqlDialect({ client }),
-         plugins,
+      super({
+         excludeTables: ["libsql_wasm_func_table"],
+         dialect: LibsqlDialect,
+         dialectArgs: [{ client }],
+         additionalPlugins: [new FilterNumericKeysPlugin()],
       });
 
-      super(kysely, {}, plugins);
       this.client = client;
-      this.supported.batching = batching_enabled;
    }
 
-   getClient(): Client {
-      return this.client;
-   }
-
-   protected override async batch<Queries extends QB[]>(
-      queries: [...Queries],
-   ): Promise<{
-      [K in keyof Queries]: Awaited<ReturnType<Queries[K]["execute"]>>;
-   }> {
-      const stms: InStatement[] = queries.map((q) => {
-         const compiled = q.compile();
+   override async executeQueries<O extends ConnQuery[]>(...qbs: O): Promise<ConnQueryResults<O>> {
+      const compiled = this.getCompiled(...qbs);
+      const stms: InStatement[] = compiled.map((q) => {
          return {
-            sql: compiled.sql,
-            args: compiled.parameters as any[],
+            sql: q.sql,
+            args: q.parameters as any[],
          };
       });
 
-      const res = await this.client.batch(stms);
-
-      // let it run through plugins
-      const kyselyPlugins = new KyselyPluginRunner(this.plugins);
-
-      const data: any = [];
-      for (const r of res) {
-         const rows = await kyselyPlugins.transformResultRows(r.rows);
-         data.push(rows);
-      }
-
-      return data;
+      return this.withTransformedRows(await this.client.batch(stms)) as any;
    }
 }

app/src/data/connection/sqlite/SqliteConnection.ts

@@ -1,16 +1,49 @@
-import type { ColumnDataType, ColumnDefinitionBuilder, Kysely, KyselyPlugin } from "kysely";
+import {
+   ParseJSONResultsPlugin,
+   type ColumnDataType,
+   type ColumnDefinitionBuilder,
+   type Dialect,
+   Kysely,
+   type KyselyPlugin,
+} from "kysely";
 import { jsonArrayFrom, jsonBuildObject, jsonObjectFrom } from "kysely/helpers/sqlite";
 import { Connection, type DbFunctions, type FieldSpec, type SchemaResponse } from "../Connection";
+import type { Constructor } from "core";
+import { customIntrospector } from "../Connection";
+import { SqliteIntrospector } from "./SqliteIntrospector";
+
+export type SqliteConnectionConfig<
+   CustomDialect extends Constructor<Dialect> = Constructor<Dialect>,
+> = {
+   excludeTables?: string[];
+   dialect: CustomDialect;
+   dialectArgs?: ConstructorParameters<CustomDialect>;
+   additionalPlugins?: KyselyPlugin[];
+   customFn?: Partial<DbFunctions>;
+};
+
+export abstract class SqliteConnection<Client = unknown> extends Connection<Client> {
+   override name = "sqlite";
+
+   constructor(config: SqliteConnectionConfig) {
+      const { excludeTables, dialect, dialectArgs = [], additionalPlugins } = config;
+      const plugins = [new ParseJSONResultsPlugin(), ...(additionalPlugins ?? [])];
+
+      const kysely = new Kysely({
+         dialect: customIntrospector(dialect, SqliteIntrospector, {
+            excludeTables,
+            plugins,
+         }).create(...dialectArgs),
+         plugins,
+      });
 
-export class SqliteConnection extends Connection {
-   constructor(kysely: Kysely<any>, fn: Partial<DbFunctions> = {}, plugins: KyselyPlugin[] = []) {
       super(
          kysely,
          {
-            ...fn,
             jsonArrayFrom,
             jsonObjectFrom,
             jsonBuildObject,
+            ...(config.customFn ?? {}),
          },
          plugins,
       );
@@ -43,7 +76,7 @@ export class SqliteConnection extends Connection {
                if (spec.onUpdate) relCol = relCol.onUpdate(spec.onUpdate);
                return relCol;
             }
-            return spec.nullable ? col : col.notNull();
+            return col;
          },
       ] as const;
    }

app/src/data/connection/sqlite/SqliteLocalConnection.ts

@@ -1,31 +1,14 @@
-import {
-   type DatabaseIntrospector,
-   Kysely,
-   ParseJSONResultsPlugin,
-   type SqliteDatabase,
-   SqliteDialect,
-} from "kysely";
+import { type SqliteDatabase, SqliteDialect } from "kysely";
 import { SqliteConnection } from "./SqliteConnection";
-import { SqliteIntrospector } from "./SqliteIntrospector";
 
-const plugins = [new ParseJSONResultsPlugin()];
+export class SqliteLocalConnection extends SqliteConnection<SqliteDatabase> {
+   override name = "sqlite-local";
 
-class CustomSqliteDialect extends SqliteDialect {
-   override createIntrospector(db: Kysely<any>): DatabaseIntrospector {
-      return new SqliteIntrospector(db, {
-         excludeTables: ["test_table"],
-         plugins,
+   constructor(database: SqliteDatabase) {
+      super({
+         dialect: SqliteDialect,
+         dialectArgs: [{ database }],
       });
-   }
-}
-
-export class SqliteLocalConnection extends SqliteConnection {
-   constructor(private database: SqliteDatabase) {
-      const kysely = new Kysely({
-         dialect: new CustomSqliteDialect({ database }),
-         plugins,
-      });
-
-      super(kysely, {}, plugins);
+      this.client = database;
    }
 }