added BaseIntrospector and reorganized connections

app/__test__/data/specs/connection/SqliteIntrospector.spec.ts

@@ -1,8 +1,7 @@
-import { beforeEach, describe, test, expect } from "bun:test";
+import { describe, expect, test } from "bun:test";
-import { SqliteIntrospector } from "data/connection/SqliteIntrospector";
+import { SqliteIntrospector } from "data/connection";
-import { getDummyConnection, getDummyDatabase } from "../../helper";
+import { getDummyDatabase } from "../../helper";
-import { Kysely, ParseJSONResultsPlugin, SqliteDialect } from "kysely";
+import { Kysely, SqliteDialect } from "kysely";
-import { _jsonp } from "core/utils";
 
 function create() {
    const database = getDummyDatabase().dummyDb;

app/src/data/connection/BaseIntrospector.ts

@@ -0,0 +1,75 @@
+import {
+   type DatabaseMetadata,
+   type DatabaseMetadataOptions,
+   type Kysely,
+   type KyselyPlugin,
+   type RawBuilder,
+   type TableMetadata,
+   type DatabaseIntrospector,
+   type SchemaMetadata,
+   ParseJSONResultsPlugin,
+   DEFAULT_MIGRATION_TABLE,
+   DEFAULT_MIGRATION_LOCK_TABLE,
+} from "kysely";
+import { KyselyPluginRunner } from "data/plugins/KyselyPluginRunner";
+import type { IndexMetadata } from "data/connection/Connection";
+
+export type TableSpec = TableMetadata & {
+   indices: IndexMetadata[];
+};
+export type SchemaSpec = TableSpec[];
+
+export type BaseIntrospectorConfig = {
+   excludeTables?: string[];
+   plugins?: KyselyPlugin[];
+};
+
+export abstract class BaseIntrospector implements DatabaseIntrospector {
+   readonly _excludeTables: string[] = [];
+   readonly _plugins: KyselyPlugin[];
+
+   constructor(
+      protected readonly db: Kysely<any>,
+      config: BaseIntrospectorConfig = {},
+   ) {
+      this._excludeTables = config.excludeTables ?? [];
+      this._plugins = config.plugins ?? [new ParseJSONResultsPlugin()];
+   }
+
+   abstract getSchemaSpec(): Promise<SchemaSpec>;
+   abstract getSchemas(): Promise<SchemaMetadata[]>;
+
+   protected getExcludedTableNames(): string[] {
+      return [...this._excludeTables, DEFAULT_MIGRATION_TABLE, DEFAULT_MIGRATION_LOCK_TABLE];
+   }
+
+   protected async executeWithPlugins<T>(query: RawBuilder<any>): Promise<T> {
+      const result = await query.execute(this.db);
+      const runner = new KyselyPluginRunner(this._plugins ?? []);
+      return (await runner.transformResultRows(result.rows)) as unknown as T;
+   }
+
+   async getMetadata(options?: DatabaseMetadataOptions): Promise<DatabaseMetadata> {
+      return {
+         tables: await this.getTables(options),
+      };
+   }
+
+   async getIndices(tbl_name?: string): Promise<IndexMetadata[]> {
+      const schema = await this.getSchemaSpec();
+      return schema
+         .flatMap((table) => table.indices)
+         .filter((index) => !tbl_name || index.table === tbl_name);
+   }
+
+   async getTables(
+      options: DatabaseMetadataOptions = { withInternalKyselyTables: false },
+   ): Promise<TableMetadata[]> {
+      const schema = await this.getSchemaSpec();
+      return schema.map((table) => ({
+         name: table.name,
+         isView: table.isView,
+         columns: table.columns,
+      }));
+   }
+}

app/src/data/connection/Connection.ts

@@ -13,6 +13,7 @@ import {
    type Simplify,
    sql,
 } from "kysely";
+import type { BaseIntrospector } from "./BaseIntrospector";
 
 export type QB = SelectQueryBuilder<any, any, any>;
 
@@ -23,10 +24,6 @@ export type IndexMetadata = {
    columns: { name: string; order: number }[];
 };
 
-export interface ConnectionIntrospector extends DatabaseIntrospector {
-   getIndices(tbl_name?: string): Promise<IndexMetadata[]>;
-}
-
 export interface SelectQueryBuilderExpression<O> extends AliasableExpression<O> {
    get isSelectQueryBuilder(): true;
    toOperationNode(): SelectQueryNode;
@@ -100,8 +97,8 @@ export abstract class Connection<DB = any> {
       return conn[CONN_SYMBOL] === true;
    }
 
-   getIntrospector(): ConnectionIntrospector {
+   getIntrospector(): BaseIntrospector {
-      return this.kysely.introspection as ConnectionIntrospector;
+      return this.kysely.introspection as any;
    }
 
    supportsBatching(): boolean {

app/src/data/connection/DummyConnection.ts

@@ -1,7 +1,11 @@
-import { Connection } from "./Connection";
+import { Connection, type FieldSpec, type SchemaResponse } from "./Connection";
 
 export class DummyConnection extends Connection {
    constructor() {
       super(undefined as any);
    }
+
+   override getFieldSchema(spec: FieldSpec, strict?: boolean): SchemaResponse {
+      throw new Error("Method not implemented.");
+   }
 }

app/src/data/connection/SqliteIntrospector.ts

@@ -1,151 +0,0 @@
-import {
-   type DatabaseIntrospector,
-   type DatabaseMetadata,
-   type DatabaseMetadataOptions,
-   type Kysely,
-   ParseJSONResultsPlugin,
-   type SchemaMetadata,
-   type TableMetadata,
-   type KyselyPlugin,
-} from "kysely";
-import { DEFAULT_MIGRATION_LOCK_TABLE, DEFAULT_MIGRATION_TABLE, sql } from "kysely";
-import type { ConnectionIntrospector, IndexMetadata } from "./Connection";
-import { KyselyPluginRunner } from "data";
-
-export type SqliteIntrospectorConfig = {
-   excludeTables?: string[];
-   plugins?: KyselyPlugin[];
-};
-
-export class SqliteIntrospector implements DatabaseIntrospector, ConnectionIntrospector {
-   readonly #db: Kysely<any>;
-   readonly _excludeTables: string[] = [];
-   readonly _plugins: KyselyPlugin[];
-
-   constructor(db: Kysely<any>, config: SqliteIntrospectorConfig = {}) {
-      this.#db = db;
-      this._excludeTables = config.excludeTables ?? [];
-      this._plugins = config.plugins ?? [new ParseJSONResultsPlugin()];
-   }
-
-   async getSchemas(): Promise<SchemaMetadata[]> {
-      // Sqlite doesn't support schemas.
-      return [];
-   }
-
-   async getSchema() {
-      const excluded = [
-         ...this._excludeTables,
-         DEFAULT_MIGRATION_TABLE,
-         DEFAULT_MIGRATION_LOCK_TABLE,
-      ];
-      const query = sql`
-         SELECT m.name, m.type, m.sql,
-            (SELECT json_group_array(
-               json_object(
-                  'name', p.name,
-                  'type', p.type,
-                  'notnull', p."notnull",
-                  'default', p.dflt_value,
-                  'primary_key', p.pk
-               )) FROM pragma_table_info(m.name) p) AS columns,
-            (SELECT json_group_array(
-               json_object(
-                  'name', i.name,
-                  'origin', i.origin,
-                  'partial', i.partial,
-                  'sql', im.sql,
-                  'columns', (SELECT json_group_array(
-                     json_object(
-                        'name', ii.name,
-                        'seqno', ii.seqno
-                     )) FROM pragma_index_info(i.name) ii)
-               )) FROM pragma_index_list(m.name) i
-                 LEFT JOIN sqlite_master im ON im.name = i.name
-                  AND im.type = 'index'
-            ) AS indices
-         FROM sqlite_master m
-         WHERE m.type IN ('table', 'view')
-           and m.name not like 'sqlite_%'
-           and m.name not in (${excluded.join(", ")})
-      `;
-
-      const result = await query.execute(this.#db);
-      const runner = new KyselyPluginRunner(this._plugins ?? []);
-      const tables = (await runner.transformResultRows(result.rows)) as unknown as {
-         name: string;
-         type: string;
-         sql: string;
-         columns: {
-            name: string;
-            type: string;
-            notnull: number;
-            dflt_value: any;
-            pk: number;
-         }[];
-         indices: {
-            name: string;
-            origin: string;
-            partial: number;
-            sql: string;
-            columns: { name: string; seqno: number }[];
-         }[];
-      }[];
-
-      //console.log("tables", tables);
-      return tables.map((table) => ({
-         name: table.name,
-         isView: table.type === "view",
-         columns: table.columns.map((col) => {
-            const autoIncrementCol = table.sql
-               ?.split(/[\(\),]/)
-               ?.find((it) => it.toLowerCase().includes("autoincrement"))
-               ?.trimStart()
-               ?.split(/\s+/)?.[0]
-               ?.replace(/["`]/g, "");
-
-            return {
-               name: col.name,
-               dataType: col.type,
-               isNullable: !col.notnull,
-               isAutoIncrementing: col.name === autoIncrementCol,
-               hasDefaultValue: col.dflt_value != null,
-               comment: undefined,
-            };
-         }),
-         indices: table.indices.map((index) => ({
-            name: index.name,
-            table: table.name,
-            isUnique: index.sql?.match(/unique/i) != null,
-            columns: index.columns.map((col) => ({
-               name: col.name,
-               order: col.seqno,
-            })),
-         })),
-      }));
-   }
-
-   async getMetadata(options?: DatabaseMetadataOptions): Promise<DatabaseMetadata> {
-      return {
-         tables: await this.getTables(options),
-      };
-   }
-
-   async getIndices(tbl_name?: string): Promise<IndexMetadata[]> {
-      const schema = await this.getSchema();
-      return schema
-         .flatMap((table) => table.indices)
-         .filter((index) => !tbl_name || index.table === tbl_name);
-   }
-
-   async getTables(
-      options: DatabaseMetadataOptions = { withInternalKyselyTables: false },
-   ): Promise<TableMetadata[]> {
-      const schema = await this.getSchema();
-      return schema.map((table) => ({
-         name: table.name,
-         isView: table.isView,
-         columns: table.columns,
-      }));
-   }
-}

app/src/data/connection/index.ts

@@ -0,0 +1,12 @@
+export { Connection } from "./Connection";
+export { BaseIntrospector } from "./BaseIntrospector";
+
+// sqlite
+export { LibsqlConnection, type LibSqlCredentials } from "./sqlite/LibsqlConnection";
+export { SqliteConnection } from "./sqlite/SqliteConnection";
+export { SqliteLocalConnection } from "./sqlite/SqliteLocalConnection";
+export { SqliteIntrospector } from "./sqlite/SqliteIntrospector";
+
+// postgres
+export { PostgresConnection, type PostgresConnectionConfig } from "./postgres/PostgresConnection";
+export { PostgresIntrospector } from "./postgres/PostgresIntrospector";

app/src/data/connection/postgres/PostgresConnection.ts

@@ -42,11 +42,16 @@ export class PostgresConnection extends Connection {
       let type: ColumnDataType = spec.primary ? "serial" : spec.type;
 
       switch (spec.type) {
+         case "blob":
+            type = "bytea";
+            break;
          case "date":
          case "datetime":
+            // https://www.postgresql.org/docs/17/datatype-datetime.html
             type = "timestamp";
             break;
          case "text":
+            // https://www.postgresql.org/docs/17/datatype-character.html
             type = "varchar";
             break;
       }

app/src/data/connection/postgres/PostgresIntrospector.ts

@@ -1,55 +1,37 @@
-import {
+import { type SchemaMetadata, sql } from "kysely";
-   type DatabaseIntrospector,
+import { BaseIntrospector } from "data/connection/BaseIntrospector";
-   type DatabaseMetadata,
-   type DatabaseMetadataOptions,
-   type SchemaMetadata,
-   type TableMetadata,
-   type Kysely,
-   type KyselyPlugin,
-   ParseJSONResultsPlugin,
-} from "kysely";
-import { DEFAULT_MIGRATION_LOCK_TABLE, DEFAULT_MIGRATION_TABLE, sql } from "kysely";
-import { KyselyPluginRunner } from "data";
-import type { IndexMetadata } from "data/connection/Connection";
 
-export type PostgresIntrospectorConfig = {
+type PostgresSchemaSpec = {
-   excludeTables?: string[];
+   name: string;
-   plugins?: KyselyPlugin[];
+   type: "VIEW" | "BASE TABLE";
+   columns: {
+      name: string;
+      type: string;
+      notnull: number;
+      dflt: string;
+      pk: boolean;
+   }[];
+   indices: {
+      name: string;
+      origin: string;
+      partial: number;
+      sql: string;
+      columns: { name: string; seqno: number }[];
+   }[];
 };
 
-export class PostgresIntrospector implements DatabaseIntrospector {
+export class PostgresIntrospector extends BaseIntrospector {
-   readonly #db: Kysely<any>;
-   readonly _excludeTables: string[] = [];
-   readonly _plugins: KyselyPlugin[];
-
-   constructor(db: Kysely<any>, config: PostgresIntrospectorConfig = {}) {
-      this.#db = db;
-      this._excludeTables = config.excludeTables ?? [];
-      this._plugins = config.plugins ?? [new ParseJSONResultsPlugin()];
-   }
-
    async getSchemas(): Promise<SchemaMetadata[]> {
-      const rawSchemas = await this.#db
+      const rawSchemas = await this.db
          .selectFrom("pg_catalog.pg_namespace")
          .select("nspname")
-         .$castTo<RawSchemaMetadata>()
+         .$castTo<{ nspname: string }>()
          .execute();
 
       return rawSchemas.map((it) => ({ name: it.nspname }));
    }
 
-   async getMetadata(options?: DatabaseMetadataOptions): Promise<DatabaseMetadata> {
+   async getSchemaSpec() {
-      return {
-         tables: await this.getTables(options),
-      };
-   }
-
-   async getSchema() {
-      const excluded = [
-         ...this._excludeTables,
-         DEFAULT_MIGRATION_TABLE,
-         DEFAULT_MIGRATION_LOCK_TABLE,
-      ];
       const query = sql`
          WITH tables_and_views AS (
             SELECT table_name AS name,
@@ -58,7 +40,7 @@ export class PostgresIntrospector implements DatabaseIntrospector {
             WHERE table_schema = 'public'
               AND table_type IN ('BASE TABLE', 'VIEW')
               AND table_name NOT LIKE 'pg_%'
-              AND table_name NOT IN (${excluded.join(", ")})
+              AND table_name NOT IN (${this.getExcludedTableNames().join(", ")})
          ),
 
             columns_info AS (
@@ -115,26 +97,7 @@ export class PostgresIntrospector implements DatabaseIntrospector {
             LEFT JOIN indices_info ii ON tv.name = ii.table_name;
       `;
 
-      const result = await query.execute(this.#db);
+      const tables = await this.executeWithPlugins<PostgresSchemaSpec[]>(query);
-      const runner = new KyselyPluginRunner(this._plugins ?? []);
-      const tables = (await runner.transformResultRows(result.rows)) as unknown as {
-         name: string;
-         type: "VIEW" | "BASE TABLE";
-         columns: {
-            name: string;
-            type: string;
-            notnull: number;
-            dflt: string;
-            pk: boolean;
-         }[];
-         indices: {
-            name: string;
-            origin: string;
-            partial: number;
-            sql: string;
-            columns: { name: string; seqno: number }[];
-         }[];
-      }[];
 
       return tables.map((table) => ({
          name: table.name,
@@ -144,6 +107,7 @@ export class PostgresIntrospector implements DatabaseIntrospector {
                name: col.name,
                dataType: col.type,
                isNullable: !col.notnull,
+               // @todo: check default value on 'nextval' see https://www.postgresql.org/docs/17/datatype-numeric.html#DATATYPE-SERIAL
                isAutoIncrementing: true, // just for now
                hasDefaultValue: col.dflt != null,
                comment: undefined,
@@ -160,26 +124,4 @@ export class PostgresIntrospector implements DatabaseIntrospector {
          })),
       }));
    }
-
-   async getIndices(tbl_name?: string): Promise<IndexMetadata[]> {
-      const schema = await this.getSchema();
-      return schema
-         .flatMap((table) => table.indices)
-         .filter((index) => !tbl_name || index.table === tbl_name);
-   }
-
-   async getTables(
-      options: DatabaseMetadataOptions = { withInternalKyselyTables: false },
-   ): Promise<TableMetadata[]> {
-      const schema = await this.getSchema();
-      return schema.map((table) => ({
-         name: table.name,
-         isView: table.isView,
-         columns: table.columns,
-      }));
-   }
-}
-
-interface RawSchemaMetadata {
-   nspname: string;
 }

app/src/data/connection/sqlite/LibsqlConnection.ts

@@ -1,9 +1,9 @@
 import { type Client, type Config, type InStatement, createClient } from "@libsql/client";
 import { LibsqlDialect } from "@libsql/kysely-libsql";
 import { type DatabaseIntrospector, Kysely, ParseJSONResultsPlugin } from "kysely";
-import { FilterNumericKeysPlugin } from "../plugins/FilterNumericKeysPlugin";
+import { FilterNumericKeysPlugin } from "data/plugins/FilterNumericKeysPlugin";
-import { KyselyPluginRunner } from "../plugins/KyselyPluginRunner";
+import { KyselyPluginRunner } from "data/plugins/KyselyPluginRunner";
-import type { QB } from "./Connection";
+import type { QB } from "../Connection";
 import { SqliteConnection } from "./SqliteConnection";
 import { SqliteIntrospector } from "./SqliteIntrospector";
 

app/src/data/connection/sqlite/SqliteConnection.ts

@@ -1,6 +1,6 @@
 import type { ColumnDataType, ColumnDefinitionBuilder, Kysely, KyselyPlugin } from "kysely";
 import { jsonArrayFrom, jsonBuildObject, jsonObjectFrom } from "kysely/helpers/sqlite";
-import { Connection, type DbFunctions, type FieldSpec, type SchemaResponse } from "./Connection";
+import { Connection, type DbFunctions, type FieldSpec, type SchemaResponse } from "../Connection";
 
 export class SqliteConnection extends Connection {
    constructor(kysely: Kysely<any>, fn: Partial<DbFunctions> = {}, plugins: KyselyPlugin[] = []) {

app/src/data/connection/sqlite/SqliteIntrospector.ts

@@ -0,0 +1,95 @@
+import { type SchemaMetadata, sql } from "kysely";
+import { BaseIntrospector } from "../BaseIntrospector";
+
+export type SqliteSchemaSpec = {
+   name: string;
+   type: "table" | "view";
+   sql: string;
+   columns: {
+      name: string;
+      type: string;
+      notnull: number;
+      dflt_value: any;
+      pk: number;
+   }[];
+   indices: {
+      name: string;
+      origin: string;
+      partial: number;
+      sql: string;
+      columns: { name: string; seqno: number }[];
+   }[];
+};
+
+export class SqliteIntrospector extends BaseIntrospector {
+   async getSchemas(): Promise<SchemaMetadata[]> {
+      // Sqlite doesn't support schemas.
+      return [];
+   }
+
+   async getSchemaSpec() {
+      const query = sql`
+         SELECT m.name, m.type, m.sql,
+            (SELECT json_group_array(
+               json_object(
+                  'name', p.name,
+                  'type', p.type,
+                  'notnull', p."notnull",
+                  'default', p.dflt_value,
+                  'primary_key', p.pk
+               )) FROM pragma_table_info(m.name) p) AS columns,
+            (SELECT json_group_array(
+               json_object(
+                  'name', i.name,
+                  'origin', i.origin,
+                  'partial', i.partial,
+                  'sql', im.sql,
+                  'columns', (SELECT json_group_array(
+                     json_object(
+                        'name', ii.name,
+                        'seqno', ii.seqno
+                     )) FROM pragma_index_info(i.name) ii)
+               )) FROM pragma_index_list(m.name) i
+                 LEFT JOIN sqlite_master im ON im.name = i.name
+                  AND im.type = 'index'
+            ) AS indices
+         FROM sqlite_master m
+         WHERE m.type IN ('table', 'view')
+           and m.name not like 'sqlite_%'
+           and m.name not in (${this.getExcludedTableNames().join(", ")})
+      `;
+
+      const tables = await this.executeWithPlugins<SqliteSchemaSpec[]>(query);
+
+      return tables.map((table) => ({
+         name: table.name,
+         isView: table.type === "view",
+         columns: table.columns.map((col) => {
+            const autoIncrementCol = table.sql
+               ?.split(/[\(\),]/)
+               ?.find((it) => it.toLowerCase().includes("autoincrement"))
+               ?.trimStart()
+               ?.split(/\s+/)?.[0]
+               ?.replace(/["`]/g, "");
+
+            return {
+               name: col.name,
+               dataType: col.type,
+               isNullable: !col.notnull,
+               isAutoIncrementing: col.name === autoIncrementCol,
+               hasDefaultValue: col.dflt_value != null,
+               comment: undefined,
+            };
+         }),
+         indices: table.indices.map((index) => ({
+            name: index.name,
+            table: table.name,
+            isUnique: index.sql?.match(/unique/i) != null,
+            columns: index.columns.map((col) => ({
+               name: col.name,
+               order: col.seqno,
+            })),
+         })),
+      }));
+   }
+}

app/src/data/index.ts

@@ -5,6 +5,7 @@ export * from "./entities";
 export * from "./relations";
 export * from "./schema/SchemaManager";
 export * from "./prototype";
+export * from "./connection";
 
 export {
    type RepoQuery,
@@ -14,11 +15,6 @@ export {
    whereSchema,
 } from "./server/data-query-impl";
 
-export { Connection } from "./connection/Connection";
-export { LibsqlConnection, type LibSqlCredentials } from "./connection/LibsqlConnection";
-export { SqliteConnection } from "./connection/SqliteConnection";
-export { SqliteLocalConnection } from "./connection/SqliteLocalConnection";
-export { SqliteIntrospector } from "./connection/SqliteIntrospector";
 export { KyselyPluginRunner } from "./plugins/KyselyPluginRunner";
 
 export { constructEntity, constructRelation } from "./schema/constructor";

app/src/data/schema/SchemaManager.ts

@@ -1,4 +1,4 @@
-import type { AlterTableColumnAlteringBuilder, CompiledQuery, TableMetadata } from "kysely";
+import type { CompiledQuery, TableMetadata } from "kysely";
 import type { IndexMetadata, SchemaResponse } from "../connection/Connection";
 import type { Entity, EntityManager } from "../entities";
 import { PrimaryField } from "../fields";

app/src/media/MediaField.ts

@@ -47,7 +47,7 @@ export class MediaField<
       return this.config.min_items;
    }
 
-   schema() {
+   override schema() {
       return undefined;
    }