shishantbiswas/bknd
1
0
Fork 0
mirror of https://github.com/shishantbiswas/bknd.git synced 2026-03-15 20:17:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

added BaseIntrospector and reorganized connections

Browse Source
This commit is contained in:
dswbx
2025-03-07 17:20:37 +01:00
parent 25a3cb9655
commit 3704cad53a
15 changed files with 232 additions and 258 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
app/__test__/data/specs/connection/SqliteIntrospector.spec.ts
View File

@@ -1,8 +1,7 @@
import { beforeEach, describe, test, expect } from "bun:test";
import { SqliteIntrospector } from "data/connection/SqliteIntrospector";
import { getDummyConnection, getDummyDatabase } from "../../helper";
import { Kysely, ParseJSONResultsPlugin, SqliteDialect } from "kysely";
import { _jsonp } from "core/utils";
import { describe, expect, test } from "bun:test";
import { SqliteIntrospector } from "data/connection";
import { getDummyDatabase } from "../../helper";
import { Kysely, SqliteDialect } from "kysely";
function create() {
const database = getDummyDatabase().dummyDb;

75
app/src/data/connection/BaseIntrospector.ts Normal file
View File

@@ -0,0 +1,75 @@
import {
type DatabaseMetadata,
type DatabaseMetadataOptions,
type Kysely,
type KyselyPlugin,
type RawBuilder,
type TableMetadata,
type DatabaseIntrospector,
type SchemaMetadata,
ParseJSONResultsPlugin,
DEFAULT_MIGRATION_TABLE,
DEFAULT_MIGRATION_LOCK_TABLE,
} from "kysely";
import { KyselyPluginRunner } from "data/plugins/KyselyPluginRunner";
import type { IndexMetadata } from "data/connection/Connection";
export type TableSpec = TableMetadata & {
indices: IndexMetadata[];
};
export type SchemaSpec = TableSpec[];
export type BaseIntrospectorConfig = {
excludeTables?: string[];
plugins?: KyselyPlugin[];
};
export abstract class BaseIntrospector implements DatabaseIntrospector {
readonly _excludeTables: string[] = [];
readonly _plugins: KyselyPlugin[];
constructor(
protected readonly db: Kysely<any>,
config: BaseIntrospectorConfig = {},
) {
this._excludeTables = config.excludeTables ?? [];
this._plugins = config.plugins ?? [new ParseJSONResultsPlugin()];
}
abstract getSchemaSpec(): Promise<SchemaSpec>;
abstract getSchemas(): Promise<SchemaMetadata[]>;
protected getExcludedTableNames(): string[] {
return [...this._excludeTables, DEFAULT_MIGRATION_TABLE, DEFAULT_MIGRATION_LOCK_TABLE];
}
protected async executeWithPlugins<T>(query: RawBuilder<any>): Promise<T> {
const result = await query.execute(this.db);
const runner = new KyselyPluginRunner(this._plugins ?? []);
return (await runner.transformResultRows(result.rows)) as unknown as T;
}
async getMetadata(options?: DatabaseMetadataOptions): Promise<DatabaseMetadata> {
return {
tables: await this.getTables(options),
};
}
async getIndices(tbl_name?: string): Promise<IndexMetadata[]> {
const schema = await this.getSchemaSpec();
return schema
.flatMap((table) => table.indices)
.filter((index) => !tbl_name || index.table === tbl_name);
}
async getTables(
options: DatabaseMetadataOptions = { withInternalKyselyTables: false },
): Promise<TableMetadata[]> {
const schema = await this.getSchemaSpec();
return schema.map((table) => ({
name: table.name,
isView: table.isView,
columns: table.columns,
}));
}
}

9
app/src/data/connection/Connection.ts
View File

@@ -13,6 +13,7 @@ import {
type Simplify,
sql,
} from "kysely";
import type { BaseIntrospector } from "./BaseIntrospector";
export type QB = SelectQueryBuilder<any, any, any>;
@@ -23,10 +24,6 @@ export type IndexMetadata = {
columns: { name: string; order: number }[];
};
export interface ConnectionIntrospector extends DatabaseIntrospector {
getIndices(tbl_name?: string): Promise<IndexMetadata[]>;
}
export interface SelectQueryBuilderExpression<O> extends AliasableExpression<O> {
get isSelectQueryBuilder(): true;
toOperationNode(): SelectQueryNode;
@@ -100,8 +97,8 @@ export abstract class Connection<DB = any> {
return conn[CONN_SYMBOL] === true;
}
getIntrospector(): ConnectionIntrospector {
return this.kysely.introspection as ConnectionIntrospector;
getIntrospector(): BaseIntrospector {
return this.kysely.introspection as any;
}
supportsBatching(): boolean {

6
app/src/data/connection/DummyConnection.ts
View File

@@ -1,7 +1,11 @@
import { Connection } from "./Connection";
import { Connection, type FieldSpec, type SchemaResponse } from "./Connection";
export class DummyConnection extends Connection {
constructor() {
super(undefined as any);
}
override getFieldSchema(spec: FieldSpec, strict?: boolean): SchemaResponse {
throw new Error("Method not implemented.");
}
}

151
app/src/data/connection/SqliteIntrospector.ts
View File

@@ -1,151 +0,0 @@
import {
type DatabaseIntrospector,
type DatabaseMetadata,
type DatabaseMetadataOptions,
type Kysely,
ParseJSONResultsPlugin,
type SchemaMetadata,
type TableMetadata,
type KyselyPlugin,
} from "kysely";
import { DEFAULT_MIGRATION_LOCK_TABLE, DEFAULT_MIGRATION_TABLE, sql } from "kysely";
import type { ConnectionIntrospector, IndexMetadata } from "./Connection";
import { KyselyPluginRunner } from "data";
export type SqliteIntrospectorConfig = {
excludeTables?: string[];
plugins?: KyselyPlugin[];
};
export class SqliteIntrospector implements DatabaseIntrospector, ConnectionIntrospector {
readonly #db: Kysely<any>;
readonly _excludeTables: string[] = [];
readonly _plugins: KyselyPlugin[];
constructor(db: Kysely<any>, config: SqliteIntrospectorConfig = {}) {
this.#db = db;
this._excludeTables = config.excludeTables ?? [];
this._plugins = config.plugins ?? [new ParseJSONResultsPlugin()];
}
async getSchemas(): Promise<SchemaMetadata[]> {
// Sqlite doesn't support schemas.
return [];
}
async getSchema() {
const excluded = [
...this._excludeTables,
DEFAULT_MIGRATION_TABLE,
DEFAULT_MIGRATION_LOCK_TABLE,
];
const query = sql`
SELECT m.name, m.type, m.sql,
(SELECT json_group_array(
json_object(
'name', p.name,
'type', p.type,
'notnull', p."notnull",
'default', p.dflt_value,
'primary_key', p.pk
)) FROM pragma_table_info(m.name) p) AS columns,
(SELECT json_group_array(
json_object(
'name', i.name,
'origin', i.origin,
'partial', i.partial,
'sql', im.sql,
'columns', (SELECT json_group_array(
json_object(
'name', ii.name,
'seqno', ii.seqno
)) FROM pragma_index_info(i.name) ii)
)) FROM pragma_index_list(m.name) i
LEFT JOIN sqlite_master im ON im.name = i.name
AND im.type = 'index'
) AS indices
FROM sqlite_master m
WHERE m.type IN ('table', 'view')
and m.name not like 'sqlite_%'
and m.name not in (${excluded.join(", ")})
`;
const result = await query.execute(this.#db);
const runner = new KyselyPluginRunner(this._plugins ?? []);
const tables = (await runner.transformResultRows(result.rows)) as unknown as {
name: string;
type: string;
sql: string;
columns: {
name: string;
type: string;
notnull: number;
dflt_value: any;
pk: number;
}[];
indices: {
name: string;
origin: string;
partial: number;
sql: string;
columns: { name: string; seqno: number }[];
}[];
}[];
//console.log("tables", tables);
return tables.map((table) => ({
name: table.name,
isView: table.type === "view",
columns: table.columns.map((col) => {
const autoIncrementCol = table.sql
?.split(/[\(\),]/)
?.find((it) => it.toLowerCase().includes("autoincrement"))
?.trimStart()
?.split(/\s+/)?.[0]
?.replace(/["`]/g, "");
return {
name: col.name,
dataType: col.type,
isNullable: !col.notnull,
isAutoIncrementing: col.name === autoIncrementCol,
hasDefaultValue: col.dflt_value != null,
comment: undefined,
};
}),
indices: table.indices.map((index) => ({
name: index.name,
table: table.name,
isUnique: index.sql?.match(/unique/i) != null,
columns: index.columns.map((col) => ({
name: col.name,
order: col.seqno,
})),
})),
}));
}
async getMetadata(options?: DatabaseMetadataOptions): Promise<DatabaseMetadata> {
return {
tables: await this.getTables(options),
};
}
async getIndices(tbl_name?: string): Promise<IndexMetadata[]> {
const schema = await this.getSchema();
return schema
.flatMap((table) => table.indices)
.filter((index) => !tbl_name || index.table === tbl_name);
}
async getTables(
options: DatabaseMetadataOptions = { withInternalKyselyTables: false },
): Promise<TableMetadata[]> {
const schema = await this.getSchema();
return schema.map((table) => ({
name: table.name,
isView: table.isView,
columns: table.columns,
}));
}
}

12
app/src/data/connection/index.ts Normal file
View File

@@ -0,0 +1,12 @@
export { Connection } from "./Connection";
export { BaseIntrospector } from "./BaseIntrospector";
// sqlite
export { LibsqlConnection, type LibSqlCredentials } from "./sqlite/LibsqlConnection";
export { SqliteConnection } from "./sqlite/SqliteConnection";
export { SqliteLocalConnection } from "./sqlite/SqliteLocalConnection";
export { SqliteIntrospector } from "./sqlite/SqliteIntrospector";
// postgres
export { PostgresConnection, type PostgresConnectionConfig } from "./postgres/PostgresConnection";
export { PostgresIntrospector } from "./postgres/PostgresIntrospector";

5
app/src/data/connection/postgres/PostgresConnection.ts
View File

@@ -42,11 +42,16 @@ export class PostgresConnection extends Connection {
let type: ColumnDataType = spec.primary ? "serial" : spec.type;
switch (spec.type) {
case "blob":
type = "bytea";
break;
case "date":
case "datetime":
// https://www.postgresql.org/docs/17/datatype-datetime.html
type = "timestamp";
break;
case "text":
// https://www.postgresql.org/docs/17/datatype-character.html
type = "varchar";
break;
}

110
app/src/data/connection/postgres/PostgresIntrospector.ts
View File

@@ -1,55 +1,37 @@
import {
type DatabaseIntrospector,
type DatabaseMetadata,
type DatabaseMetadataOptions,
type SchemaMetadata,
type TableMetadata,
type Kysely,
type KyselyPlugin,
ParseJSONResultsPlugin,
} from "kysely";
import { DEFAULT_MIGRATION_LOCK_TABLE, DEFAULT_MIGRATION_TABLE, sql } from "kysely";
import { KyselyPluginRunner } from "data";
import type { IndexMetadata } from "data/connection/Connection";
import { type SchemaMetadata, sql } from "kysely";
import { BaseIntrospector } from "data/connection/BaseIntrospector";
export type PostgresIntrospectorConfig = {
excludeTables?: string[];
plugins?: KyselyPlugin[];
type PostgresSchemaSpec = {
name: string;
type: "VIEW" | "BASE TABLE";
columns: {
name: string;
type: string;
notnull: number;
dflt: string;
pk: boolean;
}[];
indices: {
name: string;
origin: string;
partial: number;
sql: string;
columns: { name: string; seqno: number }[];
}[];
};
export class PostgresIntrospector implements DatabaseIntrospector {
readonly #db: Kysely<any>;
readonly _excludeTables: string[] = [];
readonly _plugins: KyselyPlugin[];
constructor(db: Kysely<any>, config: PostgresIntrospectorConfig = {}) {
this.#db = db;
this._excludeTables = config.excludeTables ?? [];
this._plugins = config.plugins ?? [new ParseJSONResultsPlugin()];
}
export class PostgresIntrospector extends BaseIntrospector {
async getSchemas(): Promise<SchemaMetadata[]> {
const rawSchemas = await this.#db
const rawSchemas = await this.db
.selectFrom("pg_catalog.pg_namespace")
.select("nspname")
.$castTo<RawSchemaMetadata>()
.$castTo<{ nspname: string }>()
.execute();
return rawSchemas.map((it) => ({ name: it.nspname }));
}
async getMetadata(options?: DatabaseMetadataOptions): Promise<DatabaseMetadata> {
return {
tables: await this.getTables(options),
};
}
async getSchema() {
const excluded = [
...this._excludeTables,
DEFAULT_MIGRATION_TABLE,
DEFAULT_MIGRATION_LOCK_TABLE,
];
async getSchemaSpec() {
const query = sql`
WITH tables_and_views AS (
SELECT table_name AS name,
@@ -58,7 +40,7 @@ export class PostgresIntrospector implements DatabaseIntrospector {
WHERE table_schema = 'public'
AND table_type IN ('BASE TABLE', 'VIEW')
AND table_name NOT LIKE 'pg_%'
AND table_name NOT IN (${excluded.join(", ")})
AND table_name NOT IN (${this.getExcludedTableNames().join(", ")})
),
columns_info AS (
@@ -115,26 +97,7 @@ export class PostgresIntrospector implements DatabaseIntrospector {
LEFT JOIN indices_info ii ON tv.name = ii.table_name;
`;
const result = await query.execute(this.#db);
const runner = new KyselyPluginRunner(this._plugins ?? []);
const tables = (await runner.transformResultRows(result.rows)) as unknown as {
name: string;
type: "VIEW" | "BASE TABLE";
columns: {
name: string;
type: string;
notnull: number;
dflt: string;
pk: boolean;
}[];
indices: {
name: string;
origin: string;
partial: number;
sql: string;
columns: { name: string; seqno: number }[];
}[];
}[];
const tables = await this.executeWithPlugins<PostgresSchemaSpec[]>(query);
return tables.map((table) => ({
name: table.name,
@@ -144,6 +107,7 @@ export class PostgresIntrospector implements DatabaseIntrospector {
name: col.name,
dataType: col.type,
isNullable: !col.notnull,
// @todo: check default value on 'nextval' see https://www.postgresql.org/docs/17/datatype-numeric.html#DATATYPE-SERIAL
isAutoIncrementing: true, // just for now
hasDefaultValue: col.dflt != null,
comment: undefined,
@@ -160,26 +124,4 @@ export class PostgresIntrospector implements DatabaseIntrospector {
})),
}));
}
async getIndices(tbl_name?: string): Promise<IndexMetadata[]> {
const schema = await this.getSchema();
return schema
.flatMap((table) => table.indices)
.filter((index) => !tbl_name || index.table === tbl_name);
}
async getTables(
options: DatabaseMetadataOptions = { withInternalKyselyTables: false },
): Promise<TableMetadata[]> {
const schema = await this.getSchema();
return schema.map((table) => ({
name: table.name,
isView: table.isView,
columns: table.columns,
}));
}
}
interface RawSchemaMetadata {
nspname: string;
}

6
app/src/data/connection/LibsqlConnection.ts → app/src/data/connection/sqlite/LibsqlConnection.ts
View File

@@ -1,9 +1,9 @@
import { type Client, type Config, type InStatement, createClient } from "@libsql/client";
import { LibsqlDialect } from "@libsql/kysely-libsql";
import { type DatabaseIntrospector, Kysely, ParseJSONResultsPlugin } from "kysely";
import { FilterNumericKeysPlugin } from "../plugins/FilterNumericKeysPlugin";
import { KyselyPluginRunner } from "../plugins/KyselyPluginRunner";
import type { QB } from "./Connection";
import { FilterNumericKeysPlugin } from "data/plugins/FilterNumericKeysPlugin";
import { KyselyPluginRunner } from "data/plugins/KyselyPluginRunner";
import type { QB } from "../Connection";
import { SqliteConnection } from "./SqliteConnection";
import { SqliteIntrospector } from "./SqliteIntrospector";

2
app/src/data/connection/SqliteConnection.ts → app/src/data/connection/sqlite/SqliteConnection.ts
View File

@@ -1,6 +1,6 @@
import type { ColumnDataType, ColumnDefinitionBuilder, Kysely, KyselyPlugin } from "kysely";
import { jsonArrayFrom, jsonBuildObject, jsonObjectFrom } from "kysely/helpers/sqlite";
import { Connection, type DbFunctions, type FieldSpec, type SchemaResponse } from "./Connection";
import { Connection, type DbFunctions, type FieldSpec, type SchemaResponse } from "../Connection";
export class SqliteConnection extends Connection {
constructor(kysely: Kysely<any>, fn: Partial<DbFunctions> = {}, plugins: KyselyPlugin[] = []) {

95
app/src/data/connection/sqlite/SqliteIntrospector.ts Normal file
View File

@@ -0,0 +1,95 @@
import { type SchemaMetadata, sql } from "kysely";
import { BaseIntrospector } from "../BaseIntrospector";
export type SqliteSchemaSpec = {
name: string;
type: "table" | "view";
sql: string;
columns: {
name: string;
type: string;
notnull: number;
dflt_value: any;
pk: number;
}[];
indices: {
name: string;
origin: string;
partial: number;
sql: string;
columns: { name: string; seqno: number }[];
}[];
};
export class SqliteIntrospector extends BaseIntrospector {
async getSchemas(): Promise<SchemaMetadata[]> {
// Sqlite doesn't support schemas.
return [];
}
async getSchemaSpec() {
const query = sql`
SELECT m.name, m.type, m.sql,
(SELECT json_group_array(
json_object(
'name', p.name,
'type', p.type,
'notnull', p."notnull",
'default', p.dflt_value,
'primary_key', p.pk
)) FROM pragma_table_info(m.name) p) AS columns,
(SELECT json_group_array(
json_object(
'name', i.name,
'origin', i.origin,
'partial', i.partial,
'sql', im.sql,
'columns', (SELECT json_group_array(
json_object(
'name', ii.name,
'seqno', ii.seqno
)) FROM pragma_index_info(i.name) ii)
)) FROM pragma_index_list(m.name) i
LEFT JOIN sqlite_master im ON im.name = i.name
AND im.type = 'index'
) AS indices
FROM sqlite_master m
WHERE m.type IN ('table', 'view')
and m.name not like 'sqlite_%'
and m.name not in (${this.getExcludedTableNames().join(", ")})
`;
const tables = await this.executeWithPlugins<SqliteSchemaSpec[]>(query);
return tables.map((table) => ({
name: table.name,
isView: table.type === "view",
columns: table.columns.map((col) => {
const autoIncrementCol = table.sql
?.split(/[\(\),]/)
?.find((it) => it.toLowerCase().includes("autoincrement"))
?.trimStart()
?.split(/\s+/)?.[0]
?.replace(/["`]/g, "");
return {
name: col.name,
dataType: col.type,
isNullable: !col.notnull,
isAutoIncrementing: col.name === autoIncrementCol,
hasDefaultValue: col.dflt_value != null,
comment: undefined,
};
}),
indices: table.indices.map((index) => ({
name: index.name,
table: table.name,
isUnique: index.sql?.match(/unique/i) != null,
columns: index.columns.map((col) => ({
name: col.name,
order: col.seqno,
})),
})),
}));
}
}

0
app/src/data/connection/SqliteLocalConnection.ts → app/src/data/connection/sqlite/SqliteLocalConnection.ts
View File

6
app/src/data/index.ts
View File

@@ -5,6 +5,7 @@ export * from "./entities";
export * from "./relations";
export * from "./schema/SchemaManager";
export * from "./prototype";
export * from "./connection";
export {
type RepoQuery,
@@ -14,11 +15,6 @@ export {
whereSchema,
} from "./server/data-query-impl";
export { Connection } from "./connection/Connection";
export { LibsqlConnection, type LibSqlCredentials } from "./connection/LibsqlConnection";
export { SqliteConnection } from "./connection/SqliteConnection";
export { SqliteLocalConnection } from "./connection/SqliteLocalConnection";
export { SqliteIntrospector } from "./connection/SqliteIntrospector";
export { KyselyPluginRunner } from "./plugins/KyselyPluginRunner";
export { constructEntity, constructRelation } from "./schema/constructor";

2
app/src/data/schema/SchemaManager.ts
View File

@@ -1,4 +1,4 @@
import type { AlterTableColumnAlteringBuilder, CompiledQuery, TableMetadata } from "kysely";
import type { CompiledQuery, TableMetadata } from "kysely";
import type { IndexMetadata, SchemaResponse } from "../connection/Connection";
import type { Entity, EntityManager } from "../entities";
import { PrimaryField } from "../fields";

2
app/src/media/MediaField.ts
View File

@@ -47,7 +47,7 @@ export class MediaField<
return this.config.min_items;
}
schema() {
override schema() {
return undefined;
}