shishantbiswas/bknd
1
0
Fork 0
mirror of https://github.com/shishantbiswas/bknd.git synced 2026-03-16 04:27:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

initialized postgres support

Browse Source
This commit is contained in:
dswbx
2025-03-07 15:02:19 +01:00
parent 8550aef606
commit a5c422d45d
30 changed files with 759 additions and 220 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
app/__test__/App.spec.ts
View File

@@ -1,9 +1,9 @@
import { afterAll, describe, expect, test } from "bun:test";
import { afterAll, afterEach, describe, expect, test } from "bun:test";
import { App } from "../src";
import { getDummyConnection } from "./helper";
const { dummyConnection, afterAllCleanup } = getDummyConnection();
afterAll(afterAllCleanup);
afterEach(afterAllCleanup);
describe("App tests", async () => {
test("boots and pongs", async () => {
@@ -12,4 +12,16 @@ describe("App tests", async () => {
//expect(await app.data?.em.ping()).toBeTrue();
});
/*test.only("what", async () => {
const app = new App(dummyConnection, {
auth: {
enabled: true,
},
});
await app.module.auth.build();
await app.module.data.build();
console.log(app.em.entities.map((e) => e.name));
console.log(await app.em.schema().getDiff());
});*/
});

53
app/__test__/data/pg.spec.ts Normal file
View File

@@ -0,0 +1,53 @@
import { describe, expect, test } from "bun:test";
import { createApp } from "App";
import { PostgresConnection } from "data/connection/postgres/PostgresConnection";
import * as proto from "data/prototype";
import { PostgresIntrospector } from "data/connection/postgres/PostgresIntrospector";
import { ParseJSONResultsPlugin } from "kysely";
const connection = new PostgresConnection({
database: "test",
host: "localhost",
user: "root",
password: "1234",
port: 5433,
});
describe("postgres", () => {
test.skip("introspector", async () => {
const introspector = new PostgresIntrospector(connection.kysely, {
plugins: [new ParseJSONResultsPlugin()],
});
console.log(await introspector.getSchema());
});
test("builds", async () => {
const schema = proto.em(
{
posts: proto.entity("posts", {
title: proto.text().required(),
}),
comments: proto.entity("comments", {
text: proto.text(),
}),
},
(ctx, s) => {
ctx.relation(s.comments).manyToOne(s.posts);
ctx.index(s.posts).on(["title"], true);
ctx.index(s.comments).on(["text"]);
},
);
const app = createApp({
initialConfig: {
data: schema.toJSON(),
},
connection,
});
await app.build({ sync: true });
expect(app.version()).toBeDefined();
});
});

4
app/__test__/data/relations.test.ts
View File

@@ -27,7 +27,7 @@ describe("Relations", async () => {
const sql1 = schema
.createTable("posts")
.addColumn(...r1.schema()!)
.addColumn(...em.connection.getFieldSchema(r1.schema())!)
.compile().sql;
expect(sql1).toBe(
@@ -43,7 +43,7 @@ describe("Relations", async () => {
const sql2 = schema
.createTable("posts")
.addColumn(...r2.schema()!)
.addColumn(...em.connection.getFieldSchema(r2.schema())!)
.compile().sql;
expect(sql2).toBe(

4
app/__test__/data/specs/SchemaManager.spec.ts
View File

@@ -15,7 +15,7 @@ describe("SchemaManager tests", async () => {
const em = new EntityManager([entity], dummyConnection, [], [index]);
const schema = new SchemaManager(em);
const introspection = schema.getIntrospectionFromEntity(em.entities[0]);
const introspection = schema.getIntrospectionFromEntity(em.entities[0]!);
expect(introspection).toEqual({
name: "test",
isView: false,
@@ -109,7 +109,7 @@ describe("SchemaManager tests", async () => {
await schema.sync({ force: true, drop: true });
const diffAfter = await schema.getDiff();
console.log("diffAfter", diffAfter);
//console.log("diffAfter", diffAfter);
expect(diffAfter.length).toBe(0);
await kysely.schema.dropTable(table).execute();

108
app/__test__/data/specs/connection/SqliteIntrospector.spec.ts Normal file
View File

@@ -0,0 +1,108 @@
import { beforeEach, describe, test, expect } from "bun:test";
import { SqliteIntrospector } from "data/connection/SqliteIntrospector";
import { getDummyConnection, getDummyDatabase } from "../../helper";
import { Kysely, ParseJSONResultsPlugin, SqliteDialect } from "kysely";
import { _jsonp } from "core/utils";
function create() {
const database = getDummyDatabase().dummyDb;
return new Kysely({
dialect: new SqliteDialect({ database }),
});
}
function createLibsql() {
const database = getDummyDatabase().dummyDb;
return new Kysely({
dialect: new SqliteDialect({ database }),
});
}
describe("SqliteIntrospector", () => {
test("asdf", async () => {
const kysely = create();
await kysely.schema
.createTable("test")
.addColumn("id", "integer", (col) => col.primaryKey().autoIncrement().notNull())
.addColumn("string", "text", (col) => col.notNull())
.addColumn("number", "integer")
.execute();
await kysely.schema
.createIndex("idx_test_string")
.on("test")
.columns(["string"])
.unique()
.execute();
await kysely.schema
.createTable("test2")
.addColumn("id", "integer", (col) => col.primaryKey().autoIncrement().notNull())
.addColumn("number", "integer")
.execute();
await kysely.schema.createIndex("idx_test2_number").on("test2").columns(["number"]).execute();
const introspector = new SqliteIntrospector(kysely, {});
const result = await introspector.getTables();
//console.log(_jsonp(result));
expect(result).toEqual([
{
name: "test",
isView: false,
columns: [
{
name: "id",
dataType: "INTEGER",
isNullable: false,
isAutoIncrementing: true,
hasDefaultValue: false,
comment: undefined,
},
{
name: "string",
dataType: "TEXT",
isNullable: false,
isAutoIncrementing: false,
hasDefaultValue: false,
comment: undefined,
},
{
comment: undefined,
dataType: "INTEGER",
hasDefaultValue: false,
isAutoIncrementing: false,
isNullable: true,
name: "number",
},
],
},
{
name: "test2",
isView: false,
columns: [
{
name: "id",
dataType: "INTEGER",
isNullable: false,
isAutoIncrementing: true,
hasDefaultValue: false,
comment: undefined,
},
{
name: "number",
dataType: "INTEGER",
isNullable: true,
isAutoIncrementing: false,
hasDefaultValue: false,
comment: undefined,
},
],
},
]);
});
});

20
app/__test__/data/specs/fields/Field.spec.ts
View File

@@ -1,23 +1,29 @@
import { describe, expect, test } from "bun:test";
import { Default, parse, stripMark } from "../../../../src/core/utils";
import { Field, type SchemaResponse, TextField, baseFieldConfigSchema } from "../../../../src/data";
import { runBaseFieldTests, transformPersist } from "./inc";
import { Default, stripMark } from "../../../../src/core/utils";
import { baseFieldConfigSchema, Field } from "../../../../src/data/fields/Field";
import { runBaseFieldTests } from "./inc";
describe("[data] Field", async () => {
class FieldSpec extends Field {
schema(): SchemaResponse {
return this.useSchemaHelper("text");
}
getSchema() {
return baseFieldConfigSchema;
}
}
test("fieldSpec", () => {
expect(new FieldSpec("test").schema()).toEqual({
name: "test",
type: "text",
nullable: true, // always true
dflt: undefined, // never using default value
});
});
runBaseFieldTests(FieldSpec, { defaultValue: "test", schemaType: "text" });
test("default config", async () => {
const config = Default(baseFieldConfigSchema, {});
expect(stripMark(new FieldSpec("test").config)).toEqual(config);
expect(stripMark(new FieldSpec("test").config)).toEqual(config as any);
});
test("transformPersist (specific)", async () => {

7
app/__test__/data/specs/fields/PrimaryField.spec.ts
View File

@@ -10,7 +10,12 @@ describe("[data] PrimaryField", async () => {
test("schema", () => {
expect(field.name).toBe("primary");
expect(field.schema()).toEqual(["primary", "integer", expect.any(Function)]);
expect(field.schema()).toEqual({
name: "primary",
type: "integer" as const,
nullable: false,
primary: true,
});
});
test("hasDefault", async () => {

13
app/__test__/data/specs/fields/inc.ts
View File

@@ -34,11 +34,14 @@ export function runBaseFieldTests(
test("schema", () => {
expect(noConfigField.name).toBe("no_config");
expect(noConfigField.schema(null as any)).toEqual([
"no_config",
config.schemaType,
expect.any(Function),
]);
const { type, name, nullable, dflt } = noConfigField.schema()!;
expect({ type, name, nullable, dflt }).toEqual({
type: config.schemaType as any,
name: "no_config",
nullable: true, // always true
dflt: undefined, // never using default value
});
});
test("hasDefault", async () => {

8
app/__test__/integration/auth.integration.test.ts
View File

@@ -1,9 +1,12 @@
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "bun:test";
import { App, createApp } from "../../src";
import type { AuthResponse } from "../../src/auth";
import { auth } from "../../src/auth/middlewares";
import { randomString, secureRandomString, withDisabledConsole } from "../../src/core/utils";
import { disableConsoleLog, enableConsoleLog } from "../helper";
import { disableConsoleLog, enableConsoleLog, getDummyConnection } from "../helper";
const { dummyConnection, afterAllCleanup } = getDummyConnection();
afterEach(afterAllCleanup);
beforeAll(disableConsoleLog);
afterAll(enableConsoleLog);
@@ -64,6 +67,7 @@ const configs = {
function createAuthApp() {
const app = createApp({
connection: dummyConnection,
initialConfig: {
auth: configs.auth,
},

6
app/package.json
View File

@@ -63,16 +63,17 @@
"@aws-sdk/client-s3": "^3.613.0",
"@bluwy/giget-core": "^0.1.2",
"@dagrejs/dagre": "^1.1.4",
"@mantine/modals": "^7.13.4",
"@mantine/notifications": "^7.13.4",
"@hono/typebox-validator": "^0.2.6",
"@hono/vite-dev-server": "^0.17.0",
"@hono/zod-validator": "^0.4.1",
"@hookform/resolvers": "^3.9.1",
"@libsql/kysely-libsql": "^0.4.1",
"@mantine/modals": "^7.13.4",
"@mantine/notifications": "^7.13.4",
"@rjsf/core": "5.22.2",
"@tabler/icons-react": "3.18.0",
"@types/node": "^22.10.0",
"@types/pg": "^8.11.11",
"@types/react": "^18.3.12",
"@types/react-dom": "^18.3.1",
"@vitejs/plugin-react": "^4.3.3",
@@ -84,6 +85,7 @@
"kysely-d1": "^0.3.0",
"open": "^10.1.0",
"openapi-types": "^12.1.3",
"pg": "^8.13.3",
"postcss": "^8.4.47",
"postcss-preset-mantine": "^1.17.0",
"postcss-simple-vars": "^7.0.1",

46
app/src/data/connection/Connection.ts
View File

@@ -1,9 +1,12 @@
import {
type AliasableExpression,
type ColumnBuilderCallback,
type ColumnDataType,
type DatabaseIntrospector,
type Expression,
type Kysely,
type KyselyPlugin,
type OnModifyForeignAction,
type RawBuilder,
type SelectQueryBuilder,
type SelectQueryNode,
@@ -29,6 +32,38 @@ export interface SelectQueryBuilderExpression<O> extends AliasableExpression<O>
toOperationNode(): SelectQueryNode;
}
export type SchemaResponse = [string, ColumnDataType, ColumnBuilderCallback] | undefined;
const FieldSpecTypes = [
"text",
"integer",
"real",
"blob",
"date",
"datetime",
"timestamp",
"boolean",
"json",
] as const;
export type FieldSpec = {
type: (typeof FieldSpecTypes)[number];
name: string;
nullable?: boolean;
dflt?: any;
unique?: boolean;
primary?: boolean;
references?: string;
onDelete?: OnModifyForeignAction;
onUpdate?: OnModifyForeignAction;
};
export type IndexSpec = {
name: string;
columns: string[];
unique?: boolean;
};
export type DbFunctions = {
jsonObjectFrom<O>(expr: SelectQueryBuilderExpression<O>): RawBuilder<Simplify<O> | null>;
jsonArrayFrom<O>(expr: SelectQueryBuilderExpression<O>): RawBuilder<Simplify<O>[]>;
@@ -108,4 +143,15 @@ export abstract class Connection<DB = any> {
return await this.batch(queries);
}
protected validateFieldSpecType(type: string): type is FieldSpec["type"] {
if (!FieldSpecTypes.includes(type as any)) {
throw new Error(
`Invalid field type "${type}". Allowed types are: ${FieldSpecTypes.join(", ")}`,
);
}
return true;
}
abstract getFieldSchema(spec: FieldSpec, strict?: boolean): SchemaResponse;
}

4
app/src/data/connection/LibsqlConnection.ts
View File

@@ -12,10 +12,13 @@ export type LibSqlCredentials = Config & {
protocol?: (typeof LIBSQL_PROTOCOLS)[number];
};
const plugins = [new FilterNumericKeysPlugin(), new ParseJSONResultsPlugin()];
class CustomLibsqlDialect extends LibsqlDialect {
override createIntrospector(db: Kysely<any>): DatabaseIntrospector {
return new SqliteIntrospector(db, {
excludeTables: ["libsql_wasm_func_table"],
plugins,
});
}
}
@@ -26,7 +29,6 @@ export class LibsqlConnection extends SqliteConnection {
constructor(client: Client);
constructor(credentials: LibSqlCredentials);
constructor(clientOrCredentials: Client | LibSqlCredentials) {
const plugins = [new FilterNumericKeysPlugin(), new ParseJSONResultsPlugin()];
let client: Client;
if (clientOrCredentials && "url" in clientOrCredentials) {
let { url, authToken, protocol } = clientOrCredentials;

32
app/src/data/connection/SqliteConnection.ts
View File

@@ -1,6 +1,6 @@
import type { Kysely, KyselyPlugin } from "kysely";
import type { ColumnDataType, ColumnDefinitionBuilder, Kysely, KyselyPlugin } from "kysely";
import { jsonArrayFrom, jsonBuildObject, jsonObjectFrom } from "kysely/helpers/sqlite";
import { Connection, type DbFunctions } from "./Connection";
import { Connection, type DbFunctions, type FieldSpec, type SchemaResponse } from "./Connection";
export class SqliteConnection extends Connection {
constructor(kysely: Kysely<any>, fn: Partial<DbFunctions> = {}, plugins: KyselyPlugin[] = []) {
@@ -19,4 +19,32 @@ export class SqliteConnection extends Connection {
override supportsIndices(): boolean {
return true;
}
override getFieldSchema(spec: FieldSpec): SchemaResponse {
this.validateFieldSpecType(spec.type);
let type: ColumnDataType = spec.type;
switch (spec.type) {
case "json":
type = "text";
break;
}
return [
spec.name,
type,
(col: ColumnDefinitionBuilder) => {
if (spec.primary) {
return col.primaryKey().notNull().autoIncrement();
}
if (spec.references) {
let relCol = col.references(spec.references);
if (spec.onDelete) relCol = relCol.onDelete(spec.onDelete);
if (spec.onUpdate) relCol = relCol.onUpdate(spec.onUpdate);
return relCol;
}
return spec.nullable ? col : col.notNull();
},
] as const;
}
}

241
app/src/data/connection/SqliteIntrospector.ts
View File

@@ -1,26 +1,31 @@
import type {
DatabaseIntrospector,
DatabaseMetadata,
DatabaseMetadataOptions,
ExpressionBuilder,
Kysely,
SchemaMetadata,
TableMetadata,
import {
type DatabaseIntrospector,
type DatabaseMetadata,
type DatabaseMetadataOptions,
type Kysely,
ParseJSONResultsPlugin,
type SchemaMetadata,
type TableMetadata,
type KyselyPlugin,
} from "kysely";
import { DEFAULT_MIGRATION_LOCK_TABLE, DEFAULT_MIGRATION_TABLE, sql } from "kysely";
import type { ConnectionIntrospector, IndexMetadata } from "./Connection";
import { KyselyPluginRunner } from "data";
export type SqliteIntrospectorConfig = {
excludeTables?: string[];
plugins?: KyselyPlugin[];
};
export class SqliteIntrospector implements DatabaseIntrospector, ConnectionIntrospector {
readonly #db: Kysely<any>;
readonly _excludeTables: string[] = [];
readonly _plugins: KyselyPlugin[];
constructor(db: Kysely<any>, config: SqliteIntrospectorConfig = {}) {
this.#db = db;
this._excludeTables = config.excludeTables ?? [];
this._plugins = config.plugins ?? [new ParseJSONResultsPlugin()];
}
async getSchemas(): Promise<SchemaMetadata[]> {
@@ -28,86 +33,96 @@ export class SqliteIntrospector implements DatabaseIntrospector, ConnectionIntro
return [];
}
async getIndices(tbl_name?: string): Promise<IndexMetadata[]> {
const indices = await this.#db
.selectFrom("sqlite_master")
.where("type", "=", "index")
.$if(!!tbl_name, (eb) => eb.where("tbl_name", "=", tbl_name))
.select("name")
.$castTo<{ name: string }>()
.execute();
async getSchema() {
const excluded = [
...this._excludeTables,
DEFAULT_MIGRATION_TABLE,
DEFAULT_MIGRATION_LOCK_TABLE,
];
const query = sql`
SELECT m.name, m.type, m.sql,
(SELECT json_group_array(
json_object(
'name', p.name,
'type', p.type,
'notnull', p."notnull",
'default', p.dflt_value,
'primary_key', p.pk
)) FROM pragma_table_info(m.name) p) AS columns,
(SELECT json_group_array(
json_object(
'name', i.name,
'origin', i.origin,
'partial', i.partial,
'sql', im.sql,
'columns', (SELECT json_group_array(
json_object(
'name', ii.name,
'seqno', ii.seqno
)) FROM pragma_index_info(i.name) ii)
)) FROM pragma_index_list(m.name) i
LEFT JOIN sqlite_master im ON im.name = i.name
AND im.type = 'index'
) AS indices
FROM sqlite_master m
WHERE m.type IN ('table', 'view')
and m.name not like 'sqlite_%'
and m.name not in (${excluded.join(", ")})
`;
return Promise.all(indices.map(({ name }) => this.#getIndexMetadata(name)));
}
const result = await query.execute(this.#db);
const runner = new KyselyPluginRunner(this._plugins ?? []);
const tables = (await runner.transformResultRows(result.rows)) as unknown as {
name: string;
type: string;
sql: string;
columns: {
name: string;
type: string;
notnull: number;
dflt_value: any;
pk: number;
}[];
indices: {
name: string;
origin: string;
partial: number;
sql: string;
columns: { name: string; seqno: number }[];
}[];
}[];
async #getIndexMetadata(index: string): Promise<IndexMetadata> {
const db = this.#db;
//console.log("tables", tables);
return tables.map((table) => ({
name: table.name,
isView: table.type === "view",
columns: table.columns.map((col) => {
const autoIncrementCol = table.sql
?.split(/[\(\),]/)
?.find((it) => it.toLowerCase().includes("autoincrement"))
?.trimStart()
?.split(/\s+/)?.[0]
?.replace(/["`]/g, "");
// Get the SQL that was used to create the index.
const indexDefinition = await db
.selectFrom("sqlite_master")
.where("name", "=", index)
.select(["sql", "tbl_name", "type"])
.$castTo<{ sql: string | undefined; tbl_name: string; type: string }>()
.executeTakeFirstOrThrow();
//console.log("--indexDefinition--", indexDefinition, index);
// check unique by looking for the word "unique" in the sql
const isUnique = indexDefinition.sql?.match(/unique/i) != null;
const columns = await db
.selectFrom(
sql<{
seqno: number;
cid: number;
name: string;
}>`pragma_index_info(${index})`.as("index_info"),
)
.select(["seqno", "cid", "name"])
.orderBy("cid")
.execute();
return {
name: index,
table: indexDefinition.tbl_name,
isUnique: isUnique,
columns: columns.map((col) => ({
name: col.name,
order: col.seqno,
return {
name: col.name,
dataType: col.type,
isNullable: !col.notnull,
isAutoIncrementing: col.name === autoIncrementCol,
hasDefaultValue: col.dflt_value != null,
comment: undefined,
};
}),
indices: table.indices.map((index) => ({
name: index.name,
table: table.name,
isUnique: index.sql?.match(/unique/i) != null,
columns: index.columns.map((col) => ({
name: col.name,
order: col.seqno,
})),
})),
};
}
private excludeTables(tables: string[] = []) {
return (eb: ExpressionBuilder<any, any>) => {
const and = tables.map((t) => eb("name", "!=", t));
return eb.and(and);
};
}
async getTables(
options: DatabaseMetadataOptions = { withInternalKyselyTables: false },
): Promise<TableMetadata[]> {
let query = this.#db
.selectFrom("sqlite_master")
.where("type", "in", ["table", "view"])
.where("name", "not like", "sqlite_%")
.select("name")
.orderBy("name")
.$castTo<{ name: string }>();
if (!options.withInternalKyselyTables) {
query = query.where(
this.excludeTables([DEFAULT_MIGRATION_TABLE, DEFAULT_MIGRATION_LOCK_TABLE]),
);
}
if (this._excludeTables.length > 0) {
query = query.where(this.excludeTables(this._excludeTables));
}
const tables = await query.execute();
return Promise.all(tables.map(({ name }) => this.#getTableMetadata(name)));
}));
}
async getMetadata(options?: DatabaseMetadataOptions): Promise<DatabaseMetadata> {
@@ -116,49 +131,21 @@ export class SqliteIntrospector implements DatabaseIntrospector, ConnectionIntro
};
}
async #getTableMetadata(table: string): Promise<TableMetadata> {
const db = this.#db;
async getIndices(tbl_name?: string): Promise<IndexMetadata[]> {
const schema = await this.getSchema();
return schema
.flatMap((table) => table.indices)
.filter((index) => !tbl_name || index.table === tbl_name);
}
// Get the SQL that was used to create the table.
const tableDefinition = await db
.selectFrom("sqlite_master")
.where("name", "=", table)
.select(["sql", "type"])
.$castTo<{ sql: string | undefined; type: string }>()
.executeTakeFirstOrThrow();
// Try to find the name of the column that has `autoincrement` 🤦
const autoIncrementCol = tableDefinition.sql
?.split(/[\(\),]/)
?.find((it) => it.toLowerCase().includes("autoincrement"))
?.trimStart()
?.split(/\s+/)?.[0]
?.replace(/["`]/g, "");
const columns = await db
.selectFrom(
sql<{
name: string;
type: string;
notnull: 0 | 1;
dflt_value: any;
}>`pragma_table_info(${table})`.as("table_info"),
)
.select(["name", "type", "notnull", "dflt_value"])
.orderBy("cid")
.execute();
return {
name: table,
isView: tableDefinition.type === "view",
columns: columns.map((col) => ({
name: col.name,
dataType: col.type,
isNullable: !col.notnull,
isAutoIncrementing: col.name === autoIncrementCol,
hasDefaultValue: col.dflt_value != null,
comment: undefined,
})),
};
async getTables(
options: DatabaseMetadataOptions = { withInternalKyselyTables: false },
): Promise<TableMetadata[]> {
const schema = await this.getSchema();
return schema.map((table) => ({
name: table.name,
isView: table.isView,
columns: table.columns,
}));
}
}

8
app/src/data/connection/SqliteLocalConnection.ts
View File

@@ -3,25 +3,25 @@ import { Kysely, SqliteDialect } from "kysely";
import { SqliteConnection } from "./SqliteConnection";
import { SqliteIntrospector } from "./SqliteIntrospector";
const plugins = [new ParseJSONResultsPlugin()];
class CustomSqliteDialect extends SqliteDialect {
override createIntrospector(db: Kysely<any>): DatabaseIntrospector {
return new SqliteIntrospector(db, {
excludeTables: ["test_table"],
plugins,
});
}
}
export class SqliteLocalConnection extends SqliteConnection {
constructor(private database: SqliteDatabase) {
const plugins = [new ParseJSONResultsPlugin()];
const kysely = new Kysely({
dialect: new CustomSqliteDialect({ database }),
plugins,
//log: ["query"],
});
super(kysely);
this.plugins = plugins;
super(kysely, {}, plugins);
}
override supportsIndices(): boolean {

71
app/src/data/connection/postgres/PostgresConnection.ts Normal file
View File

@@ -0,0 +1,71 @@
import {
Kysely,
PostgresDialect,
type DatabaseIntrospector,
type ColumnDataType,
type ColumnDefinitionBuilder,
ParseJSONResultsPlugin,
} from "kysely";
import pg from "pg";
import { PostgresIntrospector } from "./PostgresIntrospector";
import { type FieldSpec, type SchemaResponse, Connection } from "data/connection/Connection";
export type PostgresConnectionConfig = pg.PoolConfig;
const plugins = [new ParseJSONResultsPlugin()];
class CustomPostgresDialect extends PostgresDialect {
override createIntrospector(db: Kysely<any>): DatabaseIntrospector {
return new PostgresIntrospector(db);
}
}
export class PostgresConnection extends Connection {
constructor(config: PostgresConnectionConfig) {
const kysely = new Kysely({
dialect: new CustomPostgresDialect({
pool: new pg.Pool(config),
}),
plugins,
//log: ["query", "error"],
});
super(kysely, {}, plugins);
}
override supportsIndices(): boolean {
return true;
}
override getFieldSchema(spec: FieldSpec): SchemaResponse {
this.validateFieldSpecType(spec.type);
let type: ColumnDataType = spec.primary ? "serial" : spec.type;
switch (spec.type) {
case "date":
case "datetime":
type = "timestamp";
break;
case "text":
type = "varchar";
break;
}
return [
spec.name,
type,
(col: ColumnDefinitionBuilder) => {
if (spec.primary) {
return col.primaryKey();
}
if (spec.references) {
return col
.references(spec.references)
.onDelete(spec.onDelete ?? "set null")
.onUpdate(spec.onUpdate ?? "no action");
}
return spec.nullable ? col : col.notNull();
},
];
}
}

185
app/src/data/connection/postgres/PostgresIntrospector.ts Normal file
View File

@@ -0,0 +1,185 @@
import {
type DatabaseIntrospector,
type DatabaseMetadata,
type DatabaseMetadataOptions,
type SchemaMetadata,
type TableMetadata,
type Kysely,
type KyselyPlugin,
ParseJSONResultsPlugin,
} from "kysely";
import { DEFAULT_MIGRATION_LOCK_TABLE, DEFAULT_MIGRATION_TABLE, sql } from "kysely";
import { KyselyPluginRunner } from "data";
import type { IndexMetadata } from "data/connection/Connection";
export type PostgresIntrospectorConfig = {
excludeTables?: string[];
plugins?: KyselyPlugin[];
};
export class PostgresIntrospector implements DatabaseIntrospector {
readonly #db: Kysely<any>;
readonly _excludeTables: string[] = [];
readonly _plugins: KyselyPlugin[];
constructor(db: Kysely<any>, config: PostgresIntrospectorConfig = {}) {
this.#db = db;
this._excludeTables = config.excludeTables ?? [];
this._plugins = config.plugins ?? [new ParseJSONResultsPlugin()];
}
async getSchemas(): Promise<SchemaMetadata[]> {
const rawSchemas = await this.#db
.selectFrom("pg_catalog.pg_namespace")
.select("nspname")
.$castTo<RawSchemaMetadata>()
.execute();
return rawSchemas.map((it) => ({ name: it.nspname }));
}
async getMetadata(options?: DatabaseMetadataOptions): Promise<DatabaseMetadata> {
return {
tables: await this.getTables(options),
};
}
async getSchema() {
const excluded = [
...this._excludeTables,
DEFAULT_MIGRATION_TABLE,
DEFAULT_MIGRATION_LOCK_TABLE,
];
const query = sql`
WITH tables_and_views AS (
SELECT table_name AS name,
table_type AS type
FROM information_schema.tables
WHERE table_schema = 'public'
AND table_type IN ('BASE TABLE', 'VIEW')
AND table_name NOT LIKE 'pg_%'
AND table_name NOT IN (${excluded.join(", ")})
),
columns_info AS (
SELECT table_name AS name,
json_agg(json_build_object(
'name', column_name,
'type', data_type,
'notnull', (CASE WHEN is_nullable = 'NO' THEN true ELSE false END),
'dflt', column_default,
'pk', (SELECT COUNT(*) > 0
FROM information_schema.table_constraints tc
INNER JOIN information_schema.key_column_usage kcu
ON tc.constraint_name = kcu.constraint_name
WHERE tc.table_name = c.table_name
AND tc.constraint_type = 'PRIMARY KEY'
AND kcu.column_name = c.column_name)
)) AS columns
FROM information_schema.columns c
WHERE table_schema = 'public'
GROUP BY table_name
),
indices_info AS (
SELECT
t.relname AS table_name,
json_agg(json_build_object(
'name', i.relname,
'origin', pg_get_indexdef(i.oid),
'partial', (CASE WHEN ix.indisvalid THEN false ELSE true END),
'sql', pg_get_indexdef(i.oid),
'columns', (
SELECT json_agg(json_build_object(
'name', a.attname,
'seqno', x.ordinal_position
))
FROM unnest(ix.indkey) WITH ORDINALITY AS x(attnum, ordinal_position)
JOIN pg_attribute a ON a.attnum = x.attnum AND a.attrelid = t.oid
))) AS indices
FROM pg_class t
LEFT JOIN pg_index ix ON t.oid = ix.indrelid
LEFT JOIN pg_class i ON i.oid = ix.indexrelid
WHERE t.relkind IN ('r', 'v') -- r = table, v = view
AND t.relname NOT LIKE 'pg_%'
GROUP BY t.relname
)
SELECT
tv.name,
tv.type,
ci.columns,
ii.indices
FROM tables_and_views tv
LEFT JOIN columns_info ci ON tv.name = ci.name
LEFT JOIN indices_info ii ON tv.name = ii.table_name;
`;
const result = await query.execute(this.#db);
const runner = new KyselyPluginRunner(this._plugins ?? []);
const tables = (await runner.transformResultRows(result.rows)) as unknown as {
name: string;
type: "VIEW" | "BASE TABLE";
columns: {
name: string;
type: string;
notnull: number;
dflt: string;
pk: boolean;
}[];
indices: {
name: string;
origin: string;
partial: number;
sql: string;
columns: { name: string; seqno: number }[];
}[];
}[];
return tables.map((table) => ({
name: table.name,
isView: table.type === "VIEW",
columns: table.columns.map((col) => {
return {
name: col.name,
dataType: col.type,
isNullable: !col.notnull,
isAutoIncrementing: true, // just for now
hasDefaultValue: col.dflt != null,
comment: undefined,
};
}),
indices: table.indices.map((index) => ({
name: index.name,
table: table.name,
isUnique: index.sql?.match(/unique/i) != null,
columns: index.columns.map((col) => ({
name: col.name,
order: col.seqno,
})),
})),
}));
}
async getIndices(tbl_name?: string): Promise<IndexMetadata[]> {
const schema = await this.getSchema();
return schema
.flatMap((table) => table.indices)
.filter((index) => !tbl_name || index.table === tbl_name);
}
async getTables(
options: DatabaseMetadataOptions = { withInternalKyselyTables: false },
): Promise<TableMetadata[]> {
const schema = await this.getSchema();
return schema.map((table) => ({
name: table.name,
isView: table.isView,
columns: table.columns,
}));
}
}
interface RawSchemaMetadata {
nspname: string;
}

8
app/src/data/fields/BooleanField.ts
View File

@@ -32,9 +32,11 @@ export class BooleanField<Required extends true | false = false> extends Field<
}
}
schema() {
// @todo: potentially use "integer" instead
return this.useSchemaHelper("boolean");
override schema() {
return Object.freeze({
...super.schema()!,
type: "boolean",
});
}
override getHtmlConfig() {

6
app/src/data/fields/DateField.ts
View File

@@ -32,8 +32,10 @@ export class DateField<Required extends true | false = false> extends Field<
}
override schema() {
const type = this.config.type === "datetime" ? "datetime" : "date";
return this.useSchemaHelper(type);
return Object.freeze({
...super.schema()!,
type: this.config.type === "datetime" ? "datetime" : "date",
});
}
override getHtmlConfig() {

4
app/src/data/fields/EnumField.ts
View File

@@ -66,10 +66,6 @@ export class EnumField<Required extends true | false = false, TypeOverride = str
return enumFieldConfigSchema;
}
override schema() {
return this.useSchemaHelper("text");
}
getOptions(): { label: string; value: string }[] {
const options = this.config?.options ?? { type: "strings", values: [] };

31
app/src/data/fields/Field.ts
View File

@@ -1,16 +1,16 @@
import {
parse,
snakeToPascalWithSpaces,
type Static,
StringEnum,
type TSchema,
Type,
TypeInvalidError,
parse,
snakeToPascalWithSpaces,
} from "core/utils";
import type { ColumnBuilderCallback, ColumnDataType, ColumnDefinitionBuilder } from "kysely";
import type { HTMLInputTypeAttribute, InputHTMLAttributes } from "react";
import type { EntityManager } from "../entities";
import { InvalidFieldConfigException, TransformPersistFailedException } from "../errors";
import type { FieldSpec } from "data/connection/Connection";
// @todo: contexts need to be reworked
// e.g. "table" is irrelevant, because if read is not given, it fails
@@ -67,8 +67,6 @@ export const baseFieldConfigSchema = Type.Object(
);
export type BaseFieldConfig = Static<typeof baseFieldConfigSchema>;
export type SchemaResponse = [string, ColumnDataType, ColumnBuilderCallback] | undefined;
export abstract class Field<
Config extends BaseFieldConfig = BaseFieldConfig,
Type = any,
@@ -106,25 +104,18 @@ export abstract class Field<
protected abstract getSchema(): TSchema;
protected useSchemaHelper(
type: ColumnDataType,
builder?: (col: ColumnDefinitionBuilder) => ColumnDefinitionBuilder,
): SchemaResponse {
return [
this.name,
type,
(col: ColumnDefinitionBuilder) => {
if (builder) return builder(col);
return col;
},
];
}
/**
* Used in SchemaManager.ts
* @param em
*/
abstract schema(em: EntityManager<any>): SchemaResponse;
schema(): FieldSpec | undefined {
return Object.freeze({
name: this.name,
type: "text",
nullable: true,
dflt: this.getDefault(),
});
}
hasDefault() {
return this.config.default_value !== undefined;

4
app/src/data/fields/JsonField.ts
View File

@@ -18,10 +18,6 @@ export class JsonField<Required extends true | false = false, TypeOverride = obj
return jsonFieldConfigSchema;
}
override schema() {
return this.useSchemaHelper("text");
}
/**
* Transform value after retrieving from database
* @param value

4
app/src/data/fields/JsonSchemaField.ts
View File

@@ -36,10 +36,6 @@ export class JsonSchemaField<
return jsonSchemaFieldConfigSchema;
}
override schema() {
return this.useSchemaHelper("text");
}
getJsonSchema(): JsonSchema {
return this.config?.schema as JsonSchema;
}

7
app/src/data/fields/NumberField.ts
View File

@@ -44,8 +44,11 @@ export class NumberField<Required extends true | false = false> extends Field<
};
}
schema() {
return this.useSchemaHelper("integer");
override schema() {
return Object.freeze({
...super.schema()!,
type: "integer",
});
}
override getValue(value: any, context?: TRenderContext): any {

9
app/src/data/fields/PrimaryField.ts
View File

@@ -30,9 +30,12 @@ export class PrimaryField<Required extends true | false = false> extends Field<
return baseFieldConfigSchema;
}
schema() {
return this.useSchemaHelper("integer", (col) => {
return col.primaryKey().notNull().autoIncrement();
override schema() {
return Object.freeze({
type: "integer",
name: this.name,
primary: true,
nullable: false,
});
}

4
app/src/data/fields/TextField.ts
View File

@@ -47,10 +47,6 @@ export class TextField<Required extends true | false = false> extends Field<
return textFieldConfigSchema;
}
override schema() {
return this.useSchemaHelper("text");
}
override getHtmlConfig() {
if (this.config.html_config) {
return this.config.html_config as any;

2
app/src/data/fields/VirtualField.ts
View File

@@ -17,7 +17,7 @@ export class VirtualField extends Field<VirtualFieldConfig> {
return virtualFieldConfigSchema;
}
schema() {
override schema() {
return undefined;
}

16
app/src/data/relations/RelationField.ts
View File

@@ -1,6 +1,6 @@
import { type Static, StringEnum, Type } from "core/utils";
import type { EntityManager } from "../entities";
import { Field, type SchemaResponse, baseFieldConfigSchema } from "../fields";
import { Field, baseFieldConfigSchema } from "../fields";
import type { EntityRelation } from "./EntityRelation";
import type { EntityRelationAnchor } from "./EntityRelationAnchor";
@@ -72,14 +72,12 @@ export class RelationField extends Field<RelationFieldConfig> {
return this.config.target_field!;
}
override schema(): SchemaResponse {
return this.useSchemaHelper("integer", (col) => {
//col.references('person.id').onDelete('cascade').notNull()
// @todo: implement cascading?
return col
.references(`${this.config.target}.${this.config.target_field}`)
.onDelete(this.config.on_delete ?? "set null");
override schema() {
return Object.freeze({
...super.schema()!,
type: "integer",
references: `${this.config.target}.${this.config.target_field}`,
onDelete: this.config.on_delete ?? "set null",
});
}

11
app/src/data/schema/SchemaManager.ts
View File

@@ -1,7 +1,7 @@
import type { AlterTableColumnAlteringBuilder, CompiledQuery, TableMetadata } from "kysely";
import type { IndexMetadata } from "../connection/Connection";
import type { IndexMetadata, SchemaResponse } from "../connection/Connection";
import type { Entity, EntityManager } from "../entities";
import { PrimaryField, type SchemaResponse } from "../fields";
import { PrimaryField } from "../fields";
type IntrospectedTable = TableMetadata & {
indices: IndexMetadata[];
@@ -239,10 +239,9 @@ export class SchemaManager {
for (const column of columns) {
const field = this.em.entity(table).getField(column)!;
const fieldSchema = field.schema(this.em);
if (Array.isArray(fieldSchema) && fieldSchema.length === 3) {
schemas.push(fieldSchema);
//throw new Error(`Field "${field.name}" on entity "${table}" has no schema`);
const fieldSchema = field.schema();
if (fieldSchema) {
schemas.push(this.em.connection.getFieldSchema(fieldSchema));
}
}