Merge remote-tracking branch 'origin/release/0.20' into feat/opfs-and-sqlocal

app/src/data/api/DataController.ts

@@ -96,6 +96,9 @@ export class DataController extends Controller {
       // read entity schema
       hono.get(
          "/schema.json",
+         permission(SystemPermissions.schemaRead, {
+            context: (_c) => ({ module: "data" }),
+         }),
          permission(DataPermissions.entityRead, {
             context: (c) => ({ entity: c.req.param("entity") }),
          }),
@@ -124,6 +127,9 @@ export class DataController extends Controller {
       // read schema
       hono.get(
          "/schemas/:entity/:context?",
+         permission(SystemPermissions.schemaRead, {
+            context: (_c) => ({ module: "data" }),
+         }),
          permission(DataPermissions.entityRead, {
             context: (c) => ({ entity: c.req.param("entity") }),
          }),
@@ -161,7 +167,7 @@ export class DataController extends Controller {
       hono.get(
          "/types",
          permission(SystemPermissions.schemaRead, {
-            context: (c) => ({ module: "data" }),
+            context: (_c) => ({ module: "data" }),
          }),
          describeRoute({
             summary: "Retrieve data typescript definitions",
@@ -182,6 +188,9 @@ export class DataController extends Controller {
        */
       hono.get(
          "/info/:entity",
+         permission(SystemPermissions.schemaRead, {
+            context: (_c) => ({ module: "data" }),
+         }),
          permission(DataPermissions.entityRead, {
             context: (c) => ({ entity: c.req.param("entity") }),
          }),

app/src/data/connection/connection-test-suite.ts

@@ -14,27 +14,31 @@ export function connectionTestSuite(
    {
       makeConnection,
       rawDialectDetails,
+      disableConsoleLog: _disableConsoleLog = true,
    }: {
       makeConnection: () => MaybePromise<{
          connection: Connection;
          dispose: () => MaybePromise<void>;
       }>;
       rawDialectDetails: string[];
+      disableConsoleLog?: boolean;
    },
 ) {
    const { test, expect, describe, beforeEach, afterEach, afterAll, beforeAll } = testRunner;
-   beforeAll(() => disableConsoleLog());
-   afterAll(() => enableConsoleLog());
+   if (_disableConsoleLog) {
+      beforeAll(() => disableConsoleLog());
+      afterAll(() => enableConsoleLog());
+   }
 
-   describe("base", () => {
-      let ctx: Awaited<ReturnType<typeof makeConnection>>;
-      beforeEach(async () => {
-         ctx = await makeConnection();
-      });
-      afterEach(async () => {
-         await ctx.dispose();
-      });
+   let ctx: Awaited<ReturnType<typeof makeConnection>>;
+   beforeEach(async () => {
+      ctx = await makeConnection();
+   });
+   afterEach(async () => {
+      await ctx.dispose();
+   });
 
+   describe("base", async () => {
       test("pings", async () => {
          const res = await ctx.connection.ping();
          expect(res).toBe(true);
@@ -98,52 +102,54 @@ export function connectionTestSuite(
    });
 
    describe("schema", async () => {
-      const { connection, dispose } = await makeConnection();
-      afterAll(async () => {
-         await dispose();
-      });
+      const makeSchema = async () => {
+         const fields = [
+            {
+               type: "integer",
+               name: "id",
+               primary: true,
+            },
+            {
+               type: "text",
+               name: "text",
+            },
+            {
+               type: "json",
+               name: "json",
+            },
+         ] as const satisfies FieldSpec[];
 
-      const fields = [
-         {
-            type: "integer",
-            name: "id",
-            primary: true,
-         },
-         {
-            type: "text",
-            name: "text",
-         },
-         {
-            type: "json",
-            name: "json",
-         },
-      ] as const satisfies FieldSpec[];
+         let b = ctx.connection.kysely.schema.createTable("test");
+         for (const field of fields) {
+            // @ts-expect-error
+            b = b.addColumn(...ctx.connection.getFieldSchema(field));
+         }
+         await b.execute();
 
-      let b = connection.kysely.schema.createTable("test");
-      for (const field of fields) {
-         // @ts-expect-error
-         b = b.addColumn(...connection.getFieldSchema(field));
-      }
-      await b.execute();
-
-      // add index
-      await connection.kysely.schema.createIndex("test_index").on("test").columns(["id"]).execute();
+         // add index
+         await ctx.connection.kysely.schema
+            .createIndex("test_index")
+            .on("test")
+            .columns(["id"])
+            .execute();
+      };
 
       test("executes query", async () => {
-         await connection.kysely
+         await makeSchema();
+         await ctx.connection.kysely
             .insertInto("test")
             .values({ id: 1, text: "test", json: JSON.stringify({ a: 1 }) })
             .execute();
 
          const expected = { id: 1, text: "test", json: { a: 1 } };
 
-         const qb = connection.kysely.selectFrom("test").selectAll();
-         const res = await connection.executeQuery(qb);
+         const qb = ctx.connection.kysely.selectFrom("test").selectAll();
+         const res = await ctx.connection.executeQuery(qb);
          expect(res.rows).toEqual([expected]);
          expect(rawDialectDetails.every((detail) => getPath(res, detail) !== undefined)).toBe(true);
 
          {
-            const res = await connection.executeQueries(qb, qb);
+            const res = await ctx.connection.executeQueries(qb, qb);
             expect(res.length).toBe(2);
             res.map((r) => {
                expect(r.rows).toEqual([expected]);
@@ -155,15 +161,21 @@ export function connectionTestSuite(
       });
 
       test("introspects", async () => {
-         const tables = await connection.getIntrospector().getTables({
+         await makeSchema();
+         const tables = await ctx.connection.getIntrospector().getTables({
             withInternalKyselyTables: false,
          });
          const clean = tables.map((t) => ({
             ...t,
-            columns: t.columns.map((c) => ({
-               ...c,
-               dataType: undefined,
-            })),
+            columns: t.columns
+               .map((c) => ({
+                  ...c,
+                  // ignore data type
+                  dataType: undefined,
+                  // ignore default value if "id"
+                  hasDefaultValue: c.name !== "id" ? c.hasDefaultValue : undefined,
+               }))
+               .sort((a, b) => a.name.localeCompare(b.name)),
          }));
 
          expect(clean).toEqual([
@@ -176,14 +188,8 @@ export function connectionTestSuite(
                      dataType: undefined,
                      isNullable: false,
                      isAutoIncrementing: true,
-                     hasDefaultValue: false,
-                  },
-                  {
-                     name: "text",
-                     dataType: undefined,
-                     isNullable: true,
-                     isAutoIncrementing: false,
-                     hasDefaultValue: false,
+                     hasDefaultValue: undefined,
+                     comment: undefined,
                   },
                   {
                      name: "json",
@@ -191,25 +197,34 @@ export function connectionTestSuite(
                      isNullable: true,
                      isAutoIncrementing: false,
                      hasDefaultValue: false,
+                     comment: undefined,
+                  },
+                  {
+                     name: "text",
+                     dataType: undefined,
+                     isNullable: true,
+                     isAutoIncrementing: false,
+                     hasDefaultValue: false,
+                     comment: undefined,
+                  },
+               ],
+            },
+         ]);
+
+         expect(await ctx.connection.getIntrospector().getIndices()).toEqual([
+            {
+               name: "test_index",
+               table: "test",
+               isUnique: false,
+               columns: [
+                  {
+                     name: "id",
+                     order: 0,
                   },
                ],
             },
          ]);
       });
-
-      expect(await connection.getIntrospector().getIndices()).toEqual([
-         {
-            name: "test_index",
-            table: "test",
-            isUnique: false,
-            columns: [
-               {
-                  name: "id",
-                  order: 0,
-               },
-            ],
-         },
-      ]);
    });
 
    describe("integration", async () => {

app/src/data/connection/postgres/PgPostgresConnection.ts

@@ -0,0 +1,33 @@
+import { Kysely, PostgresDialect, type PostgresDialectConfig as KyselyPostgresDialectConfig } from "kysely";
+import { PostgresIntrospector } from "./PostgresIntrospector";
+import { PostgresConnection, plugins } from "./PostgresConnection";
+import { customIntrospector } from "../Connection";
+import type { Pool } from "pg";
+
+export type PostgresDialectConfig = Omit<KyselyPostgresDialectConfig, "pool"> & {
+   pool: Pool;
+};
+
+export class PgPostgresConnection extends PostgresConnection<Pool> {
+   override name = "pg";
+
+   constructor(config: PostgresDialectConfig) {
+      const kysely = new Kysely({
+         dialect: customIntrospector(PostgresDialect, PostgresIntrospector, {
+            excludeTables: [],
+         }).create(config),
+         plugins,
+      });
+
+      super(kysely);
+      this.client = config.pool;
+   }
+
+   override async close(): Promise<void> {
+      await this.client.end();
+   }
+}
+
+export function pg(config: PostgresDialectConfig): PgPostgresConnection {
+   return new PgPostgresConnection(config);
+}

app/src/data/connection/postgres/PostgresConnection.ts

@@ -0,0 +1,89 @@
+import {
+   Connection,
+   type DbFunctions,
+   type FieldSpec,
+   type SchemaResponse,
+   type ConnQuery,
+   type ConnQueryResults,
+} from "../Connection";
+import {
+   ParseJSONResultsPlugin,
+   type ColumnDataType,
+   type ColumnDefinitionBuilder,
+   type Kysely,
+   type KyselyPlugin,
+   type SelectQueryBuilder,
+} from "kysely";
+import { jsonArrayFrom, jsonBuildObject, jsonObjectFrom } from "kysely/helpers/postgres";
+
+export type QB = SelectQueryBuilder<any, any, any>;
+
+export const plugins = [new ParseJSONResultsPlugin()];
+
+export abstract class PostgresConnection<Client = unknown> extends Connection<Client> {
+   protected override readonly supported = {
+      batching: true,
+      softscans: true,
+   };
+
+   constructor(kysely: Kysely<any>, fn?: Partial<DbFunctions>, _plugins?: KyselyPlugin[]) {
+      super(
+         kysely,
+         fn ?? {
+            jsonArrayFrom,
+            jsonBuildObject,
+            jsonObjectFrom,
+         },
+         _plugins ?? plugins,
+      );
+   }
+
+   override getFieldSchema(spec: FieldSpec): SchemaResponse {
+      this.validateFieldSpecType(spec.type);
+      let type: ColumnDataType = spec.type;
+
+      if (spec.primary) {
+         if (spec.type === "integer") {
+            type = "serial";
+         }
+      }
+
+      switch (spec.type) {
+         case "blob":
+            type = "bytea";
+            break;
+         case "date":
+         case "datetime":
+            // https://www.postgresql.org/docs/17/datatype-datetime.html
+            type = "timestamp";
+            break;
+         case "text":
+            // https://www.postgresql.org/docs/17/datatype-character.html
+            type = "varchar";
+            break;
+      }
+
+      return [
+         spec.name,
+         type,
+         (col: ColumnDefinitionBuilder) => {
+            if (spec.primary) {
+               return col.primaryKey().notNull();
+            }
+            if (spec.references) {
+               return col
+                  .references(spec.references)
+                  .onDelete(spec.onDelete ?? "set null")
+                  .onUpdate(spec.onUpdate ?? "no action");
+            }
+            return col;
+         },
+      ];
+   }
+
+   override async executeQueries<O extends ConnQuery[]>(...qbs: O): Promise<ConnQueryResults<O>> {
+      return this.kysely.transaction().execute(async (trx) => {
+         return Promise.all(qbs.map((q) => trx.executeQuery(q)));
+      }) as any;
+   }
+}

app/src/data/connection/postgres/PostgresIntrospector.ts

@@ -0,0 +1,128 @@
+import { type SchemaMetadata, sql } from "kysely";
+import { BaseIntrospector } from "../BaseIntrospector";
+
+type PostgresSchemaSpec = {
+   name: string;
+   type: "VIEW" | "BASE TABLE";
+   columns: {
+      name: string;
+      type: string;
+      notnull: number;
+      dflt: string;
+      pk: boolean;
+   }[];
+   indices: {
+      name: string;
+      origin: string;
+      partial: number;
+      sql: string;
+      columns: { name: string; seqno: number }[];
+   }[];
+};
+
+export class PostgresIntrospector extends BaseIntrospector {
+   async getSchemas(): Promise<SchemaMetadata[]> {
+      const rawSchemas = await this.db
+         .selectFrom("pg_catalog.pg_namespace")
+         .select("nspname")
+         .$castTo<{ nspname: string }>()
+         .execute();
+
+      return rawSchemas.map((it) => ({ name: it.nspname }));
+   }
+
+   async getSchemaSpec() {
+      const query = sql`
+         WITH tables_and_views AS (
+            SELECT table_name AS name,
+               table_type AS type
+            FROM information_schema.tables
+            WHERE table_schema = 'public'
+              AND table_type IN ('BASE TABLE', 'VIEW')
+              AND table_name NOT LIKE 'pg_%'
+              AND table_name NOT IN (${this.getExcludedTableNames().join(", ")})
+         ),
+
+            columns_info AS (
+               SELECT table_name AS name,
+                  json_agg(json_build_object(
+                     'name', column_name,
+                     'type', data_type,
+                     'notnull', (CASE WHEN is_nullable = 'NO' THEN true ELSE false END),
+                     'dflt', column_default,
+                     'pk', (SELECT COUNT(*) > 0
+                        FROM information_schema.table_constraints tc
+                        INNER JOIN information_schema.key_column_usage kcu
+                          ON tc.constraint_name = kcu.constraint_name
+                        WHERE tc.table_name = c.table_name
+                          AND tc.constraint_type = 'PRIMARY KEY'
+                          AND kcu.column_name = c.column_name)
+                     )) AS columns
+               FROM information_schema.columns c
+               WHERE table_schema = 'public'
+               GROUP BY table_name
+            ),
+
+            indices_info AS (
+               SELECT
+                  t.relname AS table_name,
+                  json_agg(json_build_object(
+                     'name', i.relname,
+                     'origin', pg_get_indexdef(i.oid),
+                     'partial', (CASE WHEN ix.indisvalid THEN false ELSE true END),
+                     'sql', pg_get_indexdef(i.oid),
+                     'columns', (
+                        SELECT json_agg(json_build_object(
+                           'name', a.attname,
+                           'seqno', x.ordinal_position
+                         ))
+                        FROM unnest(ix.indkey) WITH ORDINALITY AS x(attnum, ordinal_position)
+                            JOIN pg_attribute a ON a.attnum = x.attnum AND a.attrelid = t.oid
+                     ))) AS indices
+               FROM pg_class t
+                  LEFT JOIN pg_index ix ON t.oid = ix.indrelid
+                  LEFT JOIN pg_class i ON i.oid = ix.indexrelid
+               WHERE t.relkind IN ('r', 'v')  -- r = table, v = view
+                 AND t.relname NOT LIKE 'pg_%'
+               GROUP BY t.relname
+            )
+
+         SELECT
+            tv.name,
+            tv.type,
+            ci.columns,
+            ii.indices
+         FROM tables_and_views tv
+            LEFT JOIN columns_info ci ON tv.name = ci.name
+            LEFT JOIN indices_info ii ON tv.name = ii.table_name;
+      `;
+
+      const tables = await this.executeWithPlugins<PostgresSchemaSpec[]>(query);
+
+      return tables.map((table) => ({
+         name: table.name,
+         isView: table.type === "VIEW",
+         columns: table.columns.map((col) => ({
+            name: col.name,
+            dataType: col.type,
+            isNullable: !col.notnull,
+            isAutoIncrementing: col.dflt?.toLowerCase().includes("nextval") ?? false,
+            hasDefaultValue: col.dflt != null,
+            comment: undefined,
+         })),
+         indices: table.indices
+            // filter out db-managed primary key index
+            .filter((index) => index.name !== `${table.name}_pkey`)
+            .map((index) => ({
+               name: index.name,
+               table: table.name,
+               isUnique: index.sql?.match(/unique/i) != null,
+               columns: index.columns.map((col) => ({
+                  name: col.name,
+                  // seqno starts at 1
+                  order: col.seqno - 1,
+               })),
+            })),
+      }));
+   }
+}

app/src/data/connection/postgres/PostgresJsConnection.ts

@@ -0,0 +1,31 @@
+import { Kysely } from "kysely";
+import { PostgresIntrospector } from "./PostgresIntrospector";
+import { PostgresConnection, plugins } from "./PostgresConnection";
+import { customIntrospector } from "../Connection";
+import { PostgresJSDialect, type PostgresJSDialectConfig } from "kysely-postgres-js";
+
+export class PostgresJsConnection extends PostgresConnection<PostgresJSDialectConfig["postgres"]> {
+   override name = "postgres-js";
+
+   constructor(config: PostgresJSDialectConfig) {
+      const kysely = new Kysely({
+         dialect: customIntrospector(PostgresJSDialect, PostgresIntrospector, {
+            excludeTables: [],
+         }).create(config),
+         plugins,
+      });
+
+      super(kysely);
+      this.client = config.postgres;
+   }
+
+   override async close(): Promise<void> {
+      await this.client.end();
+   }
+}
+
+export function postgresJs(
+   config: PostgresJSDialectConfig,
+): PostgresJsConnection {
+   return new PostgresJsConnection(config);
+}

app/src/data/connection/postgres/custom.ts

@@ -0,0 +1,46 @@
+import { customIntrospector, type DbFunctions } from "../Connection";
+import { Kysely, type Dialect, type KyselyPlugin } from "kysely";
+import { plugins, PostgresConnection } from "./PostgresConnection";
+import { PostgresIntrospector } from "./PostgresIntrospector";
+
+export type Constructor<T> = new (...args: any[]) => T;
+
+export type CustomPostgresConnection = {
+   supports?: Partial<PostgresConnection["supported"]>;
+   fn?: Partial<DbFunctions>;
+   plugins?: KyselyPlugin[];
+   excludeTables?: string[];
+};
+
+export function createCustomPostgresConnection<
+   T extends Constructor<Dialect>,
+   C extends ConstructorParameters<T>[0],
+>(
+   name: string,
+   dialect: Constructor<Dialect>,
+   options?: CustomPostgresConnection,
+): (config: C) => PostgresConnection {
+   const supported = {
+      batching: true,
+      ...((options?.supports ?? {}) as any),
+   };
+
+   return (config: C) =>
+      new (class extends PostgresConnection {
+         override name = name;
+         override readonly supported = supported;
+
+         constructor(config: C) {
+            super(
+               new Kysely({
+                  dialect: customIntrospector(dialect, PostgresIntrospector, {
+                     excludeTables: options?.excludeTables ?? [],
+                  }).create(config),
+                  plugins: options?.plugins ?? plugins,
+               }),
+               options?.fn,
+               options?.plugins,
+            );
+         }
+      })(config);
+}

app/src/data/connection/postgres/index.ts

@@ -0,0 +1,5 @@
+export { pg, PgPostgresConnection, type PgPostgresConnectionConfig } from "./PgPostgresConnection";
+export { PostgresIntrospector } from "./PostgresIntrospector";
+export { PostgresConnection, type QB, plugins } from "./PostgresConnection";
+export { postgresJs, PostgresJsConnection, type PostgresJsConfig } from "./PostgresJsConnection";
+export { createCustomPostgresConnection } from "./custom";

app/src/data/connection/sqlite/SqliteConnection.ts

@@ -13,7 +13,6 @@ import { customIntrospector } from "../Connection";
 import { SqliteIntrospector } from "./SqliteIntrospector";
 import type { Field } from "data/fields/Field";
 
-// @todo: add pragmas
 export type SqliteConnectionConfig<
    CustomDialect extends Constructor<Dialect> = Constructor<Dialect>,
 > = {

app/src/data/connection/sqlite/SqliteIntrospector.ts

@@ -83,7 +83,7 @@ export class SqliteIntrospector extends BaseIntrospector {
                   dataType: col.type,
                   isNullable: !col.notnull,
                   isAutoIncrementing: col.name === autoIncrementCol,
-                  hasDefaultValue: col.dflt_value != null,
+                  hasDefaultValue: col.name === autoIncrementCol ? true : col.dflt_value != null,
                   comment: undefined,
                };
             }) ?? [],

app/src/data/entities/query/Repository.ts

@@ -103,6 +103,7 @@ export class Repository<TBD extends object = DefaultDB, TB extends keyof TBD = a
          validated.with = options.with;
       }
 
+      // add explicit joins. Implicit joins are added in `where` builder
       if (options.join && options.join.length > 0) {
          for (const entry of options.join) {
             const related = this.em.relationOf(entity.name, entry);
@@ -127,12 +128,28 @@ export class Repository<TBD extends object = DefaultDB, TB extends keyof TBD = a
          const invalid = WhereBuilder.getPropertyNames(options.where).filter((field) => {
             if (field.includes(".")) {
                const [alias, prop] = field.split(".") as [string, string];
-               if (!aliases.includes(alias)) {
+               // check aliases first (added joins)
+               if (aliases.includes(alias)) {
+                  this.checkIndex(alias, prop, "where");
+                  return !this.em.entity(alias).getField(prop);
+               }
+               // check if alias (entity) exists
+               if (!this.em.hasEntity(alias)) {
                   return true;
                }
+               // check related fields for auto join
+               const related = this.em.relationOf(entity.name, alias);
+               if (related) {
+                  const other = related.other(entity);
+                  if (other.entity.getField(prop)) {
+                     // if related field is found, add join to validated options
+                     validated.join?.push(alias);
+                     this.checkIndex(alias, prop, "where");
+                     return false;
+                  }
+               }
 
-               this.checkIndex(alias, prop, "where");
-               return !this.em.entity(alias).getField(prop);
+               return true;
             }
 
             this.checkIndex(entity.name, field, "where");

app/src/data/fields/NumberField.ts

@@ -52,7 +52,7 @@ export class NumberField<Required extends true | false = false> extends Field<
 
       switch (context) {
          case "submit":
-            return Number.parseInt(value);
+            return Number.parseInt(value, 10);
       }
 
       return value;

app/src/data/helper.ts

@@ -28,7 +28,7 @@ export function getChangeSet(
          const value = _value === "" ? null : _value;
 
          // normalize to null if undefined
-         const newValue = field.getValue(value, "submit") || null;
+         const newValue = field.getValue(value, "submit") ?? null;
          // @todo: add typing for "action"
          if (action === "create" || newValue !== data[key]) {
             acc[key] = newValue;

app/src/data/prototype/index.ts

@@ -289,7 +289,7 @@ class EntityManagerPrototype<Entities extends Record<string, Entity>> extends En
       super(Object.values(__entities), new DummyConnection(), relations, indices);
    }
 
-   withConnection(connection: Connection): EntityManager<Schema<Entities>> {
+   withConnection(connection: Connection): EntityManager<Schemas<Entities>> {
       return new EntityManager(this.entities, connection, this.relations.all, this.indices);
    }
 }

app/src/data/server/query.spec.ts

@@ -1,8 +1,9 @@
-import { test, describe, expect } from "bun:test";
+import { test, describe, expect, beforeAll, afterAll } from "bun:test";
 import * as q from "./query";
 import { parse as $parse, type ParseOptions } from "bknd/utils";
 import type { PrimaryFieldType } from "modules";
 import type { Generated } from "kysely";
+import { disableConsoleLog, enableConsoleLog } from "core/utils/test";
 
 const parse = (v: unknown, o: ParseOptions = {}) =>
    $parse(q.repoQuery, v, {
@@ -15,6 +16,9 @@ const decode = (input: any, output: any) => {
    expect(parse(input)).toEqual(output);
 };
 
+beforeAll(() => disableConsoleLog());
+afterAll(() => enableConsoleLog());
+
 describe("server/query", () => {
    test("limit & offset", () => {
       //expect(() => parse({ limit: false })).toThrow();