public commit

app/src/data/AppData.ts

@@ -0,0 +1,122 @@
+import { transformObject } from "core/utils";
+import { DataPermissions, Entity, EntityIndex, type EntityManager, type Field } from "data";
+import { Module } from "modules/Module";
+import { DataController } from "./api/DataController";
+import {
+   type AppDataConfig,
+   FIELDS,
+   RELATIONS,
+   type TAppDataEntity,
+   type TAppDataRelation,
+   dataConfigSchema
+} from "./data-schema";
+
+export class AppData<DB> extends Module<typeof dataConfigSchema> {
+   static constructEntity(name: string, entityConfig: TAppDataEntity) {
+      const fields = transformObject(entityConfig.fields ?? {}, (fieldConfig, name) => {
+         const { type } = fieldConfig;
+         if (!(type in FIELDS)) {
+            throw new Error(`Field type "${type}" not found`);
+         }
+
+         const { field } = FIELDS[type as any];
+         const returnal = new field(name, fieldConfig.config) as Field;
+         return returnal;
+      });
+
+      // @todo: entity must be migrated to typebox
+      return new Entity(
+         name,
+         Object.values(fields),
+         entityConfig.config as any,
+         entityConfig.type as any
+      );
+   }
+
+   static constructRelation(
+      relationConfig: TAppDataRelation,
+      resolver: (name: Entity | string) => Entity
+   ) {
+      return new RELATIONS[relationConfig.type].cls(
+         resolver(relationConfig.source),
+         resolver(relationConfig.target),
+         relationConfig.config
+      );
+   }
+
+   override async build() {
+      const entities = transformObject(this.config.entities ?? {}, (entityConfig, name) => {
+         return AppData.constructEntity(name, entityConfig);
+      });
+
+      const _entity = (_e: Entity | string): Entity => {
+         const name = typeof _e === "string" ? _e : _e.name;
+         const entity = entities[name];
+         if (entity) return entity;
+         throw new Error(`Entity "${name}" not found`);
+      };
+
+      const relations = transformObject(this.config.relations ?? {}, (relation) =>
+         AppData.constructRelation(relation, _entity)
+      );
+
+      const indices = transformObject(this.config.indices ?? {}, (index, name) => {
+         const entity = _entity(index.entity)!;
+         const fields = index.fields.map((f) => entity.field(f)!);
+         return new EntityIndex(entity, fields, index.unique, name);
+      });
+
+      for (const entity of Object.values(entities)) {
+         this.ctx.em.addEntity(entity);
+      }
+
+      for (const relation of Object.values(relations)) {
+         this.ctx.em.addRelation(relation);
+      }
+
+      for (const index of Object.values(indices)) {
+         this.ctx.em.addIndex(index);
+      }
+
+      this.ctx.server.route(
+         this.basepath,
+         new DataController(this.ctx, this.config).getController()
+      );
+      this.ctx.guard.registerPermissions(Object.values(DataPermissions));
+
+      this.setBuilt();
+   }
+
+   getSchema() {
+      return dataConfigSchema;
+   }
+
+   get em(): EntityManager<DB> {
+      this.throwIfNotBuilt();
+      return this.ctx.em;
+   }
+
+   private get basepath() {
+      return this.config.basepath ?? "/api/data";
+   }
+
+   override getOverwritePaths() {
+      return [
+         /^entities\..*\.config$/,
+         /^entities\..*\.fields\..*\.config$/
+         ///^entities\..*\.fields\..*\.config\.schema$/
+      ];
+   }
+
+   /*registerController(server: AppServer) {
+      console.log("adding data controller to", this.basepath);
+      server.add(this.basepath, new DataController(this.em));
+   }*/
+
+   override toJSON(secrets?: boolean): AppDataConfig {
+      return {
+         ...this.config,
+         ...this.em.toJSON()
+      };
+   }
+}

app/src/data/api/DataApi.ts

@@ -0,0 +1,63 @@
+import type { EntityData, RepoQuery, RepositoryResponse } from "data";
+import { type BaseModuleApiOptions, ModuleApi, type PrimaryFieldType } from "modules";
+
+export type DataApiOptions = BaseModuleApiOptions & {
+   defaultQuery?: Partial<RepoQuery>;
+};
+
+export class DataApi extends ModuleApi<DataApiOptions> {
+   protected override getDefaultOptions(): Partial<DataApiOptions> {
+      return {
+         basepath: "/api/data",
+         defaultQuery: {
+            limit: 10
+         }
+      };
+   }
+
+   async readOne(
+      entity: string,
+      id: PrimaryFieldType,
+      query: Partial<Omit<RepoQuery, "where" | "limit" | "offset">> = {}
+   ) {
+      return this.get<RepositoryResponse<EntityData>>([entity, id], query);
+   }
+
+   async readMany(entity: string, query: Partial<RepoQuery> = {}) {
+      return this.get<Pick<RepositoryResponse, "meta" | "data">>(
+         [entity],
+         query ?? this.options.defaultQuery
+      );
+   }
+
+   async readManyByReference(
+      entity: string,
+      id: PrimaryFieldType,
+      reference: string,
+      query: Partial<RepoQuery> = {}
+   ) {
+      return this.get<Pick<RepositoryResponse, "meta" | "data">>(
+         [entity, id, reference],
+         query ?? this.options.defaultQuery
+      );
+   }
+
+   async createOne(entity: string, input: EntityData) {
+      return this.post<RepositoryResponse<EntityData>>([entity], input);
+   }
+
+   async updateOne(entity: string, id: PrimaryFieldType, input: EntityData) {
+      return this.patch<RepositoryResponse<EntityData>>([entity, id], input);
+   }
+
+   async deleteOne(entity: string, id: PrimaryFieldType) {
+      return this.delete<RepositoryResponse<EntityData>>([entity, id]);
+   }
+
+   async count(entity: string, where: RepoQuery["where"] = {}) {
+      return this.post<RepositoryResponse<{ entity: string; count: number }>>(
+         [entity, "fn", "count"],
+         where
+      );
+   }
+}

app/src/data/api/DataController.ts

@@ -0,0 +1,384 @@
+import { type ClassController, isDebug, tbValidator as tb } from "core";
+import { Type, objectCleanEmpty, objectTransform } from "core/utils";
+import {
+   DataPermissions,
+   type EntityData,
+   type EntityManager,
+   FieldClassMap,
+   type MutatorResponse,
+   PrimaryField,
+   type RepoQuery,
+   type RepositoryResponse,
+   TextField,
+   querySchema
+} from "data";
+import { Hono } from "hono";
+import type { Handler } from "hono/types";
+import type { ModuleBuildContext } from "modules";
+import { AppData } from "../AppData";
+import { type AppDataConfig, FIELDS } from "../data-schema";
+
+export class DataController implements ClassController {
+   constructor(
+      private readonly ctx: ModuleBuildContext,
+      private readonly config: AppDataConfig
+   ) {
+      /*console.log(
+         "data controller",
+         this.em.entities.map((e) => e.name)
+      );*/
+   }
+
+   get em(): EntityManager<any> {
+      return this.ctx.em;
+   }
+
+   get guard() {
+      return this.ctx.guard;
+   }
+
+   repoResult<T extends RepositoryResponse<any> = RepositoryResponse>(
+      res: T
+   ): Pick<T, "meta" | "data"> {
+      let meta: Partial<RepositoryResponse["meta"]> = {};
+
+      if ("meta" in res) {
+         const { query, ...rest } = res.meta;
+         meta = rest;
+         if (isDebug()) meta.query = query;
+      }
+
+      const template = { data: res.data, meta };
+
+      // @todo: this works but it breaks in FE (need to improve DataTable)
+      //return objectCleanEmpty(template) as any;
+      // filter empty
+      return Object.fromEntries(
+         Object.entries(template).filter(([_, v]) => typeof v !== "undefined" && v !== null)
+      ) as any;
+   }
+
+   mutatorResult(res: MutatorResponse | MutatorResponse<EntityData>) {
+      const template = { data: res.data };
+
+      // filter empty
+      //return objectCleanEmpty(template);
+      return Object.fromEntries(Object.entries(template).filter(([_, v]) => v !== undefined));
+   }
+
+   entityExists(entity: string) {
+      try {
+         return !!this.em.entity(entity);
+      } catch (e) {
+         return false;
+      }
+   }
+
+   getController(): Hono<any> {
+      const hono = new Hono();
+      const definedEntities = this.em.entities.map((e) => e.name);
+      const tbNumber = Type.Transform(Type.String({ pattern: "^[1-9][0-9]{0,}$" }))
+         .Decode(Number.parseInt)
+         .Encode(String);
+
+      // @todo: sample implementation how to augment handler with additional info
+      function handler<HH extends Handler>(name: string, h: HH): any {
+         const func = h;
+         // @ts-ignore
+         func.description = name;
+         return func;
+      }
+
+      // add timing
+      /*hono.use("*", async (c, next) => {
+         startTime(c, "data");
+         await next();
+         endTime(c, "data");
+      });*/
+
+      // info
+      hono.get(
+         "/",
+         handler("data info", (c) => {
+            // sample implementation
+            return c.json(this.em.toJSON());
+         })
+      );
+
+      // sync endpoint
+      hono.get("/sync", async (c) => {
+         this.guard.throwUnlessGranted(DataPermissions.databaseSync);
+
+         const force = c.req.query("force") === "1";
+         const drop = c.req.query("drop") === "1";
+         //console.log("force", force);
+         const tables = await this.em.schema().introspect();
+         //console.log("tables", tables);
+         const changes = await this.em.schema().sync({
+            force,
+            drop
+         });
+         return c.json({ tables: tables.map((t) => t.name), changes });
+      });
+
+      /**
+       * Function endpoints
+       */
+      hono
+         // fn: count
+         .post(
+            "/:entity/fn/count",
+            tb("param", Type.Object({ entity: Type.String() })),
+            async (c) => {
+               this.guard.throwUnlessGranted(DataPermissions.entityRead);
+
+               const { entity } = c.req.valid("param");
+               if (!this.entityExists(entity)) {
+                  return c.notFound();
+               }
+
+               const where = c.req.json() as any;
+               const result = await this.em.repository(entity).count(where);
+               return c.json({ entity, count: result.count });
+            }
+         )
+         // fn: exists
+         .post(
+            "/:entity/fn/exists",
+            tb("param", Type.Object({ entity: Type.String() })),
+            async (c) => {
+               this.guard.throwUnlessGranted(DataPermissions.entityRead);
+
+               const { entity } = c.req.valid("param");
+               if (!this.entityExists(entity)) {
+                  return c.notFound();
+               }
+
+               const where = c.req.json() as any;
+               const result = await this.em.repository(entity).exists(where);
+               return c.json({ entity, exists: result.exists });
+            }
+         );
+
+      /**
+       * Read endpoints
+       */
+      hono
+         // read entity schema
+         .get("/schema.json", async (c) => {
+            this.guard.throwUnlessGranted(DataPermissions.entityRead);
+            const url = new URL(c.req.url);
+            const $id = `${url.origin}${this.config.basepath}/schema.json`;
+            const schemas = Object.fromEntries(
+               this.em.entities.map((e) => [
+                  e.name,
+                  {
+                     $ref: `schemas/${e.name}`
+                  }
+               ])
+            );
+            return c.json({
+               $schema: "https://json-schema.org/draft/2020-12/schema",
+               $id,
+               properties: schemas
+            });
+         })
+         // read schema
+         .get(
+            "/schemas/:entity",
+            tb("param", Type.Object({ entity: Type.String() })),
+            async (c) => {
+               this.guard.throwUnlessGranted(DataPermissions.entityRead);
+
+               //console.log("request", c.req.raw);
+               const { entity } = c.req.param();
+               if (!this.entityExists(entity)) {
+                  console.log("not found", entity, definedEntities);
+                  return c.notFound();
+               }
+               const _entity = this.em.entity(entity);
+               const schema = _entity.toSchema();
+               const url = new URL(c.req.url);
+               const base = `${url.origin}${this.config.basepath}`;
+               const $id = `${base}/schemas/${entity}`;
+               return c.json({
+                  $schema: `${base}/schema.json`,
+                  $id,
+                  title: _entity.label,
+                  $comment: _entity.config.description,
+                  ...schema
+               });
+            }
+         )
+         // read many
+         .get(
+            "/:entity",
+            tb("param", Type.Object({ entity: Type.String() })),
+            tb("query", querySchema),
+            async (c) => {
+               this.guard.throwUnlessGranted(DataPermissions.entityRead);
+
+               //console.log("request", c.req.raw);
+               const { entity } = c.req.param();
+               if (!this.entityExists(entity)) {
+                  console.log("not found", entity, definedEntities);
+                  return c.notFound();
+               }
+               const options = c.req.valid("query") as RepoQuery;
+               //console.log("before", this.ctx.emgr.Events);
+               const result = await this.em.repository(entity).findMany(options);
+
+               return c.json(this.repoResult(result), { status: result.data ? 200 : 404 });
+            }
+         )
+
+         // read one
+         .get(
+            "/:entity/:id",
+            tb(
+               "param",
+               Type.Object({
+                  entity: Type.String(),
+                  id: tbNumber
+               })
+            ),
+            tb("query", querySchema),
+            /*zValidator("param", z.object({ entity: z.string(), id: z.coerce.number() })),
+            zValidator("query", repoQuerySchema),*/
+            async (c) => {
+               this.guard.throwUnlessGranted(DataPermissions.entityRead);
+
+               const { entity, id } = c.req.param();
+               if (!this.entityExists(entity)) {
+                  return c.notFound();
+               }
+               const options = c.req.valid("query") as RepoQuery;
+               const result = await this.em.repository(entity).findId(Number(id), options);
+
+               return c.json(this.repoResult(result), { status: result.data ? 200 : 404 });
+            }
+         )
+         // read many by reference
+         .get(
+            "/:entity/:id/:reference",
+            tb(
+               "param",
+               Type.Object({
+                  entity: Type.String(),
+                  id: tbNumber,
+                  reference: Type.String()
+               })
+            ),
+            tb("query", querySchema),
+            async (c) => {
+               this.guard.throwUnlessGranted(DataPermissions.entityRead);
+
+               const { entity, id, reference } = c.req.param();
+               if (!this.entityExists(entity)) {
+                  return c.notFound();
+               }
+
+               const options = c.req.valid("query") as RepoQuery;
+               const result = await this.em
+                  .repository(entity)
+                  .findManyByReference(Number(id), reference, options);
+
+               return c.json(this.repoResult(result), { status: result.data ? 200 : 404 });
+            }
+         )
+         // func query
+         .post(
+            "/:entity/query",
+            tb("param", Type.Object({ entity: Type.String() })),
+            tb("json", querySchema),
+            async (c) => {
+               this.guard.throwUnlessGranted(DataPermissions.entityRead);
+
+               const { entity } = c.req.param();
+               if (!this.entityExists(entity)) {
+                  return c.notFound();
+               }
+               const options = (await c.req.valid("json")) as RepoQuery;
+               console.log("options", options);
+               const result = await this.em.repository(entity).findMany(options);
+
+               return c.json(this.repoResult(result), { status: result.data ? 200 : 404 });
+            }
+         );
+
+      /**
+       * Mutation endpoints
+       */
+      // insert one
+      hono
+         .post("/:entity", tb("param", Type.Object({ entity: Type.String() })), async (c) => {
+            this.guard.throwUnlessGranted(DataPermissions.entityCreate);
+
+            const { entity } = c.req.param();
+            if (!this.entityExists(entity)) {
+               return c.notFound();
+            }
+            const body = (await c.req.json()) as EntityData;
+            const result = await this.em.mutator(entity).insertOne(body);
+
+            return c.json(this.mutatorResult(result), 201);
+         })
+         // update one
+         .patch(
+            "/:entity/:id",
+            tb("param", Type.Object({ entity: Type.String(), id: tbNumber })),
+            async (c) => {
+               this.guard.throwUnlessGranted(DataPermissions.entityUpdate);
+
+               const { entity, id } = c.req.param();
+               if (!this.entityExists(entity)) {
+                  return c.notFound();
+               }
+               const body = (await c.req.json()) as EntityData;
+               const result = await this.em.mutator(entity).updateOne(Number(id), body);
+
+               return c.json(this.mutatorResult(result));
+            }
+         )
+         // delete one
+         .delete(
+            "/:entity/:id",
+            tb("param", Type.Object({ entity: Type.String(), id: tbNumber })),
+            async (c) => {
+               this.guard.throwUnlessGranted(DataPermissions.entityDelete);
+
+               const { entity, id } = c.req.param();
+               if (!this.entityExists(entity)) {
+                  return c.notFound();
+               }
+               const result = await this.em.mutator(entity).deleteOne(Number(id));
+
+               return c.json(this.mutatorResult(result));
+            }
+         )
+
+         // delete many
+         .delete(
+            "/:entity",
+            tb("param", Type.Object({ entity: Type.String() })),
+            tb("json", querySchema.properties.where),
+            async (c) => {
+               this.guard.throwUnlessGranted(DataPermissions.entityDelete);
+
+               //console.log("request", c.req.raw);
+               const { entity } = c.req.param();
+               if (!this.entityExists(entity)) {
+                  return c.notFound();
+               }
+               const where = c.req.valid("json") as RepoQuery["where"];
+               console.log("where", where);
+
+               const result = await this.em.mutator(entity).deleteMany(where);
+
+               return c.json(this.mutatorResult(result));
+            }
+         );
+
+      return hono;
+   }
+}

app/src/data/connection/Connection.ts

@@ -0,0 +1,97 @@
+import {
+   type AliasableExpression,
+   type DatabaseIntrospector,
+   type Expression,
+   type Kysely,
+   type KyselyPlugin,
+   type RawBuilder,
+   type SelectQueryBuilder,
+   type SelectQueryNode,
+   type Simplify,
+   sql
+} from "kysely";
+
+export type QB = SelectQueryBuilder<any, any, any>;
+
+export type IndexMetadata = {
+   name: string;
+   table: string;
+   isUnique: boolean;
+   columns: { name: string; order: number }[];
+};
+
+export interface ConnectionIntrospector extends DatabaseIntrospector {
+   getIndices(tbl_name?: string): Promise<IndexMetadata[]>;
+}
+
+export interface SelectQueryBuilderExpression<O> extends AliasableExpression<O> {
+   get isSelectQueryBuilder(): true;
+   toOperationNode(): SelectQueryNode;
+}
+
+export type DbFunctions = {
+   jsonObjectFrom<O>(expr: SelectQueryBuilderExpression<O>): RawBuilder<Simplify<O> | null>;
+   jsonArrayFrom<O>(expr: SelectQueryBuilderExpression<O>): RawBuilder<Simplify<O>[]>;
+   jsonBuildObject<O extends Record<string, Expression<unknown>>>(
+      obj: O
+   ): RawBuilder<
+      Simplify<{
+         [K in keyof O]: O[K] extends Expression<infer V> ? V : never;
+      }>
+   >;
+};
+
+export abstract class Connection {
+   kysely: Kysely<any>;
+
+   constructor(
+      kysely: Kysely<any>,
+      public fn: Partial<DbFunctions> = {},
+      protected plugins: KyselyPlugin[] = []
+   ) {
+      this.kysely = kysely;
+   }
+
+   getIntrospector(): ConnectionIntrospector {
+      return this.kysely.introspection as ConnectionIntrospector;
+   }
+
+   supportsBatching(): boolean {
+      return false;
+   }
+
+   supportsIndices(): boolean {
+      return false;
+   }
+
+   async ping(): Promise<boolean> {
+      const res = await sql`SELECT 1`.execute(this.kysely);
+      return res.rows.length > 0;
+   }
+
+   protected async batch<Queries extends QB[]>(
+      queries: [...Queries]
+   ): Promise<{
+      [K in keyof Queries]: Awaited<ReturnType<Queries[K]["execute"]>>;
+   }> {
+      throw new Error("Batching not supported");
+   }
+
+   async batchQuery<Queries extends QB[]>(
+      queries: [...Queries]
+   ): Promise<{
+      [K in keyof Queries]: Awaited<ReturnType<Queries[K]["execute"]>>;
+   }> {
+      // bypass if no client support
+      if (!this.supportsBatching()) {
+         const data: any = [];
+         for (const q of queries) {
+            const result = await q.execute();
+            data.push(result);
+         }
+         return data;
+      }
+
+      return await this.batch(queries);
+   }
+}

app/src/data/connection/LibsqlConnection.ts

@@ -0,0 +1,100 @@
+import { type Client, type InStatement, createClient } from "@libsql/client/web";
+import { LibsqlDialect } from "@libsql/kysely-libsql";
+import { type DatabaseIntrospector, Kysely, ParseJSONResultsPlugin, sql } from "kysely";
+import { FilterNumericKeysPlugin } from "../plugins/FilterNumericKeysPlugin";
+import { KyselyPluginRunner } from "../plugins/KyselyPluginRunner";
+import type { QB } from "./Connection";
+import { SqliteConnection } from "./SqliteConnection";
+import { SqliteIntrospector } from "./SqliteIntrospector";
+
+export const LIBSQL_PROTOCOLS = ["wss", "https", "libsql"] as const;
+export type LibSqlCredentials = {
+   url: string;
+   authToken?: string;
+   protocol?: (typeof LIBSQL_PROTOCOLS)[number];
+};
+
+class CustomLibsqlDialect extends LibsqlDialect {
+   override createIntrospector(db: Kysely<any>): DatabaseIntrospector {
+      return new SqliteIntrospector(db, {
+         excludeTables: ["libsql_wasm_func_table"]
+      });
+   }
+}
+
+export class LibsqlConnection extends SqliteConnection {
+   private client: Client;
+
+   constructor(client: Client);
+   constructor(credentials: LibSqlCredentials);
+   constructor(clientOrCredentials: Client | LibSqlCredentials) {
+      const plugins = [new FilterNumericKeysPlugin(), new ParseJSONResultsPlugin()];
+      let client: Client;
+      if ("url" in clientOrCredentials) {
+         let { url, authToken, protocol } = clientOrCredentials;
+         if (protocol && LIBSQL_PROTOCOLS.includes(protocol)) {
+            console.log("changing protocol to", protocol);
+            const [, rest] = url.split("://");
+            url = `${protocol}://${rest}`;
+         }
+
+         //console.log("using", url, { protocol });
+
+         client = createClient({ url, authToken });
+      } else {
+         //console.log("-- client provided");
+         client = clientOrCredentials;
+      }
+
+      const kysely = new Kysely({
+         // @ts-expect-error libsql has type issues
+         dialect: new CustomLibsqlDialect({ client }),
+         plugins
+         //log: ["query"],
+      });
+
+      super(kysely, {}, plugins);
+      this.client = client;
+   }
+
+   override supportsBatching(): boolean {
+      return true;
+   }
+
+   override supportsIndices(): boolean {
+      return true;
+   }
+
+   getClient(): Client {
+      return this.client;
+   }
+
+   protected override async batch<Queries extends QB[]>(
+      queries: [...Queries]
+   ): Promise<{
+      [K in keyof Queries]: Awaited<ReturnType<Queries[K]["execute"]>>;
+   }> {
+      const stms: InStatement[] = queries.map((q) => {
+         const compiled = q.compile();
+         //console.log("compiled", compiled.sql, compiled.parameters);
+         return {
+            sql: compiled.sql,
+            args: compiled.parameters as any[]
+         };
+      });
+
+      const res = await this.client.batch(stms);
+
+      // let it run through plugins
+      const kyselyPlugins = new KyselyPluginRunner(this.plugins);
+
+      const data: any = [];
+      for (const r of res) {
+         const rows = await kyselyPlugins.transformResultRows(r.rows);
+         data.push(rows);
+      }
+      //console.log("data", data);
+
+      return data;
+   }
+}

app/src/data/connection/SqliteConnection.ts

@@ -0,0 +1,22 @@
+import type { Kysely, KyselyPlugin } from "kysely";
+import { jsonArrayFrom, jsonBuildObject, jsonObjectFrom } from "kysely/helpers/sqlite";
+import { Connection, type DbFunctions } from "./Connection";
+
+export class SqliteConnection extends Connection {
+   constructor(kysely: Kysely<any>, fn: Partial<DbFunctions> = {}, plugins: KyselyPlugin[] = []) {
+      super(
+         kysely,
+         {
+            ...fn,
+            jsonArrayFrom,
+            jsonObjectFrom,
+            jsonBuildObject
+         },
+         plugins
+      );
+   }
+
+   override supportsIndices(): boolean {
+      return true;
+   }
+}

app/src/data/connection/SqliteIntrospector.ts

@@ -0,0 +1,164 @@
+import type {
+   DatabaseIntrospector,
+   DatabaseMetadata,
+   DatabaseMetadataOptions,
+   ExpressionBuilder,
+   Kysely,
+   SchemaMetadata,
+   TableMetadata,
+} from "kysely";
+import { DEFAULT_MIGRATION_LOCK_TABLE, DEFAULT_MIGRATION_TABLE, sql } from "kysely";
+import type { ConnectionIntrospector, IndexMetadata } from "./Connection";
+
+export type SqliteIntrospectorConfig = {
+   excludeTables?: string[];
+};
+
+export class SqliteIntrospector implements DatabaseIntrospector, ConnectionIntrospector {
+   readonly #db: Kysely<any>;
+   readonly _excludeTables: string[] = [];
+
+   constructor(db: Kysely<any>, config: SqliteIntrospectorConfig = {}) {
+      this.#db = db;
+      this._excludeTables = config.excludeTables ?? [];
+   }
+
+   async getSchemas(): Promise<SchemaMetadata[]> {
+      // Sqlite doesn't support schemas.
+      return [];
+   }
+
+   async getIndices(tbl_name?: string): Promise<IndexMetadata[]> {
+      const indices = await this.#db
+         .selectFrom("sqlite_master")
+         .where("type", "=", "index")
+         .$if(!!tbl_name, (eb) => eb.where("tbl_name", "=", tbl_name))
+         .select("name")
+         .$castTo<{ name: string }>()
+         .execute();
+
+      return Promise.all(indices.map(({ name }) => this.#getIndexMetadata(name)));
+   }
+
+   async #getIndexMetadata(index: string): Promise<IndexMetadata> {
+      const db = this.#db;
+
+      // Get the SQL that was used to create the index.
+      const indexDefinition = await db
+         .selectFrom("sqlite_master")
+         .where("name", "=", index)
+         .select(["sql", "tbl_name", "type"])
+         .$castTo<{ sql: string | undefined; tbl_name: string; type: string }>()
+         .executeTakeFirstOrThrow();
+
+      //console.log("--indexDefinition--", indexDefinition, index);
+
+      // check unique by looking for the word "unique" in the sql
+      const isUnique = indexDefinition.sql?.match(/unique/i) != null;
+
+      const columns = await db
+         .selectFrom(
+            sql<{
+               seqno: number;
+               cid: number;
+               name: string;
+            }>`pragma_index_info(${index})`.as("index_info"),
+         )
+         .select(["seqno", "cid", "name"])
+         .orderBy("cid")
+         .execute();
+
+      return {
+         name: index,
+         table: indexDefinition.tbl_name,
+         isUnique: isUnique,
+         columns: columns.map((col) => ({
+            name: col.name,
+            order: col.seqno,
+         })),
+      };
+   }
+
+   private excludeTables(tables: string[] = []) {
+      return (eb: ExpressionBuilder<any, any>) => {
+         const and = tables.map((t) => eb("name", "!=", t));
+         return eb.and(and);
+      };
+   }
+
+   async getTables(
+      options: DatabaseMetadataOptions = { withInternalKyselyTables: false },
+   ): Promise<TableMetadata[]> {
+      let query = this.#db
+         .selectFrom("sqlite_master")
+         .where("type", "in", ["table", "view"])
+         .where("name", "not like", "sqlite_%")
+         .select("name")
+         .orderBy("name")
+         .$castTo<{ name: string }>();
+
+      if (!options.withInternalKyselyTables) {
+         query = query.where(
+            this.excludeTables([DEFAULT_MIGRATION_TABLE, DEFAULT_MIGRATION_LOCK_TABLE]),
+         );
+      }
+      if (this._excludeTables.length > 0) {
+         query = query.where(this.excludeTables(this._excludeTables));
+      }
+
+      const tables = await query.execute();
+      return Promise.all(tables.map(({ name }) => this.#getTableMetadata(name)));
+   }
+
+   async getMetadata(options?: DatabaseMetadataOptions): Promise<DatabaseMetadata> {
+      return {
+         tables: await this.getTables(options),
+      };
+   }
+
+   async #getTableMetadata(table: string): Promise<TableMetadata> {
+      const db = this.#db;
+
+      // Get the SQL that was used to create the table.
+      const tableDefinition = await db
+         .selectFrom("sqlite_master")
+         .where("name", "=", table)
+         .select(["sql", "type"])
+         .$castTo<{ sql: string | undefined; type: string }>()
+         .executeTakeFirstOrThrow();
+
+      // Try to find the name of the column that has `autoincrement` 🤦
+      const autoIncrementCol = tableDefinition.sql
+         ?.split(/[\(\),]/)
+         ?.find((it) => it.toLowerCase().includes("autoincrement"))
+         ?.trimStart()
+         ?.split(/\s+/)?.[0]
+         ?.replace(/["`]/g, "");
+
+      const columns = await db
+         .selectFrom(
+            sql<{
+               name: string;
+               type: string;
+               notnull: 0 | 1;
+               dflt_value: any;
+            }>`pragma_table_info(${table})`.as("table_info"),
+         )
+         .select(["name", "type", "notnull", "dflt_value"])
+         .orderBy("cid")
+         .execute();
+
+      return {
+         name: table,
+         isView: tableDefinition.type === "view",
+         columns: columns.map((col) => ({
+            name: col.name,
+            dataType: col.type,
+            isNullable: !col.notnull,
+            isAutoIncrementing: col.name === autoIncrementCol,
+            hasDefaultValue: col.dflt_value != null,
+            comment: undefined,
+         })),
+      };
+   }
+}

app/src/data/connection/SqliteLocalConnection.ts

@@ -0,0 +1,31 @@
+import type { DatabaseIntrospector, SqliteDatabase } from "kysely";
+import { Kysely, SqliteDialect } from "kysely";
+import { DeserializeJsonValuesPlugin } from "../plugins/DeserializeJsonValuesPlugin";
+import { SqliteConnection } from "./SqliteConnection";
+import { SqliteIntrospector } from "./SqliteIntrospector";
+
+class CustomSqliteDialect extends SqliteDialect {
+   override createIntrospector(db: Kysely<any>): DatabaseIntrospector {
+      return new SqliteIntrospector(db, {
+         excludeTables: ["test_table"]
+      });
+   }
+}
+
+export class SqliteLocalConnection extends SqliteConnection {
+   constructor(private database: SqliteDatabase) {
+      const plugins = [new DeserializeJsonValuesPlugin()];
+      const kysely = new Kysely({
+         dialect: new CustomSqliteDialect({ database }),
+         plugins
+         //log: ["query"],
+      });
+
+      super(kysely);
+      this.plugins = plugins;
+   }
+
+   override supportsIndices(): boolean {
+      return true;
+   }
+}

app/src/data/data-schema.ts

@@ -0,0 +1,83 @@
+import { type Static, StringRecord, Type, objectTransform } from "core/utils";
+import {
+   FieldClassMap,
+   RelationClassMap,
+   RelationFieldClassMap,
+   entityConfigSchema,
+   entityTypes
+} from "data";
+import { MediaField, mediaFieldConfigSchema } from "../media/MediaField";
+
+export const FIELDS = {
+   ...FieldClassMap,
+   ...RelationFieldClassMap,
+   media: { schema: mediaFieldConfigSchema, field: MediaField }
+};
+export type FieldType = keyof typeof FIELDS;
+
+export const RELATIONS = RelationClassMap;
+
+export const fieldsSchemaObject = objectTransform(FIELDS, (field, name) => {
+   return Type.Object(
+      {
+         type: Type.Const(name, { default: name, readOnly: true }),
+         config: Type.Optional(field.schema)
+      },
+      {
+         title: name
+      }
+   );
+});
+export const fieldsSchema = Type.Union(Object.values(fieldsSchemaObject));
+export const entityFields = StringRecord(fieldsSchema);
+export type TAppDataField = Static<typeof fieldsSchema>;
+export type TAppDataEntityFields = Static<typeof entityFields>;
+
+export const entitiesSchema = Type.Object({
+   //name: Type.String(),
+   type: Type.Optional(Type.String({ enum: entityTypes, default: "regular", readOnly: true })),
+   config: Type.Optional(entityConfigSchema),
+   fields: Type.Optional(entityFields)
+});
+export type TAppDataEntity = Static<typeof entitiesSchema>;
+
+export const relationsSchema = Object.entries(RelationClassMap).map(([name, relationClass]) => {
+   return Type.Object(
+      {
+         type: Type.Const(name, { default: name, readOnly: true }),
+         source: Type.String(),
+         target: Type.String(),
+         config: Type.Optional(relationClass.schema)
+      },
+      {
+         title: name
+      }
+   );
+});
+export type TAppDataRelation = Static<(typeof relationsSchema)[number]>;
+
+export const indicesSchema = Type.Object(
+   {
+      entity: Type.String(),
+      fields: Type.Array(Type.String(), { minItems: 1 }),
+      //name: Type.Optional(Type.String()),
+      unique: Type.Optional(Type.Boolean({ default: false }))
+   },
+   {
+      additionalProperties: false
+   }
+);
+
+export const dataConfigSchema = Type.Object(
+   {
+      basepath: Type.Optional(Type.String({ default: "/api/data" })),
+      entities: Type.Optional(StringRecord(entitiesSchema, { default: {} })),
+      relations: Type.Optional(StringRecord(Type.Union(relationsSchema), { default: {} })),
+      indices: Type.Optional(StringRecord(indicesSchema, { default: {} }))
+   },
+   {
+      additionalProperties: false
+   }
+);
+
+export type AppDataConfig = Static<typeof dataConfigSchema>;

app/src/data/entities/Entity.ts

@@ -0,0 +1,238 @@
+import { config } from "core";
+import {
+   type Static,
+   StringEnum,
+   Type,
+   parse,
+   snakeToPascalWithSpaces,
+   transformObject
+} from "core/utils";
+import { type Field, PrimaryField, type TActionContext, type TRenderContext } from "../fields";
+
+// @todo: entity must be migrated to typebox
+export const entityConfigSchema = Type.Object(
+   {
+      name: Type.Optional(Type.String()),
+      name_singular: Type.Optional(Type.String()),
+      description: Type.Optional(Type.String()),
+      sort_field: Type.Optional(Type.String({ default: config.data.default_primary_field })),
+      sort_dir: Type.Optional(StringEnum(["asc", "desc"], { default: "asc" }))
+   },
+   {
+      additionalProperties: false
+   }
+);
+
+export type EntityConfig = Static<typeof entityConfigSchema>;
+
+export type EntityData = Record<string, any>;
+export type EntityJSON = ReturnType<Entity["toJSON"]>;
+
+/**
+ * regular: normal defined entity
+ * system: generated by the system, e.g. "users" from auth
+ * generated: result of a relation, e.g. many-to-many relation's connection entity
+ */
+export const entityTypes = ["regular", "system", "generated"] as const;
+export type TEntityType = (typeof entityTypes)[number];
+
+/**
+ * @todo: add check for adding fields (primary and relation not allowed)
+ * @todo: add option to disallow api deletes (or api actions in general)
+ */
+export class Entity<
+   EntityName extends string = string,
+   Fields extends Record<string, Field<any, any, any>> = Record<string, Field<any, any, any>>
+> {
+   readonly #_name!: EntityName;
+   readonly #_fields!: Fields; // only for types
+
+   readonly name: string;
+   readonly fields: Field[];
+   readonly config: EntityConfig;
+   protected data: EntityData[] | undefined;
+   readonly type: TEntityType = "regular";
+
+   constructor(name: string, fields?: Field[], config?: EntityConfig, type?: TEntityType) {
+      if (typeof name !== "string" || name.length === 0) {
+         throw new Error("Entity name must be a non-empty string");
+      }
+
+      this.name = name;
+      this.config = parse(entityConfigSchema, config || {}) as EntityConfig;
+
+      // add id field if not given
+      // @todo: add test
+      const primary_count = fields?.filter((field) => field instanceof PrimaryField).length ?? 0;
+      if (primary_count > 1) {
+         throw new Error(`Entity "${name}" has more than one primary field`);
+      }
+      this.fields = primary_count === 1 ? [] : [new PrimaryField()];
+
+      if (fields) {
+         fields.forEach((field) => this.addField(field));
+      }
+
+      if (type) this.type = type;
+   }
+
+   static create(args: {
+      name: string;
+      fields?: Field[];
+      config?: EntityConfig;
+      type?: TEntityType;
+   }) {
+      return new Entity(args.name, args.fields, args.config, args.type);
+   }
+
+   // @todo: add test
+   getType(): TEntityType {
+      return this.type;
+   }
+
+   getSelect(alias?: string, context?: TActionContext | TRenderContext): string[] {
+      return this.getFields()
+         .filter((field) => !field.isHidden(context ?? "read"))
+         .map((field) => (alias ? `${alias}.${field.name} as ${field.name}` : field.name));
+   }
+
+   getDefaultSort() {
+      return {
+         by: this.config.sort_field,
+         dir: this.config.sort_dir
+      };
+   }
+
+   getAliasedSelectFrom(
+      select: string[],
+      _alias?: string,
+      context?: TActionContext | TRenderContext
+   ): string[] {
+      const alias = _alias ?? this.name;
+      return this.getFields()
+         .filter(
+            (field) =>
+               !field.isVirtual() &&
+               !field.isHidden(context ?? "read") &&
+               select.includes(field.name)
+         )
+         .map((field) => (alias ? `${alias}.${field.name} as ${field.name}` : field.name));
+   }
+
+   getFillableFields(context?: TActionContext, include_virtual?: boolean): Field[] {
+      return this.getFields(include_virtual).filter((field) => field.isFillable(context));
+   }
+
+   getRequiredFields(): Field[] {
+      return this.getFields().filter((field) => field.isRequired());
+   }
+
+   getDefaultObject(): EntityData {
+      return this.getFields().reduce((acc, field) => {
+         if (field.hasDefault()) {
+            acc[field.name] = field.getDefault();
+         }
+         return acc;
+      }, {} as EntityData);
+   }
+
+   getField(name: string): Field | undefined {
+      return this.fields.find((field) => field.name === name);
+   }
+
+   __experimental_replaceField(name: string, field: Field) {
+      const index = this.fields.findIndex((f) => f.name === name);
+      if (index === -1) {
+         throw new Error(`Field "${name}" not found on entity "${this.name}"`);
+      }
+
+      this.fields[index] = field;
+   }
+
+   getPrimaryField(): PrimaryField {
+      return this.fields[0] as PrimaryField;
+   }
+
+   id(): PrimaryField {
+      return this.getPrimaryField();
+   }
+
+   get label(): string {
+      return snakeToPascalWithSpaces(this.config.name ?? this.name);
+   }
+
+   field(name: string): Field | undefined {
+      return this.getField(name);
+   }
+
+   getFields(include_virtual: boolean = false): Field[] {
+      if (include_virtual) return this.fields;
+      return this.fields.filter((f) => !f.isVirtual());
+   }
+
+   addField(field: Field) {
+      const existing = this.getField(field.name);
+      // make unique name check
+      if (existing) {
+         // @todo: for now adding a graceful method
+         if (JSON.stringify(existing) === JSON.stringify(field)) {
+            /*console.warn(
+               `Field "${field.name}" already exists on entity "${this.name}", but it's the same, so skipping.`,
+            );*/
+            return;
+         }
+
+         throw new Error(`Field "${field.name}" already exists on entity "${this.name}"`);
+      }
+
+      this.fields.push(field);
+   }
+
+   __setData(data: EntityData[]) {
+      this.data = data;
+   }
+
+   isValidData(data: EntityData, context: TActionContext, explain?: boolean): boolean {
+      const fields = this.getFillableFields(context, false);
+      //const fields = this.fields;
+      //console.log("data", data);
+      for (const field of fields) {
+         if (!field.isValid(data[field.name], context)) {
+            console.log("Entity.isValidData:invalid", context, field.name, data[field.name]);
+            if (explain) {
+               throw new Error(`Field "${field.name}" has invalid data: "${data[field.name]}"`);
+            }
+
+            return false;
+         }
+      }
+
+      return true;
+   }
+
+   toSchema(clean?: boolean): object {
+      const fields = Object.fromEntries(this.fields.map((field) => [field.name, field]));
+      const schema = Type.Object(
+         transformObject(fields, (field) => ({
+            title: field.config.label,
+            $comment: field.config.description,
+            $field: field.type,
+            readOnly: !field.isFillable("update") ? true : undefined,
+            writeOnly: !field.isFillable("create") ? true : undefined,
+            ...field.toJsonSchema()
+         }))
+      );
+
+      return clean ? JSON.parse(JSON.stringify(schema)) : schema;
+   }
+
+   toJSON() {
+      return {
+         //name: this.name,
+         type: this.type,
+         //fields: transformObject(this.fields, (field) => field.toJSON()),
+         fields: Object.fromEntries(this.fields.map((field) => [field.name, field.toJSON()])),
+         config: this.config
+      };
+   }
+}

app/src/data/entities/EntityManager.ts

@@ -0,0 +1,266 @@
+import { EventManager } from "core/events";
+import { sql } from "kysely";
+import { Connection } from "../connection/Connection";
+import {
+   EntityNotDefinedException,
+   TransformRetrieveFailedException,
+   UnableToConnectException
+} from "../errors";
+import { MutatorEvents, RepositoryEvents } from "../events";
+import type { EntityIndex } from "../fields/indices/EntityIndex";
+import type { EntityRelation } from "../relations";
+import { RelationAccessor } from "../relations/RelationAccessor";
+import { SchemaManager } from "../schema/SchemaManager";
+import { Entity } from "./Entity";
+import { type EntityData, Mutator, Repository } from "./index";
+
+export class EntityManager<DB> {
+   connection: Connection;
+
+   private _entities: Entity[] = [];
+   private _relations: EntityRelation[] = [];
+   private _indices: EntityIndex[] = [];
+   private _schema?: SchemaManager;
+   readonly emgr: EventManager<typeof EntityManager.Events>;
+   static readonly Events = { ...MutatorEvents, ...RepositoryEvents };
+
+   constructor(
+      entities: Entity[],
+      connection: Connection,
+      relations: EntityRelation[] = [],
+      indices: EntityIndex[] = [],
+      emgr?: EventManager<any>
+   ) {
+      // add entities & relations
+      entities.forEach((entity) => this.addEntity(entity));
+      relations.forEach((relation) => this.addRelation(relation));
+      indices.forEach((index) => this.addIndex(index));
+
+      if (!(connection instanceof Connection)) {
+         throw new UnableToConnectException("");
+      }
+
+      this.connection = connection;
+      this.emgr = emgr ?? new EventManager();
+      //console.log("registering events", EntityManager.Events);
+      this.emgr.registerEvents(EntityManager.Events);
+   }
+
+   /**
+    * Forks the EntityManager without the EventManager.
+    * This is useful when used inside an event handler.
+    */
+   fork(): EntityManager<DB> {
+      return new EntityManager(this._entities, this.connection, this._relations, this._indices);
+   }
+
+   get entities(): Entity[] {
+      return this._entities;
+   }
+
+   get relations(): RelationAccessor {
+      return new RelationAccessor(this._relations);
+   }
+
+   get indices(): EntityIndex[] {
+      return this._indices;
+   }
+
+   async ping(): Promise<boolean> {
+      const res = await sql`SELECT 1`.execute(this.connection.kysely);
+      return res.rows.length > 0;
+   }
+
+   addEntity(entity: Entity) {
+      const existing = this.entities.find((e) => e.name === entity.name);
+      // check if already exists by name
+      if (existing) {
+         // @todo: for now adding a graceful method
+         if (JSON.stringify(existing) === JSON.stringify(entity)) {
+            //console.warn(`Entity "${entity.name}" already exists, but it's the same, so skipping.`);
+            return;
+         }
+
+         throw new Error(`Entity "${entity.name}" already exists`);
+      }
+
+      this.entities.push(entity);
+   }
+
+   entity(name: string): Entity {
+      const entity = this.entities.find((e) => e.name === name);
+      if (!entity) {
+         throw new EntityNotDefinedException(name);
+      }
+
+      return entity;
+   }
+
+   hasEntity(entity: string): boolean;
+   hasEntity(entity: Entity): boolean;
+   hasEntity(nameOrEntity: string | Entity): boolean {
+      const name = typeof nameOrEntity === "string" ? nameOrEntity : nameOrEntity.name;
+      return this.entities.some((e) => e.name === name);
+   }
+
+   hasIndex(index: string): boolean;
+   hasIndex(index: EntityIndex): boolean;
+   hasIndex(nameOrIndex: string | EntityIndex): boolean {
+      const name = typeof nameOrIndex === "string" ? nameOrIndex : nameOrIndex.name;
+      return this.indices.some((e) => e.name === name);
+   }
+
+   addRelation(relation: EntityRelation) {
+      // check if entities are registered
+      if (!this.entity(relation.source.entity.name) || !this.entity(relation.target.entity.name)) {
+         throw new Error("Relation source or target entity not found");
+      }
+
+      // @todo: potentially add name to relation in order to have multiple
+      const found = this._relations.find((r) => {
+         const equalSourceTarget =
+            r.source.entity.name === relation.source.entity.name &&
+            r.target.entity.name === relation.target.entity.name;
+         const equalReferences =
+            r.source.reference === relation.source.reference &&
+            r.target.reference === relation.target.reference;
+
+         return (
+            //r.type === relation.type && // ignore type for now
+            equalSourceTarget && equalReferences
+         );
+      });
+
+      if (found) {
+         throw new Error(
+            `Relation "${relation.type}" between "${relation.source.entity.name}" ` +
+               `and "${relation.target.entity.name}" already exists`
+         );
+      }
+
+      this._relations.push(relation);
+      relation.initialize(this);
+   }
+
+   relationsOf(entity_name: string): EntityRelation[] {
+      return this.relations.relationsOf(this.entity(entity_name));
+   }
+
+   relationOf(entity_name: string, reference: string): EntityRelation | undefined {
+      return this.relations.relationOf(this.entity(entity_name), reference);
+   }
+
+   hasRelations(entity_name: string): boolean {
+      return this.relations.hasRelations(this.entity(entity_name));
+   }
+
+   relatedEntitiesOf(entity_name: string): Entity[] {
+      return this.relations.relatedEntitiesOf(this.entity(entity_name));
+   }
+
+   relationReferencesOf(entity_name: string): string[] {
+      return this.relations.relationReferencesOf(this.entity(entity_name));
+   }
+
+   repository(_entity: Entity | string) {
+      const entity = _entity instanceof Entity ? _entity : this.entity(_entity);
+      return new Repository(this, entity, this.emgr);
+   }
+
+   repo<E extends Entity>(
+      _entity: E
+   ): Repository<
+      DB,
+      E extends Entity<infer Name> ? (Name extends keyof DB ? Name : never) : never
+   > {
+      return new Repository(this, _entity, this.emgr);
+   }
+
+   _repo<TB extends keyof DB>(_entity: TB): Repository<DB, TB> {
+      const entity = this.entity(_entity as any);
+      return new Repository(this, entity, this.emgr);
+   }
+
+   mutator(_entity: Entity | string) {
+      const entity = _entity instanceof Entity ? _entity : this.entity(_entity);
+      return new Mutator(this, entity, this.emgr);
+   }
+
+   addIndex(index: EntityIndex, force = false) {
+      // check if already exists by name
+      if (this.indices.find((e) => e.name === index.name)) {
+         if (force) {
+            throw new Error(`Index "${index.name}" already exists`);
+         }
+         return;
+      }
+
+      this._indices.push(index);
+   }
+
+   getIndicesOf(_entity: Entity | string): EntityIndex[] {
+      const entity = _entity instanceof Entity ? _entity : this.entity(_entity);
+      return this.indices.filter((index) => index.entity.name === entity.name);
+   }
+
+   schema() {
+      if (!this._schema) {
+         this._schema = new SchemaManager(this);
+      }
+
+      return this._schema;
+   }
+
+   // @todo: centralize and add tests
+   hydrate(entity_name: string, _data: EntityData[]) {
+      const entity = this.entity(entity_name);
+      const data: EntityData[] = [];
+
+      for (const row of _data) {
+         for (let [key, value] of Object.entries(row)) {
+            const field = entity.getField(key);
+
+            if (!field || field.isVirtual()) {
+               // if relation, use related entity to hydrate
+               const relation = this.relationOf(entity_name, key);
+               if (relation) {
+                  if (!value) continue;
+
+                  value = relation.hydrate(key, Array.isArray(value) ? value : [value], this);
+                  row[key] = value;
+                  continue;
+               } else if (field?.isVirtual()) {
+                  continue;
+               }
+
+               throw new Error(`Field "${key}" not found on entity "${entity.name}"`);
+            }
+
+            try {
+               if (value === null && field.hasDefault()) {
+                  row[key] = field.getDefault();
+               }
+
+               row[key] = field.transformRetrieve(value as any);
+            } catch (e: any) {
+               throw new TransformRetrieveFailedException(
+                  `"${field.type}" field "${key}" on entity "${entity.name}": ${e.message}`
+               );
+            }
+         }
+
+         data.push(row);
+      }
+
+      return data;
+   }
+
+   toJSON() {
+      return {
+         entities: Object.fromEntries(this.entities.map((e) => [e.name, e.toJSON()])),
+         relations: Object.fromEntries(this.relations.all.map((r) => [r.getName(), r.toJSON()])),
+         //relations: this.relations.all.map((r) => r.toJSON()),
+         indices: Object.fromEntries(this.indices.map((i) => [i.name, i.toJSON()]))
+      };
+   }
+}

app/src/data/entities/Mutator.ts

@@ -0,0 +1,270 @@
+import type { PrimaryFieldType } from "core";
+import { type EmitsEvents, EventManager } from "core/events";
+import type { DeleteQueryBuilder, InsertQueryBuilder, UpdateQueryBuilder } from "kysely";
+import { type TActionContext, WhereBuilder } from "..";
+import type { Entity, EntityData, EntityManager } from "../entities";
+import { InvalidSearchParamsException } from "../errors";
+import { MutatorEvents } from "../events";
+import { RelationMutator } from "../relations";
+import type { RepoQuery } from "../server/data-query-impl";
+
+type MutatorQB =
+   | InsertQueryBuilder<any, any, any>
+   | UpdateQueryBuilder<any, any, any, any>
+   | DeleteQueryBuilder<any, any, any>;
+
+type MutatorUpdateOrDelete =
+   | UpdateQueryBuilder<any, any, any, any>
+   | DeleteQueryBuilder<any, any, any>;
+
+export type MutatorResponse<T = EntityData[]> = {
+   entity: Entity;
+   sql: string;
+   parameters: any[];
+   result: EntityData[];
+   data: T;
+};
+
+export class Mutator<DB> implements EmitsEvents {
+   em: EntityManager<DB>;
+   entity: Entity;
+   static readonly Events = MutatorEvents;
+   emgr: EventManager<typeof MutatorEvents>;
+
+   // @todo: current hacky workaround to disable creation of system entities
+   __unstable_disable_system_entity_creation = true;
+   __unstable_toggleSystemEntityCreation(value: boolean) {
+      this.__unstable_disable_system_entity_creation = value;
+   }
+
+   constructor(em: EntityManager<DB>, entity: Entity, emgr?: EventManager<any>) {
+      this.em = em;
+      this.entity = entity;
+      this.emgr = emgr ?? new EventManager(MutatorEvents);
+   }
+
+   private get conn() {
+      return this.em.connection.kysely;
+   }
+
+   async getValidatedData(data: EntityData, context: TActionContext): Promise<EntityData> {
+      const entity = this.entity;
+      if (!context) {
+         throw new Error("Context must be provided for validation");
+      }
+
+      const keys = Object.keys(data);
+      const validatedData: EntityData = {};
+
+      // get relational references/keys
+      const relationMutator = new RelationMutator(entity, this.em);
+      const relational_keys = relationMutator.getRelationalKeys();
+
+      for (const key of keys) {
+         if (relational_keys.includes(key)) {
+            const result = await relationMutator.persistRelation(key, data[key]);
+
+            // if relation field (include key and value in validatedData)
+            if (Array.isArray(result)) {
+               //console.log("--- (instructions)", result);
+               const [relation_key, relation_value] = result;
+               validatedData[relation_key] = relation_value;
+            }
+            continue;
+         }
+
+         const field = entity.getField(key);
+         if (!field) {
+            throw new Error(
+               `Field "${key}" not found on entity "${entity.name}". Fields: ${entity
+                  .getFillableFields()
+                  .map((f) => f.name)
+                  .join(", ")}`
+            );
+         }
+
+         // we should never get here, but just to be sure (why?)
+         if (!field.isFillable(context)) {
+            throw new Error(`Field "${key}" is not fillable on entity "${entity.name}"`);
+         }
+
+         validatedData[key] = await field.transformPersist(data[key], this.em, context);
+      }
+
+      if (Object.keys(validatedData).length === 0) {
+         throw new Error(`No data left to update "${entity.name}"`);
+      }
+
+      return validatedData;
+   }
+
+   protected async many(qb: MutatorQB): Promise<MutatorResponse> {
+      const entity = this.entity;
+      const { sql, parameters } = qb.compile();
+      //console.log("mutatoar:exec", sql, parameters);
+      const result = await qb.execute();
+
+      const data = this.em.hydrate(entity.name, result) as EntityData[];
+
+      return {
+         entity,
+         sql,
+         parameters: [...parameters],
+         result: result,
+         data
+      };
+   }
+
+   protected async single(qb: MutatorQB): Promise<MutatorResponse<EntityData>> {
+      const { data, ...response } = await this.many(qb);
+      return { ...response, data: data[0]! };
+   }
+
+   async insertOne(data: EntityData): Promise<MutatorResponse<EntityData>> {
+      const entity = this.entity;
+      if (entity.type === "system" && this.__unstable_disable_system_entity_creation) {
+         throw new Error(`Creation of system entity "${entity.name}" is disabled`);
+      }
+
+      // @todo: establish the original order from "data"
+      const validatedData = {
+         ...entity.getDefaultObject(),
+         ...(await this.getValidatedData(data, "create"))
+      };
+
+      await this.emgr.emit(new Mutator.Events.MutatorInsertBefore({ entity, data: validatedData }));
+
+      // check if required fields are present
+      const required = entity.getRequiredFields();
+      for (const field of required) {
+         if (
+            typeof validatedData[field.name] === "undefined" ||
+            validatedData[field.name] === null
+         ) {
+            throw new Error(`Field "${field.name}" is required`);
+         }
+      }
+
+      const query = this.conn
+         .insertInto(entity.name)
+         .values(validatedData)
+         .returning(entity.getSelect());
+
+      const res = await this.single(query);
+
+      await this.emgr.emit(new Mutator.Events.MutatorInsertAfter({ entity, data: res.data }));
+
+      return res;
+   }
+
+   async updateOne(id: PrimaryFieldType, data: EntityData): Promise<MutatorResponse<EntityData>> {
+      const entity = this.entity;
+      if (!Number.isInteger(id)) {
+         throw new Error("ID must be provided for update");
+      }
+
+      const validatedData = await this.getValidatedData(data, "update");
+
+      await this.emgr.emit(
+         new Mutator.Events.MutatorUpdateBefore({ entity, entityId: id, data: validatedData })
+      );
+
+      const query = this.conn
+         .updateTable(entity.name)
+         .set(validatedData)
+         .where(entity.id().name, "=", id)
+         .returning(entity.getSelect());
+
+      const res = await this.single(query);
+
+      await this.emgr.emit(
+         new Mutator.Events.MutatorUpdateAfter({ entity, entityId: id, data: res.data })
+      );
+
+      return res;
+   }
+
+   async deleteOne(id: PrimaryFieldType): Promise<MutatorResponse<EntityData>> {
+      const entity = this.entity;
+      if (!Number.isInteger(id)) {
+         throw new Error("ID must be provided for deletion");
+      }
+
+      await this.emgr.emit(new Mutator.Events.MutatorDeleteBefore({ entity, entityId: id }));
+
+      const query = this.conn
+         .deleteFrom(entity.name)
+         .where(entity.id().name, "=", id)
+         .returning(entity.getSelect());
+
+      const res = await this.single(query);
+
+      await this.emgr.emit(
+         new Mutator.Events.MutatorDeleteAfter({ entity, entityId: id, data: res.data })
+      );
+
+      return res;
+   }
+
+   private getValidOptions(options?: Partial<RepoQuery>): Partial<RepoQuery> {
+      const entity = this.entity;
+      const validated: Partial<RepoQuery> = {};
+
+      if (options?.where) {
+         // @todo: add tests for aliased fields in where
+         const invalid = WhereBuilder.getPropertyNames(options.where).filter((field) => {
+            return typeof entity.getField(field) === "undefined";
+         });
+
+         if (invalid.length > 0) {
+            throw new InvalidSearchParamsException(`Invalid where field(s): ${invalid.join(", ")}`);
+         }
+
+         validated.where = options.where;
+      }
+
+      return validated;
+   }
+
+   private appendWhere<QB extends MutatorUpdateOrDelete>(qb: QB, _where?: RepoQuery["where"]): QB {
+      const entity = this.entity;
+
+      const alias = entity.name;
+      const aliased = (field: string) => `${alias}.${field}`;
+
+      // add where if present
+      if (_where) {
+         // @todo: add tests for aliased fields in where
+         const invalid = WhereBuilder.getPropertyNames(_where).filter((field) => {
+            return typeof entity.getField(field) === "undefined";
+         });
+
+         if (invalid.length > 0) {
+            throw new InvalidSearchParamsException(`Invalid where field(s): ${invalid.join(", ")}`);
+         }
+
+         return WhereBuilder.addClause(qb, _where);
+      }
+
+      return qb;
+   }
+
+   // @todo: decide whether entries should be deleted all at once or one by one (for events)
+   async deleteMany(where?: RepoQuery["where"]): Promise<MutatorResponse<EntityData>> {
+      const entity = this.entity;
+
+      const qb = this.appendWhere(this.conn.deleteFrom(entity.name), where).returning(
+         entity.getSelect()
+      );
+
+      //await this.emgr.emit(new Mutator.Events.MutatorDeleteBefore({ entity, entityId: id }));
+
+      const res = await this.many(qb);
+
+      /*await this.emgr.emit(
+         new Mutator.Events.MutatorDeleteAfter({ entity, entityId: id, data: res.data })
+      );*/
+
+      return res;
+   }
+}

app/src/data/entities/index.ts

@@ -0,0 +1,6 @@
+export * from "./Entity";
+export * from "./EntityManager";
+export * from "./Mutator";
+export * from "./query/Repository";
+export * from "./query/WhereBuilder";
+export * from "./query/WithBuilder";

app/src/data/entities/query/JoinBuilder.ts

@@ -0,0 +1,51 @@
+import { ManyToManyRelation, ManyToOneRelation } from "../../relations";
+import type { Entity } from "../Entity";
+import type { EntityManager } from "../EntityManager";
+import type { RepositoryQB } from "./Repository";
+
+export class JoinBuilder {
+   private static buildClause(
+      em: EntityManager<any>,
+      qb: RepositoryQB,
+      entity: Entity,
+      withString: string,
+   ) {
+      const relation = em.relationOf(entity.name, withString);
+      if (!relation) {
+         throw new Error(`Relation "${withString}" not found`);
+      }
+
+      return relation.buildJoin(entity, qb, withString);
+   }
+
+   // @todo: returns multiple on manytomany (edit: so?)
+   static getJoinedEntityNames(em: EntityManager<any>, entity: Entity, joins: string[]): string[] {
+      return joins.flatMap((join) => {
+         const relation = em.relationOf(entity.name, join);
+         if (!relation) {
+            throw new Error(`Relation "${join}" not found`);
+         }
+
+         const other = relation.other(entity);
+
+         if (relation instanceof ManyToOneRelation) {
+            return [other.entity.name];
+         } else if (relation instanceof ManyToManyRelation) {
+            return [other.entity.name, relation.connectionEntity.name];
+         }
+
+         return [];
+      });
+   }
+
+   static addClause(em: EntityManager<any>, qb: RepositoryQB, entity: Entity, joins: string[]) {
+      if (joins.length === 0) return qb;
+
+      let newQb = qb;
+      for (const entry of joins) {
+         newQb = JoinBuilder.buildClause(em, newQb, entity, entry);
+      }
+
+      return newQb;
+   }
+}

app/src/data/entities/query/Repository.ts

@@ -0,0 +1,407 @@
+import type { PrimaryFieldType } from "core";
+import { type EmitsEvents, EventManager } from "core/events";
+import { type SelectQueryBuilder, sql } from "kysely";
+import { cloneDeep } from "lodash-es";
+import { InvalidSearchParamsException } from "../../errors";
+import { MutatorEvents, RepositoryEvents, RepositoryFindManyBefore } from "../../events";
+import { type RepoQuery, defaultQuerySchema } from "../../server/data-query-impl";
+import {
+   type Entity,
+   type EntityData,
+   type EntityManager,
+   WhereBuilder,
+   WithBuilder
+} from "../index";
+import { JoinBuilder } from "./JoinBuilder";
+
+export type RepositoryQB = SelectQueryBuilder<any, any, any>;
+
+export type RepositoryRawResponse = {
+   sql: string;
+   parameters: any[];
+   result: EntityData[];
+};
+export type RepositoryResponse<T = EntityData[]> = RepositoryRawResponse & {
+   entity: Entity;
+   data: T;
+   meta: {
+      total: number;
+      count: number;
+      items: number;
+      time?: number;
+      query?: {
+         sql: string;
+         parameters: readonly any[];
+      };
+   };
+};
+
+export type RepositoryCountResponse = RepositoryRawResponse & {
+   count: number;
+};
+export type RepositoryExistsResponse = RepositoryRawResponse & {
+   exists: boolean;
+};
+
+export class Repository<DB = any, TB extends keyof DB = any> implements EmitsEvents {
+   em: EntityManager<DB>;
+   entity: Entity;
+   static readonly Events = RepositoryEvents;
+   emgr: EventManager<typeof Repository.Events>;
+
+   constructor(em: EntityManager<DB>, entity: Entity, emgr?: EventManager<any>) {
+      this.em = em;
+      this.entity = entity;
+      this.emgr = emgr ?? new EventManager(MutatorEvents);
+   }
+
+   private cloneFor(entity: Entity) {
+      return new Repository(this.em, entity, this.emgr);
+   }
+
+   private get conn() {
+      return this.em.connection.kysely;
+   }
+
+   private getValidOptions(options?: Partial<RepoQuery>): RepoQuery {
+      const entity = this.entity;
+      // @todo: if not cloned deep, it will keep references and error if multiple requests come in
+      const validated = {
+         ...cloneDeep(defaultQuerySchema),
+         sort: entity.getDefaultSort(),
+         select: entity.getSelect()
+      };
+      //console.log("validated", validated);
+
+      if (!options) return validated;
+
+      if (options.sort) {
+         if (!validated.select.includes(options.sort.by)) {
+            throw new InvalidSearchParamsException(`Invalid sort field "${options.sort.by}"`);
+         }
+         if (!["asc", "desc"].includes(options.sort.dir)) {
+            throw new InvalidSearchParamsException(`Invalid sort direction "${options.sort.dir}"`);
+         }
+
+         validated.sort = options.sort;
+      }
+
+      if (options.select && options.select.length > 0) {
+         const invalid = options.select.filter((field) => !validated.select.includes(field));
+
+         if (invalid.length > 0) {
+            throw new InvalidSearchParamsException(
+               `Invalid select field(s): ${invalid.join(", ")}`
+            );
+         }
+
+         validated.select = options.select;
+      }
+
+      if (options.with && options.with.length > 0) {
+         for (const entry of options.with) {
+            const related = this.em.relationOf(entity.name, entry);
+            if (!related) {
+               throw new InvalidSearchParamsException(
+                  `WITH: "${entry}" is not a relation of "${entity.name}"`
+               );
+            }
+
+            validated.with.push(entry);
+         }
+      }
+
+      if (options.join && options.join.length > 0) {
+         for (const entry of options.join) {
+            const related = this.em.relationOf(entity.name, entry);
+            if (!related) {
+               throw new InvalidSearchParamsException(
+                  `JOIN: "${entry}" is not a relation of "${entity.name}"`
+               );
+            }
+
+            validated.join.push(entry);
+         }
+      }
+
+      if (options.where) {
+         // @todo: auto-alias base entity when using joins! otherwise "id" is ambiguous
+         const aliases = [entity.name];
+         if (validated.join.length > 0) {
+            aliases.push(...JoinBuilder.getJoinedEntityNames(this.em, entity, validated.join));
+         }
+
+         // @todo: add tests for aliased fields in where
+         const invalid = WhereBuilder.getPropertyNames(options.where).filter((field) => {
+            if (field.includes(".")) {
+               const [alias, prop] = field.split(".") as [string, string];
+               if (!aliases.includes(alias)) {
+                  return true;
+               }
+
+               return !this.em.entity(alias).getField(prop);
+            }
+
+            return typeof entity.getField(field) === "undefined";
+         });
+
+         if (invalid.length > 0) {
+            throw new InvalidSearchParamsException(`Invalid where field(s): ${invalid.join(", ")}`);
+         }
+
+         validated.where = options.where;
+      }
+
+      // pass unfiltered
+      if (options.limit) validated.limit = options.limit;
+      if (options.offset) validated.offset = options.offset;
+
+      return validated;
+   }
+
+   protected async performQuery(qb: RepositoryQB): Promise<RepositoryResponse> {
+      const entity = this.entity;
+      const compiled = qb.compile();
+      /*const { sql, parameters } = qb.compile();
+      console.log("many", sql, parameters);*/
+
+      const start = performance.now();
+      const selector = (as = "count") => this.conn.fn.countAll<number>().as(as);
+      const countQuery = qb
+         .clearSelect()
+         .select(selector())
+         .clearLimit()
+         .clearOffset()
+         .clearGroupBy()
+         .clearOrderBy();
+      const totalQuery = this.conn.selectFrom(entity.name).select(selector("total"));
+
+      try {
+         const [_count, _total, result] = await this.em.connection.batchQuery([
+            countQuery,
+            totalQuery,
+            qb
+         ]);
+         //console.log("result", { _count, _total });
+
+         const time = Number.parseFloat((performance.now() - start).toFixed(2));
+         const data = this.em.hydrate(entity.name, result);
+
+         return {
+            entity,
+            sql: compiled.sql,
+            parameters: [...compiled.parameters],
+            result,
+            data,
+            meta: {
+               total: _total[0]?.total ?? 0,
+               count: _count[0]?.count ?? 0, // @todo: better graceful method
+               items: result.length,
+               time,
+               query: { sql: compiled.sql, parameters: compiled.parameters }
+            }
+         };
+      } catch (e) {
+         console.error("many error", e, compiled);
+         throw e;
+      }
+   }
+
+   protected async single(
+      qb: RepositoryQB,
+      options: RepoQuery
+   ): Promise<RepositoryResponse<EntityData>> {
+      await this.emgr.emit(
+         new Repository.Events.RepositoryFindOneBefore({ entity: this.entity, options })
+      );
+
+      const { data, ...response } = await this.performQuery(qb);
+
+      await this.emgr.emit(
+         new Repository.Events.RepositoryFindOneAfter({
+            entity: this.entity,
+            options,
+            data: data[0]!
+         })
+      );
+
+      return { ...response, data: data[0]! };
+   }
+
+   private buildQuery(
+      _options?: Partial<RepoQuery>,
+      exclude_options: (keyof RepoQuery)[] = []
+   ): { qb: RepositoryQB; options: RepoQuery } {
+      const entity = this.entity;
+      const options = this.getValidOptions(_options);
+
+      const alias = entity.name;
+      const aliased = (field: string) => `${alias}.${field}`;
+      let qb = this.conn
+         .selectFrom(entity.name)
+         .select(entity.getAliasedSelectFrom(options.select, alias));
+
+      //console.log("build query options", options);
+      if (!exclude_options.includes("with") && options.with) {
+         qb = WithBuilder.addClause(this.em, qb, entity, options.with);
+      }
+
+      if (!exclude_options.includes("join") && options.join) {
+         qb = JoinBuilder.addClause(this.em, qb, entity, options.join);
+      }
+
+      // add where if present
+      if (!exclude_options.includes("where") && options.where) {
+         qb = WhereBuilder.addClause(qb, options.where);
+      }
+
+      if (!exclude_options.includes("limit")) qb = qb.limit(options.limit);
+      if (!exclude_options.includes("offset")) qb = qb.offset(options.offset);
+
+      // sorting
+      if (!exclude_options.includes("sort")) {
+         qb = qb.orderBy(aliased(options.sort.by), options.sort.dir);
+      }
+
+      return { qb, options };
+   }
+
+   async findId(
+      id: PrimaryFieldType,
+      _options?: Partial<Omit<RepoQuery, "where" | "limit" | "offset">>
+   ): Promise<RepositoryResponse<DB[TB]>> {
+      const { qb, options } = this.buildQuery(
+         {
+            ..._options,
+            where: { [this.entity.getPrimaryField().name]: id },
+            limit: 1
+         },
+         ["offset", "sort"]
+      );
+
+      return this.single(qb, options) as any;
+   }
+
+   async findOne(
+      where: RepoQuery["where"],
+      _options?: Partial<Omit<RepoQuery, "where" | "limit" | "offset">>
+   ): Promise<RepositoryResponse<DB[TB] | undefined>> {
+      const { qb, options } = this.buildQuery(
+         {
+            ..._options,
+            where,
+            limit: 1
+         },
+         ["offset", "sort"]
+      );
+
+      return this.single(qb, options) as any;
+   }
+
+   async findMany(_options?: Partial<RepoQuery>): Promise<RepositoryResponse<DB[TB][]>> {
+      const { qb, options } = this.buildQuery(_options);
+      //console.log("findMany:options", options);
+
+      await this.emgr.emit(
+         new Repository.Events.RepositoryFindManyBefore({ entity: this.entity, options })
+      );
+
+      const res = await this.performQuery(qb);
+
+      await this.emgr.emit(
+         new Repository.Events.RepositoryFindManyAfter({
+            entity: this.entity,
+            options,
+            data: res.data
+         })
+      );
+
+      return res as any;
+   }
+
+   // @todo: add unit tests, specially for many to many
+   async findManyByReference(
+      id: PrimaryFieldType,
+      reference: string,
+      _options?: Partial<Omit<RepoQuery, "limit" | "offset">>
+   ): Promise<RepositoryResponse<EntityData>> {
+      const entity = this.entity;
+      const listable_relations = this.em.relations.listableRelationsOf(entity);
+      const relation = listable_relations.find((r) => r.ref(reference).reference === reference);
+
+      if (!relation) {
+         throw new Error(
+            `Relation "${reference}" not found or not listable on entity "${entity.name}"`
+         );
+      }
+
+      const newEntity = relation.other(entity).entity;
+      const refQueryOptions = relation.getReferenceQuery(newEntity, id as number, reference);
+      if (!("where" in refQueryOptions) || Object.keys(refQueryOptions.where as any).length === 0) {
+         throw new Error(
+            `Invalid reference query for "${reference}" on entity "${newEntity.name}"`
+         );
+      }
+
+      const findManyOptions = {
+         ..._options,
+         ...refQueryOptions,
+         where: {
+            ...refQueryOptions.where,
+            ..._options?.where
+         }
+      };
+
+      //console.log("findManyOptions", newEntity.name, findManyOptions);
+      return this.cloneFor(newEntity).findMany(findManyOptions);
+   }
+
+   async count(where?: RepoQuery["where"]): Promise<RepositoryCountResponse> {
+      const entity = this.entity;
+      const options = this.getValidOptions({ where });
+
+      const selector = this.conn.fn.count<number>(sql`*`).as("count");
+      let qb = this.conn.selectFrom(entity.name).select(selector);
+
+      // add where if present
+      if (options.where) {
+         qb = WhereBuilder.addClause(qb, options.where);
+      }
+
+      const compiled = qb.compile();
+      const result = await qb.execute();
+
+      return {
+         sql: compiled.sql,
+         parameters: [...compiled.parameters],
+         result,
+         count: result[0]?.count ?? 0
+      };
+   }
+
+   async exists(where: Required<RepoQuery["where"]>): Promise<RepositoryExistsResponse> {
+      const entity = this.entity;
+      const options = this.getValidOptions({ where });
+
+      const selector = this.conn.fn.count<number>(sql`*`).as("count");
+      let qb = this.conn.selectFrom(entity.name).select(selector);
+
+      // add mandatory where
+      qb = WhereBuilder.addClause(qb, options.where);
+
+      // we only need 1
+      qb = qb.limit(1);
+
+      const compiled = qb.compile();
+      //console.log("exists query", compiled.sql, compiled.parameters);
+      const result = await qb.execute();
+      //console.log("result", result);
+
+      return {
+         sql: compiled.sql,
+         parameters: [...compiled.parameters],
+         result,
+         exists: result[0]!.count > 0
+      };
+   }
+}

app/src/data/entities/query/WhereBuilder.ts

@@ -0,0 +1,132 @@
+import {
+   type BooleanLike,
+   type FilterQuery,
+   type Primitive,
+   type TExpression,
+   exp,
+   isBooleanLike,
+   isPrimitive,
+   makeValidator
+} from "core";
+import type {
+   DeleteQueryBuilder,
+   ExpressionBuilder,
+   ExpressionWrapper,
+   SelectQueryBuilder,
+   UpdateQueryBuilder
+} from "kysely";
+import type { RepositoryQB } from "./Repository";
+
+type Builder = ExpressionBuilder<any, any>;
+type Wrapper = ExpressionWrapper<any, any, any>;
+type WhereQb =
+   | SelectQueryBuilder<any, any, any>
+   | UpdateQueryBuilder<any, any, any, any>
+   | DeleteQueryBuilder<any, any, any>;
+
+function key(e: unknown): string {
+   if (typeof e !== "string") {
+      throw new Error(`Invalid key: ${e}`);
+   }
+   return e as string;
+}
+
+const expressions: TExpression<any, any, any>[] = [
+   exp(
+      "$eq",
+      (v: Primitive) => isPrimitive(v),
+      (v, k, eb: Builder) => eb(key(k), "=", v)
+   ),
+   exp(
+      "$ne",
+      (v: Primitive) => isPrimitive(v),
+      (v, k, eb: Builder) => eb(key(k), "!=", v)
+   ),
+   exp(
+      "$gt",
+      (v: Primitive) => isPrimitive(v),
+      (v, k, eb: Builder) => eb(key(k), ">", v)
+   ),
+   exp(
+      "$gte",
+      (v: Primitive) => isPrimitive(v),
+      (v, k, eb: Builder) => eb(key(k), ">=", v)
+   ),
+   exp(
+      "$lt",
+      (v: Primitive) => isPrimitive(v),
+      (v, k, eb: Builder) => eb(key(k), "<", v)
+   ),
+   exp(
+      "$lte",
+      (v: Primitive) => isPrimitive(v),
+      (v, k, eb: Builder) => eb(key(k), "<=", v)
+   ),
+   exp(
+      "$isnull",
+      (v: BooleanLike) => isBooleanLike(v),
+      (v, k, eb: Builder) => eb(key(k), v ? "is" : "is not", null)
+   ),
+   exp(
+      "$in",
+      (v: any[]) => Array.isArray(v),
+      (v, k, eb: Builder) => eb(key(k), "in", v)
+   ),
+   exp(
+      "$notin",
+      (v: any[]) => Array.isArray(v),
+      (v, k, eb: Builder) => eb(key(k), "not in", v)
+   ),
+   exp(
+      "$between",
+      (v: [number, number]) => Array.isArray(v) && v.length === 2,
+      (v, k, eb: Builder) => eb.between(key(k), v[0], v[1])
+   ),
+   exp(
+      "$like",
+      (v: Primitive) => isPrimitive(v),
+      (v, k, eb: Builder) => eb(key(k), "like", String(v).replace(/\*/g, "%"))
+   )
+];
+
+export type WhereQuery = FilterQuery<typeof expressions>;
+
+const validator = makeValidator(expressions);
+
+export class WhereBuilder {
+   static addClause<QB extends WhereQb>(qb: QB, query: WhereQuery) {
+      if (Object.keys(query).length === 0) {
+         return qb;
+      }
+
+      // @ts-ignore
+      return qb.where((eb) => {
+         const fns = validator.build(query, {
+            value_is_kv: true,
+            exp_ctx: eb,
+            convert: true
+         });
+
+         if (fns.$or.length > 0 && fns.$and.length > 0) {
+            return eb.and(fns.$and).or(eb.and(fns.$or));
+         } else if (fns.$or.length > 0) {
+            return eb.or(fns.$or);
+         }
+
+         return eb.and(fns.$and);
+      });
+   }
+
+   static convert(query: WhereQuery): WhereQuery {
+      return validator.convert(query);
+   }
+
+   static getPropertyNames(query: WhereQuery): string[] {
+      const { keys } = validator.build(query, {
+         value_is_kv: true,
+         exp_ctx: () => null,
+         convert: true
+      });
+      return Array.from(keys);
+   }
+}

app/src/data/entities/query/WithBuilder.ts

@@ -0,0 +1,42 @@
+import type { Entity, EntityManager, RepositoryQB } from "../../entities";
+
+export class WithBuilder {
+   private static buildClause(
+      em: EntityManager<any>,
+      qb: RepositoryQB,
+      entity: Entity,
+      withString: string
+   ) {
+      const relation = em.relationOf(entity.name, withString);
+      if (!relation) {
+         throw new Error(`Relation "${withString}" not found`);
+      }
+
+      const cardinality = relation.ref(withString).cardinality;
+      //console.log("with--builder", { entity: entity.name, withString, cardinality });
+
+      const fns = em.connection.fn;
+      const jsonFrom = cardinality === 1 ? fns.jsonObjectFrom : fns.jsonArrayFrom;
+
+      if (!jsonFrom) {
+         throw new Error("Connection does not support jsonObjectFrom/jsonArrayFrom");
+      }
+
+      try {
+         return relation.buildWith(entity, qb, jsonFrom, withString);
+      } catch (e) {
+         throw new Error(`Could not build "with" relation "${withString}": ${(e as any).message}`);
+      }
+   }
+
+   static addClause(em: EntityManager<any>, qb: RepositoryQB, entity: Entity, withs: string[]) {
+      if (withs.length === 0) return qb;
+
+      let newQb = qb;
+      for (const entry of withs) {
+         newQb = WithBuilder.buildClause(em, newQb, entity, entry);
+      }
+
+      return newQb;
+   }
+}

app/src/data/errors.ts

@@ -0,0 +1,77 @@
+import { Exception } from "core";
+import type { TypeInvalidError } from "core/utils";
+import type { Entity } from "./entities";
+import type { Field } from "./fields";
+
+export class UnableToConnectException extends Exception {
+   override name = "UnableToConnectException";
+   override code = 500;
+}
+
+export class InvalidSearchParamsException extends Exception {
+   override name = "InvalidSearchParamsException";
+   override code = 422;
+}
+
+export class TransformRetrieveFailedException extends Exception {
+   override name = "TransformRetrieveFailedException";
+   override code = 422;
+}
+
+export class TransformPersistFailedException extends Exception {
+   override name = "TransformPersistFailedException";
+   override code = 422;
+
+   static invalidType(property: string, expected: string, given: any) {
+      const givenValue = typeof given === "object" ? JSON.stringify(given) : given;
+      const message =
+         `Property "${property}" must be of type "${expected}", ` +
+         `"${givenValue}" of type "${typeof given}" given.`;
+      return new TransformPersistFailedException(message);
+   }
+
+   static required(property: string) {
+      return new TransformPersistFailedException(`Property "${property}" is required`);
+   }
+}
+
+export class InvalidFieldConfigException extends Exception {
+   override name = "InvalidFieldConfigException";
+   override code = 400;
+
+   constructor(
+      field: Field<any, any, any>,
+      public given: any,
+      error: TypeInvalidError
+   ) {
+      console.error("InvalidFieldConfigException", {
+         given,
+         error: error.firstToString()
+      });
+      super(`Invalid Field config given for field "${field.name}": ${error.firstToString()}`);
+   }
+}
+
+export class EntityNotDefinedException extends Exception {
+   override name = "EntityNotDefinedException";
+   override code = 400;
+
+   constructor(entity?: Entity | string) {
+      if (!entity) {
+         super("Cannot find an entity that is undefined");
+      } else {
+         super(`Entity "${typeof entity !== "string" ? entity.name : entity}" not defined`);
+      }
+   }
+}
+
+export class EntityNotFoundException extends Exception {
+   override name = "EntityNotFoundException";
+   override code = 404;
+
+   constructor(entity: Entity | string, id: any) {
+      super(
+         `Entity "${typeof entity !== "string" ? entity.name : entity}" with id "${id}" not found`
+      );
+   }
+}

app/src/data/events/index.ts

@@ -0,0 +1,74 @@
+import type { PrimaryFieldType } from "core";
+import { Event } from "core/events";
+import type { Entity, EntityData } from "../entities";
+import type { RepoQuery } from "../server/data-query-impl";
+
+export class MutatorInsertBefore extends Event<{ entity: Entity; data: EntityData }> {
+   static override slug = "mutator-insert-before";
+}
+export class MutatorInsertAfter extends Event<{ entity: Entity; data: EntityData }> {
+   static override slug = "mutator-insert-after";
+}
+export class MutatorUpdateBefore extends Event<{
+   entity: Entity;
+   entityId: PrimaryFieldType;
+   data: EntityData;
+}> {
+   static override slug = "mutator-update-before";
+}
+export class MutatorUpdateAfter extends Event<{
+   entity: Entity;
+   entityId: PrimaryFieldType;
+   data: EntityData;
+}> {
+   static override slug = "mutator-update-after";
+}
+export class MutatorDeleteBefore extends Event<{ entity: Entity; entityId: PrimaryFieldType }> {
+   static override slug = "mutator-delete-before";
+}
+export class MutatorDeleteAfter extends Event<{
+   entity: Entity;
+   entityId: PrimaryFieldType;
+   data: EntityData;
+}> {
+   static override slug = "mutator-delete-after";
+}
+
+export const MutatorEvents = {
+   MutatorInsertBefore,
+   MutatorInsertAfter,
+   MutatorUpdateBefore,
+   MutatorUpdateAfter,
+   MutatorDeleteBefore,
+   MutatorDeleteAfter
+};
+
+export class RepositoryFindOneBefore extends Event<{ entity: Entity; options: RepoQuery }> {
+   static override slug = "repository-find-one-before";
+}
+export class RepositoryFindOneAfter extends Event<{
+   entity: Entity;
+   options: RepoQuery;
+   data: EntityData;
+}> {
+   static override slug = "repository-find-one-after";
+}
+
+export class RepositoryFindManyBefore extends Event<{ entity: Entity; options: RepoQuery }> {
+   static override slug = "repository-find-many-before";
+   static another = "one";
+}
+export class RepositoryFindManyAfter extends Event<{
+   entity: Entity;
+   options: RepoQuery;
+   data: EntityData;
+}> {
+   static override slug = "repository-find-many-after";
+}
+
+export const RepositoryEvents = {
+   RepositoryFindOneBefore,
+   RepositoryFindOneAfter,
+   RepositoryFindManyBefore,
+   RepositoryFindManyAfter
+};

app/src/data/fields/BooleanField.ts

@@ -0,0 +1,88 @@
+import { type Static, Type } from "core/utils";
+import type { EntityManager } from "data";
+import { TransformPersistFailedException } from "../errors";
+import { Field, type TActionContext, type TRenderContext, baseFieldConfigSchema } from "./Field";
+
+export const booleanFieldConfigSchema = Type.Composite([
+   Type.Object({
+      default_value: Type.Optional(Type.Boolean({ default: false }))
+   }),
+   baseFieldConfigSchema
+]);
+
+export type BooleanFieldConfig = Static<typeof booleanFieldConfigSchema>;
+
+export class BooleanField<Required extends true | false = false> extends Field<
+   BooleanFieldConfig,
+   boolean,
+   Required
+> {
+   override readonly type = "boolean";
+
+   protected getSchema() {
+      return booleanFieldConfigSchema;
+   }
+
+   override getValue(value: unknown, context: TRenderContext) {
+      switch (context) {
+         case "table":
+            return value ? "Yes" : "No";
+         default:
+            return value;
+      }
+   }
+
+   schema() {
+      // @todo: potentially use "integer" instead
+      return this.useSchemaHelper("boolean");
+   }
+
+   override getHtmlConfig() {
+      return {
+         ...super.getHtmlConfig(),
+         element: "boolean"
+      };
+   }
+
+   override transformRetrieve(value: unknown): boolean | null {
+      //console.log("Boolean:transformRetrieve:value", value);
+      if (typeof value === "undefined" || value === null) {
+         if (this.isRequired()) return false;
+         if (this.hasDefault()) return this.getDefault();
+
+         return null;
+      }
+
+      if (typeof value === "string") {
+         return value === "1";
+      }
+
+      // cast to boolean, as it might be stored as number
+      return !!value;
+   }
+
+   override async transformPersist(
+      val: unknown,
+      em: EntityManager<any>,
+      context: TActionContext
+   ): Promise<boolean | undefined> {
+      const value = await super.transformPersist(val, em, context);
+      if (this.nullish(value)) {
+         return this.isRequired() ? Boolean(this.config.default_value) : undefined;
+      }
+
+      if (typeof value === "number") {
+         return value !== 0;
+      }
+
+      if (typeof value !== "boolean") {
+         throw TransformPersistFailedException.invalidType(this.name, "boolean", value);
+      }
+
+      return value as boolean;
+   }
+
+   override toJsonSchema() {
+      return this.toSchemaWrapIfRequired(Type.Boolean({ default: this.getDefault() }));
+   }
+}

app/src/data/fields/DateField.ts

@@ -0,0 +1,151 @@
+import { type Static, StringEnum, Type, dayjs } from "core/utils";
+import type { EntityManager } from "../entities";
+import { Field, type TActionContext, type TRenderContext, baseFieldConfigSchema } from "./Field";
+
+export const dateFieldConfigSchema = Type.Composite(
+   [
+      Type.Object({
+         //default_value: Type.Optional(Type.Date()),
+         type: StringEnum(["date", "datetime", "week"] as const, { default: "date" }),
+         timezone: Type.Optional(Type.String()),
+         min_date: Type.Optional(Type.String()),
+         max_date: Type.Optional(Type.String())
+      }),
+      baseFieldConfigSchema
+   ],
+   {
+      additionalProperties: false
+   }
+);
+
+export type DateFieldConfig = Static<typeof dateFieldConfigSchema>;
+
+export class DateField<Required extends true | false = false> extends Field<
+   DateFieldConfig,
+   Date,
+   Required
+> {
+   override readonly type = "date";
+
+   protected getSchema() {
+      return dateFieldConfigSchema;
+   }
+
+   override schema() {
+      const type = this.config.type === "datetime" ? "datetime" : "date";
+      return this.useSchemaHelper(type);
+   }
+
+   override getHtmlConfig() {
+      const htmlType = this.config.type === "datetime" ? "datetime-local" : this.config.type;
+
+      return {
+         ...super.getHtmlConfig(),
+         element: "date",
+         props: {
+            type: htmlType
+         }
+      };
+   }
+
+   private parseDateFromString(value: string): Date {
+      //console.log("parseDateFromString", value);
+      if (this.config.type === "week" && value.includes("-W")) {
+         const [year, week] = value.split("-W").map((n) => Number.parseInt(n, 10)) as [
+            number,
+            number
+         ];
+         //console.log({ year, week });
+         // @ts-ignore causes errors on build?
+         return dayjs().year(year).week(week).toDate();
+      }
+
+      return new Date(value);
+   }
+
+   override getValue(value: string, context?: TRenderContext): string | undefined {
+      if (value === null || !value) return;
+      //console.log("getValue", { value, context });
+      const date = this.parseDateFromString(value);
+      //console.log("getValue.date", date);
+
+      if (context === "submit") {
+         try {
+            return date.toISOString();
+         } catch (e) {
+            //console.warn("DateField.getValue:value/submit", value, e);
+            return undefined;
+         }
+      }
+
+      if (this.config.type === "week") {
+         try {
+            return `${date.getFullYear()}-W${dayjs(date).week()}`;
+         } catch (e) {
+            console.warn("error - DateField.getValue:week", value, e);
+            return;
+         }
+      }
+
+      try {
+         const utc = new Date();
+         const offset = utc.getTimezoneOffset();
+         //console.log("offset", offset);
+         const local = new Date(date.getTime() - offset * 60000);
+
+         return this.formatDate(local);
+      } catch (e) {
+         console.warn("DateField.getValue:value", value);
+         console.warn("DateField.getValue:e", e);
+         return;
+      }
+   }
+
+   formatDate(_date: Date): string {
+      switch (this.config.type) {
+         case "datetime":
+            return _date.toISOString().split(".")[0]!.replace("T", " ");
+         default:
+            return _date.toISOString().split("T")[0]!;
+         /*case "week": {
+            const date = dayjs(_date);
+            return `${date.year()}-W${date.week()}`;
+         }*/
+      }
+   }
+
+   override transformRetrieve(_value: string): Date | null {
+      //console.log("transformRetrieve DateField", _value);
+      const value = super.transformRetrieve(_value);
+      if (value === null) return null;
+
+      try {
+         return new Date(value);
+      } catch (e) {
+         return null;
+      }
+   }
+
+   override async transformPersist(
+      _value: any,
+      em: EntityManager<any>,
+      context: TActionContext
+   ): Promise<string | undefined> {
+      const value = await super.transformPersist(_value, em, context);
+      if (this.nullish(value)) return value;
+
+      //console.log("transformPersist DateField", value);
+      switch (this.config.type) {
+         case "date":
+         case "week":
+            return new Date(value).toISOString().split("T")[0]!;
+         default:
+            return new Date(value).toISOString();
+      }
+   }
+
+   // @todo: check this
+   override toJsonSchema() {
+      return this.toSchemaWrapIfRequired(Type.String({ default: this.getDefault() }));
+   }
+}

app/src/data/fields/EnumField.ts

@@ -0,0 +1,153 @@
+import { Const, type Static, StringEnum, StringRecord, Type } from "core/utils";
+import type { EntityManager } from "data";
+import { TransformPersistFailedException } from "../errors";
+import { Field, type TActionContext, type TRenderContext, baseFieldConfigSchema } from "./Field";
+
+export const enumFieldConfigSchema = Type.Composite(
+   [
+      Type.Object({
+         default_value: Type.Optional(Type.String()),
+         options: Type.Optional(
+            Type.Union([
+               Type.Object(
+                  {
+                     type: Const("strings"),
+                     values: Type.Array(Type.String())
+                  },
+                  { title: "Strings" }
+               ),
+               Type.Object(
+                  {
+                     type: Const("objects"),
+                     values: Type.Array(
+                        Type.Object({
+                           label: Type.String(),
+                           value: Type.String()
+                        })
+                     )
+                  },
+                  {
+                     title: "Objects",
+                     additionalProperties: false
+                  }
+               )
+            ])
+         )
+      }),
+      baseFieldConfigSchema
+   ],
+   {
+      additionalProperties: false
+   }
+);
+
+export type EnumFieldConfig = Static<typeof enumFieldConfigSchema>;
+
+export class EnumField<Required extends true | false = false, TypeOverride = string> extends Field<
+   EnumFieldConfig,
+   TypeOverride,
+   Required
+> {
+   override readonly type = "enum";
+
+   constructor(name: string, config: Partial<EnumFieldConfig>) {
+      super(name, config);
+
+      /*if (this.config.options.values.length === 0) {
+         throw new Error(`Enum field "${this.name}" requires at least one option`);
+      }*/
+
+      if (this.config.default_value && !this.isValidValue(this.config.default_value)) {
+         throw new Error(`Default value "${this.config.default_value}" is not a valid option`);
+      }
+   }
+
+   protected getSchema() {
+      return enumFieldConfigSchema;
+   }
+
+   override schema() {
+      return this.useSchemaHelper("text");
+   }
+
+   getOptions(): { label: string; value: string }[] {
+      const options = this.config?.options ?? { type: "strings", values: [] };
+
+      /*if (options.values?.length === 0) {
+         throw new Error(`Enum field "${this.name}" requires at least one option`);
+      }*/
+
+      if (options.type === "strings") {
+         return options.values?.map((option) => ({ label: option, value: option }));
+      }
+
+      return options?.values;
+   }
+
+   isValidValue(value: string): boolean {
+      const valid_values = this.getOptions().map((option) => option.value);
+      return valid_values.includes(value);
+   }
+
+   override getValue(value: any, context: TRenderContext) {
+      if (!this.isValidValue(value)) {
+         return this.hasDefault() ? this.getDefault() : null;
+      }
+
+      switch (context) {
+         case "table":
+            return this.getOptions().find((option) => option.value === value)?.label ?? value;
+      }
+
+      return value;
+   }
+
+   /**
+    * Transform value after retrieving from database
+    * @param value
+    */
+   override transformRetrieve(value: string | null): string | null {
+      const val = super.transformRetrieve(value);
+
+      if (val === null && this.hasDefault()) {
+         return this.getDefault();
+      }
+
+      if (!this.isValidValue(val)) {
+         return this.hasDefault() ? this.getDefault() : null;
+      }
+
+      return val;
+   }
+
+   override async transformPersist(
+      _value: any,
+      em: EntityManager<any>,
+      context: TActionContext
+   ): Promise<string | undefined> {
+      const value = await super.transformPersist(_value, em, context);
+      if (this.nullish(value)) return value;
+
+      if (!this.isValidValue(value)) {
+         throw new TransformPersistFailedException(
+            `Field "${this.name}" must be one of the following values: ${this.getOptions()
+               .map((o) => o.value)
+               .join(", ")}`
+         );
+      }
+
+      return value;
+   }
+
+   override toJsonSchema() {
+      const options = this.config?.options ?? { type: "strings", values: [] };
+      const values =
+         options.values?.map((option) => (typeof option === "string" ? option : option.value)) ??
+         [];
+      return this.toSchemaWrapIfRequired(
+         StringEnum(values, {
+            default: this.getDefault()
+         })
+      );
+   }
+}

app/src/data/fields/Field.ts

@@ -0,0 +1,244 @@
+import {
+   type Static,
+   StringEnum,
+   type TSchema,
+   Type,
+   TypeInvalidError,
+   parse,
+   snakeToPascalWithSpaces
+} from "core/utils";
+import type { ColumnBuilderCallback, ColumnDataType, ColumnDefinitionBuilder } from "kysely";
+import type { HTMLInputTypeAttribute, InputHTMLAttributes } from "react";
+import type { EntityManager } from "../entities";
+import { InvalidFieldConfigException, TransformPersistFailedException } from "../errors";
+
+export const ActionContext = ["create", "read", "update", "delete"] as const;
+export type TActionContext = (typeof ActionContext)[number];
+
+export const RenderContext = ["form", "table", "read", "submit"] as const;
+export type TRenderContext = (typeof RenderContext)[number];
+
+const TmpContext = ["create", "read", "update", "delete", "form", "table", "submit"] as const;
+export type TmpActionAndRenderContext = (typeof TmpContext)[number];
+
+const DEFAULT_REQUIRED = false;
+const DEFAULT_FILLABLE = true;
+const DEFAULT_HIDDEN = false;
+
+// @todo: add refine functions (e.g. if required, but not fillable, needs default value)
+export const baseFieldConfigSchema = Type.Object(
+   {
+      label: Type.Optional(Type.String()),
+      description: Type.Optional(Type.String()),
+      required: Type.Optional(Type.Boolean({ default: DEFAULT_REQUIRED })),
+      fillable: Type.Optional(
+         Type.Union(
+            [
+               Type.Boolean({ title: "Boolean", default: DEFAULT_FILLABLE }),
+               Type.Array(StringEnum(ActionContext), { title: "Context", uniqueItems: true })
+            ],
+            {
+               default: DEFAULT_FILLABLE
+            }
+         )
+      ),
+      hidden: Type.Optional(
+         Type.Union(
+            [
+               Type.Boolean({ title: "Boolean", default: DEFAULT_HIDDEN }),
+               // @todo: tmp workaround
+               Type.Array(StringEnum(TmpContext), { title: "Context", uniqueItems: true })
+            ],
+            {
+               default: DEFAULT_HIDDEN
+            }
+         )
+      ),
+      // if field is virtual, it will not call transformPersist & transformRetrieve
+      virtual: Type.Optional(Type.Boolean()),
+      default_value: Type.Optional(Type.Any())
+   },
+   {
+      additionalProperties: false
+   }
+);
+export type BaseFieldConfig = Static<typeof baseFieldConfigSchema>;
+
+export type SchemaResponse = [string, ColumnDataType, ColumnBuilderCallback] | undefined;
+
+export abstract class Field<
+   Config extends BaseFieldConfig = BaseFieldConfig,
+   Type = any,
+   Required extends true | false = false
+> {
+   _required!: Required;
+   _type!: Type;
+
+   /**
+    * Property name that gets persisted on database
+    */
+   readonly name: string;
+   readonly type: string = "field";
+   readonly config: Config;
+
+   constructor(name: string, config?: Partial<Config>) {
+      this.name = name;
+      this._type;
+      this._required;
+
+      try {
+         this.config = parse(this.getSchema(), config || {}) as Config;
+      } catch (e) {
+         if (e instanceof TypeInvalidError) {
+            throw new InvalidFieldConfigException(this, config, e);
+         }
+
+         throw e;
+      }
+   }
+
+   getType() {
+      return this.type;
+   }
+
+   protected abstract getSchema(): TSchema;
+
+   protected useSchemaHelper(
+      type: ColumnDataType,
+      builder?: (col: ColumnDefinitionBuilder) => ColumnDefinitionBuilder
+   ): SchemaResponse {
+      return [
+         this.name,
+         type,
+         (col: ColumnDefinitionBuilder) => {
+            if (builder) return builder(col);
+            return col;
+         }
+      ];
+   }
+
+   /**
+    * Used in SchemaManager.ts
+    * @param em
+    */
+   abstract schema(em: EntityManager<any>): SchemaResponse;
+
+   hasDefault() {
+      return this.config.default_value !== undefined;
+   }
+
+   getDefault() {
+      return this.config?.default_value;
+   }
+
+   isFillable(context?: TActionContext): boolean {
+      if (Array.isArray(this.config.fillable)) {
+         return context ? this.config.fillable.includes(context) : DEFAULT_FILLABLE;
+      }
+      return !!this.config.fillable;
+   }
+
+   isHidden(context?: TmpActionAndRenderContext): boolean {
+      if (Array.isArray(this.config.hidden)) {
+         return context ? this.config.hidden.includes(context as any) : DEFAULT_HIDDEN;
+      }
+      return this.config.hidden ?? false;
+   }
+
+   isRequired(): boolean {
+      return this.config?.required ?? false;
+   }
+
+   /**
+    * Virtual fields are not persisted or retrieved from database
+    * Used for MediaField, to add specifics about uploads, etc.
+    */
+   isVirtual(): boolean {
+      return this.config.virtual ?? false;
+   }
+
+   getLabel(): string {
+      return this.config.label ?? snakeToPascalWithSpaces(this.name);
+   }
+
+   getDescription(): string | undefined {
+      return this.config.description;
+   }
+
+   /**
+    * [GET] DB -> field.transformRetrieve -> [sent]
+    *   table: form.getValue("table")
+    *   form: form.getValue("form") -> modified -> form.getValue("submit") -> [sent]
+    *
+    * [PATCH] body parse json -> field.transformPersist -> [stored]
+    *
+    * @param value
+    * @param context
+    */
+   getValue(value: any, context?: TRenderContext) {
+      return value;
+   }
+
+   getHtmlConfig(): { element: HTMLInputTypeAttribute | string; props?: InputHTMLAttributes<any> } {
+      return {
+         element: "input",
+         props: { type: "text" }
+      };
+   }
+
+   isValid(value: any, context: TActionContext): boolean {
+      if (value) {
+         return this.isFillable(context);
+      } else {
+         return !this.isRequired();
+      }
+   }
+
+   /**
+    * Transform value after retrieving from database
+    * @param value
+    */
+   transformRetrieve(value: any): any {
+      return value;
+   }
+
+   /**
+    * Transform value before persisting to database
+    * @param value
+    * @param em EntityManager (optional, for relation fields)
+    */
+   async transformPersist(
+      value: unknown,
+      em: EntityManager<any>,
+      context: TActionContext
+   ): Promise<any> {
+      if (this.nullish(value)) {
+         if (this.isRequired() && !this.hasDefault()) {
+            throw TransformPersistFailedException.required(this.name);
+         }
+         return this.getDefault();
+      }
+
+      return value;
+   }
+
+   protected toSchemaWrapIfRequired<Schema extends TSchema>(schema: Schema) {
+      return this.isRequired() ? schema : Type.Optional(schema);
+   }
+
+   protected nullish(value: any) {
+      return value === null || value === undefined;
+   }
+
+   toJsonSchema(): TSchema {
+      return this.toSchemaWrapIfRequired(Type.Any());
+   }
+
+   toJSON() {
+      return {
+         //name: this.name,
+         type: this.type,
+         config: this.config
+      };
+   }
+}

app/src/data/fields/JsonField.ts

@@ -0,0 +1,104 @@
+import { type Static, Type } from "core/utils";
+import type { EntityManager } from "data";
+import { TransformPersistFailedException } from "../errors";
+import { Field, type TActionContext, type TRenderContext, baseFieldConfigSchema } from "./Field";
+
+export const jsonFieldConfigSchema = Type.Composite([baseFieldConfigSchema, Type.Object({})]);
+
+export type JsonFieldConfig = Static<typeof jsonFieldConfigSchema>;
+
+export class JsonField<Required extends true | false = false, TypeOverride = object> extends Field<
+   JsonFieldConfig,
+   TypeOverride,
+   Required
+> {
+   override readonly type = "json";
+
+   protected getSchema() {
+      return jsonFieldConfigSchema;
+   }
+
+   override schema() {
+      return this.useSchemaHelper("text");
+   }
+
+   /**
+    * Transform value after retrieving from database
+    * @param value
+    */
+   override transformRetrieve(value: any): any {
+      const val = super.transformRetrieve(value);
+
+      if (val === null && this.hasDefault()) {
+         return this.getDefault();
+      }
+
+      if (this.isSerialized(val)) {
+         return JSON.parse(val);
+      }
+
+      return val;
+   }
+
+   isSerializable(value: any) {
+      try {
+         const stringified = JSON.stringify(value);
+         if (stringified === JSON.stringify(JSON.parse(stringified))) {
+            return true;
+         }
+      } catch (e) {}
+
+      return false;
+   }
+
+   isSerialized(value: any) {
+      try {
+         if (typeof value === "string") {
+            return value === JSON.stringify(JSON.parse(value));
+         }
+      } catch (e) {}
+
+      return false;
+   }
+
+   override getValue(value: any, context: TRenderContext): any {
+      switch (context) {
+         case "form":
+            if (value === null) return "";
+            return JSON.stringify(value, null, 2);
+         case "table":
+            if (value === null) return null;
+            return JSON.stringify(value);
+         case "submit":
+            if (typeof value === "string" && value.length === 0) {
+               return null;
+            }
+
+            return JSON.parse(value);
+      }
+
+      return value;
+   }
+
+   override async transformPersist(
+      _value: any,
+      em: EntityManager<any>,
+      context: TActionContext
+   ): Promise<string | undefined> {
+      const value = await super.transformPersist(_value, em, context);
+      //console.log("value", value);
+      if (this.nullish(value)) return value;
+
+      if (!this.isSerializable(value)) {
+         throw new TransformPersistFailedException(
+            `Field "${this.name}" must be serializable to JSON.`
+         );
+      }
+
+      if (this.isSerialized(value)) {
+         return value;
+      }
+
+      return JSON.stringify(value);
+   }
+}

app/src/data/fields/JsonSchemaField.ts

@@ -0,0 +1,132 @@
+import { type Schema as JsonSchema, Validator } from "@cfworker/json-schema";
+import { Default, FromSchema, type Static, Type } from "core/utils";
+import type { EntityManager } from "data";
+import { TransformPersistFailedException } from "../errors";
+import { Field, type TActionContext, type TRenderContext, baseFieldConfigSchema } from "./Field";
+
+export const jsonSchemaFieldConfigSchema = Type.Composite(
+   [
+      Type.Object({
+         schema: Type.Object({}, { default: {} }),
+         ui_schema: Type.Optional(Type.Object({})),
+         default_from_schema: Type.Optional(Type.Boolean())
+      }),
+      baseFieldConfigSchema
+   ],
+   {
+      additionalProperties: false
+   }
+);
+
+export type JsonSchemaFieldConfig = Static<typeof jsonSchemaFieldConfigSchema>;
+
+export class JsonSchemaField<
+   Required extends true | false = false,
+   TypeOverride = object
+> extends Field<JsonSchemaFieldConfig, TypeOverride, Required> {
+   override readonly type = "jsonschema";
+   private validator: Validator;
+
+   constructor(name: string, config: Partial<JsonSchemaFieldConfig>) {
+      super(name, config);
+      this.validator = new Validator(this.getJsonSchema());
+   }
+
+   protected getSchema() {
+      return jsonSchemaFieldConfigSchema;
+   }
+
+   override schema() {
+      return this.useSchemaHelper("text");
+   }
+
+   getJsonSchema(): JsonSchema {
+      return this.config?.schema as JsonSchema;
+   }
+
+   getJsonUiSchema() {
+      return this.config.ui_schema ?? {};
+   }
+
+   override isValid(value: any, context: TActionContext = "update"): boolean {
+      const parentValid = super.isValid(value, context);
+      //console.log("jsonSchemaField:isValid", this.getJsonSchema(), this.name, value, parentValid);
+
+      if (parentValid) {
+         // already checked in parent
+         if (!value || typeof value !== "object") {
+            //console.log("jsonschema:valid: not checking", this.name, value, context);
+            return true;
+         }
+
+         const result = this.validator.validate(value);
+         //console.log("jsonschema:errors", this.name, result.errors);
+         return result.valid;
+      } else {
+         //console.log("jsonschema:invalid", this.name, value, context);
+      }
+
+      return false;
+   }
+
+   override getValue(value: any, context: TRenderContext): any {
+      switch (context) {
+         case "form":
+            if (value === null) return "";
+            return value;
+         case "table":
+            if (value === null) return null;
+            return value;
+         case "submit":
+            break;
+      }
+
+      return value;
+   }
+
+   override transformRetrieve(value: any): any {
+      const val = super.transformRetrieve(value);
+
+      if (val === null) {
+         if (this.config.default_from_schema) {
+            try {
+               return Default(FromSchema(this.getJsonSchema()), {});
+            } catch (e) {
+               //console.error("jsonschema:transformRetrieve", e);
+               return null;
+            }
+         } else if (this.hasDefault()) {
+            return this.getDefault();
+         }
+      }
+
+      return val;
+   }
+
+   override async transformPersist(
+      _value: any,
+      em: EntityManager<any>,
+      context: TActionContext
+   ): Promise<string | undefined> {
+      const value = await super.transformPersist(_value, em, context);
+      if (this.nullish(value)) return value;
+
+      if (!this.isValid(value)) {
+         throw new TransformPersistFailedException(this.name, value);
+      }
+
+      if (!value || typeof value !== "object") return this.getDefault();
+
+      return JSON.stringify(value);
+   }
+
+   override toJsonSchema() {
+      const schema = this.getJsonSchema() ?? { type: "object" };
+      return this.toSchemaWrapIfRequired(
+         FromSchema({
+            default: this.getDefault(),
+            ...schema
+         })
+      );
+   }
+}

app/src/data/fields/NumberField.ts

@@ -0,0 +1,100 @@
+import { type Static, Type } from "core/utils";
+import type { EntityManager } from "data";
+import { TransformPersistFailedException } from "../errors";
+import { Field, type TActionContext, type TRenderContext, baseFieldConfigSchema } from "./Field";
+
+export const numberFieldConfigSchema = Type.Composite(
+   [
+      Type.Object({
+         default_value: Type.Optional(Type.Number()),
+         minimum: Type.Optional(Type.Number()),
+         maximum: Type.Optional(Type.Number()),
+         exclusiveMinimum: Type.Optional(Type.Number()),
+         exclusiveMaximum: Type.Optional(Type.Number()),
+         multipleOf: Type.Optional(Type.Number())
+      }),
+      baseFieldConfigSchema
+   ],
+   {
+      additionalProperties: false
+   }
+);
+
+export type NumberFieldConfig = Static<typeof numberFieldConfigSchema>;
+
+export class NumberField<Required extends true | false = false> extends Field<
+   NumberFieldConfig,
+   number,
+   Required
+> {
+   override readonly type = "number";
+
+   protected getSchema() {
+      return numberFieldConfigSchema;
+   }
+
+   override getHtmlConfig() {
+      return {
+         element: "input",
+         props: {
+            type: "number",
+            pattern: "d*",
+            inputMode: "numeric"
+         } as any // @todo: react expects "inputMode", but type dictates "inputmode"
+      };
+   }
+
+   schema() {
+      return this.useSchemaHelper("integer");
+   }
+
+   override getValue(value: any, context?: TRenderContext): any {
+      if (typeof value === "undefined" || value === null) return null;
+
+      switch (context) {
+         case "submit":
+            return Number.parseInt(value);
+      }
+
+      return value;
+   }
+
+   override async transformPersist(
+      _value: unknown,
+      em: EntityManager<any>,
+      context: TActionContext
+   ): Promise<number | undefined> {
+      const value = await super.transformPersist(_value, em, context);
+
+      if (!this.nullish(value) && typeof value !== "number") {
+         throw TransformPersistFailedException.invalidType(this.name, "number", value);
+      }
+
+      if (this.config.maximum && (value as number) > this.config.maximum) {
+         throw new TransformPersistFailedException(
+            `Field "${this.name}" cannot be greater than ${this.config.maximum}`
+         );
+      }
+
+      if (this.config.minimum && (value as number) < this.config.minimum) {
+         throw new TransformPersistFailedException(
+            `Field "${this.name}" cannot be less than ${this.config.minimum}`
+         );
+      }
+
+      return value as number;
+   }
+
+   override toJsonSchema() {
+      return this.toSchemaWrapIfRequired(
+         Type.Number({
+            default: this.getDefault(),
+            minimum: this.config?.minimum,
+            maximum: this.config?.maximum,
+            exclusiveMinimum: this.config?.exclusiveMinimum,
+            exclusiveMaximum: this.config?.exclusiveMaximum,
+            multipleOf: this.config?.multipleOf
+         })
+      );
+   }
+}

app/src/data/fields/PrimaryField.ts

@@ -0,0 +1,46 @@
+import { config } from "core";
+import { type Static, Type } from "core/utils";
+import { Field, baseFieldConfigSchema } from "./Field";
+
+export const primaryFieldConfigSchema = Type.Composite([
+   Type.Omit(baseFieldConfigSchema, ["required"]),
+   Type.Object({
+      required: Type.Optional(Type.Literal(false))
+   })
+]);
+
+export type PrimaryFieldConfig = Static<typeof primaryFieldConfigSchema>;
+
+export class PrimaryField<Required extends true | false = false> extends Field<
+   PrimaryFieldConfig,
+   string,
+   Required
+> {
+   override readonly type = "primary";
+
+   constructor(name: string = config.data.default_primary_field) {
+      super(name, { fillable: false, required: false });
+   }
+
+   override isRequired(): boolean {
+      return false;
+   }
+
+   protected getSchema() {
+      return baseFieldConfigSchema;
+   }
+
+   schema() {
+      return this.useSchemaHelper("integer", (col) => {
+         return col.primaryKey().notNull().autoIncrement();
+      });
+   }
+
+   override async transformPersist(value: any): Promise<number> {
+      throw new Error("This function should not be called");
+   }
+
+   override toJsonSchema() {
+      return this.toSchemaWrapIfRequired(Type.Number({ writeOnly: undefined }));
+   }
+}

app/src/data/fields/TextField.ts

@@ -0,0 +1,120 @@
+import { type Static, Type } from "core/utils";
+import type { EntityManager } from "data";
+import { TransformPersistFailedException } from "../errors";
+import { Field, type TActionContext, baseFieldConfigSchema } from "./Field";
+
+export const textFieldConfigSchema = Type.Composite(
+   [
+      Type.Object({
+         default_value: Type.Optional(Type.String()),
+         minLength: Type.Optional(Type.Number()),
+         maxLength: Type.Optional(Type.Number()),
+         pattern: Type.Optional(Type.String()),
+         html_config: Type.Optional(
+            Type.Object({
+               element: Type.Optional(Type.String({ default: "input" })),
+               props: Type.Optional(
+                  Type.Object(
+                     {},
+                     {
+                        additionalProperties: Type.Union([
+                           Type.String({ title: "String" }),
+                           Type.Number({ title: "Number" })
+                        ])
+                     }
+                  )
+               )
+            })
+         )
+      }),
+      baseFieldConfigSchema
+   ],
+   {
+      additionalProperties: false
+   }
+);
+
+export type TextFieldConfig = Static<typeof textFieldConfigSchema>;
+
+export class TextField<Required extends true | false = false> extends Field<
+   TextFieldConfig,
+   string,
+   Required
+> {
+   override readonly type = "text";
+
+   protected getSchema() {
+      return textFieldConfigSchema;
+   }
+
+   override schema() {
+      return this.useSchemaHelper("text");
+   }
+
+   override getHtmlConfig() {
+      if (this.config.html_config) {
+         return this.config.html_config as any;
+      }
+
+      return super.getHtmlConfig();
+   }
+
+   /**
+    * Transform value after retrieving from database
+    * @param value
+    */
+   override transformRetrieve(value: string): string | null {
+      const val = super.transformRetrieve(value);
+
+      // @todo: now sure about these two
+      if (this.config.maxLength) {
+         return val.substring(0, this.config.maxLength);
+      }
+
+      if (this.isRequired()) {
+         return val ? val.toString() : "";
+      }
+
+      return val;
+   }
+
+   override async transformPersist(
+      _value: any,
+      em: EntityManager<any>,
+      context: TActionContext
+   ): Promise<string | undefined> {
+      let value = await super.transformPersist(_value, em, context);
+
+      if (this.nullish(value)) return value;
+
+      // transform to string
+      if (value !== null && typeof value !== "string") {
+         value = String(value);
+      }
+
+      if (this.config.maxLength && value?.length > this.config.maxLength) {
+         throw new TransformPersistFailedException(
+            `Field "${this.name}" must be at most ${this.config.maxLength} character(s)`
+         );
+      }
+
+      if (this.config.minLength && value?.length < this.config.minLength) {
+         throw new TransformPersistFailedException(
+            `Field "${this.name}" must be at least ${this.config.minLength} character(s)`
+         );
+      }
+
+      return value;
+   }
+
+   override toJsonSchema() {
+      return this.toSchemaWrapIfRequired(
+         Type.String({
+            default: this.getDefault(),
+            minLength: this.config?.minLength,
+            maxLength: this.config?.maxLength,
+            pattern: this.config?.pattern
+         })
+      );
+   }
+}

app/src/data/fields/VirtualField.ts

@@ -0,0 +1,32 @@
+import { type Static, Type } from "core/utils";
+import { Field, baseFieldConfigSchema } from "./Field";
+
+export const virtualFieldConfigSchema = Type.Composite([baseFieldConfigSchema, Type.Object({})]);
+
+export type VirtualFieldConfig = Static<typeof virtualFieldConfigSchema>;
+
+export class VirtualField extends Field<VirtualFieldConfig> {
+   override readonly type = "virtual";
+
+   constructor(name: string, config?: Partial<VirtualFieldConfig>) {
+      // field must be virtual, as it doesn't store a reference to the entity
+      super(name, { ...config, fillable: false, virtual: true });
+   }
+
+   protected getSchema() {
+      return virtualFieldConfigSchema;
+   }
+
+   schema() {
+      return undefined;
+   }
+
+   override toJsonSchema() {
+      return this.toSchemaWrapIfRequired(
+         Type.Any({
+            default: this.getDefault(),
+            readOnly: true
+         })
+      );
+   }
+}

app/src/data/fields/index.ts

@@ -0,0 +1,55 @@
+import { BooleanField, type BooleanFieldConfig, booleanFieldConfigSchema } from "./BooleanField";
+import { DateField, type DateFieldConfig, dateFieldConfigSchema } from "./DateField";
+import { EnumField, type EnumFieldConfig, enumFieldConfigSchema } from "./EnumField";
+import { JsonField, type JsonFieldConfig, jsonFieldConfigSchema } from "./JsonField";
+import {
+   JsonSchemaField,
+   type JsonSchemaFieldConfig,
+   jsonSchemaFieldConfigSchema
+} from "./JsonSchemaField";
+import { NumberField, type NumberFieldConfig, numberFieldConfigSchema } from "./NumberField";
+import { PrimaryField, type PrimaryFieldConfig, primaryFieldConfigSchema } from "./PrimaryField";
+import { TextField, type TextFieldConfig, textFieldConfigSchema } from "./TextField";
+
+export {
+   PrimaryField,
+   primaryFieldConfigSchema,
+   type PrimaryFieldConfig,
+   BooleanField,
+   booleanFieldConfigSchema,
+   type BooleanFieldConfig,
+   DateField,
+   dateFieldConfigSchema,
+   type DateFieldConfig,
+   EnumField,
+   enumFieldConfigSchema,
+   type EnumFieldConfig,
+   JsonField,
+   jsonFieldConfigSchema,
+   type JsonFieldConfig,
+   JsonSchemaField,
+   jsonSchemaFieldConfigSchema,
+   type JsonSchemaFieldConfig,
+   NumberField,
+   numberFieldConfigSchema,
+   type NumberFieldConfig,
+   TextField,
+   textFieldConfigSchema,
+   type TextFieldConfig
+};
+
+export * from "./Field";
+export * from "./PrimaryField";
+export * from "./VirtualField";
+export * from "./indices/EntityIndex";
+
+export const FieldClassMap = {
+   primary: { schema: primaryFieldConfigSchema, field: PrimaryField },
+   text: { schema: textFieldConfigSchema, field: TextField },
+   number: { schema: numberFieldConfigSchema, field: NumberField },
+   boolean: { schema: booleanFieldConfigSchema, field: BooleanField },
+   date: { schema: dateFieldConfigSchema, field: DateField },
+   enum: { schema: enumFieldConfigSchema, field: EnumField },
+   json: { schema: jsonFieldConfigSchema, field: JsonField },
+   jsonschema: { schema: jsonSchemaFieldConfigSchema, field: JsonSchemaField }
+} as const;

app/src/data/fields/indices/EntityIndex.ts

@@ -0,0 +1,46 @@
+import type { Entity } from "../../entities";
+import { Field } from "../Field";
+
+export class EntityIndex {
+   constructor(
+      public entity: Entity,
+      public fields: Field[],
+      public unique: boolean = false,
+      public name?: string
+   ) {
+      if (fields.length === 0) {
+         throw new Error("Indices must contain at least one field");
+      }
+      if (fields.some((f) => !(f instanceof Field))) {
+         throw new Error("All fields must be instances of Field");
+      }
+
+      if (unique) {
+         const firstRequired = fields[0]?.isRequired();
+         if (!firstRequired) {
+            throw new Error(
+               `Unique indices must have first field as required: ${fields
+                  .map((f) => f.name)
+                  .join(", ")}`
+            );
+         }
+      }
+
+      if (!name) {
+         this.name = [
+            unique ? "idx_unique" : "idx",
+            entity.name,
+            ...fields.map((f) => f.name)
+         ].join("_");
+      }
+   }
+
+   toJSON() {
+      return {
+         entity: this.entity.name,
+         fields: this.fields.map((f) => f.name),
+         //name: this.name,
+         unique: this.unique
+      };
+   }
+}

app/src/data/helper.ts

@@ -0,0 +1,48 @@
+import type { EntityData, Field } from "data";
+import { transform } from "lodash-es";
+
+export function getDefaultValues(fields: Field[], data: EntityData): EntityData {
+   return transform(
+      fields,
+      (acc, field) => {
+         // form fields don't like "null" or "undefined", so return empty string
+         acc[field.name] = field.getValue(data?.[field.name], "form") ?? "";
+      },
+      {} as EntityData
+   );
+}
+
+export function getChangeSet(
+   action: string,
+   formData: EntityData,
+   data: EntityData,
+   fields: Field[]
+): EntityData {
+   return transform(
+      formData,
+      (acc, _value, key) => {
+         const field = fields.find((f) => f.name === key);
+         // @todo: filtering virtual here, need to check (because of media)
+         if (!field || field.isVirtual()) return;
+         const value = _value === "" ? null : _value;
+
+         const newValue = field.getValue(value, "submit");
+         // @todo: add typing for "action"
+         if (action === "create" || newValue !== data[key]) {
+            acc[key] = newValue;
+            console.log("changed", {
+               key,
+               value,
+               valueType: typeof value,
+               prev: data[key],
+               newValue,
+               new: value,
+               sent: acc[key]
+            });
+         } else {
+            //console.log("no change", key, value, data[key]);
+         }
+      },
+      {} as typeof formData
+   );
+}

app/src/data/index.ts

@@ -0,0 +1,28 @@
+import { MutatorEvents, RepositoryEvents } from "./events";
+
+export * from "./fields";
+export * from "./entities";
+export * from "./relations";
+export * from "./schema/SchemaManager";
+
+export {
+   type RepoQuery,
+   defaultQuerySchema,
+   querySchema,
+   whereSchema
+} from "./server/data-query-impl";
+
+export { whereRepoSchema as deprecated__whereRepoSchema } from "./server/query";
+
+export { Connection } from "./connection/Connection";
+export { LibsqlConnection, type LibSqlCredentials } from "./connection/LibsqlConnection";
+export { SqliteConnection } from "./connection/SqliteConnection";
+export { SqliteLocalConnection } from "./connection/SqliteLocalConnection";
+
+export const DatabaseEvents = {
+   ...MutatorEvents,
+   ...RepositoryEvents
+};
+export { MutatorEvents, RepositoryEvents };
+
+export * as DataPermissions from "./permissions";

app/src/data/permissions/index.ts

@@ -0,0 +1,9 @@
+import { Permission } from "core";
+
+export const entityRead = new Permission("data.entity.read");
+export const entityCreate = new Permission("data.entity.create");
+export const entityUpdate = new Permission("data.entity.update");
+export const entityDelete = new Permission("data.entity.delete");
+export const databaseSync = new Permission("data.database.sync");
+export const rawQuery = new Permission("data.raw.query");
+export const rawMutate = new Permission("data.raw.mutate");

app/src/data/plugins/DeserializeJsonValuesPlugin.ts

@@ -0,0 +1,36 @@
+import type {
+   KyselyPlugin,
+   PluginTransformQueryArgs,
+   PluginTransformResultArgs,
+   QueryResult,
+   RootOperationNode,
+   UnknownRow,
+} from "kysely";
+
+type KeyValueObject = { [key: string]: any };
+
+export class DeserializeJsonValuesPlugin implements KyselyPlugin {
+   transformQuery(args: PluginTransformQueryArgs): RootOperationNode {
+      return args.node;
+   }
+   transformResult(
+      args: PluginTransformResultArgs
+   ): Promise<QueryResult<UnknownRow>> {
+      return Promise.resolve({
+         ...args.result,
+         rows: args.result.rows.map((row: KeyValueObject) => {
+            const result: KeyValueObject = {};
+            for (const key in row) {
+               try {
+                  // Attempt to parse the value as JSON
+                  result[key] = JSON.parse(row[key]);
+               } catch (error) {
+                  // If parsing fails, keep the original value
+                  result[key] = row[key];
+               }
+            }
+            return result;
+         }),
+      });
+   }
+}

app/src/data/plugins/FilterNumericKeysPlugin.ts

@@ -0,0 +1,31 @@
+import type {
+   KyselyPlugin,
+   PluginTransformQueryArgs,
+   PluginTransformResultArgs,
+   QueryResult,
+   RootOperationNode,
+   UnknownRow,
+} from "kysely";
+
+type KeyValueObject = { [key: string]: any };
+
+export class FilterNumericKeysPlugin implements KyselyPlugin {
+   transformQuery(args: PluginTransformQueryArgs): RootOperationNode {
+      return args.node;
+   }
+   transformResult(args: PluginTransformResultArgs): Promise<QueryResult<UnknownRow>> {
+      return Promise.resolve({
+         ...args.result,
+         rows: args.result.rows.map((row: KeyValueObject) => {
+            const filteredObj: KeyValueObject = {};
+            for (const key in row) {
+               if (Number.isNaN(+key)) {
+                  // Check if the key is not a number
+                  filteredObj[key] = row[key];
+               }
+            }
+            return filteredObj;
+         }),
+      });
+   }
+}

app/src/data/plugins/KyselyPluginRunner.ts

@@ -0,0 +1,23 @@
+import type { KyselyPlugin, UnknownRow } from "kysely";
+
+// @todo: add test
+export class KyselyPluginRunner {
+   protected plugins: Set<KyselyPlugin>;
+
+   constructor(plugins: KyselyPlugin[] = []) {
+      this.plugins = new Set(plugins);
+   }
+
+   async transformResultRows<T>(rows: T[]): Promise<T[]> {
+      let copy = rows;
+      for (const plugin of this.plugins) {
+         const res = await plugin.transformResult({
+            queryId: "1" as any,
+            result: { rows: copy as UnknownRow[] },
+         });
+         copy = res.rows as T[];
+      }
+
+      return copy as T[];
+   }
+}

app/src/data/prototype/index.ts

@@ -0,0 +1,295 @@
+import {
+   BooleanField,
+   type BooleanFieldConfig,
+   DateField,
+   type DateFieldConfig,
+   Entity,
+   type EntityConfig,
+   EnumField,
+   type EnumFieldConfig,
+   type Field,
+   JsonField,
+   type JsonFieldConfig,
+   JsonSchemaField,
+   type JsonSchemaFieldConfig,
+   ManyToManyRelation,
+   type ManyToManyRelationConfig,
+   ManyToOneRelation,
+   type ManyToOneRelationConfig,
+   NumberField,
+   type NumberFieldConfig,
+   OneToOneRelation,
+   type OneToOneRelationConfig,
+   PolymorphicRelation,
+   type PolymorphicRelationConfig,
+   type TEntityType,
+   TextField,
+   type TextFieldConfig
+} from "data";
+import type { Generated } from "kysely";
+import { MediaField, type MediaFieldConfig, type MediaItem } from "media/MediaField";
+
+type Options<Config = any> = {
+   entity: { name: string; fields: Record<string, Field<any, any, any>> };
+   field_name: string;
+   config: Config;
+   is_required: boolean;
+};
+
+const FieldMap = {
+   text: (o: Options) => new TextField(o.field_name, { ...o.config, required: o.is_required }),
+   number: (o: Options) => new NumberField(o.field_name, { ...o.config, required: o.is_required }),
+   date: (o: Options) => new DateField(o.field_name, { ...o.config, required: o.is_required }),
+   datetime: (o: Options) => new DateField(o.field_name, { ...o.config, required: o.is_required }),
+   boolean: (o: Options) =>
+      new BooleanField(o.field_name, { ...o.config, required: o.is_required }),
+   enumm: (o: Options) => new EnumField(o.field_name, { ...o.config, required: o.is_required }),
+   json: (o: Options) => new JsonField(o.field_name, { ...o.config, required: o.is_required }),
+   jsonSchema: (o: Options) =>
+      new JsonSchemaField(o.field_name, { ...o.config, required: o.is_required }),
+   media: (o: Options) =>
+      new MediaField(o.field_name, { ...o.config, entity: o.entity.name, required: o.is_required }),
+   medium: (o: Options) =>
+      new MediaField(o.field_name, { ...o.config, entity: o.entity.name, required: o.is_required })
+} as const;
+type TFieldType = keyof typeof FieldMap;
+
+export function text(
+   config?: Omit<TextFieldConfig, "required">
+): TextField<false> & { required: () => TextField<true> } {
+   return new FieldPrototype("text", config, false) as any;
+}
+export function number(
+   config?: Omit<NumberFieldConfig, "required">
+): NumberField<false> & { required: () => NumberField<true> } {
+   return new FieldPrototype("number", config, false) as any;
+}
+export function date(
+   config?: Omit<DateFieldConfig, "required" | "type">
+): DateField<false> & { required: () => DateField<true> } {
+   return new FieldPrototype("date", { ...config, type: "date" }, false) as any;
+}
+export function datetime(
+   config?: Omit<DateFieldConfig, "required" | "type">
+): DateField<false> & { required: () => DateField<true> } {
+   return new FieldPrototype("date", { ...config, type: "datetime" }, false) as any;
+}
+export function week(
+   config?: Omit<DateFieldConfig, "required" | "type">
+): DateField<false> & { required: () => DateField<true> } {
+   return new FieldPrototype("date", { ...config, type: "week" }, false) as any;
+}
+export function boolean(
+   config?: Omit<BooleanFieldConfig, "required">
+): BooleanField<false> & { required: () => BooleanField<true> } {
+   return new FieldPrototype("boolean", config, false) as any;
+}
+export function enumm<TypeOverride = string>(
+   config?: Omit<EnumFieldConfig, "required" | "options"> & {
+      enum: string[] | { label: string; value: string }[];
+   }
+): EnumField<false, TypeOverride> & {
+   required: () => EnumField<true, TypeOverride>;
+} {
+   const type = typeof config?.enum?.[0] !== "string" ? "objects" : "strings";
+   const actual_config = {
+      options: {
+         type,
+         values: config?.enum ?? []
+      }
+   };
+   return new FieldPrototype("enumm", actual_config, false) as any;
+}
+export function json<TypeOverride = object>(
+   config?: Omit<JsonFieldConfig, "required">
+): JsonField<false, TypeOverride> & { required: () => JsonField<true, TypeOverride> } {
+   return new FieldPrototype("json", config, false) as any;
+}
+export function jsonSchema<TypeOverride = object>(
+   config?: Omit<JsonSchemaFieldConfig, "required">
+): JsonField<false, TypeOverride> & { required: () => JsonSchemaField<true, TypeOverride> } {
+   return new FieldPrototype("jsonSchema", config, false) as any;
+}
+export function media(config?: Omit<MediaFieldConfig, "entity">): MediaField<false> {
+   return new FieldPrototype("media", config, false) as any;
+}
+export function medium(
+   config?: Omit<MediaFieldConfig, "required" | "entity" | "max_items">
+): MediaField<false, MediaItem> {
+   return new FieldPrototype("media", { ...config, max_items: 1 }, false) as any;
+}
+export function make<Actual extends Field<any, any>>(name: string, field: Actual): Actual {
+   if (field instanceof FieldPrototype) {
+      return field.make(name) as Actual;
+   }
+   throw new Error("Invalid field");
+}
+
+export class FieldPrototype {
+   constructor(
+      public type: TFieldType,
+      public config: any,
+      public is_required: boolean
+   ) {}
+
+   required() {
+      this.is_required = true;
+      return this;
+   }
+
+   getField(o: Options): Field {
+      if (!FieldMap[this.type]) {
+         throw new Error(`Unknown field type: ${this.type}`);
+      }
+      try {
+         return FieldMap[this.type](o) as unknown as Field;
+      } catch (e) {
+         throw new Error(`Faild to construct field "${this.type}": ${e}`);
+      }
+   }
+
+   make(field_name: string): Field {
+      if (!FieldMap[this.type]) {
+         throw new Error(`Unknown field type: ${this.type}`);
+      }
+      try {
+         return FieldMap[this.type]({
+            entity: { name: "unknown", fields: {} },
+            field_name,
+            config: this.config,
+            is_required: this.is_required
+         }) as unknown as Field;
+      } catch (e) {
+         throw new Error(`Faild to construct field "${this.type}": ${e}`);
+      }
+   }
+}
+
+//type Entity<Fields extends Record<string, Field<any, any>> = {}> = { name: string; fields: Fields };
+
+export function entity<
+   EntityName extends string,
+   Fields extends Record<string, Field<any, any, any>>
+>(
+   name: EntityName,
+   fields: Fields,
+   config?: EntityConfig,
+   type?: TEntityType
+): Entity<EntityName, Fields> {
+   const _fields: Field[] = [];
+   for (const [field_name, field] of Object.entries(fields)) {
+      const f = field as unknown as FieldPrototype;
+      const o: Options = {
+         entity: { name, fields },
+         field_name,
+         config: f.config,
+         is_required: f.is_required
+      };
+      _fields.push(f.getField(o));
+   }
+   return new Entity(name, _fields, config, type);
+}
+
+export function relation<Local extends Entity>(local: Local) {
+   return {
+      manyToOne: <Foreign extends Entity>(foreign: Foreign, config?: ManyToOneRelationConfig) => {
+         return new ManyToOneRelation(local, foreign, config);
+      },
+      oneToOne: <Foreign extends Entity>(foreign: Foreign, config?: OneToOneRelationConfig) => {
+         return new OneToOneRelation(local, foreign, config);
+      },
+      manyToMany: <Foreign extends Entity>(
+         foreign: Foreign,
+         config?: ManyToManyRelationConfig,
+         additionalFields?: Record<string, Field<any, any, any>>
+      ) => {
+         const add_fields: Field[] = [];
+         if (additionalFields) {
+            const fields = additionalFields!;
+            const _fields: Field[] = [];
+            const entity_name =
+               config?.connectionTable ?? ManyToManyRelation.defaultConnectionTable(local, foreign);
+            for (const [field_name, field] of Object.entries(additionalFields)) {
+               const f = field as unknown as FieldPrototype;
+               const o: Options = {
+                  entity: { name: entity_name, fields },
+                  field_name,
+                  config: f.config,
+                  is_required: f.is_required
+               };
+               _fields.push(f.getField(o));
+            }
+            add_fields.push(_fields as any);
+         }
+
+         return new ManyToManyRelation(local, foreign, config as any, add_fields);
+      },
+      polyToOne: <Foreign extends Entity>(
+         foreign: Foreign,
+         config?: Omit<PolymorphicRelationConfig, "targetCardinality">
+      ) => {
+         return new PolymorphicRelation(local, foreign, { ...config, targetCardinality: 1 });
+      },
+      polyToMany: <Foreign extends Entity>(
+         foreign: Foreign,
+         config?: PolymorphicRelationConfig
+      ) => {
+         return new PolymorphicRelation(local, foreign, config);
+      }
+   };
+}
+
+type InferEntityFields<T> = T extends Entity<infer _N, infer Fields>
+   ? {
+        [K in keyof Fields]: Fields[K] extends { _type: infer Type; _required: infer Required }
+           ? Required extends true
+              ? Type
+              : Type | undefined
+           : never;
+     }
+   : never;
+
+export type InferFields<Fields> = Fields extends Record<string, Field<any, any, any>>
+   ? {
+        [K in keyof Fields]: Fields[K] extends { _type: infer Type; _required: infer Required }
+           ? Required extends true
+              ? Type
+              : Type | undefined
+           : never;
+     }
+   : never;
+
+type Prettify<T> = {
+   [K in keyof T]: T[K];
+};
+export type Simplify<T> = { [KeyType in keyof T]: T[KeyType] } & {};
+
+// from https://github.com/type-challenges/type-challenges/issues/28200
+type Merge<T> = {
+   [K in keyof T]: T[K];
+};
+type OptionalUndefined<
+   T,
+   Props extends keyof T = keyof T,
+   OptionsProps extends keyof T = Props extends keyof T
+      ? undefined extends T[Props]
+         ? Props
+         : never
+      : never
+> = Merge<
+   {
+      [K in OptionsProps]?: T[K];
+   } & {
+      [K in Exclude<keyof T, OptionsProps>]: T[K];
+   }
+>;
+
+type InferField<Field> = Field extends { _type: infer Type; _required: infer Required }
+   ? Required extends true
+      ? Type
+      : Type | undefined
+   : never;
+
+export type InsertSchema<T> = Simplify<OptionalUndefined<InferEntityFields<T>>>;
+export type Schema<T> = { id: Generated<number> } & InsertSchema<T>;
+export type FieldSchema<T> = Simplify<OptionalUndefined<InferFields<T>>>;

app/src/data/relations/EntityRelation.ts

@@ -0,0 +1,231 @@
+import { type Static, Type, parse } from "core/utils";
+import type { SelectQueryBuilder } from "kysely";
+import type { Entity, EntityData, EntityManager } from "../entities";
+import {
+   type EntityRelationAnchor,
+   type MutationInstructionResponse,
+   RelationHelper
+} from "../relations";
+import type { RepoQuery } from "../server/data-query-impl";
+import type { RelationType } from "./relation-types";
+
+export type KyselyJsonFrom = any;
+export type KyselyQueryBuilder = SelectQueryBuilder<any, any, any>;
+
+/*export type RelationConfig = {
+   mappedBy?: string;
+   inversedBy?: string;
+   sourceCardinality?: number;
+   connectionTable?: string;
+   connectionTableMappedName?: string;
+   required?: boolean;
+};*/
+
+export type BaseRelationConfig = Static<typeof EntityRelation.schema>;
+
+// @todo: add generic type for relation config
+export abstract class EntityRelation<
+   Schema extends typeof EntityRelation.schema = typeof EntityRelation.schema
+> {
+   config: Static<Schema>;
+
+   source: EntityRelationAnchor;
+   target: EntityRelationAnchor;
+
+   // @todo: add unit tests
+   // allowed directions, used in RelationAccessor for visibility
+   directions: ("source" | "target")[] = ["source", "target"];
+
+   static schema = Type.Object({
+      mappedBy: Type.Optional(Type.String()),
+      inversedBy: Type.Optional(Type.String()),
+      required: Type.Optional(Type.Boolean())
+   });
+
+   // don't make protected, App requires it to instantiatable
+   constructor(
+      source: EntityRelationAnchor,
+      target: EntityRelationAnchor,
+      config: Partial<Static<Schema>> = {}
+   ) {
+      this.source = source;
+      this.target = target;
+
+      const schema = (this.constructor as typeof EntityRelation).schema;
+      // @ts-ignore for now
+      this.config = parse(schema, config);
+   }
+
+   abstract initialize(em: EntityManager<any>): void;
+
+   /**
+    * Build the "with" part of the query.
+    *
+    * @param entity requesting entity, so target needs to be added
+    * @param qb
+    * @param jsonFrom
+    */
+   abstract buildWith(
+      entity: Entity,
+      qb: KyselyQueryBuilder,
+      jsonFrom: KyselyJsonFrom,
+      reference: string
+   ): KyselyQueryBuilder;
+
+   abstract buildJoin(
+      entity: Entity,
+      qb: KyselyQueryBuilder,
+      reference: string
+   ): KyselyQueryBuilder;
+
+   getReferenceQuery(entity: Entity, id: number, reference: string): Partial<RepoQuery> {
+      return {};
+   }
+
+   /** @deprecated */
+   helper(entity_name: string): RelationHelper {
+      return new RelationHelper(this, entity_name);
+   }
+
+   /**
+    * Get the other side of the relation quickly
+    * @param entity
+    */
+   other(entity: Entity | string): EntityRelationAnchor {
+      const entity_name = typeof entity === "string" ? entity : entity.name;
+
+      // special case for self referencing, check which side is not cardinality 1
+      if (this.source.entity.name === this.target.entity.name) {
+         return this.source.cardinality === 1 ? this.target : this.source;
+      }
+
+      if (this.source.entity.name === entity_name) {
+         return this.target;
+      } else if (this.target.entity.name === entity_name) {
+         return this.source;
+      }
+
+      throw new Error(
+         `Entity "${entity_name}" is not part of the relation ` +
+            `"${this.source.entity.name} <-> ${this.target.entity.name}"`
+      );
+   }
+
+   ref(reference: string): EntityRelationAnchor {
+      return this.source.reference === reference ? this.source : this.target;
+   }
+
+   otherRef(reference: string): EntityRelationAnchor {
+      return this.source.reference === reference ? this.target : this.source;
+   }
+
+   // @todo: add unit tests
+   visibleFrom(from: "source" | "target"): boolean {
+      return this.directions.includes(from);
+   }
+
+   /**
+    * Hydrate the relation. "entity" represents where the payload belongs to.
+    * E.g. if entity is "categories", then value is the result of categories
+    *
+    * IMPORTANT: This method is called from EM, high potential of recursion!
+    *
+    * @param entity
+    * @param value
+    * @param em
+    */
+   hydrate(entity: Entity | string, value: EntityData[], em: EntityManager<any>) {
+      const entity_name = typeof entity === "string" ? entity : entity.name;
+      const anchor = this.ref(entity_name);
+      const hydrated = em.hydrate(anchor.entity.name, value);
+
+      if (anchor.cardinality === 1) {
+         if (Array.isArray(hydrated) && hydrated.length > 1) {
+            throw new Error(
+               `Failed to hydrate "${anchor.entity.name}" ` +
+                  `with value: ${JSON.stringify(value)} (cardinality: 1)`
+            );
+         }
+
+         return hydrated[0];
+      }
+
+      if (!hydrated) {
+         throw new Error(
+            `Failed to hydrate "${anchor.entity.name}" ` +
+               `with value: ${JSON.stringify(value)} (cardinality: -)`
+         );
+      }
+
+      return hydrated;
+   }
+
+   /**
+    * Determines if the relation is listable for the given entity
+    * If the given entity is the one with the local reference, then it's not listable
+    * Only if there are multiple, which is generally the other side (except for 1:1)
+    * @param entity
+    */
+   isListableFor(entity: Entity): boolean {
+      //console.log("isListableFor", entity.name, this.source.entity.name, this.target.entity.name);
+      return this.target.entity.name === entity.name;
+   }
+
+   abstract type(): RelationType;
+
+   get required(): boolean {
+      return !!this.config.required;
+   }
+
+   async $set(
+      em: EntityManager<any>,
+      key: string,
+      value: unknown
+   ): Promise<void | MutationInstructionResponse> {
+      throw new Error("$set is not allowed");
+   }
+
+   async $create(
+      em: EntityManager<any>,
+      key: string,
+      value: unknown
+   ): Promise<void | MutationInstructionResponse> {
+      throw new Error("$create is not allowed");
+   }
+
+   async $attach(
+      em: EntityManager<any>,
+      key: string,
+      value: unknown
+   ): Promise<void | MutationInstructionResponse> {
+      throw new Error("$attach is not allowed");
+   }
+
+   async $detach(
+      em: EntityManager<any>,
+      key: string,
+      value: unknown
+   ): Promise<void | MutationInstructionResponse> {
+      throw new Error("$detach is not allowed");
+   }
+
+   getName(): string {
+      const parts = [
+         this.type().replace(":", ""),
+         this.source.entity.name,
+         this.target.entity.name,
+         this.config.mappedBy,
+         this.config.inversedBy
+      ].filter(Boolean);
+      return parts.join("_");
+   }
+
+   toJSON() {
+      return {
+         type: this.type(),
+         source: this.source.entity.name,
+         target: this.target.entity.name,
+         config: this.config
+      };
+   }
+}

app/src/data/relations/EntityRelationAnchor.ts

@@ -0,0 +1,25 @@
+import type { Entity } from "../entities";
+
+export class EntityRelationAnchor {
+   entity: Entity;
+   cardinality?: number;
+
+   /**
+    * The name that the other entity will use to reference this entity
+    */
+   reference: string;
+
+   constructor(entity: Entity, name: string, cardinality?: number) {
+      this.entity = entity;
+      this.cardinality = cardinality;
+      this.reference = name;
+   }
+
+   toJSON() {
+      return {
+         entity: this.entity.name,
+         cardinality: this.cardinality,
+         name: this.reference,
+      };
+   }
+}

app/src/data/relations/ManyToManyRelation.ts

@@ -0,0 +1,189 @@
+import { type Static, Type } from "core/utils";
+import { Entity, type EntityManager } from "../entities";
+import { type Field, PrimaryField, VirtualField } from "../fields";
+import type { RepoQuery } from "../server/data-query-impl";
+import { EntityRelation, type KyselyJsonFrom, type KyselyQueryBuilder } from "./EntityRelation";
+import { EntityRelationAnchor } from "./EntityRelationAnchor";
+import { RelationField } from "./RelationField";
+import { type RelationType, RelationTypes } from "./relation-types";
+
+export type ManyToManyRelationConfig = Static<typeof ManyToManyRelation.schema>;
+
+export class ManyToManyRelation extends EntityRelation<typeof ManyToManyRelation.schema> {
+   connectionEntity: Entity;
+   additionalFields: Field[] = [];
+   connectionTableMappedName: string;
+   private em?: EntityManager<any>;
+
+   static override schema = Type.Composite(
+      [
+         EntityRelation.schema,
+         Type.Object({
+            connectionTable: Type.Optional(Type.String()),
+            connectionTableMappedName: Type.Optional(Type.String())
+         })
+      ],
+      {
+         additionalProperties: false
+      }
+   );
+
+   constructor(
+      source: Entity,
+      target: Entity,
+      config?: ManyToManyRelationConfig,
+      additionalFields?: Field[]
+   ) {
+      const connectionTable =
+         config?.connectionTable || ManyToManyRelation.defaultConnectionTable(source, target);
+
+      const sourceAnchor = new EntityRelationAnchor(source, source.name);
+      const targetAnchor = new EntityRelationAnchor(target, target.name);
+      super(sourceAnchor, targetAnchor, config);
+
+      this.connectionEntity = new Entity(connectionTable, additionalFields, undefined, "generated");
+
+      this.connectionTableMappedName = config?.connectionTableMappedName || connectionTable;
+      this.additionalFields = additionalFields || [];
+      //this.connectionTable = connectionTable;
+   }
+
+   static defaultConnectionTable(source: Entity, target: Entity) {
+      return `${source.name}_${target.name}`;
+   }
+
+   type(): RelationType {
+      return RelationTypes.ManyToMany;
+   }
+
+   /**
+    * Many to many is always listable in both directions
+    */
+   override isListableFor(): boolean {
+      return true;
+   }
+
+   getField(entity: Entity): RelationField {
+      const conn = this.connectionEntity;
+      const selfField = conn.fields.find(
+         (f) => f instanceof RelationField && f.target() === entity.name
+      )!;
+
+      if (!selfField || !(selfField instanceof RelationField)) {
+         throw new Error(
+            `Connection entity "${conn.name}" does not have a relation to "${entity.name}"`
+         );
+      }
+
+      return selfField;
+   }
+
+   private getQueryInfo(entity: Entity) {
+      const other = this.other(entity);
+      const conn = this.connectionEntity;
+      const entityField = this.getField(entity);
+      const otherField = this.getField(other.entity);
+      const join = [
+         conn.name,
+         `${other.entity.name}.${other.entity.getPrimaryField().name}`,
+         `${conn.name}.${otherField.name}`
+      ] as const;
+
+      const entityRef = `${entity.name}.${entity.getPrimaryField().name}`;
+      const otherRef = `${conn.name}.${entityField.name}`;
+
+      const groupBy = `${entity.name}.${entity.getPrimaryField().name}`;
+
+      return {
+         other,
+         join,
+         entityRef,
+         otherRef,
+         groupBy
+      };
+   }
+
+   override getReferenceQuery(entity: Entity, id: number): Partial<RepoQuery> {
+      const conn = this.connectionEntity;
+
+      return {
+         where: {
+            [`${conn.name}.${entity.name}_${entity.getPrimaryField().name}`]: id
+         },
+         join: [this.target.reference]
+      };
+   }
+
+   buildJoin(entity: Entity, qb: KyselyQueryBuilder) {
+      const { other, join, entityRef, otherRef, groupBy } = this.getQueryInfo(entity);
+
+      return qb
+         .innerJoin(other.entity.name, entityRef, otherRef)
+         .innerJoin(...join)
+         .groupBy(groupBy);
+   }
+
+   buildWith(entity: Entity, qb: KyselyQueryBuilder, jsonFrom: KyselyJsonFrom) {
+      if (!this.em) {
+         throw new Error("EntityManager not set, can't build");
+      }
+      const jsonBuildObject = this.em.connection.fn.jsonBuildObject;
+      if (!jsonBuildObject) {
+         throw new Error("Connection does not support jsonBuildObject");
+      }
+
+      const limit = 5;
+      const { other, join, entityRef, otherRef } = this.getQueryInfo(entity);
+      const additionalFields = this.connectionEntity.fields.filter(
+         (f) => !(f instanceof RelationField || f instanceof PrimaryField)
+      );
+
+      return qb.select((eb) => {
+         const select: any[] = other.entity.getSelect(other.entity.name);
+         // @todo: also add to find by references
+         if (additionalFields.length > 0) {
+            const conn = this.connectionEntity.name;
+            select.push(
+               jsonBuildObject(
+                  Object.fromEntries(
+                     additionalFields.map((f) => [f.name, eb.ref(`${conn}.${f.name}`)])
+                  )
+               ).as(this.connectionTableMappedName)
+            );
+         }
+
+         return jsonFrom(
+            eb
+               .selectFrom(other.entity.name)
+               .select(select)
+               .whereRef(entityRef, "=", otherRef)
+               .innerJoin(...join)
+               .limit(limit)
+         ).as(other.reference);
+      });
+   }
+
+   initialize(em: EntityManager<any>) {
+      this.em = em;
+
+      //this.connectionEntity.addField(new RelationField(this.source.entity));
+      //this.connectionEntity.addField(new RelationField(this.target.entity));
+      this.connectionEntity.addField(RelationField.create(this, this.source));
+      this.connectionEntity.addField(RelationField.create(this, this.target));
+
+      // @todo: check this
+      for (const field of this.additionalFields) {
+         this.source.entity.addField(new VirtualField(this.connectionTableMappedName));
+         this.target.entity.addField(new VirtualField(this.connectionTableMappedName));
+      }
+
+      em.addEntity(this.connectionEntity);
+   }
+
+   override getName(): string {
+      return [
+         super.getName(),
+         [this.connectionEntity.name, this.connectionTableMappedName].filter(Boolean)
+      ].join("_");
+   }
+}

app/src/data/relations/ManyToOneRelation.ts

@@ -0,0 +1,228 @@
+import type { PrimaryFieldType } from "core";
+import { snakeToPascalWithSpaces } from "core/utils";
+import { type Static, Type } from "core/utils";
+import type { Entity, EntityManager } from "../entities";
+import type { RepoQuery } from "../server/data-query-impl";
+import { EntityRelation, type KyselyJsonFrom, type KyselyQueryBuilder } from "./EntityRelation";
+import { EntityRelationAnchor } from "./EntityRelationAnchor";
+import { RelationField, type RelationFieldBaseConfig } from "./RelationField";
+import type { MutationInstructionResponse } from "./RelationMutator";
+import { type RelationType, RelationTypes } from "./relation-types";
+
+/**
+ * Source entity receives the mapping field
+ *
+ * Many-to-one (many) [sources] has (one) [target]
+ * Example: [posts] has (one) [user]
+ *    posts gets a users_id field
+ */
+
+export type ManyToOneRelationConfig = Static<typeof ManyToOneRelation.schema>;
+
+export class ManyToOneRelation extends EntityRelation<typeof ManyToOneRelation.schema> {
+   private fieldConfig?: RelationFieldBaseConfig;
+   static DEFAULTS = {
+      with_limit: 5
+   };
+
+   static override schema = Type.Composite(
+      [
+         EntityRelation.schema,
+         Type.Object({
+            sourceCardinality: Type.Optional(Type.Number()),
+            with_limit: Type.Optional(
+               Type.Number({ default: ManyToOneRelation.DEFAULTS.with_limit })
+            ),
+            fieldConfig: Type.Optional(
+               Type.Object({
+                  label: Type.String()
+               })
+            )
+         })
+      ],
+      {
+         additionalProperties: false
+      }
+   );
+
+   constructor(
+      source: Entity,
+      target: Entity,
+      config: Partial<Static<typeof ManyToOneRelation.schema>> = {}
+   ) {
+      const mappedBy = config.mappedBy || target.name;
+      const inversedBy = config.inversedBy || source.name;
+
+      // if source can be multiple, allow it. otherwise unlimited
+      const sourceCardinality =
+         typeof config.sourceCardinality === "number" && config.sourceCardinality > 0
+            ? config.sourceCardinality
+            : undefined;
+      const sourceAnchor = new EntityRelationAnchor(source, inversedBy, sourceCardinality);
+      const targetAnchor = new EntityRelationAnchor(target, mappedBy, 1);
+      super(sourceAnchor, targetAnchor, config);
+
+      this.fieldConfig = config.fieldConfig ?? {};
+      // set relation required or not
+      //this.required = !!config.required;
+   }
+
+   type(): RelationType {
+      return RelationTypes.ManyToOne;
+   }
+
+   override initialize(em: EntityManager<any>) {
+      const defaultLabel = snakeToPascalWithSpaces(this.target.reference);
+
+      // add required mapping field on source
+      const field = RelationField.create(this, this.target, {
+         label: defaultLabel,
+         ...this.fieldConfig
+      });
+
+      if (!this.source.entity.field(field.name)) {
+         this.source.entity.addField(
+            RelationField.create(this, this.target, {
+               label: defaultLabel,
+               ...this.fieldConfig
+            })
+         );
+      }
+   }
+
+   /**
+    * Retrieve the RelationField
+    */
+   getField(): RelationField {
+      const id = this.target.entity.getPrimaryField().name;
+      const field = this.source.entity.getField(`${this.target.reference}_${id}`);
+
+      if (!(field instanceof RelationField)) {
+         throw new Error(
+            `Field "${this.target.reference}_${id}" not found on entity "${this.source.entity.name}"`
+         );
+      }
+
+      return field;
+   }
+
+   private queryInfo(entity: Entity, reference: string) {
+      const side = this.source.reference === reference ? "source" : "target";
+      const self = this[side];
+      const other = this[side === "source" ? "target" : "source"];
+      let relationRef: string;
+      let entityRef: string;
+      let otherRef: string;
+      if (side === "source") {
+         relationRef = this.source.reference;
+         entityRef = `${relationRef}.${this.getField().name}`;
+         otherRef = `${entity.name}.${self.entity.getPrimaryField().name}`;
+      } else {
+         relationRef = this.target.reference;
+         entityRef = `${relationRef}.${entity.getPrimaryField().name}`;
+         otherRef = `${entity.name}.${this.getField().name}`;
+      }
+
+      const groupBy = `${entity.name}.${entity.getPrimaryField().name}`;
+      //console.log("queryInfo", entity.name, { reference, side, relationRef, entityRef, otherRef });
+
+      return {
+         other,
+         self,
+         relationRef,
+         entityRef,
+         otherRef,
+         groupBy
+      };
+   }
+
+   override getReferenceQuery(entity: Entity, id: number, reference: string): Partial<RepoQuery> {
+      const side = this.source.reference === reference ? "source" : "target";
+      const self = this[side];
+      const other = this[side === "source" ? "target" : "source"];
+      const otherRef = `${other.reference}_${other.entity.getPrimaryField().name}`;
+
+      return {
+         where: {
+            [otherRef]: id
+         },
+         join: other.entity.name === self.entity.name ? [] : [other.entity.name]
+      };
+   }
+
+   buildJoin(entity: Entity, qb: KyselyQueryBuilder, reference: string) {
+      const { self, entityRef, otherRef, groupBy } = this.queryInfo(entity, reference);
+      return qb.innerJoin(self.entity.name, entityRef, otherRef).groupBy(groupBy);
+   }
+
+   buildWith(entity: Entity, qb: KyselyQueryBuilder, jsonFrom: KyselyJsonFrom, reference: string) {
+      const { self, entityRef, otherRef, relationRef } = this.queryInfo(entity, reference);
+      const limit =
+         self.cardinality === 1
+            ? 1
+            : this.config.with_limit ?? ManyToOneRelation.DEFAULTS.with_limit;
+      //console.log("buildWith", entity.name, reference, { limit });
+
+      return qb.select((eb) =>
+         jsonFrom(
+            eb
+               .selectFrom(`${self.entity.name} as ${relationRef}`)
+               .select(self.entity.getSelect(relationRef))
+               .whereRef(entityRef, "=", otherRef)
+               .limit(limit)
+         ).as(relationRef)
+      );
+   }
+
+   /**
+    * $set is performed using the reference:
+    * { [reference]: { $set: { id: 1 } } }
+    *
+    * It must resolve from [reference] ("users") to field ("user_id")
+    * -> returns instructions
+    */
+   override async $set(
+      em: EntityManager<any>,
+      key: string,
+      value: object
+   ): Promise<void | MutationInstructionResponse> {
+      if (typeof value !== "object") {
+         throw new Error(`Invalid value for relation field "${key}" given, expected object.`);
+      }
+
+      const entity = this.source.entity;
+      const helper = this.helper(entity.name);
+      const info = helper.getMutationInfo();
+      if (!info.$set) {
+         throw new Error(`Cannot perform $set for relation "${key}"`);
+      }
+
+      const local_field = info.local_field;
+      const field = this.getField();
+      // @ts-ignore
+      const primaryReference = value[Object.keys(value)[0]] as PrimaryFieldType;
+
+      if (!local_field || !(field instanceof RelationField)) {
+         throw new Error(`Cannot perform $set for relation "${key}"`);
+      }
+
+      // if "{ $set: { id: null } }" given, and not required, allow it
+      if (primaryReference === null && !field.isRequired()) {
+         return [local_field, null] satisfies MutationInstructionResponse;
+      }
+
+      const query = await em.repository(field.target()).exists({
+         [field.targetField()]: primaryReference as any
+      });
+
+      if (!query.exists) {
+         const idProp = field.targetField();
+         throw new Error(
+            `Cannot connect "${entity.name}.${key}" to ` +
+               `"${field.target()}.${idProp}" = "${primaryReference}": not found.`
+         );
+      }
+
+      return [local_field, primaryReference] satisfies MutationInstructionResponse;
+   }
+}

app/src/data/relations/OneToOneRelation.ts

@@ -0,0 +1,77 @@
+import type { Entity, EntityManager } from "../entities";
+import { ManyToOneRelation, type ManyToOneRelationConfig } from "./ManyToOneRelation";
+import type { MutationInstructionResponse } from "./RelationMutator";
+import { type RelationType, RelationTypes } from "./relation-types";
+
+/**
+ * Both source and target receive a mapping field
+ * @todo: determine if it should be removed
+ */
+export type OneToOneRelationConfig = ManyToOneRelationConfig;
+
+/* export type OneToOneRelationConfig = {
+   mappedBy?: string; // author|users
+   inversedBy?: string; // posts
+   required?: boolean;
+}; */
+
+export class OneToOneRelation extends ManyToOneRelation {
+   constructor(source: Entity, target: Entity, config?: OneToOneRelationConfig) {
+      const { mappedBy, inversedBy, required } = config || {};
+      super(source, target, {
+         mappedBy,
+         inversedBy,
+         sourceCardinality: 1,
+         required
+      });
+   }
+
+   override type(): RelationType {
+      return RelationTypes.OneToOne;
+   }
+
+   /**
+    * One-to-one relations are not listable in either direction
+    */
+   override isListableFor(): boolean {
+      return false;
+   }
+
+   // need to override since it inherits manytoone
+   override async $set(
+      em: EntityManager<any>,
+      key: string,
+      value: object
+   ): Promise<MutationInstructionResponse> {
+      throw new Error("$set is not allowed");
+   }
+
+   override async $create(
+      em: EntityManager<any>,
+      key: string,
+      value: unknown
+   ): Promise<void | MutationInstructionResponse> {
+      if (value === null || typeof value !== "object") {
+         throw new Error(`Invalid value for relation field "${key}" given, expected object.`);
+      }
+
+      const target = this.other(this.source.entity).entity;
+      const helper = this.helper(this.source.entity.name);
+      const info = helper.getMutationInfo();
+      const primary = info.primary;
+      const local_field = info.local_field;
+      if (!info.$create || !primary || !local_field) {
+         throw new Error(`Cannot perform $create for relation "${key}"`);
+      }
+
+      // create the relational entity
+      try {
+         const { data } = await em.mutator(target).insertOne(value);
+
+         const retrieved_value = data[primary];
+         return [local_field, retrieved_value] satisfies MutationInstructionResponse;
+      } catch (e) {
+         throw new Error(`Error performing $create on "${target.name}".`);
+      }
+   }
+}

app/src/data/relations/PolymorphicRelation.ts

@@ -0,0 +1,130 @@
+import { type Static, Type } from "core/utils";
+import type { Entity, EntityManager } from "../entities";
+import { NumberField, TextField } from "../fields";
+import type { RepoQuery } from "../server/data-query-impl";
+import { EntityRelation, type KyselyJsonFrom, type KyselyQueryBuilder } from "./EntityRelation";
+import { EntityRelationAnchor } from "./EntityRelationAnchor";
+import { type RelationType, RelationTypes } from "./relation-types";
+
+export type PolymorphicRelationConfig = Static<typeof PolymorphicRelation.schema>;
+
+// @todo: what about cascades?
+export class PolymorphicRelation extends EntityRelation<typeof PolymorphicRelation.schema> {
+   static override schema = Type.Composite(
+      [
+         EntityRelation.schema,
+         Type.Object({
+            targetCardinality: Type.Optional(Type.Number())
+         })
+      ],
+      {
+         additionalProperties: false
+      }
+   );
+
+   constructor(source: Entity, target: Entity, config: Partial<PolymorphicRelationConfig> = {}) {
+      const mappedBy = config.mappedBy || target.name;
+      const inversedBy = config.inversedBy || source.name;
+
+      // if target can be multiple, allow it. otherwise unlimited
+      const targetCardinality =
+         typeof config.targetCardinality === "number" && config.targetCardinality > 0
+            ? config.targetCardinality
+            : undefined;
+      const sourceAnchor = new EntityRelationAnchor(source, inversedBy, 1);
+      const targetAnchor = new EntityRelationAnchor(target, mappedBy, targetCardinality);
+      super(sourceAnchor, targetAnchor, config);
+
+      this.directions = ["source"];
+   }
+
+   type(): RelationType {
+      return RelationTypes.Polymorphic;
+   }
+
+   private queryInfo(entity: Entity) {
+      const other = this.other(entity);
+      const whereLhs = `${other.entity.name}.${this.getReferenceField().name}`;
+      const reference = `${entity.name}.${this.config.mappedBy}`;
+
+      // this is used for "getReferenceQuery"
+      const reference_other = `${other.entity.name}.${this.config.mappedBy}`;
+
+      const entityRef = `${entity.name}.${entity.getPrimaryField().name}`;
+      const otherRef = `${other.entity.name}.${this.getEntityIdField().name}`;
+
+      const groupBy = `${entity.name}.${entity.getPrimaryField().name}`;
+
+      return {
+         other,
+         whereLhs,
+         reference,
+         reference_other,
+         entityRef,
+         otherRef,
+         groupBy
+      };
+   }
+
+   buildJoin(entity: Entity, qb: KyselyQueryBuilder) {
+      const { other, whereLhs, reference, entityRef, otherRef, groupBy } = this.queryInfo(entity);
+
+      return qb
+         .innerJoin(other.entity.name, (join) =>
+            join.onRef(entityRef, "=", otherRef).on(whereLhs, "=", reference)
+         )
+         .groupBy(groupBy);
+   }
+
+   override getReferenceQuery(entity: Entity, id: number): Partial<RepoQuery> {
+      const info = this.queryInfo(entity);
+
+      return {
+         where: {
+            [this.getReferenceField().name]: info.reference_other,
+            [this.getEntityIdField().name]: id
+         }
+      };
+   }
+
+   buildWith(entity: Entity, qb: KyselyQueryBuilder, jsonFrom: KyselyJsonFrom) {
+      const { other, whereLhs, reference, entityRef, otherRef } = this.queryInfo(entity);
+      const limit = other.cardinality === 1 ? 1 : 5;
+
+      return qb.select((eb) =>
+         jsonFrom(
+            eb
+               .selectFrom(other.entity.name)
+               .select(other.entity.getSelect(other.entity.name))
+               .where(whereLhs, "=", reference)
+               .whereRef(entityRef, "=", otherRef)
+               .limit(limit)
+         ).as(other.reference)
+      );
+   }
+
+   override isListableFor(entity: Entity): boolean {
+      // @todo: only make listable if many? check cardinality
+      return this.source.entity.name === entity.name && this.target.cardinality !== 1;
+   }
+
+   getReferenceField(): TextField {
+      return new TextField("reference", { hidden: true, fillable: ["create"] });
+   }
+
+   getEntityIdField(): NumberField {
+      return new NumberField("entity_id", { hidden: true, fillable: ["create"] });
+   }
+
+   initialize(em: EntityManager<any>) {
+      const referenceField = this.getReferenceField();
+      const entityIdField = this.getEntityIdField();
+
+      if (!this.target.entity.field(referenceField.name)) {
+         this.target.entity.addField(referenceField);
+      }
+      if (!this.target.entity.field(entityIdField.name)) {
+         this.target.entity.addField(entityIdField);
+      }
+   }
+}

app/src/data/relations/RelationAccessor.ts

@@ -0,0 +1,74 @@
+import type { Entity } from "../entities";
+import type { EntityRelation } from "../relations";
+
+export class RelationAccessor {
+   private readonly _relations: EntityRelation[] = [];
+
+   constructor(relations: EntityRelation[]) {
+      this._relations = relations;
+   }
+
+   get all(): EntityRelation[] {
+      return this._relations;
+   }
+
+   /**
+    * Searches for the relations of [entity_name]
+    */
+   relationsOf(entity: Entity): EntityRelation[] {
+      return this._relations.filter((relation) => {
+         return (
+            (relation.visibleFrom("source") && relation.source.entity.name === entity.name) ||
+            (relation.visibleFrom("target") && relation.target.entity.name === entity.name)
+         );
+      });
+   }
+
+   sourceRelationsOf(entity: Entity): EntityRelation[] {
+      return this._relations.filter((relation) => {
+         return relation.source.entity.name === entity.name;
+      });
+   }
+
+   /**
+    * Search for relations that have [entity] as target
+    * - meaning it returns entities that holds a local reference field
+    */
+   targetRelationsOf(entity: Entity): EntityRelation[] {
+      return this._relations.filter((relation) => {
+         return relation.visibleFrom("target") && relation.target.entity.name === entity.name;
+      });
+   }
+
+   listableRelationsOf(entity: Entity): EntityRelation[] {
+      return this.relationsOf(entity).filter((relation) => relation.isListableFor(entity));
+   }
+
+   /**
+    * Searches for the relations of [entity_name] and
+    * return the one that has [reference] as source or target.
+    */
+   relationOf(entity: Entity, reference: string): EntityRelation | undefined {
+      return this.relationsOf(entity).find((r) => {
+         return r.source.reference === reference || r.target.reference === reference;
+      });
+   }
+
+   hasRelations(entity: Entity): boolean {
+      return this.relationsOf(entity).length > 0;
+   }
+
+   /**
+    * Get a list of related entities of [entity_name]
+    */
+   relatedEntitiesOf(entity: Entity): Entity[] {
+      return this.relationsOf(entity).map((r) => r.other(entity).entity);
+   }
+
+   /**
+    * Get relation names of [entity_name]
+    */
+   relationReferencesOf(entity): string[] {
+      return this.relationsOf(entity).map((r) => r.other(entity).reference);
+   }
+}

app/src/data/relations/RelationField.ts

@@ -0,0 +1,101 @@
+import { type Static, StringEnum, Type } from "core/utils";
+import type { EntityManager } from "../entities";
+import { Field, type SchemaResponse, baseFieldConfigSchema } from "../fields";
+import type { EntityRelation } from "./EntityRelation";
+import type { EntityRelationAnchor } from "./EntityRelationAnchor";
+
+const CASCADES = ["cascade", "set null", "set default", "restrict", "no action"] as const;
+
+export const relationFieldConfigSchema = Type.Composite([
+   baseFieldConfigSchema,
+   Type.Object({
+      reference: Type.String(),
+      target: Type.String(), // @todo: potentially has to be an instance!
+      target_field: Type.Optional(Type.String({ default: "id" })),
+      on_delete: Type.Optional(StringEnum(CASCADES, { default: "set null" }))
+   })
+]);
+/*export const relationFieldConfigSchema = baseFieldConfigSchema.extend({
+   reference: z.string(),
+   target: z.string(),
+   target_field: z.string().catch("id"),
+});*/
+
+export type RelationFieldConfig = Static<typeof relationFieldConfigSchema>;
+export type RelationFieldBaseConfig = { label?: string };
+
+export class RelationField extends Field<RelationFieldConfig> {
+   override readonly type = "relation";
+
+   protected getSchema() {
+      return relationFieldConfigSchema;
+   }
+
+   /*constructor(name: string, config?: Partial<RelationFieldConfig>) {
+      //relation_name = relation_name || target.name;
+      //const name = [relation_name, target.getPrimaryField().name].join("_");
+      super(name, config);
+
+      //console.log(this.config);
+      //this.relation.target = target;
+      //this.relation.name = relation_name;
+   }*/
+
+   static create(
+      relation: EntityRelation,
+      target: EntityRelationAnchor,
+      config?: RelationFieldBaseConfig
+   ) {
+      const name = [
+         target.reference ?? target.entity.name,
+         target.entity.getPrimaryField().name
+      ].join("_");
+      //console.log('name', name);
+      return new RelationField(name, {
+         ...config,
+         required: relation.required,
+         reference: target.reference,
+         target: target.entity.name,
+         target_field: target.entity.getPrimaryField().name
+      });
+   }
+
+   reference() {
+      return this.config.reference;
+   }
+
+   target() {
+      return this.config.target;
+   }
+
+   targetField(): string {
+      return this.config.target_field!;
+   }
+
+   override schema(): SchemaResponse {
+      return this.useSchemaHelper("integer", (col) => {
+         //col.references('person.id').onDelete('cascade').notNull()
+         // @todo: implement cascading?
+
+         return col
+            .references(`${this.config.target}.${this.config.target_field}`)
+            .onDelete(this.config.on_delete ?? "set null");
+      });
+   }
+
+   override transformRetrieve(value: any): any {
+      return value;
+   }
+
+   override async transformPersist(value: any, em: EntityManager<any>): Promise<any> {
+      throw new Error("This function should not be called");
+   }
+
+   override toJsonSchema() {
+      return this.toSchemaWrapIfRequired(
+         Type.Number({
+            $ref: `${this.config?.target}#/properties/${this.config?.target_field}`
+         })
+      );
+   }
+}

app/src/data/relations/RelationHelper.ts

@@ -0,0 +1,86 @@
+import {
+   type EntityRelation,
+   type EntityRelationAnchor,
+   type ManyToOneRelation,
+   type OneToOneRelation,
+   RelationTypes,
+} from "../relations";
+
+export const MutationOperations = ["$set", "$create", "$attach", "$detach"] as const;
+export type MutationOperation = (typeof MutationOperations)[number];
+
+export class RelationHelper {
+   relation: EntityRelation;
+   access: "source" | "target";
+   self: EntityRelationAnchor;
+   other: EntityRelationAnchor;
+
+   constructor(relation: EntityRelation, entity_name: string) {
+      this.relation = relation;
+
+      if (relation.source.entity.name === entity_name) {
+         this.access = "source";
+         this.self = relation.source;
+         this.other = relation.target;
+      } else if (relation.target.entity.name === entity_name) {
+         this.access = "target";
+         this.self = relation.target;
+         this.other = relation.source;
+      } else {
+         throw new Error(
+            `Entity "${entity_name}" is not part of the relation ` +
+               `"${relation.source.entity.name} <-> ${relation.target.entity.name}"`,
+         );
+      }
+   }
+
+   // @todo: add to respective relations
+   getMutationInfo() {
+      const ops: Record<MutationOperation, boolean> = {
+         $set: false,
+         $create: false,
+         $attach: false,
+         $detach: false,
+      };
+
+      let local_field: string | undefined;
+      let primary: string | undefined;
+
+      switch (this.relation.type()) {
+         case RelationTypes.ManyToOne:
+            // only if owning side (source), target is always single (just to assure)
+            if (typeof this.self.cardinality === "undefined" && this.other.cardinality === 1) {
+               ops.$set = true;
+               local_field = (this.relation as ManyToOneRelation).getField()?.name;
+               primary = this.other.entity.getPrimaryField().name;
+            }
+
+            break;
+         case RelationTypes.OneToOne:
+            // only if owning side (source)
+            if (this.access === "source") {
+               ops.$create = true;
+               ops.$set = true; // @todo: for now allow
+               local_field = (this.relation as OneToOneRelation).getField()?.name;
+               primary = this.other.entity.getPrimaryField().name;
+            }
+            break;
+         case RelationTypes.ManyToMany:
+            if (this.access === "source") {
+               ops.$attach = true;
+               ops.$detach = true;
+               primary = this.other.entity.getPrimaryField().name;
+            }
+            break;
+      }
+
+      return {
+         reference: this.other.reference,
+         local_field,
+         ...ops,
+         primary,
+         cardinality: this.other.cardinality,
+         relation_type: this.relation.type(),
+      };
+   }
+}

app/src/data/relations/RelationMutator.ts

@@ -0,0 +1,121 @@
+import type { PrimaryFieldType } from "core";
+import type { Entity, EntityManager } from "../entities";
+import {
+   type EntityRelation,
+   type MutationOperation,
+   MutationOperations,
+   RelationField
+} from "../relations";
+
+export type MutationInstructionResponse = [string, PrimaryFieldType | null];
+
+export class RelationMutator {
+   constructor(
+      protected entity: Entity,
+      protected em: EntityManager<any>
+   ) {}
+
+   /**
+    * Returns all keys that are somehow relational.
+    * Includes local fields (users_id) and references (users|author)
+    *
+    * @param em
+    * @param entity_name
+    *
+    * @returns string[]
+    */
+   getRelationalKeys(): string[] {
+      const references: string[] = [];
+      this.em.relationsOf(this.entity.name).map((r) => {
+         const info = r.helper(this.entity.name).getMutationInfo();
+         references.push(info.reference);
+         info.local_field && references.push(info.local_field);
+      });
+      return references;
+   }
+
+   async persistRelationField(
+      field: RelationField,
+      key: string,
+      value: PrimaryFieldType
+   ): Promise<MutationInstructionResponse> {
+      // allow empty if field is not required
+      if (value === null && !field.isRequired()) {
+         return [key, value];
+      }
+
+      // make sure it's a primitive value
+      // @todo: this is not a good way of checking primitives. Null is also an object
+      if (typeof value === "object") {
+         console.log("value", value);
+         throw new Error(`Invalid value for relation field "${key}" given, expected primitive.`);
+      }
+
+      const query = await this.em.repository(field.target()).exists({
+         [field.targetField()]: value
+      });
+
+      if (!query.exists) {
+         const idProp = field.targetField();
+         throw new Error(
+            `Cannot connect "${this.entity.name}.${key}" to ` +
+               `"${field.target()}.${idProp}" = "${value}": not found.`
+         );
+      }
+
+      return [key, value];
+   }
+
+   async persistReference(
+      relation: EntityRelation,
+      key: string,
+      value: unknown
+   ): Promise<void | MutationInstructionResponse> {
+      if (typeof value !== "object" || value === null || typeof value === "undefined") {
+         throw new Error(
+            `Invalid value for relation "${key}" given, expected object to persist reference. Like '{$set: {id: 1}}'.`
+         );
+      }
+
+      const operation = Object.keys(value)[0] as MutationOperation;
+      if (!MutationOperations.includes(operation)) {
+         throw new Error(
+            `Invalid operation "${operation}" for relation "${key}". ` +
+               `Allowed: ${MutationOperations.join(", ")}`
+         );
+      }
+
+      // @ts-ignore
+      const payload = value[operation];
+      return await relation[operation](this.em, key, payload);
+   }
+
+   async persistRelation(key: string, value: unknown): Promise<void | MutationInstructionResponse> {
+      // if field (e.g. 'user_id')
+      // relation types: n:1, 1:1 (mapping entity)
+      const field = this.entity.getField(key);
+      if (field instanceof RelationField) {
+         return this.persistRelationField(field, key, value as PrimaryFieldType);
+      }
+
+      /**
+       * If reference given, value operations are given
+       *
+       * Could be:
+       * { $set: { id: 1 } }
+       * { $set: [{ id: 1 }, { id: 2 }] }
+       * { $create: { theme: "dark" } }
+       * { $attach: [{ id: 1 }, { id: 2 }] }
+       * { $detach: [{ id: 1 }, { id: 2 }] }
+       */
+      const relation = this.em.relationOf(this.entity.name, key);
+      if (relation) {
+         return this.persistReference(relation, key, value);
+      }
+
+      throw new Error(
+         `Relation "${key}" failed to resolve on entity "${this.entity.name}": ` +
+            "Unable to resolve relation origin."
+      );
+   }
+}

app/src/data/relations/index.ts

@@ -0,0 +1,50 @@
+import { ManyToManyRelation, type ManyToManyRelationConfig } from "./ManyToManyRelation";
+import { ManyToOneRelation, type ManyToOneRelationConfig } from "./ManyToOneRelation";
+import { OneToOneRelation, type OneToOneRelationConfig } from "./OneToOneRelation";
+import { PolymorphicRelation, type PolymorphicRelationConfig } from "./PolymorphicRelation";
+import { type RelationType, RelationTypes } from "./relation-types";
+
+export * from "./EntityRelation";
+export * from "./EntityRelationAnchor";
+export * from "./RelationHelper";
+export * from "./RelationMutator";
+export * from "./RelationAccessor";
+
+import {
+   RelationField,
+   type RelationFieldBaseConfig,
+   type RelationFieldConfig,
+   relationFieldConfigSchema
+} from "./RelationField";
+
+export {
+   OneToOneRelation,
+   type OneToOneRelationConfig,
+   ManyToOneRelation,
+   type ManyToOneRelationConfig,
+   ManyToManyRelation,
+   type ManyToManyRelationConfig,
+   PolymorphicRelation,
+   type PolymorphicRelationConfig,
+   RelationTypes,
+   type RelationType,
+   // field
+   RelationField,
+   relationFieldConfigSchema,
+   type RelationFieldBaseConfig,
+   type RelationFieldConfig
+};
+
+export const RelationClassMap = {
+   [RelationTypes.OneToOne]: { schema: OneToOneRelation.schema, cls: OneToOneRelation },
+   [RelationTypes.ManyToOne]: { schema: ManyToOneRelation.schema, cls: ManyToOneRelation },
+   [RelationTypes.ManyToMany]: { schema: ManyToManyRelation.schema, cls: ManyToManyRelation },
+   [RelationTypes.Polymorphic]: {
+      schema: PolymorphicRelation.schema,
+      cls: PolymorphicRelation
+   }
+} as const;
+
+export const RelationFieldClassMap = {
+   relation: { schema: relationFieldConfigSchema, field: RelationField }
+} as const;

app/src/data/relations/relation-types.ts

@@ -0,0 +1,7 @@
+export const RelationTypes = {
+   OneToOne: "1:1",
+   ManyToOne: "n:1",
+   ManyToMany: "m:n",
+   Polymorphic: "poly",
+} as const;
+export type RelationType = (typeof RelationTypes)[keyof typeof RelationTypes];

app/src/data/schema/SchemaManager.ts

@@ -0,0 +1,349 @@
+import type { AlterTableColumnAlteringBuilder, CompiledQuery, TableMetadata } from "kysely";
+import type { IndexMetadata } from "../connection/Connection";
+import type { Entity, EntityManager } from "../entities";
+import { PrimaryField, type SchemaResponse } from "../fields";
+
+type IntrospectedTable = TableMetadata & {
+   indices: IndexMetadata[];
+};
+
+type SchemaTable = {
+   name: string;
+   columns: string[];
+};
+
+type SchemaDiffTable = {
+   name: string;
+   isNew: boolean;
+   isDrop?: boolean;
+   columns: {
+      add: string[];
+      drop: string[];
+      change: string[];
+   };
+   indices: {
+      add: string[];
+      drop: string[];
+   };
+};
+
+type ColumnDiff = {
+   name: string;
+   changes: {
+      attribute: string;
+      prev: any;
+      next: any;
+   }[];
+};
+
+/**
+ * @todo: add modified fields
+ * @todo: add drop tables
+ *
+ * @todo: change exclude tables to startWith, then add "bknd_" tables
+ */
+
+export class SchemaManager {
+   static EXCLUDE_TABLES = ["libsql_wasm_func_table", "sqlite_sequence", "_cf_KV"];
+
+   constructor(private readonly em: EntityManager<any>) {}
+
+   private getIntrospector() {
+      if (!this.em.connection.supportsIndices()) {
+         throw new Error("Indices are not supported by the current connection");
+      }
+
+      return this.em.connection.getIntrospector();
+   }
+
+   async introspect(): Promise<IntrospectedTable[]> {
+      const tables = await this.getIntrospector().getTables({
+         withInternalKyselyTables: false
+      });
+
+      const indices = await this.getIntrospector().getIndices();
+
+      const cleanTables: any[] = [];
+      for (const table of tables) {
+         if (SchemaManager.EXCLUDE_TABLES.includes(table.name)) {
+            continue;
+         }
+
+         cleanTables.push({
+            ...table,
+            indices: indices.filter((index) => index.table === table.name)
+         });
+      }
+
+      return cleanTables;
+   }
+
+   getIntrospectionFromEntity(entity: Entity): IntrospectedTable {
+      const fields = entity.getFields(false);
+      const indices = this.em.getIndicesOf(entity);
+
+      // this is intentionally setting values to defaults, like "nullable" and "default"
+      // that is because sqlite is the main focus, but in the future,
+      // we might want to support full sync with extensive schema updates (e.g. postgres)
+      return {
+         name: entity.name,
+         isView: false,
+         columns: fields.map((field) => ({
+            name: field.name,
+            dataType: "TEXT", // doesn't matter
+            isNullable: true, // managed by the field
+            isAutoIncrementing: field instanceof PrimaryField,
+            hasDefaultValue: false, // managed by the field
+            comment: undefined
+         })),
+         indices: indices.map((index) => ({
+            name: index.name,
+            table: entity.name,
+            isUnique: index.unique,
+            columns: index.fields.map((f) => ({
+               name: f.name,
+               order: 0 // doesn't matter
+            }))
+         })) as any
+      };
+   }
+
+   async getDiff(): Promise<SchemaDiffTable[]> {
+      const introspection = await this.introspect();
+      const entityStates = this.em.entities.map((e) => this.getIntrospectionFromEntity(e));
+
+      const diff: SchemaDiffTable[] = [];
+      const namesFn = (c: { name: string }) => c.name;
+
+      // @todo: add drop tables (beware, there a system tables!)
+      introspection
+         .filter((table) => {
+            if (/bknd/.test(table.name) || table.isView) {
+               return false;
+            }
+            return !entityStates.map((e) => e.name).includes(table.name);
+         })
+         .forEach((t) => {
+            diff.push({
+               name: t.name,
+               isDrop: true,
+               isNew: false,
+               columns: {
+                  add: [],
+                  drop: [],
+                  change: []
+               },
+               indices: {
+                  add: [],
+                  drop: []
+               }
+            });
+         });
+
+      for (const entity of entityStates) {
+         const table = introspection.find((t) => t.name === entity.name);
+
+         if (!table) {
+            // If the table is completely new
+            diff.push({
+               name: entity.name,
+               isNew: true,
+               columns: {
+                  add: entity.columns.map(namesFn),
+                  drop: [],
+                  change: []
+               },
+               indices: {
+                  add: entity.indices.map(namesFn),
+                  drop: []
+               }
+            });
+         } else {
+            // If the table exists, check for new columns
+            const newColumns = entity.columns.filter(
+               (newColumn) => !table.columns.map(namesFn).includes(newColumn.name)
+            );
+
+            // check for columns to drop
+            const dropColumns = table.columns.filter(
+               (oldColumn) => !entity.columns.map(namesFn).includes(oldColumn.name)
+            );
+
+            // check for changed columns
+            const columnDiffs: ColumnDiff[] = [];
+            for (const entity_col of entity.columns) {
+               const db_col = table.columns.find((c) => c.name === entity_col.name);
+               const col_diffs: ColumnDiff["changes"] = [];
+               for (const [key, value] of Object.entries(entity_col)) {
+                  if (db_col && db_col[key] !== value) {
+                     col_diffs.push({
+                        attribute: key,
+                        prev: db_col[key],
+                        next: value
+                     });
+                  }
+               }
+               if (Object.keys(col_diffs).length > 0) {
+                  columnDiffs.push({
+                     name: entity_col.name,
+                     changes: col_diffs
+                  });
+               }
+            }
+
+            // new indices
+            const newIndices = entity.indices.filter(
+               (newIndex) => !table.indices.map((i) => i.name).includes(newIndex.name)
+            );
+
+            const dropIndices = table.indices.filter(
+               (oldIndex) => !entity.indices.map((i) => i.name).includes(oldIndex.name)
+            );
+
+            const anythingChanged = [
+               newColumns,
+               dropColumns,
+               //columnDiffs, // ignored
+               newIndices,
+               dropIndices
+            ].some((arr) => arr.length > 0);
+
+            if (anythingChanged) {
+               diff.push({
+                  name: entity.name,
+                  isNew: false,
+                  columns: {
+                     add: newColumns.map(namesFn),
+                     drop: dropColumns.map(namesFn),
+                     // @todo: this is ignored for now
+                     //change: columnDiffs.map(namesFn),
+                     change: []
+                  },
+                  indices: {
+                     add: newIndices.map(namesFn),
+                     drop: dropIndices.map(namesFn)
+                  }
+               });
+            }
+         }
+      }
+
+      return diff;
+   }
+
+   private collectFieldSchemas(table: string, columns: string[]) {
+      const schemas: SchemaResponse[] = [];
+      if (columns.length === 0) {
+         return schemas;
+      }
+
+      for (const column of columns) {
+         const field = this.em.entity(table).getField(column)!;
+         const fieldSchema = field.schema(this.em);
+         if (Array.isArray(fieldSchema) && fieldSchema.length === 3) {
+            schemas.push(fieldSchema);
+            //throw new Error(`Field "${field.name}" on entity "${table}" has no schema`);
+         }
+      }
+
+      return schemas;
+   }
+
+   async sync(config: { force?: boolean; drop?: boolean } = { force: false, drop: false }) {
+      const diff = await this.getDiff();
+      let updates: number = 0;
+      const statements: { sql: string; parameters: readonly unknown[] }[] = [];
+      const schema = this.em.connection.kysely.schema;
+
+      for (const table of diff) {
+         const qbs: { compile(): CompiledQuery; execute(): Promise<void> }[] = [];
+         let local_updates: number = 0;
+         const addFieldSchemas = this.collectFieldSchemas(table.name, table.columns.add);
+         const dropFields = table.columns.drop;
+         const dropIndices = table.indices.drop;
+
+         if (table.isDrop) {
+            updates++;
+            local_updates++;
+            if (config.drop) {
+               qbs.push(schema.dropTable(table.name));
+            }
+         } else if (table.isNew) {
+            let createQb = schema.createTable(table.name);
+            // add fields
+            for (const fieldSchema of addFieldSchemas) {
+               updates++;
+               local_updates++;
+               // @ts-ignore
+               createQb = createQb.addColumn(...fieldSchema);
+            }
+
+            qbs.push(createQb);
+         } else {
+            // if fields to add
+            if (addFieldSchemas.length > 0) {
+               // add fields
+               for (const fieldSchema of addFieldSchemas) {
+                  updates++;
+                  local_updates++;
+                  // @ts-ignore
+                  qbs.push(schema.alterTable(table.name).addColumn(...fieldSchema));
+               }
+            }
+
+            // if fields to drop
+            if (config.drop && dropFields.length > 0) {
+               // drop fields
+               for (const column of dropFields) {
+                  updates++;
+                  local_updates++;
+                  qbs.push(schema.alterTable(table.name).dropColumn(column));
+               }
+            }
+         }
+
+         // add indices
+         for (const index of table.indices.add) {
+            const indices = this.em.getIndicesOf(table.name);
+            const fieldIndex = indices.find((i) => i.name === index)!;
+            let qb = schema
+               .createIndex(index)
+               .on(table.name)
+               .columns(fieldIndex.fields.map((f) => f.name));
+            if (fieldIndex.unique) {
+               qb = qb.unique();
+            }
+            qbs.push(qb);
+            local_updates++;
+            updates++;
+         }
+
+         // drop indices
+         if (config.drop) {
+            for (const index of dropIndices) {
+               qbs.push(schema.dropIndex(index));
+               local_updates++;
+               updates++;
+            }
+         }
+
+         if (local_updates === 0) continue;
+
+         // iterate through built qbs
+         for (const qb of qbs) {
+            const { sql, parameters } = qb.compile();
+            statements.push({ sql, parameters });
+
+            if (config.force) {
+               try {
+                  await qb.execute();
+               } catch (e) {
+                  throw new Error(`Failed to execute query: ${sql}: ${(e as any).message}`);
+               }
+            }
+         }
+      }
+
+      return statements;
+   }
+}

app/src/data/server/data-query-impl.ts

@@ -0,0 +1,77 @@
+import {
+   type SchemaOptions,
+   type Static,
+   type StaticDecode,
+   StringEnum,
+   Type,
+   Value
+} from "core/utils";
+import type { Simplify } from "type-fest";
+import { WhereBuilder } from "../entities";
+
+const NumberOrString = (options: SchemaOptions = {}) =>
+   Type.Transform(Type.Union([Type.Number(), Type.String()], options))
+      .Decode((value) => Number.parseInt(String(value)))
+      .Encode(String);
+
+const limit = NumberOrString({ default: 10 });
+
+const offset = NumberOrString({ default: 0 });
+
+// @todo: allow "id" and "-id"
+const sort = Type.Transform(
+   Type.Union(
+      [Type.String(), Type.Object({ by: Type.String(), dir: StringEnum(["asc", "desc"]) })],
+      {
+         default: { by: "id", dir: "asc" }
+      }
+   )
+)
+   .Decode((value) => {
+      if (typeof value === "string") {
+         return JSON.parse(value);
+      }
+      return value;
+   })
+   .Encode(JSON.stringify);
+
+const stringArray = Type.Transform(
+   Type.Union([Type.String(), Type.Array(Type.String())], { default: [] })
+)
+   .Decode((value) => {
+      if (Array.isArray(value)) {
+         return value;
+      } else if (value.includes(",")) {
+         return value.split(",");
+      }
+      return [value];
+   })
+   .Encode((value) => (Array.isArray(value) ? value : [value]));
+
+export const whereSchema = Type.Transform(
+   Type.Union([Type.String(), Type.Object({})], { default: {} })
+)
+   .Decode((value) => {
+      const q = typeof value === "string" ? JSON.parse(value) : value;
+      return WhereBuilder.convert(q);
+   })
+   .Encode(JSON.stringify);
+
+export const querySchema = Type.Object(
+   {
+      limit: Type.Optional(limit),
+      offset: Type.Optional(offset),
+      sort: Type.Optional(sort),
+      select: Type.Optional(stringArray),
+      with: Type.Optional(stringArray),
+      join: Type.Optional(stringArray),
+      where: Type.Optional(whereSchema)
+   },
+   {
+      additionalProperties: false
+   }
+);
+
+export type RepoQueryIn = Simplify<Static<typeof querySchema>>;
+export type RepoQuery = Required<StaticDecode<typeof querySchema>>;
+export const defaultQuerySchema = Value.Default(querySchema, {}) as RepoQuery;

app/src/data/server/query.ts

@@ -0,0 +1,112 @@
+import { z } from "zod";
+
+const date = z.union([z.date(), z.string()]);
+const numeric = z.union([z.number(), date]);
+const boolean = z.union([z.boolean(), z.literal(1), z.literal(0)]);
+const value = z.union([z.string(), boolean, numeric]);
+
+const expressionCond = z.union([
+   z.object({ $eq: value }).strict(),
+   z.object({ $ne: value }).strict(),
+   z.object({ $isnull: boolean }).strict(),
+   z.object({ $notnull: boolean }).strict(),
+   z.object({ $in: z.array(value) }).strict(),
+   z.object({ $notin: z.array(value) }).strict(),
+   z.object({ $gt: numeric }).strict(),
+   z.object({ $gte: numeric }).strict(),
+   z.object({ $lt: numeric }).strict(),
+   z.object({ $lte: numeric }).strict(),
+   z.object({ $between: z.array(numeric).min(2).max(2) }).strict()
+] as const);
+
+// prettier-ignore
+const nonOperandString = z
+   .string()
+   .regex(/^(?!\$).*/)
+   .min(1);
+
+// {name: 'Michael'}
+const literalCond = z.record(nonOperandString, value);
+
+// { status: { $eq: 1 } }
+const literalExpressionCond = z.record(nonOperandString, value.or(expressionCond));
+
+const operandCond = z
+   .object({
+      $and: z.array(literalCond.or(expressionCond).or(literalExpressionCond)).optional(),
+      $or: z.array(literalCond.or(expressionCond).or(literalExpressionCond)).optional()
+   })
+   .strict();
+
+const literalSchema = literalCond.or(literalExpressionCond);
+export type LiteralSchemaIn = z.input<typeof literalSchema>;
+export type LiteralSchema = z.output<typeof literalSchema>;
+
+export const filterSchema = literalSchema.or(operandCond);
+export type FilterSchemaIn = z.input<typeof filterSchema>;
+export type FilterSchema = z.output<typeof filterSchema>;
+
+const stringArray = z
+   .union([
+      z.string().transform((v) => {
+         if (v.includes(",")) return v.split(",");
+         return v;
+      }),
+      z.array(z.string())
+   ])
+   .default([])
+   .transform((v) => (Array.isArray(v) ? v : [v]));
+
+export const whereRepoSchema = z
+   .preprocess((v: unknown) => {
+      try {
+         return JSON.parse(v as string);
+      } catch {
+         return v;
+      }
+   }, filterSchema)
+   .default({});
+
+const repoQuerySchema = z.object({
+   limit: z.coerce.number().default(10),
+   offset: z.coerce.number().default(0),
+   sort: z
+      .preprocess(
+         (v: unknown) => {
+            try {
+               return JSON.parse(v as string);
+            } catch {
+               return v;
+            }
+         },
+         z.union([
+            z.string().transform((v) => {
+               if (v.includes(":")) {
+                  let [field, dir] = v.split(":") as [string, string];
+                  if (!["asc", "desc"].includes(dir)) dir = "asc";
+                  return { by: field, dir } as { by: string; dir: "asc" | "desc" };
+               } else {
+                  return { by: v, dir: "asc" } as { by: string; dir: "asc" | "desc" };
+               }
+            }),
+            z.object({
+               by: z.string(),
+               dir: z.enum(["asc", "desc"])
+            })
+         ])
+      )
+      .default({ by: "id", dir: "asc" }),
+   select: stringArray,
+   with: stringArray,
+   join: stringArray,
+   debug: z
+      .preprocess((v) => {
+         if (["0", "false"].includes(String(v))) return false;
+         return Boolean(v);
+      }, z.boolean())
+      .default(false), //z.coerce.boolean().catch(false),
+   where: whereRepoSchema
+});
+
+type RepoQueryIn = z.input<typeof repoQuerySchema>;
+type RepoQuery = z.output<typeof repoQuerySchema>;

app/src/data/test-types.ts

@@ -0,0 +1,78 @@
+type Field<Type, Required extends true | false> = {
+   _type: Type;
+   _required: Required;
+};
+type TextField<Required extends true | false = false> = Field<string, Required> & {
+   _type: string;
+   required: () => TextField<true>;
+};
+type NumberField<Required extends true | false = false> = Field<number, Required> & {
+   _type: number;
+   required: () => NumberField<true>;
+};
+
+type Entity<Fields extends Record<string, Field<any, any>> = {}> = { name: string; fields: Fields };
+
+function entity<Fields extends Record<string, Field<any, any>>>(
+   name: string,
+   fields: Fields,
+): Entity<Fields> {
+   return { name, fields };
+}
+
+function text(): TextField<false> {
+   return {} as any;
+}
+function number(): NumberField<false> {
+   return {} as any;
+}
+
+const field1 = text();
+const field1_req = text().required();
+const field2 = number();
+const user = entity("users", {
+   name: text().required(),
+   bio: text(),
+   age: number(),
+   some: number().required(),
+});
+
+type InferEntityFields<T> = T extends Entity<infer Fields>
+   ? {
+        [K in keyof Fields]: Fields[K] extends { _type: infer Type; _required: infer Required }
+           ? Required extends true
+              ? Type
+              : Type | undefined
+           : never;
+     }
+   : never;
+
+type Prettify<T> = {
+   [K in keyof T]: T[K];
+};
+export type Simplify<T> = { [KeyType in keyof T]: T[KeyType] } & {};
+
+// from https://github.com/type-challenges/type-challenges/issues/28200
+type Merge<T> = {
+   [K in keyof T]: T[K];
+};
+type OptionalUndefined<
+   T,
+   Props extends keyof T = keyof T,
+   OptionsProps extends keyof T = Props extends keyof T
+      ? undefined extends T[Props]
+         ? Props
+         : never
+      : never,
+> = Merge<
+   {
+      [K in OptionsProps]?: T[K];
+   } & {
+      [K in Exclude<keyof T, OptionsProps>]: T[K];
+   }
+>;
+
+type UserFields = InferEntityFields<typeof user>;
+type UserFields2 = Simplify<OptionalUndefined<UserFields>>;
+
+const obj: UserFields2 = { name: "h", age: 1, some: 1 };