public commit

app/src/data/entities/Entity.ts

@@ -0,0 +1,238 @@
+import { config } from "core";
+import {
+   type Static,
+   StringEnum,
+   Type,
+   parse,
+   snakeToPascalWithSpaces,
+   transformObject
+} from "core/utils";
+import { type Field, PrimaryField, type TActionContext, type TRenderContext } from "../fields";
+
+// @todo: entity must be migrated to typebox
+export const entityConfigSchema = Type.Object(
+   {
+      name: Type.Optional(Type.String()),
+      name_singular: Type.Optional(Type.String()),
+      description: Type.Optional(Type.String()),
+      sort_field: Type.Optional(Type.String({ default: config.data.default_primary_field })),
+      sort_dir: Type.Optional(StringEnum(["asc", "desc"], { default: "asc" }))
+   },
+   {
+      additionalProperties: false
+   }
+);
+
+export type EntityConfig = Static<typeof entityConfigSchema>;
+
+export type EntityData = Record<string, any>;
+export type EntityJSON = ReturnType<Entity["toJSON"]>;
+
+/**
+ * regular: normal defined entity
+ * system: generated by the system, e.g. "users" from auth
+ * generated: result of a relation, e.g. many-to-many relation's connection entity
+ */
+export const entityTypes = ["regular", "system", "generated"] as const;
+export type TEntityType = (typeof entityTypes)[number];
+
+/**
+ * @todo: add check for adding fields (primary and relation not allowed)
+ * @todo: add option to disallow api deletes (or api actions in general)
+ */
+export class Entity<
+   EntityName extends string = string,
+   Fields extends Record<string, Field<any, any, any>> = Record<string, Field<any, any, any>>
+> {
+   readonly #_name!: EntityName;
+   readonly #_fields!: Fields; // only for types
+
+   readonly name: string;
+   readonly fields: Field[];
+   readonly config: EntityConfig;
+   protected data: EntityData[] | undefined;
+   readonly type: TEntityType = "regular";
+
+   constructor(name: string, fields?: Field[], config?: EntityConfig, type?: TEntityType) {
+      if (typeof name !== "string" || name.length === 0) {
+         throw new Error("Entity name must be a non-empty string");
+      }
+
+      this.name = name;
+      this.config = parse(entityConfigSchema, config || {}) as EntityConfig;
+
+      // add id field if not given
+      // @todo: add test
+      const primary_count = fields?.filter((field) => field instanceof PrimaryField).length ?? 0;
+      if (primary_count > 1) {
+         throw new Error(`Entity "${name}" has more than one primary field`);
+      }
+      this.fields = primary_count === 1 ? [] : [new PrimaryField()];
+
+      if (fields) {
+         fields.forEach((field) => this.addField(field));
+      }
+
+      if (type) this.type = type;
+   }
+
+   static create(args: {
+      name: string;
+      fields?: Field[];
+      config?: EntityConfig;
+      type?: TEntityType;
+   }) {
+      return new Entity(args.name, args.fields, args.config, args.type);
+   }
+
+   // @todo: add test
+   getType(): TEntityType {
+      return this.type;
+   }
+
+   getSelect(alias?: string, context?: TActionContext | TRenderContext): string[] {
+      return this.getFields()
+         .filter((field) => !field.isHidden(context ?? "read"))
+         .map((field) => (alias ? `${alias}.${field.name} as ${field.name}` : field.name));
+   }
+
+   getDefaultSort() {
+      return {
+         by: this.config.sort_field,
+         dir: this.config.sort_dir
+      };
+   }
+
+   getAliasedSelectFrom(
+      select: string[],
+      _alias?: string,
+      context?: TActionContext | TRenderContext
+   ): string[] {
+      const alias = _alias ?? this.name;
+      return this.getFields()
+         .filter(
+            (field) =>
+               !field.isVirtual() &&
+               !field.isHidden(context ?? "read") &&
+               select.includes(field.name)
+         )
+         .map((field) => (alias ? `${alias}.${field.name} as ${field.name}` : field.name));
+   }
+
+   getFillableFields(context?: TActionContext, include_virtual?: boolean): Field[] {
+      return this.getFields(include_virtual).filter((field) => field.isFillable(context));
+   }
+
+   getRequiredFields(): Field[] {
+      return this.getFields().filter((field) => field.isRequired());
+   }
+
+   getDefaultObject(): EntityData {
+      return this.getFields().reduce((acc, field) => {
+         if (field.hasDefault()) {
+            acc[field.name] = field.getDefault();
+         }
+         return acc;
+      }, {} as EntityData);
+   }
+
+   getField(name: string): Field | undefined {
+      return this.fields.find((field) => field.name === name);
+   }
+
+   __experimental_replaceField(name: string, field: Field) {
+      const index = this.fields.findIndex((f) => f.name === name);
+      if (index === -1) {
+         throw new Error(`Field "${name}" not found on entity "${this.name}"`);
+      }
+
+      this.fields[index] = field;
+   }
+
+   getPrimaryField(): PrimaryField {
+      return this.fields[0] as PrimaryField;
+   }
+
+   id(): PrimaryField {
+      return this.getPrimaryField();
+   }
+
+   get label(): string {
+      return snakeToPascalWithSpaces(this.config.name ?? this.name);
+   }
+
+   field(name: string): Field | undefined {
+      return this.getField(name);
+   }
+
+   getFields(include_virtual: boolean = false): Field[] {
+      if (include_virtual) return this.fields;
+      return this.fields.filter((f) => !f.isVirtual());
+   }
+
+   addField(field: Field) {
+      const existing = this.getField(field.name);
+      // make unique name check
+      if (existing) {
+         // @todo: for now adding a graceful method
+         if (JSON.stringify(existing) === JSON.stringify(field)) {
+            /*console.warn(
+               `Field "${field.name}" already exists on entity "${this.name}", but it's the same, so skipping.`,
+            );*/
+            return;
+         }
+
+         throw new Error(`Field "${field.name}" already exists on entity "${this.name}"`);
+      }
+
+      this.fields.push(field);
+   }
+
+   __setData(data: EntityData[]) {
+      this.data = data;
+   }
+
+   isValidData(data: EntityData, context: TActionContext, explain?: boolean): boolean {
+      const fields = this.getFillableFields(context, false);
+      //const fields = this.fields;
+      //console.log("data", data);
+      for (const field of fields) {
+         if (!field.isValid(data[field.name], context)) {
+            console.log("Entity.isValidData:invalid", context, field.name, data[field.name]);
+            if (explain) {
+               throw new Error(`Field "${field.name}" has invalid data: "${data[field.name]}"`);
+            }
+
+            return false;
+         }
+      }
+
+      return true;
+   }
+
+   toSchema(clean?: boolean): object {
+      const fields = Object.fromEntries(this.fields.map((field) => [field.name, field]));
+      const schema = Type.Object(
+         transformObject(fields, (field) => ({
+            title: field.config.label,
+            $comment: field.config.description,
+            $field: field.type,
+            readOnly: !field.isFillable("update") ? true : undefined,
+            writeOnly: !field.isFillable("create") ? true : undefined,
+            ...field.toJsonSchema()
+         }))
+      );
+
+      return clean ? JSON.parse(JSON.stringify(schema)) : schema;
+   }
+
+   toJSON() {
+      return {
+         //name: this.name,
+         type: this.type,
+         //fields: transformObject(this.fields, (field) => field.toJSON()),
+         fields: Object.fromEntries(this.fields.map((field) => [field.name, field.toJSON()])),
+         config: this.config
+      };
+   }
+}

app/src/data/entities/EntityManager.ts

@@ -0,0 +1,266 @@
+import { EventManager } from "core/events";
+import { sql } from "kysely";
+import { Connection } from "../connection/Connection";
+import {
+   EntityNotDefinedException,
+   TransformRetrieveFailedException,
+   UnableToConnectException
+} from "../errors";
+import { MutatorEvents, RepositoryEvents } from "../events";
+import type { EntityIndex } from "../fields/indices/EntityIndex";
+import type { EntityRelation } from "../relations";
+import { RelationAccessor } from "../relations/RelationAccessor";
+import { SchemaManager } from "../schema/SchemaManager";
+import { Entity } from "./Entity";
+import { type EntityData, Mutator, Repository } from "./index";
+
+export class EntityManager<DB> {
+   connection: Connection;
+
+   private _entities: Entity[] = [];
+   private _relations: EntityRelation[] = [];
+   private _indices: EntityIndex[] = [];
+   private _schema?: SchemaManager;
+   readonly emgr: EventManager<typeof EntityManager.Events>;
+   static readonly Events = { ...MutatorEvents, ...RepositoryEvents };
+
+   constructor(
+      entities: Entity[],
+      connection: Connection,
+      relations: EntityRelation[] = [],
+      indices: EntityIndex[] = [],
+      emgr?: EventManager<any>
+   ) {
+      // add entities & relations
+      entities.forEach((entity) => this.addEntity(entity));
+      relations.forEach((relation) => this.addRelation(relation));
+      indices.forEach((index) => this.addIndex(index));
+
+      if (!(connection instanceof Connection)) {
+         throw new UnableToConnectException("");
+      }
+
+      this.connection = connection;
+      this.emgr = emgr ?? new EventManager();
+      //console.log("registering events", EntityManager.Events);
+      this.emgr.registerEvents(EntityManager.Events);
+   }
+
+   /**
+    * Forks the EntityManager without the EventManager.
+    * This is useful when used inside an event handler.
+    */
+   fork(): EntityManager<DB> {
+      return new EntityManager(this._entities, this.connection, this._relations, this._indices);
+   }
+
+   get entities(): Entity[] {
+      return this._entities;
+   }
+
+   get relations(): RelationAccessor {
+      return new RelationAccessor(this._relations);
+   }
+
+   get indices(): EntityIndex[] {
+      return this._indices;
+   }
+
+   async ping(): Promise<boolean> {
+      const res = await sql`SELECT 1`.execute(this.connection.kysely);
+      return res.rows.length > 0;
+   }
+
+   addEntity(entity: Entity) {
+      const existing = this.entities.find((e) => e.name === entity.name);
+      // check if already exists by name
+      if (existing) {
+         // @todo: for now adding a graceful method
+         if (JSON.stringify(existing) === JSON.stringify(entity)) {
+            //console.warn(`Entity "${entity.name}" already exists, but it's the same, so skipping.`);
+            return;
+         }
+
+         throw new Error(`Entity "${entity.name}" already exists`);
+      }
+
+      this.entities.push(entity);
+   }
+
+   entity(name: string): Entity {
+      const entity = this.entities.find((e) => e.name === name);
+      if (!entity) {
+         throw new EntityNotDefinedException(name);
+      }
+
+      return entity;
+   }
+
+   hasEntity(entity: string): boolean;
+   hasEntity(entity: Entity): boolean;
+   hasEntity(nameOrEntity: string | Entity): boolean {
+      const name = typeof nameOrEntity === "string" ? nameOrEntity : nameOrEntity.name;
+      return this.entities.some((e) => e.name === name);
+   }
+
+   hasIndex(index: string): boolean;
+   hasIndex(index: EntityIndex): boolean;
+   hasIndex(nameOrIndex: string | EntityIndex): boolean {
+      const name = typeof nameOrIndex === "string" ? nameOrIndex : nameOrIndex.name;
+      return this.indices.some((e) => e.name === name);
+   }
+
+   addRelation(relation: EntityRelation) {
+      // check if entities are registered
+      if (!this.entity(relation.source.entity.name) || !this.entity(relation.target.entity.name)) {
+         throw new Error("Relation source or target entity not found");
+      }
+
+      // @todo: potentially add name to relation in order to have multiple
+      const found = this._relations.find((r) => {
+         const equalSourceTarget =
+            r.source.entity.name === relation.source.entity.name &&
+            r.target.entity.name === relation.target.entity.name;
+         const equalReferences =
+            r.source.reference === relation.source.reference &&
+            r.target.reference === relation.target.reference;
+
+         return (
+            //r.type === relation.type && // ignore type for now
+            equalSourceTarget && equalReferences
+         );
+      });
+
+      if (found) {
+         throw new Error(
+            `Relation "${relation.type}" between "${relation.source.entity.name}" ` +
+               `and "${relation.target.entity.name}" already exists`
+         );
+      }
+
+      this._relations.push(relation);
+      relation.initialize(this);
+   }
+
+   relationsOf(entity_name: string): EntityRelation[] {
+      return this.relations.relationsOf(this.entity(entity_name));
+   }
+
+   relationOf(entity_name: string, reference: string): EntityRelation | undefined {
+      return this.relations.relationOf(this.entity(entity_name), reference);
+   }
+
+   hasRelations(entity_name: string): boolean {
+      return this.relations.hasRelations(this.entity(entity_name));
+   }
+
+   relatedEntitiesOf(entity_name: string): Entity[] {
+      return this.relations.relatedEntitiesOf(this.entity(entity_name));
+   }
+
+   relationReferencesOf(entity_name: string): string[] {
+      return this.relations.relationReferencesOf(this.entity(entity_name));
+   }
+
+   repository(_entity: Entity | string) {
+      const entity = _entity instanceof Entity ? _entity : this.entity(_entity);
+      return new Repository(this, entity, this.emgr);
+   }
+
+   repo<E extends Entity>(
+      _entity: E
+   ): Repository<
+      DB,
+      E extends Entity<infer Name> ? (Name extends keyof DB ? Name : never) : never
+   > {
+      return new Repository(this, _entity, this.emgr);
+   }
+
+   _repo<TB extends keyof DB>(_entity: TB): Repository<DB, TB> {
+      const entity = this.entity(_entity as any);
+      return new Repository(this, entity, this.emgr);
+   }
+
+   mutator(_entity: Entity | string) {
+      const entity = _entity instanceof Entity ? _entity : this.entity(_entity);
+      return new Mutator(this, entity, this.emgr);
+   }
+
+   addIndex(index: EntityIndex, force = false) {
+      // check if already exists by name
+      if (this.indices.find((e) => e.name === index.name)) {
+         if (force) {
+            throw new Error(`Index "${index.name}" already exists`);
+         }
+         return;
+      }
+
+      this._indices.push(index);
+   }
+
+   getIndicesOf(_entity: Entity | string): EntityIndex[] {
+      const entity = _entity instanceof Entity ? _entity : this.entity(_entity);
+      return this.indices.filter((index) => index.entity.name === entity.name);
+   }
+
+   schema() {
+      if (!this._schema) {
+         this._schema = new SchemaManager(this);
+      }
+
+      return this._schema;
+   }
+
+   // @todo: centralize and add tests
+   hydrate(entity_name: string, _data: EntityData[]) {
+      const entity = this.entity(entity_name);
+      const data: EntityData[] = [];
+
+      for (const row of _data) {
+         for (let [key, value] of Object.entries(row)) {
+            const field = entity.getField(key);
+
+            if (!field || field.isVirtual()) {
+               // if relation, use related entity to hydrate
+               const relation = this.relationOf(entity_name, key);
+               if (relation) {
+                  if (!value) continue;
+
+                  value = relation.hydrate(key, Array.isArray(value) ? value : [value], this);
+                  row[key] = value;
+                  continue;
+               } else if (field?.isVirtual()) {
+                  continue;
+               }
+
+               throw new Error(`Field "${key}" not found on entity "${entity.name}"`);
+            }
+
+            try {
+               if (value === null && field.hasDefault()) {
+                  row[key] = field.getDefault();
+               }
+
+               row[key] = field.transformRetrieve(value as any);
+            } catch (e: any) {
+               throw new TransformRetrieveFailedException(
+                  `"${field.type}" field "${key}" on entity "${entity.name}": ${e.message}`
+               );
+            }
+         }
+
+         data.push(row);
+      }
+
+      return data;
+   }
+
+   toJSON() {
+      return {
+         entities: Object.fromEntries(this.entities.map((e) => [e.name, e.toJSON()])),
+         relations: Object.fromEntries(this.relations.all.map((r) => [r.getName(), r.toJSON()])),
+         //relations: this.relations.all.map((r) => r.toJSON()),
+         indices: Object.fromEntries(this.indices.map((i) => [i.name, i.toJSON()]))
+      };
+   }
+}

app/src/data/entities/Mutator.ts

@@ -0,0 +1,270 @@
+import type { PrimaryFieldType } from "core";
+import { type EmitsEvents, EventManager } from "core/events";
+import type { DeleteQueryBuilder, InsertQueryBuilder, UpdateQueryBuilder } from "kysely";
+import { type TActionContext, WhereBuilder } from "..";
+import type { Entity, EntityData, EntityManager } from "../entities";
+import { InvalidSearchParamsException } from "../errors";
+import { MutatorEvents } from "../events";
+import { RelationMutator } from "../relations";
+import type { RepoQuery } from "../server/data-query-impl";
+
+type MutatorQB =
+   | InsertQueryBuilder<any, any, any>
+   | UpdateQueryBuilder<any, any, any, any>
+   | DeleteQueryBuilder<any, any, any>;
+
+type MutatorUpdateOrDelete =
+   | UpdateQueryBuilder<any, any, any, any>
+   | DeleteQueryBuilder<any, any, any>;
+
+export type MutatorResponse<T = EntityData[]> = {
+   entity: Entity;
+   sql: string;
+   parameters: any[];
+   result: EntityData[];
+   data: T;
+};
+
+export class Mutator<DB> implements EmitsEvents {
+   em: EntityManager<DB>;
+   entity: Entity;
+   static readonly Events = MutatorEvents;
+   emgr: EventManager<typeof MutatorEvents>;
+
+   // @todo: current hacky workaround to disable creation of system entities
+   __unstable_disable_system_entity_creation = true;
+   __unstable_toggleSystemEntityCreation(value: boolean) {
+      this.__unstable_disable_system_entity_creation = value;
+   }
+
+   constructor(em: EntityManager<DB>, entity: Entity, emgr?: EventManager<any>) {
+      this.em = em;
+      this.entity = entity;
+      this.emgr = emgr ?? new EventManager(MutatorEvents);
+   }
+
+   private get conn() {
+      return this.em.connection.kysely;
+   }
+
+   async getValidatedData(data: EntityData, context: TActionContext): Promise<EntityData> {
+      const entity = this.entity;
+      if (!context) {
+         throw new Error("Context must be provided for validation");
+      }
+
+      const keys = Object.keys(data);
+      const validatedData: EntityData = {};
+
+      // get relational references/keys
+      const relationMutator = new RelationMutator(entity, this.em);
+      const relational_keys = relationMutator.getRelationalKeys();
+
+      for (const key of keys) {
+         if (relational_keys.includes(key)) {
+            const result = await relationMutator.persistRelation(key, data[key]);
+
+            // if relation field (include key and value in validatedData)
+            if (Array.isArray(result)) {
+               //console.log("--- (instructions)", result);
+               const [relation_key, relation_value] = result;
+               validatedData[relation_key] = relation_value;
+            }
+            continue;
+         }
+
+         const field = entity.getField(key);
+         if (!field) {
+            throw new Error(
+               `Field "${key}" not found on entity "${entity.name}". Fields: ${entity
+                  .getFillableFields()
+                  .map((f) => f.name)
+                  .join(", ")}`
+            );
+         }
+
+         // we should never get here, but just to be sure (why?)
+         if (!field.isFillable(context)) {
+            throw new Error(`Field "${key}" is not fillable on entity "${entity.name}"`);
+         }
+
+         validatedData[key] = await field.transformPersist(data[key], this.em, context);
+      }
+
+      if (Object.keys(validatedData).length === 0) {
+         throw new Error(`No data left to update "${entity.name}"`);
+      }
+
+      return validatedData;
+   }
+
+   protected async many(qb: MutatorQB): Promise<MutatorResponse> {
+      const entity = this.entity;
+      const { sql, parameters } = qb.compile();
+      //console.log("mutatoar:exec", sql, parameters);
+      const result = await qb.execute();
+
+      const data = this.em.hydrate(entity.name, result) as EntityData[];
+
+      return {
+         entity,
+         sql,
+         parameters: [...parameters],
+         result: result,
+         data
+      };
+   }
+
+   protected async single(qb: MutatorQB): Promise<MutatorResponse<EntityData>> {
+      const { data, ...response } = await this.many(qb);
+      return { ...response, data: data[0]! };
+   }
+
+   async insertOne(data: EntityData): Promise<MutatorResponse<EntityData>> {
+      const entity = this.entity;
+      if (entity.type === "system" && this.__unstable_disable_system_entity_creation) {
+         throw new Error(`Creation of system entity "${entity.name}" is disabled`);
+      }
+
+      // @todo: establish the original order from "data"
+      const validatedData = {
+         ...entity.getDefaultObject(),
+         ...(await this.getValidatedData(data, "create"))
+      };
+
+      await this.emgr.emit(new Mutator.Events.MutatorInsertBefore({ entity, data: validatedData }));
+
+      // check if required fields are present
+      const required = entity.getRequiredFields();
+      for (const field of required) {
+         if (
+            typeof validatedData[field.name] === "undefined" ||
+            validatedData[field.name] === null
+         ) {
+            throw new Error(`Field "${field.name}" is required`);
+         }
+      }
+
+      const query = this.conn
+         .insertInto(entity.name)
+         .values(validatedData)
+         .returning(entity.getSelect());
+
+      const res = await this.single(query);
+
+      await this.emgr.emit(new Mutator.Events.MutatorInsertAfter({ entity, data: res.data }));
+
+      return res;
+   }
+
+   async updateOne(id: PrimaryFieldType, data: EntityData): Promise<MutatorResponse<EntityData>> {
+      const entity = this.entity;
+      if (!Number.isInteger(id)) {
+         throw new Error("ID must be provided for update");
+      }
+
+      const validatedData = await this.getValidatedData(data, "update");
+
+      await this.emgr.emit(
+         new Mutator.Events.MutatorUpdateBefore({ entity, entityId: id, data: validatedData })
+      );
+
+      const query = this.conn
+         .updateTable(entity.name)
+         .set(validatedData)
+         .where(entity.id().name, "=", id)
+         .returning(entity.getSelect());
+
+      const res = await this.single(query);
+
+      await this.emgr.emit(
+         new Mutator.Events.MutatorUpdateAfter({ entity, entityId: id, data: res.data })
+      );
+
+      return res;
+   }
+
+   async deleteOne(id: PrimaryFieldType): Promise<MutatorResponse<EntityData>> {
+      const entity = this.entity;
+      if (!Number.isInteger(id)) {
+         throw new Error("ID must be provided for deletion");
+      }
+
+      await this.emgr.emit(new Mutator.Events.MutatorDeleteBefore({ entity, entityId: id }));
+
+      const query = this.conn
+         .deleteFrom(entity.name)
+         .where(entity.id().name, "=", id)
+         .returning(entity.getSelect());
+
+      const res = await this.single(query);
+
+      await this.emgr.emit(
+         new Mutator.Events.MutatorDeleteAfter({ entity, entityId: id, data: res.data })
+      );
+
+      return res;
+   }
+
+   private getValidOptions(options?: Partial<RepoQuery>): Partial<RepoQuery> {
+      const entity = this.entity;
+      const validated: Partial<RepoQuery> = {};
+
+      if (options?.where) {
+         // @todo: add tests for aliased fields in where
+         const invalid = WhereBuilder.getPropertyNames(options.where).filter((field) => {
+            return typeof entity.getField(field) === "undefined";
+         });
+
+         if (invalid.length > 0) {
+            throw new InvalidSearchParamsException(`Invalid where field(s): ${invalid.join(", ")}`);
+         }
+
+         validated.where = options.where;
+      }
+
+      return validated;
+   }
+
+   private appendWhere<QB extends MutatorUpdateOrDelete>(qb: QB, _where?: RepoQuery["where"]): QB {
+      const entity = this.entity;
+
+      const alias = entity.name;
+      const aliased = (field: string) => `${alias}.${field}`;
+
+      // add where if present
+      if (_where) {
+         // @todo: add tests for aliased fields in where
+         const invalid = WhereBuilder.getPropertyNames(_where).filter((field) => {
+            return typeof entity.getField(field) === "undefined";
+         });
+
+         if (invalid.length > 0) {
+            throw new InvalidSearchParamsException(`Invalid where field(s): ${invalid.join(", ")}`);
+         }
+
+         return WhereBuilder.addClause(qb, _where);
+      }
+
+      return qb;
+   }
+
+   // @todo: decide whether entries should be deleted all at once or one by one (for events)
+   async deleteMany(where?: RepoQuery["where"]): Promise<MutatorResponse<EntityData>> {
+      const entity = this.entity;
+
+      const qb = this.appendWhere(this.conn.deleteFrom(entity.name), where).returning(
+         entity.getSelect()
+      );
+
+      //await this.emgr.emit(new Mutator.Events.MutatorDeleteBefore({ entity, entityId: id }));
+
+      const res = await this.many(qb);
+
+      /*await this.emgr.emit(
+         new Mutator.Events.MutatorDeleteAfter({ entity, entityId: id, data: res.data })
+      );*/
+
+      return res;
+   }
+}

app/src/data/entities/index.ts

@@ -0,0 +1,6 @@
+export * from "./Entity";
+export * from "./EntityManager";
+export * from "./Mutator";
+export * from "./query/Repository";
+export * from "./query/WhereBuilder";
+export * from "./query/WithBuilder";

app/src/data/entities/query/JoinBuilder.ts

@@ -0,0 +1,51 @@
+import { ManyToManyRelation, ManyToOneRelation } from "../../relations";
+import type { Entity } from "../Entity";
+import type { EntityManager } from "../EntityManager";
+import type { RepositoryQB } from "./Repository";
+
+export class JoinBuilder {
+   private static buildClause(
+      em: EntityManager<any>,
+      qb: RepositoryQB,
+      entity: Entity,
+      withString: string,
+   ) {
+      const relation = em.relationOf(entity.name, withString);
+      if (!relation) {
+         throw new Error(`Relation "${withString}" not found`);
+      }
+
+      return relation.buildJoin(entity, qb, withString);
+   }
+
+   // @todo: returns multiple on manytomany (edit: so?)
+   static getJoinedEntityNames(em: EntityManager<any>, entity: Entity, joins: string[]): string[] {
+      return joins.flatMap((join) => {
+         const relation = em.relationOf(entity.name, join);
+         if (!relation) {
+            throw new Error(`Relation "${join}" not found`);
+         }
+
+         const other = relation.other(entity);
+
+         if (relation instanceof ManyToOneRelation) {
+            return [other.entity.name];
+         } else if (relation instanceof ManyToManyRelation) {
+            return [other.entity.name, relation.connectionEntity.name];
+         }
+
+         return [];
+      });
+   }
+
+   static addClause(em: EntityManager<any>, qb: RepositoryQB, entity: Entity, joins: string[]) {
+      if (joins.length === 0) return qb;
+
+      let newQb = qb;
+      for (const entry of joins) {
+         newQb = JoinBuilder.buildClause(em, newQb, entity, entry);
+      }
+
+      return newQb;
+   }
+}

app/src/data/entities/query/Repository.ts

@@ -0,0 +1,407 @@
+import type { PrimaryFieldType } from "core";
+import { type EmitsEvents, EventManager } from "core/events";
+import { type SelectQueryBuilder, sql } from "kysely";
+import { cloneDeep } from "lodash-es";
+import { InvalidSearchParamsException } from "../../errors";
+import { MutatorEvents, RepositoryEvents, RepositoryFindManyBefore } from "../../events";
+import { type RepoQuery, defaultQuerySchema } from "../../server/data-query-impl";
+import {
+   type Entity,
+   type EntityData,
+   type EntityManager,
+   WhereBuilder,
+   WithBuilder
+} from "../index";
+import { JoinBuilder } from "./JoinBuilder";
+
+export type RepositoryQB = SelectQueryBuilder<any, any, any>;
+
+export type RepositoryRawResponse = {
+   sql: string;
+   parameters: any[];
+   result: EntityData[];
+};
+export type RepositoryResponse<T = EntityData[]> = RepositoryRawResponse & {
+   entity: Entity;
+   data: T;
+   meta: {
+      total: number;
+      count: number;
+      items: number;
+      time?: number;
+      query?: {
+         sql: string;
+         parameters: readonly any[];
+      };
+   };
+};
+
+export type RepositoryCountResponse = RepositoryRawResponse & {
+   count: number;
+};
+export type RepositoryExistsResponse = RepositoryRawResponse & {
+   exists: boolean;
+};
+
+export class Repository<DB = any, TB extends keyof DB = any> implements EmitsEvents {
+   em: EntityManager<DB>;
+   entity: Entity;
+   static readonly Events = RepositoryEvents;
+   emgr: EventManager<typeof Repository.Events>;
+
+   constructor(em: EntityManager<DB>, entity: Entity, emgr?: EventManager<any>) {
+      this.em = em;
+      this.entity = entity;
+      this.emgr = emgr ?? new EventManager(MutatorEvents);
+   }
+
+   private cloneFor(entity: Entity) {
+      return new Repository(this.em, entity, this.emgr);
+   }
+
+   private get conn() {
+      return this.em.connection.kysely;
+   }
+
+   private getValidOptions(options?: Partial<RepoQuery>): RepoQuery {
+      const entity = this.entity;
+      // @todo: if not cloned deep, it will keep references and error if multiple requests come in
+      const validated = {
+         ...cloneDeep(defaultQuerySchema),
+         sort: entity.getDefaultSort(),
+         select: entity.getSelect()
+      };
+      //console.log("validated", validated);
+
+      if (!options) return validated;
+
+      if (options.sort) {
+         if (!validated.select.includes(options.sort.by)) {
+            throw new InvalidSearchParamsException(`Invalid sort field "${options.sort.by}"`);
+         }
+         if (!["asc", "desc"].includes(options.sort.dir)) {
+            throw new InvalidSearchParamsException(`Invalid sort direction "${options.sort.dir}"`);
+         }
+
+         validated.sort = options.sort;
+      }
+
+      if (options.select && options.select.length > 0) {
+         const invalid = options.select.filter((field) => !validated.select.includes(field));
+
+         if (invalid.length > 0) {
+            throw new InvalidSearchParamsException(
+               `Invalid select field(s): ${invalid.join(", ")}`
+            );
+         }
+
+         validated.select = options.select;
+      }
+
+      if (options.with && options.with.length > 0) {
+         for (const entry of options.with) {
+            const related = this.em.relationOf(entity.name, entry);
+            if (!related) {
+               throw new InvalidSearchParamsException(
+                  `WITH: "${entry}" is not a relation of "${entity.name}"`
+               );
+            }
+
+            validated.with.push(entry);
+         }
+      }
+
+      if (options.join && options.join.length > 0) {
+         for (const entry of options.join) {
+            const related = this.em.relationOf(entity.name, entry);
+            if (!related) {
+               throw new InvalidSearchParamsException(
+                  `JOIN: "${entry}" is not a relation of "${entity.name}"`
+               );
+            }
+
+            validated.join.push(entry);
+         }
+      }
+
+      if (options.where) {
+         // @todo: auto-alias base entity when using joins! otherwise "id" is ambiguous
+         const aliases = [entity.name];
+         if (validated.join.length > 0) {
+            aliases.push(...JoinBuilder.getJoinedEntityNames(this.em, entity, validated.join));
+         }
+
+         // @todo: add tests for aliased fields in where
+         const invalid = WhereBuilder.getPropertyNames(options.where).filter((field) => {
+            if (field.includes(".")) {
+               const [alias, prop] = field.split(".") as [string, string];
+               if (!aliases.includes(alias)) {
+                  return true;
+               }
+
+               return !this.em.entity(alias).getField(prop);
+            }
+
+            return typeof entity.getField(field) === "undefined";
+         });
+
+         if (invalid.length > 0) {
+            throw new InvalidSearchParamsException(`Invalid where field(s): ${invalid.join(", ")}`);
+         }
+
+         validated.where = options.where;
+      }
+
+      // pass unfiltered
+      if (options.limit) validated.limit = options.limit;
+      if (options.offset) validated.offset = options.offset;
+
+      return validated;
+   }
+
+   protected async performQuery(qb: RepositoryQB): Promise<RepositoryResponse> {
+      const entity = this.entity;
+      const compiled = qb.compile();
+      /*const { sql, parameters } = qb.compile();
+      console.log("many", sql, parameters);*/
+
+      const start = performance.now();
+      const selector = (as = "count") => this.conn.fn.countAll<number>().as(as);
+      const countQuery = qb
+         .clearSelect()
+         .select(selector())
+         .clearLimit()
+         .clearOffset()
+         .clearGroupBy()
+         .clearOrderBy();
+      const totalQuery = this.conn.selectFrom(entity.name).select(selector("total"));
+
+      try {
+         const [_count, _total, result] = await this.em.connection.batchQuery([
+            countQuery,
+            totalQuery,
+            qb
+         ]);
+         //console.log("result", { _count, _total });
+
+         const time = Number.parseFloat((performance.now() - start).toFixed(2));
+         const data = this.em.hydrate(entity.name, result);
+
+         return {
+            entity,
+            sql: compiled.sql,
+            parameters: [...compiled.parameters],
+            result,
+            data,
+            meta: {
+               total: _total[0]?.total ?? 0,
+               count: _count[0]?.count ?? 0, // @todo: better graceful method
+               items: result.length,
+               time,
+               query: { sql: compiled.sql, parameters: compiled.parameters }
+            }
+         };
+      } catch (e) {
+         console.error("many error", e, compiled);
+         throw e;
+      }
+   }
+
+   protected async single(
+      qb: RepositoryQB,
+      options: RepoQuery
+   ): Promise<RepositoryResponse<EntityData>> {
+      await this.emgr.emit(
+         new Repository.Events.RepositoryFindOneBefore({ entity: this.entity, options })
+      );
+
+      const { data, ...response } = await this.performQuery(qb);
+
+      await this.emgr.emit(
+         new Repository.Events.RepositoryFindOneAfter({
+            entity: this.entity,
+            options,
+            data: data[0]!
+         })
+      );
+
+      return { ...response, data: data[0]! };
+   }
+
+   private buildQuery(
+      _options?: Partial<RepoQuery>,
+      exclude_options: (keyof RepoQuery)[] = []
+   ): { qb: RepositoryQB; options: RepoQuery } {
+      const entity = this.entity;
+      const options = this.getValidOptions(_options);
+
+      const alias = entity.name;
+      const aliased = (field: string) => `${alias}.${field}`;
+      let qb = this.conn
+         .selectFrom(entity.name)
+         .select(entity.getAliasedSelectFrom(options.select, alias));
+
+      //console.log("build query options", options);
+      if (!exclude_options.includes("with") && options.with) {
+         qb = WithBuilder.addClause(this.em, qb, entity, options.with);
+      }
+
+      if (!exclude_options.includes("join") && options.join) {
+         qb = JoinBuilder.addClause(this.em, qb, entity, options.join);
+      }
+
+      // add where if present
+      if (!exclude_options.includes("where") && options.where) {
+         qb = WhereBuilder.addClause(qb, options.where);
+      }
+
+      if (!exclude_options.includes("limit")) qb = qb.limit(options.limit);
+      if (!exclude_options.includes("offset")) qb = qb.offset(options.offset);
+
+      // sorting
+      if (!exclude_options.includes("sort")) {
+         qb = qb.orderBy(aliased(options.sort.by), options.sort.dir);
+      }
+
+      return { qb, options };
+   }
+
+   async findId(
+      id: PrimaryFieldType,
+      _options?: Partial<Omit<RepoQuery, "where" | "limit" | "offset">>
+   ): Promise<RepositoryResponse<DB[TB]>> {
+      const { qb, options } = this.buildQuery(
+         {
+            ..._options,
+            where: { [this.entity.getPrimaryField().name]: id },
+            limit: 1
+         },
+         ["offset", "sort"]
+      );
+
+      return this.single(qb, options) as any;
+   }
+
+   async findOne(
+      where: RepoQuery["where"],
+      _options?: Partial<Omit<RepoQuery, "where" | "limit" | "offset">>
+   ): Promise<RepositoryResponse<DB[TB] | undefined>> {
+      const { qb, options } = this.buildQuery(
+         {
+            ..._options,
+            where,
+            limit: 1
+         },
+         ["offset", "sort"]
+      );
+
+      return this.single(qb, options) as any;
+   }
+
+   async findMany(_options?: Partial<RepoQuery>): Promise<RepositoryResponse<DB[TB][]>> {
+      const { qb, options } = this.buildQuery(_options);
+      //console.log("findMany:options", options);
+
+      await this.emgr.emit(
+         new Repository.Events.RepositoryFindManyBefore({ entity: this.entity, options })
+      );
+
+      const res = await this.performQuery(qb);
+
+      await this.emgr.emit(
+         new Repository.Events.RepositoryFindManyAfter({
+            entity: this.entity,
+            options,
+            data: res.data
+         })
+      );
+
+      return res as any;
+   }
+
+   // @todo: add unit tests, specially for many to many
+   async findManyByReference(
+      id: PrimaryFieldType,
+      reference: string,
+      _options?: Partial<Omit<RepoQuery, "limit" | "offset">>
+   ): Promise<RepositoryResponse<EntityData>> {
+      const entity = this.entity;
+      const listable_relations = this.em.relations.listableRelationsOf(entity);
+      const relation = listable_relations.find((r) => r.ref(reference).reference === reference);
+
+      if (!relation) {
+         throw new Error(
+            `Relation "${reference}" not found or not listable on entity "${entity.name}"`
+         );
+      }
+
+      const newEntity = relation.other(entity).entity;
+      const refQueryOptions = relation.getReferenceQuery(newEntity, id as number, reference);
+      if (!("where" in refQueryOptions) || Object.keys(refQueryOptions.where as any).length === 0) {
+         throw new Error(
+            `Invalid reference query for "${reference}" on entity "${newEntity.name}"`
+         );
+      }
+
+      const findManyOptions = {
+         ..._options,
+         ...refQueryOptions,
+         where: {
+            ...refQueryOptions.where,
+            ..._options?.where
+         }
+      };
+
+      //console.log("findManyOptions", newEntity.name, findManyOptions);
+      return this.cloneFor(newEntity).findMany(findManyOptions);
+   }
+
+   async count(where?: RepoQuery["where"]): Promise<RepositoryCountResponse> {
+      const entity = this.entity;
+      const options = this.getValidOptions({ where });
+
+      const selector = this.conn.fn.count<number>(sql`*`).as("count");
+      let qb = this.conn.selectFrom(entity.name).select(selector);
+
+      // add where if present
+      if (options.where) {
+         qb = WhereBuilder.addClause(qb, options.where);
+      }
+
+      const compiled = qb.compile();
+      const result = await qb.execute();
+
+      return {
+         sql: compiled.sql,
+         parameters: [...compiled.parameters],
+         result,
+         count: result[0]?.count ?? 0
+      };
+   }
+
+   async exists(where: Required<RepoQuery["where"]>): Promise<RepositoryExistsResponse> {
+      const entity = this.entity;
+      const options = this.getValidOptions({ where });
+
+      const selector = this.conn.fn.count<number>(sql`*`).as("count");
+      let qb = this.conn.selectFrom(entity.name).select(selector);
+
+      // add mandatory where
+      qb = WhereBuilder.addClause(qb, options.where);
+
+      // we only need 1
+      qb = qb.limit(1);
+
+      const compiled = qb.compile();
+      //console.log("exists query", compiled.sql, compiled.parameters);
+      const result = await qb.execute();
+      //console.log("result", result);
+
+      return {
+         sql: compiled.sql,
+         parameters: [...compiled.parameters],
+         result,
+         exists: result[0]!.count > 0
+      };
+   }
+}

app/src/data/entities/query/WhereBuilder.ts

@@ -0,0 +1,132 @@
+import {
+   type BooleanLike,
+   type FilterQuery,
+   type Primitive,
+   type TExpression,
+   exp,
+   isBooleanLike,
+   isPrimitive,
+   makeValidator
+} from "core";
+import type {
+   DeleteQueryBuilder,
+   ExpressionBuilder,
+   ExpressionWrapper,
+   SelectQueryBuilder,
+   UpdateQueryBuilder
+} from "kysely";
+import type { RepositoryQB } from "./Repository";
+
+type Builder = ExpressionBuilder<any, any>;
+type Wrapper = ExpressionWrapper<any, any, any>;
+type WhereQb =
+   | SelectQueryBuilder<any, any, any>
+   | UpdateQueryBuilder<any, any, any, any>
+   | DeleteQueryBuilder<any, any, any>;
+
+function key(e: unknown): string {
+   if (typeof e !== "string") {
+      throw new Error(`Invalid key: ${e}`);
+   }
+   return e as string;
+}
+
+const expressions: TExpression<any, any, any>[] = [
+   exp(
+      "$eq",
+      (v: Primitive) => isPrimitive(v),
+      (v, k, eb: Builder) => eb(key(k), "=", v)
+   ),
+   exp(
+      "$ne",
+      (v: Primitive) => isPrimitive(v),
+      (v, k, eb: Builder) => eb(key(k), "!=", v)
+   ),
+   exp(
+      "$gt",
+      (v: Primitive) => isPrimitive(v),
+      (v, k, eb: Builder) => eb(key(k), ">", v)
+   ),
+   exp(
+      "$gte",
+      (v: Primitive) => isPrimitive(v),
+      (v, k, eb: Builder) => eb(key(k), ">=", v)
+   ),
+   exp(
+      "$lt",
+      (v: Primitive) => isPrimitive(v),
+      (v, k, eb: Builder) => eb(key(k), "<", v)
+   ),
+   exp(
+      "$lte",
+      (v: Primitive) => isPrimitive(v),
+      (v, k, eb: Builder) => eb(key(k), "<=", v)
+   ),
+   exp(
+      "$isnull",
+      (v: BooleanLike) => isBooleanLike(v),
+      (v, k, eb: Builder) => eb(key(k), v ? "is" : "is not", null)
+   ),
+   exp(
+      "$in",
+      (v: any[]) => Array.isArray(v),
+      (v, k, eb: Builder) => eb(key(k), "in", v)
+   ),
+   exp(
+      "$notin",
+      (v: any[]) => Array.isArray(v),
+      (v, k, eb: Builder) => eb(key(k), "not in", v)
+   ),
+   exp(
+      "$between",
+      (v: [number, number]) => Array.isArray(v) && v.length === 2,
+      (v, k, eb: Builder) => eb.between(key(k), v[0], v[1])
+   ),
+   exp(
+      "$like",
+      (v: Primitive) => isPrimitive(v),
+      (v, k, eb: Builder) => eb(key(k), "like", String(v).replace(/\*/g, "%"))
+   )
+];
+
+export type WhereQuery = FilterQuery<typeof expressions>;
+
+const validator = makeValidator(expressions);
+
+export class WhereBuilder {
+   static addClause<QB extends WhereQb>(qb: QB, query: WhereQuery) {
+      if (Object.keys(query).length === 0) {
+         return qb;
+      }
+
+      // @ts-ignore
+      return qb.where((eb) => {
+         const fns = validator.build(query, {
+            value_is_kv: true,
+            exp_ctx: eb,
+            convert: true
+         });
+
+         if (fns.$or.length > 0 && fns.$and.length > 0) {
+            return eb.and(fns.$and).or(eb.and(fns.$or));
+         } else if (fns.$or.length > 0) {
+            return eb.or(fns.$or);
+         }
+
+         return eb.and(fns.$and);
+      });
+   }
+
+   static convert(query: WhereQuery): WhereQuery {
+      return validator.convert(query);
+   }
+
+   static getPropertyNames(query: WhereQuery): string[] {
+      const { keys } = validator.build(query, {
+         value_is_kv: true,
+         exp_ctx: () => null,
+         convert: true
+      });
+      return Array.from(keys);
+   }
+}

app/src/data/entities/query/WithBuilder.ts

@@ -0,0 +1,42 @@
+import type { Entity, EntityManager, RepositoryQB } from "../../entities";
+
+export class WithBuilder {
+   private static buildClause(
+      em: EntityManager<any>,
+      qb: RepositoryQB,
+      entity: Entity,
+      withString: string
+   ) {
+      const relation = em.relationOf(entity.name, withString);
+      if (!relation) {
+         throw new Error(`Relation "${withString}" not found`);
+      }
+
+      const cardinality = relation.ref(withString).cardinality;
+      //console.log("with--builder", { entity: entity.name, withString, cardinality });
+
+      const fns = em.connection.fn;
+      const jsonFrom = cardinality === 1 ? fns.jsonObjectFrom : fns.jsonArrayFrom;
+
+      if (!jsonFrom) {
+         throw new Error("Connection does not support jsonObjectFrom/jsonArrayFrom");
+      }
+
+      try {
+         return relation.buildWith(entity, qb, jsonFrom, withString);
+      } catch (e) {
+         throw new Error(`Could not build "with" relation "${withString}": ${(e as any).message}`);
+      }
+   }
+
+   static addClause(em: EntityManager<any>, qb: RepositoryQB, entity: Entity, withs: string[]) {
+      if (withs.length === 0) return qb;
+
+      let newQb = qb;
+      for (const entry of withs) {
+         newQb = WithBuilder.buildClause(em, newQb, entity, entry);
+      }
+
+      return newQb;
+   }
+}