updated repo query schema, repo and withbuilder validation, and reworked relation with building

app/__test__/data/data-query-impl.spec.ts

@@ -1,8 +1,14 @@
 import { describe, expect, test } from "bun:test";
-import { Value } from "../../src/core/utils";
+import { Value, _jsonp } from "../../src/core/utils";
-import { WhereBuilder, type WhereQuery, querySchema } from "../../src/data";
+import { type RepoQuery, WhereBuilder, type WhereQuery, querySchema } from "../../src/data";
+import type { RepoQueryIn } from "../../src/data/server/data-query-impl";
 import { getDummyConnection } from "./helper";
 
+const decode = (input: RepoQueryIn, expected: RepoQuery) => {
+   const result = Value.Decode(querySchema, input);
+   expect(result).toEqual(expected);
+};
+
 describe("data-query-impl", () => {
    function qb() {
       const c = getDummyConnection();
@@ -88,21 +94,47 @@ describe("data-query-impl", () => {
          expect(keys).toEqual(expectedKeys);
       }
    });
+
+   test("with", () => {
+      decode({ with: ["posts"] }, { with: { posts: {} } });
+      decode({ with: { posts: {} } }, { with: { posts: {} } });
+      decode({ with: { posts: { limit: "1" } } }, { with: { posts: { limit: 1 } } });
+      decode(
+         {
+            with: {
+               posts: {
+                  with: {
+                     images: {
+                        select: "id"
+                     }
+                  }
+               }
+            }
+         },
+         {
+            with: {
+               posts: {
+                  with: {
+                     images: {
+                        select: ["id"]
+                     }
+                  }
+               }
+            }
+         }
+      );
+   });
 });
 
 describe("data-query-impl: Typebox", () => {
    test("sort", async () => {
-      const decode = (input: any, expected: any) => {
+      const _dflt = { sort: { by: "id", dir: "asc" } };
-         const result = Value.Decode(querySchema, input);
-         expect(result.sort).toEqual(expected);
-      };
-      const _dflt = { by: "id", dir: "asc" };
 
       decode({ sort: "" }, _dflt);
-      decode({ sort: "name" }, { by: "name", dir: "asc" });
+      decode({ sort: "name" }, { sort: { by: "name", dir: "asc" } });
-      decode({ sort: "-name" }, { by: "name", dir: "desc" });
+      decode({ sort: "-name" }, { sort: { by: "name", dir: "desc" } });
-      decode({ sort: "-posts.name" }, { by: "posts.name", dir: "desc" });
+      decode({ sort: "-posts.name" }, { sort: { by: "posts.name", dir: "desc" } });
       decode({ sort: "-1name" }, _dflt);
-      decode({ sort: { by: "name", dir: "desc" } }, { by: "name", dir: "desc" });
+      decode({ sort: { by: "name", dir: "desc" } }, { sort: { by: "name", dir: "desc" } });
    });
 });

app/__test__/data/relations.test.ts

@@ -119,12 +119,9 @@ describe("Relations", async () => {
        - select: users.*
        - cardinality: 1
        */
-      const selectPostsFromUsers = postAuthorRel.buildWith(
+      const selectPostsFromUsers = kysely
-         users,
+         .selectFrom(users.name)
-         kysely.selectFrom(users.name),
+         .select((eb) => postAuthorRel.buildWith(users, "posts")(eb).as("posts"));
-         jsonFrom,
-         "posts"
-      );
       expect(selectPostsFromUsers.compile().sql).toBe(
          'select (select "posts"."id" as "id", "posts"."title" as "title", "posts"."author_id" as "author_id" from "posts" as "posts" where "posts"."author_id" = "users"."id" limit ?) as "posts" from "users"'
       );
@@ -141,12 +138,9 @@ describe("Relations", async () => {
        - select: posts.*
        - cardinality:
        */
-      const selectUsersFromPosts = postAuthorRel.buildWith(
+      const selectUsersFromPosts = kysely
-         posts,
+         .selectFrom(posts.name)
-         kysely.selectFrom(posts.name),
+         .select((eb) => postAuthorRel.buildWith(posts, "author")(eb).as("author"));
-         jsonFrom,
-         "author"
-      );
 
       expect(selectUsersFromPosts.compile().sql).toBe(
          'select (select "author"."id" as "id", "author"."username" as "username" from "users" as "author" where "author"."id" = "posts"."author_id" limit ?) as "author" from "posts"'
@@ -315,20 +309,16 @@ describe("Relations", async () => {
        - select: users.*
        - cardinality: 1
        */
-      const selectCategoriesFromPosts = postCategoriesRel.buildWith(
+      const selectCategoriesFromPosts = kysely
-         posts,
+         .selectFrom(posts.name)
-         kysely.selectFrom(posts.name),
+         .select((eb) => postCategoriesRel.buildWith(posts)(eb).as("categories"));
-         jsonFrom
-      );
       expect(selectCategoriesFromPosts.compile().sql).toBe(
          'select (select "categories"."id" as "id", "categories"."label" as "label" from "categories" inner join "posts_categories" on "categories"."id" = "posts_categories"."categories_id" where "posts"."id" = "posts_categories"."posts_id" limit ?) as "categories" from "posts"'
       );
 
-      const selectPostsFromCategories = postCategoriesRel.buildWith(
+      const selectPostsFromCategories = kysely
-         categories,
+         .selectFrom(categories.name)
-         kysely.selectFrom(categories.name),
+         .select((eb) => postCategoriesRel.buildWith(categories)(eb).as("posts"));
-         jsonFrom
-      );
       expect(selectPostsFromCategories.compile().sql).toBe(
          'select (select "posts"."id" as "id", "posts"."title" as "title" from "posts" inner join "posts_categories" on "posts"."id" = "posts_categories"."posts_id" where "categories"."id" = "posts_categories"."categories_id" limit ?) as "posts" from "categories"'
       );

app/__test__/data/specs/WithBuilder.spec.ts

@@ -8,19 +8,60 @@ import {
    TextField,
    WithBuilder
 } from "../../../src/data";
+import * as proto from "../../../src/data/prototype";
 import { getDummyConnection } from "../helper";
 
 const { dummyConnection, afterAllCleanup } = getDummyConnection();
 afterAll(afterAllCleanup);
 
+function schemaToEm(s: ReturnType<(typeof proto)["em"]>): EntityManager<any> {
+   return new EntityManager(Object.values(s.entities), dummyConnection, s.relations, s.indices);
+}
+
 describe("[data] WithBuilder", async () => {
+   test("validate withs", async () => {
+      const schema = proto.em(
+         {
+            posts: proto.entity("posts", {}),
+            users: proto.entity("users", {}),
+            media: proto.entity("media", {})
+         },
+         ({ relation }, { posts, users, media }) => {
+            relation(posts).manyToOne(users);
+            relation(users).polyToOne(media, { mappedBy: "avatar" });
+         }
+      );
+      const em = schemaToEm(schema);
+
+      expect(WithBuilder.validateWiths(em, "posts", undefined)).toBe(0);
+      expect(WithBuilder.validateWiths(em, "posts", {})).toBe(0);
+      expect(WithBuilder.validateWiths(em, "posts", { users: {} })).toBe(1);
+      expect(
+         WithBuilder.validateWiths(em, "posts", {
+            users: {
+               with: { avatar: {} }
+            }
+         })
+      ).toBe(2);
+      expect(() => WithBuilder.validateWiths(em, "posts", { author: {} })).toThrow();
+      expect(() =>
+         WithBuilder.validateWiths(em, "posts", {
+            users: {
+               with: { glibberish: {} }
+            }
+         })
+      ).toThrow();
+   });
+
    test("missing relation", async () => {
       const users = new Entity("users", [new TextField("username")]);
       const em = new EntityManager([users], dummyConnection);
 
       expect(() =>
-         WithBuilder.addClause(em, em.connection.kysely.selectFrom("users"), users, ["posts"])
+         WithBuilder.addClause(em, em.connection.kysely.selectFrom("users"), users, {
-      ).toThrow('Relation "posts" not found');
+            posts: {}
+         })
+      ).toThrow('Relation "users<>posts" not found');
    });
 
    test("addClause: ManyToOne", async () => {
@@ -29,9 +70,9 @@ describe("[data] WithBuilder", async () => {
       const relations = [new ManyToOneRelation(posts, users, { mappedBy: "author" })];
       const em = new EntityManager([users, posts], dummyConnection, relations);
 
-      const qb = WithBuilder.addClause(em, em.connection.kysely.selectFrom("users"), users, [
+      const qb = WithBuilder.addClause(em, em.connection.kysely.selectFrom("users"), users, {
-         "posts"
+         posts: {}
-      ]);
+      });
 
       const res = qb.compile();
 
@@ -44,7 +85,9 @@ describe("[data] WithBuilder", async () => {
          em,
          em.connection.kysely.selectFrom("posts"),
          posts, // @todo: try with "users", it gives output!
-         ["author"]
+         {
+            author: {}
+         }
       );
 
       const res2 = qb2.compile();
@@ -56,9 +99,10 @@ describe("[data] WithBuilder", async () => {
    });
 
    test("test with empty join", async () => {
+      const em = new EntityManager([], dummyConnection);
       const qb = { qb: 1 } as any;
 
-      expect(WithBuilder.addClause(null as any, qb, null as any, [])).toBe(qb);
+      expect(WithBuilder.addClause(em, qb, null as any, {})).toBe(qb);
    });
 
    test("test manytomany", async () => {
@@ -89,7 +133,7 @@ describe("[data] WithBuilder", async () => {
 
       //console.log((await em.repository().findMany("posts_categories")).result);
 
-      const res = await em.repository(posts).findMany({ with: ["categories"] });
+      const res = await em.repository(posts).findMany({ with: { categories: {} } });
 
       expect(res.data).toEqual([
          {
@@ -107,7 +151,7 @@ describe("[data] WithBuilder", async () => {
          }
       ]);
 
-      const res2 = await em.repository(categories).findMany({ with: ["posts"] });
+      const res2 = await em.repository(categories).findMany({ with: { posts: {} } });
 
       //console.log(res2.sql, res2.data);
 
@@ -150,7 +194,7 @@ describe("[data] WithBuilder", async () => {
          em,
          em.connection.kysely.selectFrom("categories"),
          categories,
-         ["single"]
+         { single: {} }
       );
       const res = qb.compile();
       expect(res.sql).toBe(
@@ -162,7 +206,7 @@ describe("[data] WithBuilder", async () => {
          em,
          em.connection.kysely.selectFrom("categories"),
          categories,
-         ["multiple"]
+         { multiple: {} }
       );
       const res2 = qb2.compile();
       expect(res2.sql).toBe(

app/src/data/entities/query/Repository.ts

@@ -103,17 +103,10 @@ export class Repository<TBD extends object = DefaultDB, TB extends keyof TBD = a
          validated.select = options.select;
       }
 
-      if (options.with && options.with.length > 0) {
+      if (options.with) {
-         for (const entry of options.with) {
+         const depth = WithBuilder.validateWiths(this.em, entity.name, options.with);
-            const related = this.em.relationOf(entity.name, entry);
+         // @todo: determine allowed depth
-            if (!related) {
+         validated.with = options.with;
-               throw new InvalidSearchParamsException(
-                  `WITH: "${entry}" is not a relation of "${entity.name}"`
-               );
-            }
-
-            validated.with.push(entry);
-         }
       }
 
       if (options.join && options.join.length > 0) {

app/src/data/entities/query/WithBuilder.ts

@@ -1,11 +1,16 @@
+import { isObject } from "core/utils";
+import type { KyselyJsonFrom, RepoQuery } from "data";
+import { InvalidSearchParamsException } from "data/errors";
+import type { RepoWithSchema } from "data/server/data-query-impl";
 import type { Entity, EntityManager, RepositoryQB } from "../../entities";
 
 export class WithBuilder {
-   private static buildClause(
+   /*private static buildClause(
       em: EntityManager<any>,
       qb: RepositoryQB,
       entity: Entity,
-      withString: string
+      ref: string,
+      config?: RepoQuery
    ) {
       const relation = em.relationOf(entity.name, withString);
       if (!relation) {
@@ -15,7 +20,6 @@ export class WithBuilder {
       const cardinality = relation.ref(withString).cardinality;
       //console.log("with--builder", { entity: entity.name, withString, cardinality });
 
-      const fns = em.connection.fn;
       const jsonFrom = cardinality === 1 ? fns.jsonObjectFrom : fns.jsonArrayFrom;
 
       if (!jsonFrom) {
@@ -27,16 +31,69 @@ export class WithBuilder {
       } catch (e) {
          throw new Error(`Could not build "with" relation "${withString}": ${(e as any).message}`);
       }
+   }*/
+
+   static addClause(
+      em: EntityManager<any>,
+      qb: RepositoryQB,
+      entity: Entity,
+      withs: RepoQuery["with"]
+   ) {
+      if (!withs || !isObject(withs)) {
+         console.warn(`'withs' undefined or invalid, given: ${JSON.stringify(withs)}`);
+         return qb;
       }
 
-   static addClause(em: EntityManager<any>, qb: RepositoryQB, entity: Entity, withs: string[]) {
+      const fns = em.connection.fn;
-      if (withs.length === 0) return qb;
-
       let newQb = qb;
-      for (const entry of withs) {
+
-         newQb = WithBuilder.buildClause(em, newQb, entity, entry);
+      for (const [ref, query] of Object.entries(withs)) {
+         const relation = em.relationOf(entity.name, ref);
+         if (!relation) {
+            throw new Error(`Relation "${entity.name}<>${ref}" not found`);
+         }
+         const cardinality = relation.ref(ref).cardinality;
+         const jsonFrom: KyselyJsonFrom =
+            cardinality === 1 ? fns.jsonObjectFrom : fns.jsonArrayFrom;
+         if (!jsonFrom) {
+            throw new Error("Connection does not support jsonObjectFrom/jsonArrayFrom");
+         }
+
+         const alias = relation.other(entity).reference;
+         newQb = newQb.select((eb) => {
+            return jsonFrom(relation.buildWith(entity, ref)(eb)).as(alias);
+         });
+         //newQb = relation.buildWith(entity, qb, jsonFrom, ref);
       }
 
       return newQb;
    }
+
+   static validateWiths(em: EntityManager<any>, entity: string, withs: RepoQuery["with"]) {
+      let depth = 0;
+      if (!withs || !isObject(withs)) {
+         console.warn(`'withs' undefined or invalid, given: ${JSON.stringify(withs)}`);
+         return depth;
+      }
+
+      const child_depths: number[] = [];
+      for (const [ref, query] of Object.entries(withs)) {
+         const related = em.relationOf(entity, ref);
+         if (!related) {
+            throw new InvalidSearchParamsException(
+               `WITH: "${ref}" is not a relation of "${entity}"`
+            );
+         }
+         depth++;
+
+         if ("with" in query) {
+            child_depths.push(WithBuilder.validateWiths(em, ref, query.with as any));
+         }
+      }
+      if (child_depths.length > 0) {
+         depth += Math.max(...child_depths);
+      }
+
+      return depth;
+   }
 }

app/src/data/relations/EntityRelation.ts

@@ -1,5 +1,5 @@
 import { type Static, Type, parse } from "core/utils";
-import type { SelectQueryBuilder } from "kysely";
+import type { ExpressionBuilder, SelectQueryBuilder } from "kysely";
 import type { Entity, EntityData, EntityManager } from "../entities";
 import {
    type EntityRelationAnchor,
@@ -67,10 +67,8 @@ export abstract class EntityRelation<
     */
    abstract buildWith(
       entity: Entity,
-      qb: KyselyQueryBuilder,
-      jsonFrom: KyselyJsonFrom,
       reference: string
-   ): KyselyQueryBuilder;
+   ): (eb: ExpressionBuilder<any, any>) => KyselyQueryBuilder;
 
    abstract buildJoin(
       entity: Entity,

app/src/data/relations/ManyToManyRelation.ts

@@ -1,4 +1,5 @@
 import { type Static, Type } from "core/utils";
+import type { ExpressionBuilder } from "kysely";
 import { Entity, type EntityManager } from "../entities";
 import { type Field, PrimaryField, VirtualField } from "../fields";
 import type { RepoQuery } from "../server/data-query-impl";
@@ -123,7 +124,7 @@ export class ManyToManyRelation extends EntityRelation<typeof ManyToManyRelation
          .groupBy(groupBy);
    }
 
-   buildWith(entity: Entity, qb: KyselyQueryBuilder, jsonFrom: KyselyJsonFrom) {
+   buildWith(entity: Entity) {
       if (!this.em) {
          throw new Error("EntityManager not set, can't build");
       }
@@ -138,7 +139,29 @@ export class ManyToManyRelation extends EntityRelation<typeof ManyToManyRelation
          (f) => !(f instanceof RelationField || f instanceof PrimaryField)
       );
 
-      return qb.select((eb) => {
+      return (eb: ExpressionBuilder<any, any>) =>
+         eb
+            .selectFrom(other.entity.name)
+            .select((eb2) => {
+               const select: any[] = other.entity.getSelect(other.entity.name);
+               if (additionalFields.length > 0) {
+                  const conn = this.connectionEntity.name;
+                  select.push(
+                     jsonBuildObject(
+                        Object.fromEntries(
+                           additionalFields.map((f) => [f.name, eb2.ref(`${conn}.${f.name}`)])
+                        )
+                     ).as(this.connectionTableMappedName)
+                  );
+               }
+
+               return select;
+            })
+            .whereRef(entityRef, "=", otherRef)
+            .innerJoin(...join)
+            .limit(limit);
+
+      /*return qb.select((eb) => {
          const select: any[] = other.entity.getSelect(other.entity.name);
          // @todo: also add to find by references
          if (additionalFields.length > 0) {
@@ -160,7 +183,7 @@ export class ManyToManyRelation extends EntityRelation<typeof ManyToManyRelation
                .innerJoin(...join)
                .limit(limit)
          ).as(other.reference);
-      });
+      });*/
    }
 
    initialize(em: EntityManager<any>) {

app/src/data/relations/ManyToOneRelation.ts

@@ -1,6 +1,7 @@
 import type { PrimaryFieldType } from "core";
 import { snakeToPascalWithSpaces } from "core/utils";
 import { type Static, Type } from "core/utils";
+import type { ExpressionBuilder } from "kysely";
 import type { Entity, EntityManager } from "../entities";
 import type { RepoQuery } from "../server/data-query-impl";
 import { EntityRelation, type KyselyJsonFrom, type KyselyQueryBuilder } from "./EntityRelation";
@@ -155,15 +156,22 @@ export class ManyToOneRelation extends EntityRelation<typeof ManyToOneRelation.s
       return qb.innerJoin(self.entity.name, entityRef, otherRef).groupBy(groupBy);
    }
 
-   buildWith(entity: Entity, qb: KyselyQueryBuilder, jsonFrom: KyselyJsonFrom, reference: string) {
+   buildWith(entity: Entity, reference: string) {
       const { self, entityRef, otherRef, relationRef } = this.queryInfo(entity, reference);
       const limit =
          self.cardinality === 1
             ? 1
-            : this.config.with_limit ?? ManyToOneRelation.DEFAULTS.with_limit;
+            : (this.config.with_limit ?? ManyToOneRelation.DEFAULTS.with_limit);
       //console.log("buildWith", entity.name, reference, { limit });
 
-      return qb.select((eb) =>
+      return (eb: ExpressionBuilder<any, any>) =>
+         eb
+            .selectFrom(`${self.entity.name} as ${relationRef}`)
+            .select(self.entity.getSelect(relationRef))
+            .whereRef(entityRef, "=", otherRef)
+            .limit(limit);
+
+      /*return qb.select((eb) =>
          jsonFrom(
             eb
                .selectFrom(`${self.entity.name} as ${relationRef}`)
@@ -171,7 +179,7 @@ export class ManyToOneRelation extends EntityRelation<typeof ManyToOneRelation.s
                .whereRef(entityRef, "=", otherRef)
                .limit(limit)
          ).as(relationRef)
-      );
+      );*/
    }
 
    /**

app/src/data/relations/PolymorphicRelation.ts

@@ -1,4 +1,5 @@
 import { type Static, Type } from "core/utils";
+import type { ExpressionBuilder } from "kysely";
 import type { Entity, EntityManager } from "../entities";
 import { NumberField, TextField } from "../fields";
 import type { RepoQuery } from "../server/data-query-impl";
@@ -87,11 +88,19 @@ export class PolymorphicRelation extends EntityRelation<typeof PolymorphicRelati
       };
    }
 
-   buildWith(entity: Entity, qb: KyselyQueryBuilder, jsonFrom: KyselyJsonFrom) {
+   buildWith(entity: Entity) {
       const { other, whereLhs, reference, entityRef, otherRef } = this.queryInfo(entity);
       const limit = other.cardinality === 1 ? 1 : 5;
 
-      return qb.select((eb) =>
+      return (eb: ExpressionBuilder<any, any>) =>
+         eb
+            .selectFrom(other.entity.name)
+            .select(other.entity.getSelect(other.entity.name))
+            .where(whereLhs, "=", reference)
+            .whereRef(entityRef, "=", otherRef)
+            .limit(limit);
+
+      /*return qb.select((eb) =>
          jsonFrom(
             eb
                .selectFrom(other.entity.name)
@@ -100,7 +109,7 @@ export class PolymorphicRelation extends EntityRelation<typeof PolymorphicRelati
                .whereRef(entityRef, "=", otherRef)
                .limit(limit)
          ).as(other.reference)
-      );
+      );*/
    }
 
    override isListableFor(entity: Entity): boolean {

app/src/data/server/data-query-impl.ts

@@ -1,3 +1,4 @@
+import type { TThis } from "@sinclair/typebox";
 import {
    type SchemaOptions,
    type Static,
@@ -64,19 +65,60 @@ export const whereSchema = Type.Transform(
    })
    .Encode(JSON.stringify);
 
-export const querySchema = Type.Object(
+export type ShallowRepoQuery = {
+   limit?: number;
+   offset?: number;
+   sort?: string | { by: string; dir: "asc" | "desc" };
+   select?: string[];
+   with?: string[] | Record<string, ShallowRepoQuery>;
+   join?: string[];
+   where?: any;
+};
+export type RepoWithSchema = Record<
+   string,
+   Omit<ShallowRepoQuery, "with"> & {
+      with?: unknown;
+   }
+>;
+export const withSchema = <TSelf extends TThis>(Self: TSelf) =>
+   Type.Transform(Type.Union([stringArray, Type.Record(Type.String(), Self)]))
+      .Decode((value) => {
+         let _value = value;
+         if (Array.isArray(value)) {
+            if (!value.every((v) => typeof v === "string")) {
+               throw new Error("Invalid 'with' schema");
+            }
+
+            _value = value.reduce((acc, v) => {
+               acc[v] = {};
+               return acc;
+            }, {} as RepoWithSchema);
+         }
+
+         return _value as RepoWithSchema;
+      })
+      .Encode((value) => value);
+
+export const querySchema = Type.Recursive(
+   (Self) =>
+      Type.Partial(
+         Type.Object(
             {
-      limit: Type.Optional(limit),
+               limit: limit,
-      offset: Type.Optional(offset),
+               offset: offset,
-      sort: Type.Optional(sort),
+               sort: sort,
-      select: Type.Optional(stringArray),
+               select: stringArray,
-      with: Type.Optional(stringArray),
+               with: withSchema(Self),
-      join: Type.Optional(stringArray),
+               join: stringArray,
-      where: Type.Optional(whereSchema)
+               where: whereSchema
             },
             {
+               // @todo: determine if unknown is allowed, it's ignore anyway
                additionalProperties: false
             }
+         )
+      ),
+   { $id: "query-schema" }
 );
 
 export type RepoQueryIn = Static<typeof querySchema>;