Merge pull request #48 from bknd-io/feat/recursive-with

recursive `with`

app/__test__/data/data-query-impl.spec.ts

@@ -1,8 +1,14 @@
 import { describe, expect, test } from "bun:test";
-import { Value } from "../../src/core/utils";
+import { Value, _jsonp } from "../../src/core/utils";
-import { WhereBuilder, type WhereQuery, querySchema } from "../../src/data";
+import { type RepoQuery, WhereBuilder, type WhereQuery, querySchema } from "../../src/data";
+import type { RepoQueryIn } from "../../src/data/server/data-query-impl";
 import { getDummyConnection } from "./helper";
 
+const decode = (input: RepoQueryIn, expected: RepoQuery) => {
+   const result = Value.Decode(querySchema, input);
+   expect(result).toEqual(expected);
+};
+
 describe("data-query-impl", () => {
    function qb() {
       const c = getDummyConnection();
@@ -88,21 +94,47 @@ describe("data-query-impl", () => {
          expect(keys).toEqual(expectedKeys);
       }
    });
+
+   test("with", () => {
+      decode({ with: ["posts"] }, { with: { posts: {} } });
+      decode({ with: { posts: {} } }, { with: { posts: {} } });
+      decode({ with: { posts: { limit: "1" } } }, { with: { posts: { limit: 1 } } });
+      decode(
+         {
+            with: {
+               posts: {
+                  with: {
+                     images: {
+                        select: "id"
+                     }
+                  }
+               }
+            }
+         },
+         {
+            with: {
+               posts: {
+                  with: {
+                     images: {
+                        select: ["id"]
+                     }
+                  }
+               }
+            }
+         }
+      );
+   });
 });
 
 describe("data-query-impl: Typebox", () => {
    test("sort", async () => {
-      const decode = (input: any, expected: any) => {
+      const _dflt = { sort: { by: "id", dir: "asc" } };
-         const result = Value.Decode(querySchema, input);
-         expect(result.sort).toEqual(expected);
-      };
-      const _dflt = { by: "id", dir: "asc" };
 
       decode({ sort: "" }, _dflt);
-      decode({ sort: "name" }, { by: "name", dir: "asc" });
+      decode({ sort: "name" }, { sort: { by: "name", dir: "asc" } });
-      decode({ sort: "-name" }, { by: "name", dir: "desc" });
+      decode({ sort: "-name" }, { sort: { by: "name", dir: "desc" } });
-      decode({ sort: "-posts.name" }, { by: "posts.name", dir: "desc" });
+      decode({ sort: "-posts.name" }, { sort: { by: "posts.name", dir: "desc" } });
       decode({ sort: "-1name" }, _dflt);
-      decode({ sort: { by: "name", dir: "desc" } }, { by: "name", dir: "desc" });
+      decode({ sort: { by: "name", dir: "desc" } }, { sort: { by: "name", dir: "desc" } });
    });
 });

app/__test__/data/relations.test.ts

@@ -106,7 +106,6 @@ describe("Relations", async () => {
       expect(postAuthorRel?.other(posts).entity).toBe(users);
 
       const kysely = em.connection.kysely;
-      const jsonFrom = (e) => e;
       /**
        * Relation Helper
        */
@@ -119,14 +118,11 @@ describe("Relations", async () => {
        - select: users.*
        - cardinality: 1
        */
-      const selectPostsFromUsers = postAuthorRel.buildWith(
+      const selectPostsFromUsers = kysely
-         users,
+         .selectFrom(users.name)
-         kysely.selectFrom(users.name),
+         .select((eb) => postAuthorRel.buildWith(users, "posts")(eb).as("posts"));
-         jsonFrom,
-         "posts"
-      );
       expect(selectPostsFromUsers.compile().sql).toBe(
-         'select (select "posts"."id" as "id", "posts"."title" as "title", "posts"."author_id" as "author_id" from "posts" as "posts" where "posts"."author_id" = "users"."id" limit ?) as "posts" from "users"'
+         'select (select from "posts" as "posts" where "posts"."author_id" = "users"."id") as "posts" from "users"'
       );
       expect(postAuthorRel!.getField()).toBeInstanceOf(RelationField);
       const userObj = { id: 1, username: "test" };
@@ -141,15 +137,12 @@ describe("Relations", async () => {
        - select: posts.*
        - cardinality:
        */
-      const selectUsersFromPosts = postAuthorRel.buildWith(
+      const selectUsersFromPosts = kysely
-         posts,
+         .selectFrom(posts.name)
-         kysely.selectFrom(posts.name),
+         .select((eb) => postAuthorRel.buildWith(posts, "author")(eb).as("author"));
-         jsonFrom,
-         "author"
-      );
 
       expect(selectUsersFromPosts.compile().sql).toBe(
-         'select (select "author"."id" as "id", "author"."username" as "username" from "users" as "author" where "author"."id" = "posts"."author_id" limit ?) as "author" from "posts"'
+         'select (select from "users" as "author" where "author"."id" = "posts"."author_id" limit ?) as "author" from "posts"'
       );
       expect(postAuthorRel.getField()).toBeInstanceOf(RelationField);
       const postObj = { id: 1, title: "test" };
@@ -315,20 +308,16 @@ describe("Relations", async () => {
        - select: users.*
        - cardinality: 1
        */
-      const selectCategoriesFromPosts = postCategoriesRel.buildWith(
+      const selectCategoriesFromPosts = kysely
-         posts,
+         .selectFrom(posts.name)
-         kysely.selectFrom(posts.name),
+         .select((eb) => postCategoriesRel.buildWith(posts)(eb).as("categories"));
-         jsonFrom
-      );
       expect(selectCategoriesFromPosts.compile().sql).toBe(
          'select (select "categories"."id" as "id", "categories"."label" as "label" from "categories" inner join "posts_categories" on "categories"."id" = "posts_categories"."categories_id" where "posts"."id" = "posts_categories"."posts_id" limit ?) as "categories" from "posts"'
       );
 
-      const selectPostsFromCategories = postCategoriesRel.buildWith(
+      const selectPostsFromCategories = kysely
-         categories,
+         .selectFrom(categories.name)
-         kysely.selectFrom(categories.name),
+         .select((eb) => postCategoriesRel.buildWith(categories)(eb).as("posts"));
-         jsonFrom
-      );
       expect(selectPostsFromCategories.compile().sql).toBe(
          'select (select "posts"."id" as "id", "posts"."title" as "title" from "posts" inner join "posts_categories" on "posts"."id" = "posts_categories"."posts_id" where "categories"."id" = "posts_categories"."categories_id" limit ?) as "posts" from "categories"'
       );

app/__test__/data/specs/WithBuilder.spec.ts

@@ -1,4 +1,5 @@
 import { afterAll, describe, expect, test } from "bun:test";
+import { _jsonp } from "../../../src/core/utils";
 import {
    Entity,
    EntityManager,
@@ -8,19 +9,61 @@ import {
    TextField,
    WithBuilder
 } from "../../../src/data";
+import * as proto from "../../../src/data/prototype";
+import { compileQb, prettyPrintQb } from "../../helper";
 import { getDummyConnection } from "../helper";
 
-const { dummyConnection, afterAllCleanup } = getDummyConnection();
+const { dummyConnection } = getDummyConnection();
-afterAll(afterAllCleanup);
+
+function schemaToEm(s: ReturnType<(typeof proto)["em"]>): EntityManager<any> {
+   const { dummyConnection } = getDummyConnection();
+   return new EntityManager(Object.values(s.entities), dummyConnection, s.relations, s.indices);
+}
 
 describe("[data] WithBuilder", async () => {
+   test("validate withs", async () => {
+      const schema = proto.em(
+         {
+            posts: proto.entity("posts", {}),
+            users: proto.entity("users", {}),
+            media: proto.entity("media", {})
+         },
+         ({ relation }, { posts, users, media }) => {
+            relation(posts).manyToOne(users);
+            relation(users).polyToOne(media, { mappedBy: "avatar" });
+         }
+      );
+      const em = schemaToEm(schema);
+
+      expect(WithBuilder.validateWiths(em, "posts", undefined)).toBe(0);
+      expect(WithBuilder.validateWiths(em, "posts", {})).toBe(0);
+      expect(WithBuilder.validateWiths(em, "posts", { users: {} })).toBe(1);
+      expect(
+         WithBuilder.validateWiths(em, "posts", {
+            users: {
+               with: { avatar: {} }
+            }
+         })
+      ).toBe(2);
+      expect(() => WithBuilder.validateWiths(em, "posts", { author: {} })).toThrow();
+      expect(() =>
+         WithBuilder.validateWiths(em, "posts", {
+            users: {
+               with: { glibberish: {} }
+            }
+         })
+      ).toThrow();
+   });
+
    test("missing relation", async () => {
       const users = new Entity("users", [new TextField("username")]);
       const em = new EntityManager([users], dummyConnection);
 
       expect(() =>
-         WithBuilder.addClause(em, em.connection.kysely.selectFrom("users"), users, ["posts"])
+         WithBuilder.addClause(em, em.connection.kysely.selectFrom("users"), users, {
-      ).toThrow('Relation "posts" not found');
+            posts: {}
+         })
+      ).toThrow('Relation "users<>posts" not found');
    });
 
    test("addClause: ManyToOne", async () => {
@@ -29,36 +72,39 @@ describe("[data] WithBuilder", async () => {
       const relations = [new ManyToOneRelation(posts, users, { mappedBy: "author" })];
       const em = new EntityManager([users, posts], dummyConnection, relations);
 
-      const qb = WithBuilder.addClause(em, em.connection.kysely.selectFrom("users"), users, [
+      const qb = WithBuilder.addClause(em, em.connection.kysely.selectFrom("users"), users, {
-         "posts"
+         posts: {}
-      ]);
+      });
 
       const res = qb.compile();
 
       expect(res.sql).toBe(
-         'select (select coalesce(json_group_array(json_object(\'id\', "agg"."id", \'content\', "agg"."content", \'author_id\', "agg"."author_id")), \'[]\') from (select "posts"."id" as "id", "posts"."content" as "content", "posts"."author_id" as "author_id" from "posts" as "posts" where "posts"."author_id" = "users"."id" limit ?) as agg) as "posts" from "users"'
+         'select (select coalesce(json_group_array(json_object(\'id\', "agg"."id", \'content\', "agg"."content", \'author_id\', "agg"."author_id")), \'[]\') from (select "posts"."id" as "id", "posts"."content" as "content", "posts"."author_id" as "author_id" from "posts" as "posts" where "posts"."author_id" = "users"."id" order by "posts"."id" asc limit ? offset ?) as agg) as "posts" from "users"'
       );
-      expect(res.parameters).toEqual([5]);
+      expect(res.parameters).toEqual([10, 0]);
 
       const qb2 = WithBuilder.addClause(
          em,
          em.connection.kysely.selectFrom("posts"),
          posts, // @todo: try with "users", it gives output!
-         ["author"]
+         {
+            author: {}
+         }
       );
 
       const res2 = qb2.compile();
 
       expect(res2.sql).toBe(
-         'select (select json_object(\'id\', "obj"."id", \'username\', "obj"."username") from (select "author"."id" as "id", "author"."username" as "username" from "users" as "author" where "author"."id" = "posts"."author_id" limit ?) as obj) as "author" from "posts"'
+         'select (select json_object(\'id\', "obj"."id", \'username\', "obj"."username") from (select "users"."id" as "id", "users"."username" as "username" from "users" as "author" where "author"."id" = "posts"."author_id" order by "users"."id" asc limit ? offset ?) as obj) as "author" from "posts"'
       );
-      expect(res2.parameters).toEqual([1]);
+      expect(res2.parameters).toEqual([1, 0]);
    });
 
    test("test with empty join", async () => {
+      const em = new EntityManager([], dummyConnection);
       const qb = { qb: 1 } as any;
 
-      expect(WithBuilder.addClause(null as any, qb, null as any, [])).toBe(qb);
+      expect(WithBuilder.addClause(em, qb, null as any, {})).toBe(qb);
    });
 
    test("test manytomany", async () => {
@@ -89,7 +135,7 @@ describe("[data] WithBuilder", async () => {
 
       //console.log((await em.repository().findMany("posts_categories")).result);
 
-      const res = await em.repository(posts).findMany({ with: ["categories"] });
+      const res = await em.repository(posts).findMany({ with: { categories: {} } });
 
       expect(res.data).toEqual([
          {
@@ -107,7 +153,7 @@ describe("[data] WithBuilder", async () => {
          }
       ]);
 
-      const res2 = await em.repository(categories).findMany({ with: ["posts"] });
+      const res2 = await em.repository(categories).findMany({ with: { posts: {} } });
 
       //console.log(res2.sql, res2.data);
 
@@ -121,8 +167,8 @@ describe("[data] WithBuilder", async () => {
             id: 2,
             label: "beauty",
             posts: [
-               { id: 2, title: "beauty post" },
+               { id: 1, title: "fashion post" },
-               { id: 1, title: "fashion post" }
+               { id: 2, title: "beauty post" }
             ]
          },
          {
@@ -150,25 +196,25 @@ describe("[data] WithBuilder", async () => {
          em,
          em.connection.kysely.selectFrom("categories"),
          categories,
-         ["single"]
+         { single: {} }
       );
       const res = qb.compile();
       expect(res.sql).toBe(
-         'select (select json_object(\'id\', "obj"."id", \'path\', "obj"."path") from (select "media"."id" as "id", "media"."path" as "path" from "media" where "media"."reference" = ? and "categories"."id" = "media"."entity_id" limit ?) as obj) as "single" from "categories"'
+         'select (select json_object(\'id\', "obj"."id", \'path\', "obj"."path") from (select "media"."id" as "id", "media"."path" as "path" from "media" where "media"."reference" = ? and "categories"."id" = "media"."entity_id" order by "media"."id" asc limit ? offset ?) as obj) as "single" from "categories"'
       );
-      expect(res.parameters).toEqual(["categories.single", 1]);
+      expect(res.parameters).toEqual(["categories.single", 1, 0]);
 
       const qb2 = WithBuilder.addClause(
          em,
          em.connection.kysely.selectFrom("categories"),
          categories,
-         ["multiple"]
+         { multiple: {} }
       );
       const res2 = qb2.compile();
       expect(res2.sql).toBe(
-         'select (select coalesce(json_group_array(json_object(\'id\', "agg"."id", \'path\', "agg"."path")), \'[]\') from (select "media"."id" as "id", "media"."path" as "path" from "media" where "media"."reference" = ? and "categories"."id" = "media"."entity_id" limit ?) as agg) as "multiple" from "categories"'
+         'select (select coalesce(json_group_array(json_object(\'id\', "agg"."id", \'path\', "agg"."path")), \'[]\') from (select "media"."id" as "id", "media"."path" as "path" from "media" where "media"."reference" = ? and "categories"."id" = "media"."entity_id" order by "media"."id" asc limit ? offset ?) as agg) as "multiple" from "categories"'
       );
-      expect(res2.parameters).toEqual(["categories.multiple", 5]);
+      expect(res2.parameters).toEqual(["categories.multiple", 10, 0]);
    });
 
    /*test("test manytoone", async () => {
@@ -192,4 +238,205 @@ describe("[data] WithBuilder", async () => {
       const res = await em.repository().findMany("posts", { join: ["author"] });
       console.log(res.sql, res.parameters, res.result);
    });*/
+
+   describe("recursive", () => {
+      test("compiles with singles", async () => {
+         const schema = proto.em(
+            {
+               posts: proto.entity("posts", {}),
+               users: proto.entity("users", {
+                  username: proto.text()
+               }),
+               media: proto.entity("media", {
+                  path: proto.text()
+               })
+            },
+            ({ relation }, { posts, users, media }) => {
+               relation(posts).manyToOne(users);
+               relation(users).polyToOne(media, { mappedBy: "avatar" });
+            }
+         );
+         const em = schemaToEm(schema);
+
+         const qb = WithBuilder.addClause(
+            em,
+            em.connection.kysely.selectFrom("posts"),
+            schema.entities.posts,
+            {
+               users: {
+                  limit: 5, // ignored
+                  select: ["id", "username"],
+                  sort: { by: "username", dir: "asc" },
+                  with: {
+                     avatar: {
+                        select: ["id", "path"],
+                        limit: 2 // ignored
+                     }
+                  }
+               }
+            }
+         );
+
+         //prettyPrintQb(qb);
+         expect(qb.compile().sql).toBe(
+            'select (select json_object(\'id\', "obj"."id", \'username\', "obj"."username", \'avatar\', "obj"."avatar") from (select "users"."id" as "id", "users"."username" as "username", (select json_object(\'id\', "obj"."id", \'path\', "obj"."path") from (select "media"."id" as "id", "media"."path" as "path" from "media" where "media"."reference" = ? and "users"."id" = "media"."entity_id" order by "media"."id" asc limit ? offset ?) as obj) as "avatar" from "users" as "users" where "users"."id" = "posts"."users_id" order by "users"."username" asc limit ? offset ?) as obj) as "users" from "posts"'
+         );
+         expect(qb.compile().parameters).toEqual(["users.avatar", 1, 0, 1, 0]);
+      });
+
+      test("compiles with many", async () => {
+         const schema = proto.em(
+            {
+               posts: proto.entity("posts", {}),
+               comments: proto.entity("comments", {}),
+               users: proto.entity("users", {
+                  username: proto.text()
+               }),
+               media: proto.entity("media", {
+                  path: proto.text()
+               })
+            },
+            ({ relation }, { posts, comments, users, media }) => {
+               relation(posts).manyToOne(users).polyToOne(media, { mappedBy: "images" });
+               relation(users).polyToOne(media, { mappedBy: "avatar" });
+               relation(comments).manyToOne(posts).manyToOne(users);
+            }
+         );
+         const em = schemaToEm(schema);
+
+         const qb = WithBuilder.addClause(
+            em,
+            em.connection.kysely.selectFrom("posts"),
+            schema.entities.posts,
+            {
+               comments: {
+                  limit: 12,
+                  with: {
+                     users: {
+                        select: ["username"]
+                     }
+                  }
+               }
+            }
+         );
+
+         expect(qb.compile().sql).toBe(
+            'select (select coalesce(json_group_array(json_object(\'id\', "agg"."id", \'posts_id\', "agg"."posts_id", \'users_id\', "agg"."users_id", \'users\', "agg"."users")), \'[]\') from (select "comments"."id" as "id", "comments"."posts_id" as "posts_id", "comments"."users_id" as "users_id", (select json_object(\'username\', "obj"."username") from (select "users"."username" as "username" from "users" as "users" where "users"."id" = "comments"."users_id" order by "users"."id" asc limit ? offset ?) as obj) as "users" from "comments" as "comments" where "comments"."posts_id" = "posts"."id" order by "comments"."id" asc limit ? offset ?) as agg) as "comments" from "posts"'
+         );
+         expect(qb.compile().parameters).toEqual([1, 0, 12, 0]);
+      });
+
+      test("returns correct result", async () => {
+         const schema = proto.em(
+            {
+               posts: proto.entity("posts", {
+                  title: proto.text()
+               }),
+               comments: proto.entity("comments", {
+                  content: proto.text()
+               }),
+               users: proto.entity("users", {
+                  username: proto.text()
+               }),
+               media: proto.entity("media", {
+                  path: proto.text()
+               })
+            },
+            ({ relation }, { posts, comments, users, media }) => {
+               relation(posts).manyToOne(users).polyToOne(media, { mappedBy: "images" });
+               relation(users).polyToOne(media, { mappedBy: "avatar" });
+               relation(comments).manyToOne(posts).manyToOne(users);
+            }
+         );
+         const em = schemaToEm(schema);
+         await em.schema().sync({ force: true });
+
+         // add data
+         await em.mutator("users").insertMany([{ username: "user1" }, { username: "user2" }]);
+         await em.mutator("posts").insertMany([
+            { title: "post1", users_id: 1 },
+            { title: "post2", users_id: 1 },
+            { title: "post3", users_id: 2 }
+         ]);
+         await em.mutator("comments").insertMany([
+            { content: "comment1", posts_id: 1, users_id: 1 },
+            { content: "comment1-1", posts_id: 1, users_id: 1 },
+            { content: "comment2", posts_id: 1, users_id: 2 },
+            { content: "comment3", posts_id: 2, users_id: 1 },
+            { content: "comment4", posts_id: 2, users_id: 2 },
+            { content: "comment5", posts_id: 3, users_id: 1 },
+            { content: "comment6", posts_id: 3, users_id: 2 }
+         ]);
+
+         const result = await em.repo("posts").findMany({
+            select: ["title"],
+            with: {
+               comments: {
+                  limit: 2,
+                  select: ["content"],
+                  with: {
+                     users: {
+                        select: ["username"]
+                     }
+                  }
+               }
+            }
+         });
+
+         expect(result.data).toEqual([
+            {
+               title: "post1",
+               comments: [
+                  {
+                     content: "comment1",
+                     users: {
+                        username: "user1"
+                     }
+                  },
+                  {
+                     content: "comment1-1",
+                     users: {
+                        username: "user1"
+                     }
+                  }
+               ]
+            },
+            {
+               title: "post2",
+               comments: [
+                  {
+                     content: "comment3",
+                     users: {
+                        username: "user1"
+                     }
+                  },
+                  {
+                     content: "comment4",
+                     users: {
+                        username: "user2"
+                     }
+                  }
+               ]
+            },
+            {
+               title: "post3",
+               comments: [
+                  {
+                     content: "comment5",
+                     users: {
+                        username: "user1"
+                     }
+                  },
+                  {
+                     content: "comment6",
+                     users: {
+                        username: "user2"
+                     }
+                  }
+               ]
+            }
+         ]);
+         //console.log(_jsonp(result.data));
+      });
+   });
 });

app/__test__/helper.ts

@@ -1,6 +1,7 @@
 import { unlink } from "node:fs/promises";
-import type { SqliteDatabase } from "kysely";
+import type { SelectQueryBuilder, SqliteDatabase } from "kysely";
 import Database from "libsql";
+import { format as sqlFormat } from "sql-formatter";
 import { SqliteLocalConnection } from "../src/data";
 
 export function getDummyDatabase(memory: boolean = true): {
@@ -51,3 +52,13 @@ export function enableConsoleLog() {
       console[severity as ConsoleSeverity] = fn;
    });
 }
+
+export function compileQb(qb: SelectQueryBuilder<any, any, any>) {
+   const { sql, parameters } = qb.compile();
+   return { sql, parameters };
+}
+
+export function prettyPrintQb(qb: SelectQueryBuilder<any, any, any>) {
+   const { sql, parameters } = qb.compile();
+   console.log("$", sqlFormat(sql), "\n[params]", parameters);
+}

app/package.json

@@ -74,6 +74,7 @@
     "react-hook-form": "^7.53.1",
     "react-icons": "5.2.1",
     "react-json-view-lite": "^2.0.1",
+    "sql-formatter": "^15.4.9",
     "tailwind-merge": "^2.5.4",
     "tailwindcss": "^3.4.14",
     "tailwindcss-animate": "^1.0.7",

app/src/auth/middlewares.ts

@@ -26,25 +26,28 @@ export const auth = (options?: {
    skip?: (string | RegExp)[];
 }) =>
    createMiddleware<ServerEnv>(async (c, next) => {
-      // make sure to only register once
-      if (c.get("auth_registered")) {
-         throw new Error(`auth middleware already registered for ${getPath(c)}`);
-      }
-      c.set("auth_registered", true);
-
       const app = c.get("app");
-      const skipped = shouldSkip(c, options?.skip) || !app?.module.auth.enabled;
       const guard = app?.modules.ctx().guard;
       const authenticator = app?.module.auth.authenticator;
 
-      if (!skipped) {
+      let skipped = shouldSkip(c, options?.skip) || !app?.module.auth.enabled;
-         const resolved = c.get("auth_resolved");
+
-         if (!resolved) {
+      // make sure to only register once
-            if (!app.module.auth.enabled) {
+      if (c.get("auth_registered")) {
-               guard?.setUserContext(undefined);
+         skipped = true;
-            } else {
+         console.warn(`auth middleware already registered for ${getPath(c)}`);
-               guard?.setUserContext(await authenticator?.resolveAuthFromRequest(c));
+      } else {
-               c.set("auth_resolved", true);
+         c.set("auth_registered", true);
+
+         if (!skipped) {
+            const resolved = c.get("auth_resolved");
+            if (!resolved) {
+               if (!app?.module.auth.enabled) {
+                  guard?.setUserContext(undefined);
+               } else {
+                  guard?.setUserContext(await authenticator?.resolveAuthFromRequest(c));
+                  c.set("auth_resolved", true);
+               }
             }
          }
       }

app/src/data/api/DataController.ts

@@ -70,7 +70,7 @@ export class DataController extends Controller {
 
    override getController() {
       const { permission, auth } = this.middlewares;
-      const hono = this.create().use(auth());
+      const hono = this.create().use(auth(), permission(SystemPermissions.accessApi));
 
       const definedEntities = this.em.entities.map((e) => e.name);
       const tbNumber = Type.Transform(Type.String({ pattern: "^[1-9][0-9]{0,}$" }))
@@ -85,8 +85,6 @@ export class DataController extends Controller {
          return func;
       }
 
-      hono.use("*", permission(SystemPermissions.accessApi));
-
       // info
       hono.get(
          "/",

app/src/data/connection/SqliteLocalConnection.ts

@@ -1,6 +1,5 @@
-import type { DatabaseIntrospector, SqliteDatabase } from "kysely";
+import { type DatabaseIntrospector, ParseJSONResultsPlugin, type SqliteDatabase } from "kysely";
 import { Kysely, SqliteDialect } from "kysely";
-import { DeserializeJsonValuesPlugin } from "../plugins/DeserializeJsonValuesPlugin";
 import { SqliteConnection } from "./SqliteConnection";
 import { SqliteIntrospector } from "./SqliteIntrospector";
 
@@ -14,7 +13,7 @@ class CustomSqliteDialect extends SqliteDialect {
 
 export class SqliteLocalConnection extends SqliteConnection {
    constructor(private database: SqliteDatabase) {
-      const plugins = [new DeserializeJsonValuesPlugin()];
+      const plugins = [new ParseJSONResultsPlugin()];
       const kysely = new Kysely({
          dialect: new CustomSqliteDialect({ database }),
          plugins

app/src/data/entities/query/Repository.ts

@@ -65,7 +65,7 @@ export class Repository<TBD extends object = DefaultDB, TB extends keyof TBD = a
       return this.em.connection.kysely;
    }
 
-   private getValidOptions(options?: Partial<RepoQuery>): RepoQuery {
+   getValidOptions(options?: Partial<RepoQuery>): RepoQuery {
       const entity = this.entity;
       // @todo: if not cloned deep, it will keep references and error if multiple requests come in
       const validated = {
@@ -103,17 +103,10 @@ export class Repository<TBD extends object = DefaultDB, TB extends keyof TBD = a
          validated.select = options.select;
       }
 
-      if (options.with && options.with.length > 0) {
+      if (options.with) {
-         for (const entry of options.with) {
+         const depth = WithBuilder.validateWiths(this.em, entity.name, options.with);
-            const related = this.em.relationOf(entity.name, entry);
+         // @todo: determine allowed depth
-            if (!related) {
+         validated.with = options.with;
-               throw new InvalidSearchParamsException(
-                  `WITH: "${entry}" is not a relation of "${entity.name}"`
-               );
-            }
-
-            validated.with.push(entry);
-         }
       }
 
       if (options.join && options.join.length > 0) {
@@ -235,43 +228,79 @@ export class Repository<TBD extends object = DefaultDB, TB extends keyof TBD = a
       return { ...response, data: data[0]! };
    }
 
-   private buildQuery(
+   addOptionsToQueryBuilder(
+      _qb?: RepositoryQB,
       _options?: Partial<RepoQuery>,
-      exclude_options: (keyof RepoQuery)[] = []
+      config?: {
-   ): { qb: RepositoryQB; options: RepoQuery } {
+         validate?: boolean;
+         ignore?: (keyof RepoQuery)[];
+         alias?: string;
+         defaults?: Pick<RepoQuery, "limit" | "offset">;
+      }
+   ) {
       const entity = this.entity;
-      const options = this.getValidOptions(_options);
+      let qb = _qb ?? (this.conn.selectFrom(entity.name) as RepositoryQB);
 
-      const alias = entity.name;
+      const options = config?.validate !== false ? this.getValidOptions(_options) : _options;
+      if (!options) return qb;
+
+      const alias = config?.alias ?? entity.name;
       const aliased = (field: string) => `${alias}.${field}`;
-      let qb = this.conn
+      const ignore = config?.ignore ?? [];
-         .selectFrom(entity.name)
+      const defaults = {
-         .select(entity.getAliasedSelectFrom(options.select, alias));
+         limit: 10,
+         offset: 0,
+         ...config?.defaults
+      };
 
-      //console.log("build query options", options);
+      /*console.log("build query options", {
-      if (!exclude_options.includes("with") && options.with) {
+         entity: entity.name,
+         options,
+         config
+      });*/
+
+      if (!ignore.includes("select") && options.select) {
+         qb = qb.select(entity.getAliasedSelectFrom(options.select, alias));
+      }
+
+      if (!ignore.includes("with") && options.with) {
          qb = WithBuilder.addClause(this.em, qb, entity, options.with);
       }
 
-      if (!exclude_options.includes("join") && options.join) {
+      if (!ignore.includes("join") && options.join) {
          qb = JoinBuilder.addClause(this.em, qb, entity, options.join);
       }
 
       // add where if present
-      if (!exclude_options.includes("where") && options.where) {
+      if (!ignore.includes("where") && options.where) {
          qb = WhereBuilder.addClause(qb, options.where);
       }
 
-      if (!exclude_options.includes("limit")) qb = qb.limit(options.limit);
+      if (!ignore.includes("limit")) qb = qb.limit(options.limit ?? defaults.limit);
-      if (!exclude_options.includes("offset")) qb = qb.offset(options.offset);
+      if (!ignore.includes("offset")) qb = qb.offset(options.offset ?? defaults.offset);
 
       // sorting
-      if (!exclude_options.includes("sort")) {
+      if (!ignore.includes("sort")) {
-         qb = qb.orderBy(aliased(options.sort.by), options.sort.dir);
+         qb = qb.orderBy(aliased(options.sort?.by ?? "id"), options.sort?.dir ?? "asc");
       }
 
-      //console.log("options", { _options, options, exclude_options });
+      return qb as RepositoryQB;
-      return { qb, options };
+   }
+
+   private buildQuery(
+      _options?: Partial<RepoQuery>,
+      ignore: (keyof RepoQuery)[] = []
+   ): { qb: RepositoryQB; options: RepoQuery } {
+      const entity = this.entity;
+      const options = this.getValidOptions(_options);
+
+      return {
+         qb: this.addOptionsToQueryBuilder(undefined, options, {
+            ignore,
+            alias: entity.name
+         }),
+         options
+      };
    }
 
    async findId(

app/src/data/entities/query/WithBuilder.ts

@@ -1,42 +1,82 @@
+import { isObject } from "core/utils";
+import type { KyselyJsonFrom, RepoQuery } from "data";
+import { InvalidSearchParamsException } from "data/errors";
 import type { Entity, EntityManager, RepositoryQB } from "../../entities";
 
 export class WithBuilder {
-   private static buildClause(
+   static addClause(
       em: EntityManager<any>,
       qb: RepositoryQB,
       entity: Entity,
-      withString: string
+      withs: RepoQuery["with"]
    ) {
-      const relation = em.relationOf(entity.name, withString);
+      if (!withs || !isObject(withs)) {
-      if (!relation) {
+         console.warn(`'withs' undefined or invalid, given: ${JSON.stringify(withs)}`);
-         throw new Error(`Relation "${withString}" not found`);
+         return qb;
       }
 
-      const cardinality = relation.ref(withString).cardinality;
-      //console.log("with--builder", { entity: entity.name, withString, cardinality });
-
       const fns = em.connection.fn;
-      const jsonFrom = cardinality === 1 ? fns.jsonObjectFrom : fns.jsonArrayFrom;
-
-      if (!jsonFrom) {
-         throw new Error("Connection does not support jsonObjectFrom/jsonArrayFrom");
-      }
-
-      try {
-         return relation.buildWith(entity, qb, jsonFrom, withString);
-      } catch (e) {
-         throw new Error(`Could not build "with" relation "${withString}": ${(e as any).message}`);
-      }
-   }
-
-   static addClause(em: EntityManager<any>, qb: RepositoryQB, entity: Entity, withs: string[]) {
-      if (withs.length === 0) return qb;
-
       let newQb = qb;
-      for (const entry of withs) {
+
-         newQb = WithBuilder.buildClause(em, newQb, entity, entry);
+      for (const [ref, query] of Object.entries(withs)) {
+         const relation = em.relationOf(entity.name, ref);
+         if (!relation) {
+            throw new Error(`Relation "${entity.name}<>${ref}" not found`);
+         }
+         const cardinality = relation.ref(ref).cardinality;
+         const jsonFrom: KyselyJsonFrom =
+            cardinality === 1 ? fns.jsonObjectFrom : fns.jsonArrayFrom;
+         if (!jsonFrom) {
+            throw new Error("Connection does not support jsonObjectFrom/jsonArrayFrom");
+         }
+
+         const other = relation.other(entity);
+         newQb = newQb.select((eb) => {
+            let subQuery = relation.buildWith(entity, ref)(eb);
+            if (query) {
+               subQuery = em.repo(other.entity).addOptionsToQueryBuilder(subQuery, query as any, {
+                  ignore: ["with", "join", cardinality === 1 ? "limit" : undefined].filter(
+                     Boolean
+                  ) as any
+               });
+            }
+
+            if (query.with) {
+               subQuery = WithBuilder.addClause(em, subQuery, other.entity, query.with as any);
+            }
+
+            return jsonFrom(subQuery).as(other.reference);
+         });
       }
 
       return newQb;
    }
+
+   static validateWiths(em: EntityManager<any>, entity: string, withs: RepoQuery["with"]) {
+      let depth = 0;
+      if (!withs || !isObject(withs)) {
+         withs && console.warn(`'withs' invalid, given: ${JSON.stringify(withs)}`);
+         return depth;
+      }
+
+      const child_depths: number[] = [];
+      for (const [ref, query] of Object.entries(withs)) {
+         const related = em.relationOf(entity, ref);
+         if (!related) {
+            throw new InvalidSearchParamsException(
+               `WITH: "${ref}" is not a relation of "${entity}"`
+            );
+         }
+         depth++;
+
+         if ("with" in query) {
+            child_depths.push(WithBuilder.validateWiths(em, ref, query.with as any));
+         }
+      }
+      if (child_depths.length > 0) {
+         depth += Math.max(...child_depths);
+      }
+
+      return depth;
+   }
 }

app/src/data/relations/EntityRelation.ts

@@ -1,5 +1,5 @@
 import { type Static, Type, parse } from "core/utils";
-import type { SelectQueryBuilder } from "kysely";
+import type { ExpressionBuilder, SelectQueryBuilder } from "kysely";
 import type { Entity, EntityData, EntityManager } from "../entities";
 import {
    type EntityRelationAnchor,
@@ -67,10 +67,8 @@ export abstract class EntityRelation<
     */
    abstract buildWith(
       entity: Entity,
-      qb: KyselyQueryBuilder,
-      jsonFrom: KyselyJsonFrom,
       reference: string
-   ): KyselyQueryBuilder;
+   ): (eb: ExpressionBuilder<any, any>) => KyselyQueryBuilder;
 
    abstract buildJoin(
       entity: Entity,

app/src/data/relations/ManyToManyRelation.ts

@@ -1,4 +1,5 @@
 import { type Static, Type } from "core/utils";
+import type { ExpressionBuilder } from "kysely";
 import { Entity, type EntityManager } from "../entities";
 import { type Field, PrimaryField, VirtualField } from "../fields";
 import type { RepoQuery } from "../server/data-query-impl";
@@ -123,7 +124,7 @@ export class ManyToManyRelation extends EntityRelation<typeof ManyToManyRelation
          .groupBy(groupBy);
    }
 
-   buildWith(entity: Entity, qb: KyselyQueryBuilder, jsonFrom: KyselyJsonFrom) {
+   buildWith(entity: Entity) {
       if (!this.em) {
          throw new Error("EntityManager not set, can't build");
       }
@@ -138,7 +139,29 @@ export class ManyToManyRelation extends EntityRelation<typeof ManyToManyRelation
          (f) => !(f instanceof RelationField || f instanceof PrimaryField)
       );
 
-      return qb.select((eb) => {
+      return (eb: ExpressionBuilder<any, any>) =>
+         eb
+            .selectFrom(other.entity.name)
+            .select((eb2) => {
+               const select: any[] = other.entity.getSelect(other.entity.name);
+               if (additionalFields.length > 0) {
+                  const conn = this.connectionEntity.name;
+                  select.push(
+                     jsonBuildObject(
+                        Object.fromEntries(
+                           additionalFields.map((f) => [f.name, eb2.ref(`${conn}.${f.name}`)])
+                        )
+                     ).as(this.connectionTableMappedName)
+                  );
+               }
+
+               return select;
+            })
+            .whereRef(entityRef, "=", otherRef)
+            .innerJoin(...join)
+            .limit(limit);
+
+      /*return qb.select((eb) => {
          const select: any[] = other.entity.getSelect(other.entity.name);
          // @todo: also add to find by references
          if (additionalFields.length > 0) {
@@ -160,7 +183,7 @@ export class ManyToManyRelation extends EntityRelation<typeof ManyToManyRelation
                .innerJoin(...join)
                .limit(limit)
          ).as(other.reference);
-      });
+      });*/
    }
 
    initialize(em: EntityManager<any>) {

app/src/data/relations/ManyToOneRelation.ts

@@ -1,6 +1,7 @@
 import type { PrimaryFieldType } from "core";
 import { snakeToPascalWithSpaces } from "core/utils";
 import { type Static, Type } from "core/utils";
+import type { ExpressionBuilder } from "kysely";
 import type { Entity, EntityManager } from "../entities";
 import type { RepoQuery } from "../server/data-query-impl";
 import { EntityRelation, type KyselyJsonFrom, type KyselyQueryBuilder } from "./EntityRelation";
@@ -155,23 +156,14 @@ export class ManyToOneRelation extends EntityRelation<typeof ManyToOneRelation.s
       return qb.innerJoin(self.entity.name, entityRef, otherRef).groupBy(groupBy);
    }
 
-   buildWith(entity: Entity, qb: KyselyQueryBuilder, jsonFrom: KyselyJsonFrom, reference: string) {
+   buildWith(entity: Entity, reference: string) {
       const { self, entityRef, otherRef, relationRef } = this.queryInfo(entity, reference);
-      const limit =
-         self.cardinality === 1
-            ? 1
-            : this.config.with_limit ?? ManyToOneRelation.DEFAULTS.with_limit;
-      //console.log("buildWith", entity.name, reference, { limit });
 
-      return qb.select((eb) =>
+      return (eb: ExpressionBuilder<any, any>) =>
-         jsonFrom(
+         eb
-            eb
+            .selectFrom(`${self.entity.name} as ${relationRef}`)
-               .selectFrom(`${self.entity.name} as ${relationRef}`)
+            .whereRef(entityRef, "=", otherRef)
-               .select(self.entity.getSelect(relationRef))
+            .$if(self.cardinality === 1, (qb) => qb.limit(1));
-               .whereRef(entityRef, "=", otherRef)
-               .limit(limit)
-         ).as(relationRef)
-      );
    }
 
    /**

app/src/data/relations/PolymorphicRelation.ts

@@ -1,4 +1,5 @@
 import { type Static, Type } from "core/utils";
+import type { ExpressionBuilder } from "kysely";
 import type { Entity, EntityManager } from "../entities";
 import { NumberField, TextField } from "../fields";
 import type { RepoQuery } from "../server/data-query-impl";
@@ -87,20 +88,15 @@ export class PolymorphicRelation extends EntityRelation<typeof PolymorphicRelati
       };
    }
 
-   buildWith(entity: Entity, qb: KyselyQueryBuilder, jsonFrom: KyselyJsonFrom) {
+   buildWith(entity: Entity) {
       const { other, whereLhs, reference, entityRef, otherRef } = this.queryInfo(entity);
-      const limit = other.cardinality === 1 ? 1 : 5;
 
-      return qb.select((eb) =>
+      return (eb: ExpressionBuilder<any, any>) =>
-         jsonFrom(
+         eb
-            eb
+            .selectFrom(other.entity.name)
-               .selectFrom(other.entity.name)
+            .where(whereLhs, "=", reference)
-               .select(other.entity.getSelect(other.entity.name))
+            .whereRef(entityRef, "=", otherRef)
-               .where(whereLhs, "=", reference)
+            .$if(other.cardinality === 1, (qb) => qb.limit(1));
-               .whereRef(entityRef, "=", otherRef)
-               .limit(limit)
-         ).as(other.reference)
-      );
    }
 
    override isListableFor(entity: Entity): boolean {

app/src/data/server/data-query-impl.ts

@@ -1,3 +1,4 @@
+import type { TThis } from "@sinclair/typebox";
 import {
    type SchemaOptions,
    type Static,
@@ -64,19 +65,60 @@ export const whereSchema = Type.Transform(
    })
    .Encode(JSON.stringify);
 
-export const querySchema = Type.Object(
+export type ShallowRepoQuery = {
-   {
+   limit?: number;
-      limit: Type.Optional(limit),
+   offset?: number;
-      offset: Type.Optional(offset),
+   sort?: string | { by: string; dir: "asc" | "desc" };
-      sort: Type.Optional(sort),
+   select?: string[];
-      select: Type.Optional(stringArray),
+   with?: string[] | Record<string, ShallowRepoQuery>;
-      with: Type.Optional(stringArray),
+   join?: string[];
-      join: Type.Optional(stringArray),
+   where?: any;
-      where: Type.Optional(whereSchema)
+};
-   },
+export type RepoWithSchema = Record<
-   {
+   string,
-      additionalProperties: false
+   Omit<ShallowRepoQuery, "with"> & {
+      with?: unknown;
    }
+>;
+export const withSchema = <TSelf extends TThis>(Self: TSelf) =>
+   Type.Transform(Type.Union([stringArray, Type.Record(Type.String(), Self)]))
+      .Decode((value) => {
+         let _value = value;
+         if (Array.isArray(value)) {
+            if (!value.every((v) => typeof v === "string")) {
+               throw new Error("Invalid 'with' schema");
+            }
+
+            _value = value.reduce((acc, v) => {
+               acc[v] = {};
+               return acc;
+            }, {} as RepoWithSchema);
+         }
+
+         return _value as RepoWithSchema;
+      })
+      .Encode((value) => value);
+
+export const querySchema = Type.Recursive(
+   (Self) =>
+      Type.Partial(
+         Type.Object(
+            {
+               limit: limit,
+               offset: offset,
+               sort: sort,
+               select: stringArray,
+               with: withSchema(Self),
+               join: stringArray,
+               where: whereSchema
+            },
+            {
+               // @todo: determine if unknown is allowed, it's ignore anyway
+               additionalProperties: false
+            }
+         )
+      ),
+   { $id: "query-schema" }
 );
 
 export type RepoQueryIn = Static<typeof querySchema>;