From 69ea5a00eeb502b552ed8cd5540f09ed7b46c871 Mon Sep 17 00:00:00 2001 From: dswbx Date: Thu, 16 Jan 2025 17:18:30 +0100 Subject: [PATCH 1/3] DataApi: automatically switch to POST if the URL is too long --- app/__test__/api/DataApi.spec.ts | 16 ++++++++++++++++ app/__test__/api/ModuleApi.spec.ts | 4 ++++ app/src/data/api/DataApi.ts | 18 +++++++++++++----- 3 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 app/__test__/api/DataApi.spec.ts diff --git a/app/__test__/api/DataApi.spec.ts b/app/__test__/api/DataApi.spec.ts new file mode 100644 index 0000000..4fee5d9 --- /dev/null +++ b/app/__test__/api/DataApi.spec.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from "bun:test"; +import { DataApi } from "../../src/data/api/DataApi"; + +describe("DataApi", () => { + it("should switch to post for long url reads", async () => { + const api = new DataApi(); + + const get = api.readMany("a".repeat(100), { select: ["id", "name"] }); + expect(get.request.method).toBe("GET"); + expect(new URL(get.request.url).pathname).toBe(`/api/data/${"a".repeat(100)}`); + + const post = api.readMany("a".repeat(1000), { select: ["id", "name"] }); + expect(post.request.method).toBe("POST"); + expect(new URL(post.request.url).pathname).toBe(`/api/data/${"a".repeat(1000)}/query`); + }); +}); diff --git a/app/__test__/api/ModuleApi.spec.ts b/app/__test__/api/ModuleApi.spec.ts index caa42d0..5fb6976 100644 --- a/app/__test__/api/ModuleApi.spec.ts +++ b/app/__test__/api/ModuleApi.spec.ts @@ -28,6 +28,8 @@ describe("ModuleApi", () => { it("fetches endpoint", async () => { const app = new Hono().get("/endpoint", (c) => c.json({ foo: "bar" })); const api = new Api({ host }); + + // @ts-expect-error it's protected api.fetcher = app.request as typeof fetch; const res = await api.get("/endpoint"); @@ -40,6 +42,8 @@ describe("ModuleApi", () => { it("has accessible request", async () => { const app = new Hono().get("/endpoint", (c) => c.json({ foo: "bar" })); const api = new Api({ host }); + + // @ts-expect-error it's protected api.fetcher = app.request as typeof fetch; const promise = api.get("/endpoint"); diff --git a/app/src/data/api/DataApi.ts b/app/src/data/api/DataApi.ts index f2fe4e7..64eb9af 100644 --- a/app/src/data/api/DataApi.ts +++ b/app/src/data/api/DataApi.ts @@ -3,13 +3,15 @@ import type { EntityData, RepoQuery, RepoQueryIn, RepositoryResponse } from "dat import { type BaseModuleApiOptions, ModuleApi, type PrimaryFieldType } from "modules"; export type DataApiOptions = BaseModuleApiOptions & { - defaultQuery?: Partial; + queryLengthLimit: number; + defaultQuery: Partial; }; export class DataApi extends ModuleApi { protected override getDefaultOptions(): Partial { return { basepath: "/api/data", + queryLengthLimit: 1000, defaultQuery: { limit: 10 } @@ -28,10 +30,16 @@ export class DataApi extends ModuleApi { entity: E, query: RepoQueryIn = {} ) { - return this.get, "meta" | "data">>( - [entity as any], - query ?? this.options.defaultQuery - ); + type T = Pick, "meta" | "data">; + + const input = query ?? this.options.defaultQuery; + const exceeds = JSON.stringify([entity, input]).length > this.options.queryLengthLimit; + + if (exceeds) { + return this.post([entity as any, "query"], input); + } + + return this.get([entity as any], input); } readManyByReference< From 8a6d8329f337ae66363f23ac11a822c1ec1a44e8 Mon Sep 17 00:00:00 2001 From: dswbx Date: Thu, 16 Jan 2025 17:25:19 +0100 Subject: [PATCH 2/3] refactor for better precision --- app/__test__/api/DataApi.spec.ts | 4 ++-- app/src/data/api/DataApi.ts | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/__test__/api/DataApi.spec.ts b/app/__test__/api/DataApi.spec.ts index 4fee5d9..706a59d 100644 --- a/app/__test__/api/DataApi.spec.ts +++ b/app/__test__/api/DataApi.spec.ts @@ -5,9 +5,9 @@ describe("DataApi", () => { it("should switch to post for long url reads", async () => { const api = new DataApi(); - const get = api.readMany("a".repeat(100), { select: ["id", "name"] }); + const get = api.readMany("a".repeat(300), { select: ["id", "name"] }); expect(get.request.method).toBe("GET"); - expect(new URL(get.request.url).pathname).toBe(`/api/data/${"a".repeat(100)}`); + expect(new URL(get.request.url).pathname).toBe(`/api/data/${"a".repeat(300)}`); const post = api.readMany("a".repeat(1000), { select: ["id", "name"] }); expect(post.request.method).toBe("POST"); diff --git a/app/src/data/api/DataApi.ts b/app/src/data/api/DataApi.ts index 64eb9af..e444092 100644 --- a/app/src/data/api/DataApi.ts +++ b/app/src/data/api/DataApi.ts @@ -33,13 +33,13 @@ export class DataApi extends ModuleApi { type T = Pick, "meta" | "data">; const input = query ?? this.options.defaultQuery; - const exceeds = JSON.stringify([entity, input]).length > this.options.queryLengthLimit; + const req = this.get([entity as any], input); - if (exceeds) { - return this.post([entity as any, "query"], input); + if (req.request.url.length <= this.options.queryLengthLimit) { + return req; } - return this.get([entity as any], input); + return this.post([entity as any, "query"], input); } readManyByReference< From 2a015ba0a1e600f1a0e1ef531e754060ba2292a0 Mon Sep 17 00:00:00 2001 From: dswbx Date: Fri, 17 Jan 2025 05:02:26 +0100 Subject: [PATCH 3/3] extended dataapi tests --- app/__test__/api/DataApi.spec.ts | 56 ++++++++++++++++++++- app/__test__/data/specs/WithBuilder.spec.ts | 7 +-- app/__test__/helper.ts | 8 ++- app/src/data/api/DataController.ts | 2 +- 4 files changed, 64 insertions(+), 9 deletions(-) diff --git a/app/__test__/api/DataApi.spec.ts b/app/__test__/api/DataApi.spec.ts index 706a59d..dbbe35d 100644 --- a/app/__test__/api/DataApi.spec.ts +++ b/app/__test__/api/DataApi.spec.ts @@ -1,6 +1,16 @@ -import { describe, expect, it } from "bun:test"; +import { afterAll, beforeAll, describe, expect, it } from "bun:test"; +import { Guard } from "../../src/auth"; +import { parse } from "../../src/core/utils"; import { DataApi } from "../../src/data/api/DataApi"; +import { DataController } from "../../src/data/api/DataController"; +import { dataConfigSchema } from "../../src/data/data-schema"; +import * as proto from "../../src/data/prototype"; +import { disableConsoleLog, enableConsoleLog, schemaToEm } from "../helper"; +beforeAll(disableConsoleLog); +afterAll(enableConsoleLog); + +const dataConfig = parse(dataConfigSchema, {}); describe("DataApi", () => { it("should switch to post for long url reads", async () => { const api = new DataApi(); @@ -13,4 +23,48 @@ describe("DataApi", () => { expect(post.request.method).toBe("POST"); expect(new URL(post.request.url).pathname).toBe(`/api/data/${"a".repeat(1000)}/query`); }); + + it("returns result", async () => { + const schema = proto.em({ + posts: proto.entity("posts", { title: proto.text() }) + }); + const em = schemaToEm(schema); + await em.schema().sync({ force: true }); + + const payload = [{ title: "foo" }, { title: "bar" }, { title: "baz" }]; + await em.mutator("posts").insertMany(payload); + + const ctx: any = { em, guard: new Guard() }; + const controller = new DataController(ctx, dataConfig); + const app = controller.getController(); + + { + const res = (await app.request("/posts")) as Response; + const { data } = await res.json(); + expect(data.length).toEqual(3); + } + + // @ts-ignore tests + const api = new DataApi({ basepath: "/", queryLengthLimit: 50 }); + // @ts-ignore protected + api.fetcher = app.request as typeof fetch; + { + const req = api.readMany("posts", { select: ["title"] }); + expect(req.request.method).toBe("GET"); + const res = await req; + expect(res.data).toEqual(payload); + } + + { + const req = api.readMany("posts", { + select: ["title"], + limit: 100000, + offset: 0, + sort: "id" + }); + expect(req.request.method).toBe("POST"); + const res = await req; + expect(res.data).toEqual(payload); + } + }); }); diff --git a/app/__test__/data/specs/WithBuilder.spec.ts b/app/__test__/data/specs/WithBuilder.spec.ts index bed48a6..7b64198 100644 --- a/app/__test__/data/specs/WithBuilder.spec.ts +++ b/app/__test__/data/specs/WithBuilder.spec.ts @@ -10,16 +10,11 @@ import { WithBuilder } from "../../../src/data"; import * as proto from "../../../src/data/prototype"; -import { compileQb, prettyPrintQb } from "../../helper"; +import { compileQb, prettyPrintQb, schemaToEm } from "../../helper"; import { getDummyConnection } from "../helper"; const { dummyConnection } = getDummyConnection(); -function schemaToEm(s: ReturnType<(typeof proto)["em"]>): EntityManager { - const { dummyConnection } = getDummyConnection(); - return new EntityManager(Object.values(s.entities), dummyConnection, s.relations, s.indices); -} - describe("[data] WithBuilder", async () => { test("validate withs", async () => { const schema = proto.em( diff --git a/app/__test__/helper.ts b/app/__test__/helper.ts index de6993e..f07cd34 100644 --- a/app/__test__/helper.ts +++ b/app/__test__/helper.ts @@ -2,7 +2,8 @@ import { unlink } from "node:fs/promises"; import type { SelectQueryBuilder, SqliteDatabase } from "kysely"; import Database from "libsql"; import { format as sqlFormat } from "sql-formatter"; -import { SqliteLocalConnection } from "../src/data"; +import { type Connection, EntityManager, SqliteLocalConnection } from "../src/data"; +import type { em as protoEm } from "../src/data/prototype"; export function getDummyDatabase(memory: boolean = true): { dummyDb: SqliteDatabase; @@ -62,3 +63,8 @@ export function prettyPrintQb(qb: SelectQueryBuilder) { const { sql, parameters } = qb.compile(); console.log("$", sqlFormat(sql), "\n[params]", parameters); } + +export function schemaToEm(s: ReturnType, conn?: Connection): EntityManager { + const connection = conn ? conn : getDummyConnection().dummyConnection; + return new EntityManager(Object.values(s.entities), connection, s.relations, s.indices); +} diff --git a/app/src/data/api/DataController.ts b/app/src/data/api/DataController.ts index 6735c7a..131f3d6 100644 --- a/app/src/data/api/DataController.ts +++ b/app/src/data/api/DataController.ts @@ -281,7 +281,7 @@ export class DataController extends Controller { return c.notFound(); } const options = (await c.req.valid("json")) as RepoQuery; - console.log("options", options); + //console.log("options", options); const result = await this.em.repository(entity).findMany(options); return c.json(this.repoResult(result), { status: result.data ? 200 : 404 });