From e6ff5c3f0bf1d3eb4d9517f00d6ce6b969693a97 Mon Sep 17 00:00:00 2001 From: dswbx Date: Sat, 11 Oct 2025 20:37:14 +0200 Subject: [PATCH 1/3] fix pagination if endpoint's total is not available when using a connection that has softscans disabled (e.g. D1) pagination failed. Fixing it by overfetching and slicing --- app/src/ui/client/api/use-api.ts | 6 +- app/src/ui/components/table/DataTable.tsx | 87 ++++++++++++------- .../ui/elements/media/DropzoneContainer.tsx | 69 +++++++++------ app/src/ui/hooks/use-search.ts | 5 +- .../ui/modules/data/components/EntityForm.tsx | 14 +-- .../fields/EntityRelationalFormField.tsx | 2 +- app/src/ui/routes/data/data.$entity.index.tsx | 2 +- 7 files changed, 110 insertions(+), 75 deletions(-) diff --git a/app/src/ui/client/api/use-api.ts b/app/src/ui/client/api/use-api.ts index 72feeeb..4571bd9 100644 --- a/app/src/ui/client/api/use-api.ts +++ b/app/src/ui/client/api/use-api.ts @@ -35,7 +35,7 @@ export const useApiInfiniteQuery = < RefineFn extends (data: ResponseObject) => unknown = (data: ResponseObject) => Data, >( fn: (api: Api, page: number) => FetchPromise, - options?: SWRConfiguration & { refine?: RefineFn }, + options?: SWRConfiguration & { refine?: RefineFn; pageSize?: number }, ) => { const [endReached, setEndReached] = useState(false); const api = useApi(); @@ -47,14 +47,14 @@ export const useApiInfiniteQuery = < // @ts-ignore const swr = useSWRInfinite( (index, previousPageData: any) => { - if (previousPageData && !previousPageData.length) { + if (index > 0 && previousPageData && previousPageData.length < (options?.pageSize ?? 0)) { setEndReached(true); return null; // reached the end } return promise(index).request.url; }, (url: string) => { - return new FetchPromise(new Request(url), { fetcher: api.fetcher }, refine).execute(); + return new FetchPromise(new Request(url), { fetcher: api.fetcher }).execute(); }, { revalidateFirstPage: false, diff --git a/app/src/ui/components/table/DataTable.tsx b/app/src/ui/components/table/DataTable.tsx index 2ce2a42..ce691f9 100644 --- a/app/src/ui/components/table/DataTable.tsx +++ b/app/src/ui/components/table/DataTable.tsx @@ -53,7 +53,7 @@ export type DataTableProps = { }; export function DataTable = Record>({ - data = [], + data: _data = [], columns, checkable, onClickRow, @@ -71,11 +71,14 @@ export function DataTable = Record renderValue, onClickNew, }: DataTableProps) { + const hasTotal = !!total; + const data = Array.isArray(_data) ? _data.slice(0, perPage) : _data; total = total || data?.length || 0; page = page || 1; const select = columns && columns.length > 0 ? columns : Object.keys(data?.[0] || {}); const pages = Math.max(Math.ceil(total / perPage), 1); + const hasNext = hasTotal ? pages > page : (_data?.length || 0) > perPage; const CellRender = renderValue || CellValue; return ( @@ -202,7 +205,7 @@ export function DataTable = Record perPage={perPage} page={page} items={data?.length || 0} - total={total} + total={hasTotal ? total : undefined} />
@@ -222,11 +225,17 @@ export function DataTable = Record
)}
- Page {page} of {pages} + Page {page} + {hasTotal ? <> of {pages} : ""}
{onClickPage && (
- +
)} @@ -268,17 +277,23 @@ const SortIndicator = ({ }; const TableDisplay = ({ perPage, page, items, total }) => { - if (total === 0) { + if (items === 0 && page === 1) { return <>No rows to show; } - if (total === 1) { - return <>Showing 1 row; + const start = Math.max(perPage * (page - 1), 1); + + if (!total) { + return ( + <> + Showing {start}-{perPage * (page - 1) + items} + + ); } return ( <> - Showing {perPage * (page - 1) + 1}-{perPage * (page - 1) + items} of {total} rows + Showing {start}-{perPage * (page - 1) + items} of {total} rows ); }; @@ -287,30 +302,44 @@ type TableNavProps = { current: number; total: number; onClick?: (page: number) => void; + hasLast?: boolean; }; -const TableNav: React.FC = ({ current, total, onClick }: TableNavProps) => { +const TableNav: React.FC = ({ + current, + total, + onClick, + hasLast = true, +}: TableNavProps) => { const navMap = [ - { value: 1, Icon: TbChevronsLeft, disabled: current === 1 }, - { value: current - 1, Icon: TbChevronLeft, disabled: current === 1 }, - { value: current + 1, Icon: TbChevronRight, disabled: current === total }, - { value: total, Icon: TbChevronsRight, disabled: current === total }, + { enabled: true, value: 1, Icon: TbChevronsLeft, disabled: current === 1 }, + { enabled: true, value: current - 1, Icon: TbChevronLeft, disabled: current === 1 }, + { + enabled: true, + value: current + 1, + Icon: TbChevronRight, + disabled: current === total, + }, + { enabled: hasLast, value: total, Icon: TbChevronsRight, disabled: current === total }, ] as const; - return navMap.map((nav, key) => ( - - )); + return navMap.map( + (nav, key) => + nav.enabled && ( + + ), + ); }; diff --git a/app/src/ui/elements/media/DropzoneContainer.tsx b/app/src/ui/elements/media/DropzoneContainer.tsx index 46cef55..2a99a5f 100644 --- a/app/src/ui/elements/media/DropzoneContainer.tsx +++ b/app/src/ui/elements/media/DropzoneContainer.tsx @@ -77,7 +77,9 @@ export function DropzoneContainer({ }); const $q = infinite - ? useApiInfiniteQuery(selectApi, {}) + ? useApiInfiniteQuery(selectApi, { + pageSize, + }) : useApiQuery(selectApi, { enabled: initialItems !== false && !initialItems, revalidateOnFocus: false, @@ -108,31 +110,48 @@ export function DropzoneContainer({ []) as MediaFieldSchema[]; const _initialItems = mediaItemsToFileStates(actualItems, { baseUrl }); - const key = id + JSON.stringify(_initialItems); + const key = id + JSON.stringify(initialItems); + + // check if endpoint reeturns a total, then reaching end is easy + const total = "_data" in $q ? $q._data?.[0]?.body.meta.count : undefined; + let placeholderLength = 0; + if (infinite && "setSize" in $q) { + placeholderLength = + typeof total === "number" + ? total + : $q.endReached + ? _initialItems.length + : _initialItems.length + pageSize; + + // in case there is no total, we overfetch but SWR don't reflect an empty result + // therefore we check if it stopped loading, but has a bigger page size than the total. + // if that's the case, we assume we reached the end. + if (!total && !$q.isValidating && pageSize * $q.size >= placeholderLength) { + placeholderLength = _initialItems.length; + } + } + return ( - $q.setSize($q.size + 1)} - /> - ) - } - {...props} - /> + <> + $q.setSize($q.size + 1)} + /> + ) + } + {...props} + /> + ); } diff --git a/app/src/ui/hooks/use-search.ts b/app/src/ui/hooks/use-search.ts index 40967c1..be0cad0 100644 --- a/app/src/ui/hooks/use-search.ts +++ b/app/src/ui/hooks/use-search.ts @@ -14,10 +14,6 @@ export function useSearch( ) { const searchString = useWouterSearch(); const [location, navigate] = useLocation(); - const [value, setValue] = useState>( - options?.defaultValue ?? ({} as any), - ); - const defaults = useMemo(() => { return mergeObject( // @ts-ignore @@ -25,6 +21,7 @@ export function useSearch( options?.defaultValue ?? {}, ); }, [JSON.stringify({ schema, dflt: options?.defaultValue })]); + const [value, setValue] = useState>(defaults); useEffect(() => { const initial = diff --git a/app/src/ui/modules/data/components/EntityForm.tsx b/app/src/ui/modules/data/components/EntityForm.tsx index 63e608e..ff778a1 100644 --- a/app/src/ui/modules/data/components/EntityForm.tsx +++ b/app/src/ui/modules/data/components/EntityForm.tsx @@ -301,19 +301,9 @@ function EntityJsonFormField({ onChange={handleUpdate} onBlur={fieldApi.handleBlur} minHeight="100" - /*required={field.isRequired()}*/ {...props} /> - {/**/} ); } @@ -340,8 +330,8 @@ function EntityEnumFormField({ {...props} > {!field.isRequired() && } - {field.getOptions().map((option) => ( - ))} diff --git a/app/src/ui/modules/data/components/fields/EntityRelationalFormField.tsx b/app/src/ui/modules/data/components/fields/EntityRelationalFormField.tsx index 6b4ee77..aa331bd 100644 --- a/app/src/ui/modules/data/components/fields/EntityRelationalFormField.tsx +++ b/app/src/ui/modules/data/components/fields/EntityRelationalFormField.tsx @@ -44,7 +44,7 @@ export function EntityRelationalFormField({ const ref = useRef(null); const $q = useEntityQuery(field.target(), undefined, { select: query.select, - limit: query.limit, + limit: query.limit + 1 /* overfetch for softscan=false */, offset: (query.page - 1) * query.limit, }); const [_value, _setValue] = useState<{ id: number | undefined; [key: string]: any }>(); diff --git a/app/src/ui/routes/data/data.$entity.index.tsx b/app/src/ui/routes/data/data.$entity.index.tsx index 10b74fa..5945f37 100644 --- a/app/src/ui/routes/data/data.$entity.index.tsx +++ b/app/src/ui/routes/data/data.$entity.index.tsx @@ -61,7 +61,7 @@ function DataEntityListImpl({ params }) { (api) => api.data.readMany(entity?.name as any, { select: search.value.select, - limit: search.value.perPage, + limit: search.value.perPage + 1 /* overfetch for softscan=false */, offset: (search.value.page - 1) * search.value.perPage, sort: `${search.value.sort.dir === "asc" ? "" : "-"}${search.value.sort.by}`, }), From fd3dd310a5dbf091b812ed0312bee162749d7d45 Mon Sep 17 00:00:00 2001 From: dswbx Date: Mon, 13 Oct 2025 10:41:15 +0200 Subject: [PATCH 2/3] refactor: enhance MediaApi typing and improve vite example config handling for d1 Updated `MediaApi` to include improved generic typing for upload methods, ensuring type safety and consistency. Refactored example configuration logic in development environment setup for better modularity and maintainability. --- app/src/media/api/MediaApi.ts | 19 ++++++----- app/src/ui/client/api/use-entity.ts | 18 ++++++---- app/vite.dev.ts | 53 ++++++++++++++++++++++++----- 3 files changed, 68 insertions(+), 22 deletions(-) diff --git a/app/src/media/api/MediaApi.ts b/app/src/media/api/MediaApi.ts index db94b03..d925d4d 100644 --- a/app/src/media/api/MediaApi.ts +++ b/app/src/media/api/MediaApi.ts @@ -1,11 +1,14 @@ import type { FileListObject } from "media/storage/Storage"; import { type BaseModuleApiOptions, + type FetchPromise, + type ResponseObject, ModuleApi, type PrimaryFieldType, type TInput, } from "modules/ModuleApi"; import type { ApiFetcher } from "Api"; +import type { DB, FileUploadedEventData } from "bknd"; export type MediaApiOptions = BaseModuleApiOptions & { upload_fetcher: ApiFetcher; @@ -67,14 +70,14 @@ export class MediaApi extends ModuleApi { return new Headers(); } - protected uploadFile( + protected uploadFile( body: File | Blob | ReadableStream | Buffer, opts?: { filename?: string; path?: TInput; _init?: Omit; }, - ) { + ): FetchPromise> { const headers = { "Content-Type": "application/octet-stream", ...(opts?._init?.headers || {}), @@ -106,10 +109,10 @@ export class MediaApi extends ModuleApi { throw new Error("Invalid filename"); } - return this.post(opts?.path ?? ["upload", name], body, init); + return this.post(opts?.path ?? ["upload", name], body, init); } - async upload( + async upload( item: Request | Response | string | File | Blob | ReadableStream | Buffer, opts: { filename?: string; @@ -124,12 +127,12 @@ export class MediaApi extends ModuleApi { if (!res.ok || !res.body) { throw new Error("Failed to fetch file"); } - return this.uploadFile(res.body, opts); + return this.uploadFile(res.body, opts); } else if (item instanceof Response) { if (!item.body) { throw new Error("Invalid response"); } - return this.uploadFile(item.body, { + return this.uploadFile(item.body, { ...(opts ?? {}), _init: { ...(opts._init ?? {}), @@ -141,7 +144,7 @@ export class MediaApi extends ModuleApi { }); } - return this.uploadFile(item, opts); + return this.uploadFile(item, opts); } async uploadToEntity( @@ -153,7 +156,7 @@ export class MediaApi extends ModuleApi { _init?: Omit; fetcher?: typeof fetch; }, - ) { + ): Promise> { return this.upload(item, { ...opts, path: ["entity", entity, id, field], diff --git a/app/src/ui/client/api/use-entity.ts b/app/src/ui/client/api/use-entity.ts index b77b57d..f53798c 100644 --- a/app/src/ui/client/api/use-entity.ts +++ b/app/src/ui/client/api/use-entity.ts @@ -1,8 +1,14 @@ -import type { DB, PrimaryFieldType, EntityData, RepoQueryIn } from "bknd"; +import type { + DB, + PrimaryFieldType, + EntityData, + RepoQueryIn, + RepositoryResult, + ResponseObject, + ModuleApi, +} from "bknd"; import { objectTransform, encodeSearch } from "bknd/utils"; -import type { RepositoryResult } from "data/entities"; import type { Insertable, Selectable, Updateable } from "kysely"; -import type { FetchPromise, ModuleApi, ResponseObject } from "modules/ModuleApi"; import useSWR, { type SWRConfiguration, type SWRResponse, mutate } from "swr"; import { type Api, useApi } from "ui/client"; @@ -108,7 +114,7 @@ export function makeKey( ); } -interface UseEntityQueryReturn< +export interface UseEntityQueryReturn< Entity extends keyof DB | string, Id extends PrimaryFieldType | undefined = undefined, Data = Entity extends keyof DB ? Selectable : EntityData, @@ -136,11 +142,11 @@ export const useEntityQuery = < const fetcher = () => read(query ?? {}); type T = Awaited>; - const swr = useSWR(options?.enabled === false ? null : key, fetcher as any, { + const swr = useSWR(options?.enabled === false ? null : key, fetcher as any, { revalidateOnFocus: false, keepPreviousData: true, ...options, - }); + }) as ReturnType>; const mutateFn = async (id?: PrimaryFieldType) => { const entityKey = makeKey(api, entity as string, id); diff --git a/app/vite.dev.ts b/app/vite.dev.ts index 6f74ff1..255fe26 100644 --- a/app/vite.dev.ts +++ b/app/vite.dev.ts @@ -1,4 +1,4 @@ -import { readFile } from "node:fs/promises"; +import { readFile, writeFile } from "node:fs/promises"; import { serveStatic } from "@hono/node-server/serve-static"; import { showRoutes } from "hono/dev"; import { App, registries, type CreateAppConfig } from "./src"; @@ -9,6 +9,7 @@ import { $console } from "core/utils/console"; import { createClient } from "@libsql/client"; import util from "node:util"; import { d1Sqlite } from "adapter/cloudflare/connection/D1Connection"; +import { slugify } from "./src/core/utils/strings"; util.inspect.defaultOptions.depth = 5; registries.media.register("local", StorageLocalAdapter); @@ -21,16 +22,19 @@ $console.debug("Using db type", dbType); let dbUrl = import.meta.env.VITE_DB_URL ?? ":memory:"; const example = import.meta.env.VITE_EXAMPLE; -if (example) { - const configPath = `.configs/${example}.json`; - $console.debug("Loading config from", configPath); - const exampleConfig = JSON.parse(await readFile(configPath, "utf-8")); - config.config = exampleConfig; - dbUrl = `file:.configs/${example}.db`; +async function loadExampleConfig() { + if (example) { + const configPath = `.configs/${example}.json`; + $console.debug("Loading config from", configPath); + const exampleConfig = JSON.parse(await readFile(configPath, "utf-8")); + config.config = exampleConfig; + dbUrl = `file:.configs/${example}.db`; + } } switch (dbType) { case "libsql": { + await loadExampleConfig(); $console.debug("Using libsql connection", dbUrl); const authToken = import.meta.env.VITE_DB_LIBSQL_TOKEN; config.connection = libsql( @@ -43,15 +47,48 @@ switch (dbType) { } case "d1": { $console.debug("Using d1 connection"); + const wranglerConfig = { + name: "vite-dev", + main: "src/index.ts", + compatibility_date: "2025-08-03", + compatibility_flags: ["nodejs_compat"], + d1_databases: [ + { + binding: "DB", + database_name: "vite-dev", + database_id: "00000000-0000-0000-0000-000000000000", + }, + ], + r2_buckets: [ + { + binding: "BUCKET", + bucket_name: "vite-dev", + }, + ], + }; + let configPath = ".configs/vite.wrangler.json"; + if (example) { + const name = slugify(example); + configPath = `.configs/${slugify(example)}.wrangler.json`; + const exists = await readFile(configPath, "utf-8"); + if (!exists) { + wranglerConfig.name = name; + wranglerConfig.d1_databases[0]!.database_name = name; + wranglerConfig.d1_databases[0]!.database_id = crypto.randomUUID(); + wranglerConfig.r2_buckets[0]!.bucket_name = name; + await writeFile(configPath, JSON.stringify(wranglerConfig, null, 2)); + } + } const { getPlatformProxy } = await import("wrangler"); const platformProxy = await getPlatformProxy({ - configPath: "./vite.wrangler.json", + configPath, }); config.connection = d1Sqlite({ binding: platformProxy.env.DB as any }); break; } default: { + await loadExampleConfig(); $console.debug("Using node-sqlite connection", dbUrl); config.connection = nodeSqlite({ url: dbUrl }); break; From 3f9be3a4182ff772b833a4f7d65f8fa845b11420 Mon Sep 17 00:00:00 2001 From: dswbx Date: Mon, 13 Oct 2025 10:46:04 +0200 Subject: [PATCH 3/3] fix: refine FetchPromise execution in useApiInfiniteQuery Updated the FetchPromise execution in the useApiInfiniteQuery function to include a refine parameter, enhancing the request handling process. --- app/src/ui/client/api/use-api.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/ui/client/api/use-api.ts b/app/src/ui/client/api/use-api.ts index 4571bd9..6b6d546 100644 --- a/app/src/ui/client/api/use-api.ts +++ b/app/src/ui/client/api/use-api.ts @@ -54,7 +54,7 @@ export const useApiInfiniteQuery = < return promise(index).request.url; }, (url: string) => { - return new FetchPromise(new Request(url), { fetcher: api.fetcher }).execute(); + return new FetchPromise(new Request(url), { fetcher: api.fetcher }, refine).execute(); }, { revalidateFirstPage: false,