Merge pull request #282 from bknd-io/fix/paginate-without-totals

fix pagination if endpoint's total is not available
2026-03-15 20:17:22 +00:00 · 2025-10-13 10:52:15 +02:00
parent db58911df3 3f9be3a418
commit 66392d094d
10 changed files with 177 additions and 96 deletions
--- a/app/src/media/api/MediaApi.ts
+++ b/app/src/media/api/MediaApi.ts
@@ -1,11 +1,14 @@
 import type { FileListObject } from "media/storage/Storage";
 import {
   type BaseModuleApiOptions,
+   type FetchPromise,
+   type ResponseObject,
   ModuleApi,
   type PrimaryFieldType,
   type TInput,
 } from "modules/ModuleApi";
 import type { ApiFetcher } from "Api";
+import type { DB, FileUploadedEventData } from "bknd";

 export type MediaApiOptions = BaseModuleApiOptions & {
   upload_fetcher: ApiFetcher;
@@ -67,14 +70,14 @@ export class MediaApi extends ModuleApi<MediaApiOptions> {
      return new Headers();
   }

-   protected uploadFile(
+   protected uploadFile<T extends FileUploadedEventData>(
      body: File | Blob | ReadableStream | Buffer<ArrayBufferLike>,
      opts?: {
         filename?: string;
         path?: TInput;
         _init?: Omit<RequestInit, "body">;
      },
-   ) {
+   ): FetchPromise<ResponseObject<T>> {
      const headers = {
         "Content-Type": "application/octet-stream",
         ...(opts?._init?.headers || {}),
@@ -106,10 +109,10 @@ export class MediaApi extends ModuleApi<MediaApiOptions> {
         throw new Error("Invalid filename");
      }

-      return this.post(opts?.path ?? ["upload", name], body, init);
+      return this.post<T>(opts?.path ?? ["upload", name], body, init);
   }

-   async upload(
+   async upload<T extends FileUploadedEventData>(
      item: Request | Response | string | File | Blob | ReadableStream | Buffer<ArrayBufferLike>,
      opts: {
         filename?: string;
@@ -124,12 +127,12 @@ export class MediaApi extends ModuleApi<MediaApiOptions> {
         if (!res.ok || !res.body) {
            throw new Error("Failed to fetch file");
         }
-         return this.uploadFile(res.body, opts);
+         return this.uploadFile<T>(res.body, opts);
      } else if (item instanceof Response) {
         if (!item.body) {
            throw new Error("Invalid response");
         }
-         return this.uploadFile(item.body, {
+         return this.uploadFile<T>(item.body, {
            ...(opts ?? {}),
            _init: {
               ...(opts._init ?? {}),
@@ -141,7 +144,7 @@ export class MediaApi extends ModuleApi<MediaApiOptions> {
         });
      }

-      return this.uploadFile(item, opts);
+      return this.uploadFile<T>(item, opts);
   }

   async uploadToEntity(
@@ -153,7 +156,7 @@ export class MediaApi extends ModuleApi<MediaApiOptions> {
         _init?: Omit<RequestInit, "body">;
         fetcher?: typeof fetch;
      },
-   ) {
+   ): Promise<ResponseObject<FileUploadedEventData & { result: DB["media"] }>> {
      return this.upload(item, {
         ...opts,
         path: ["entity", entity, id, field],
--- a/app/src/ui/client/api/use-api.ts
+++ b/app/src/ui/client/api/use-api.ts
@@ -35,7 +35,7 @@ export const useApiInfiniteQuery = <
   RefineFn extends (data: ResponseObject<Data>) => unknown = (data: ResponseObject<Data>) => Data,
 >(
   fn: (api: Api, page: number) => FetchPromise<Data>,
-   options?: SWRConfiguration & { refine?: RefineFn },
+   options?: SWRConfiguration & { refine?: RefineFn; pageSize?: number },
 ) => {
   const [endReached, setEndReached] = useState(false);
   const api = useApi();
@@ -47,7 +47,7 @@ export const useApiInfiniteQuery = <
   // @ts-ignore
   const swr = useSWRInfinite<RefinedData>(
      (index, previousPageData: any) => {
-         if (previousPageData && !previousPageData.length) {
+         if (index > 0 && previousPageData && previousPageData.length < (options?.pageSize ?? 0)) {
            setEndReached(true);
            return null; // reached the end
         }
--- a/app/src/ui/client/api/use-entity.ts
+++ b/app/src/ui/client/api/use-entity.ts
@@ -1,8 +1,14 @@
-import type { DB, PrimaryFieldType, EntityData, RepoQueryIn } from "bknd";
+import type {
+   DB,
+   PrimaryFieldType,
+   EntityData,
+   RepoQueryIn,
+   RepositoryResult,
+   ResponseObject,
+   ModuleApi,
+} from "bknd";
 import { objectTransform, encodeSearch } from "bknd/utils";
-import type { RepositoryResult } from "data/entities";
 import type { Insertable, Selectable, Updateable } from "kysely";
-import type { FetchPromise, ModuleApi, ResponseObject } from "modules/ModuleApi";
 import useSWR, { type SWRConfiguration, type SWRResponse, mutate } from "swr";
 import { type Api, useApi } from "ui/client";

@@ -108,7 +114,7 @@ export function makeKey(
   );
 }

-interface UseEntityQueryReturn<
+export interface UseEntityQueryReturn<
   Entity extends keyof DB | string,
   Id extends PrimaryFieldType | undefined = undefined,
   Data = Entity extends keyof DB ? Selectable<DB[Entity]> : EntityData,
@@ -136,11 +142,11 @@ export const useEntityQuery = <
   const fetcher = () => read(query ?? {});

   type T = Awaited<ReturnType<typeof fetcher>>;
-   const swr = useSWR<T>(options?.enabled === false ? null : key, fetcher as any, {
+   const swr = useSWR(options?.enabled === false ? null : key, fetcher as any, {
      revalidateOnFocus: false,
      keepPreviousData: true,
      ...options,
-   });
+   }) as ReturnType<typeof useSWR<T>>;

   const mutateFn = async (id?: PrimaryFieldType) => {
      const entityKey = makeKey(api, entity as string, id);
--- a/app/src/ui/components/table/DataTable.tsx
+++ b/app/src/ui/components/table/DataTable.tsx
@@ -53,7 +53,7 @@ export type DataTableProps<Data> = {
 };

 export function DataTable<Data extends Record<string, any> = Record<string, any>>({
-   data = [],
+   data: _data = [],
   columns,
   checkable,
   onClickRow,
@@ -71,11 +71,14 @@ export function DataTable<Data extends Record<string, any> = Record<string, any>
   renderValue,
   onClickNew,
 }: DataTableProps<Data>) {
+   const hasTotal = !!total;
+   const data = Array.isArray(_data) ? _data.slice(0, perPage) : _data;
   total = total || data?.length || 0;
   page = page || 1;

   const select = columns && columns.length > 0 ? columns : Object.keys(data?.[0] || {});
   const pages = Math.max(Math.ceil(total / perPage), 1);
+   const hasNext = hasTotal ? pages > page : (_data?.length || 0) > perPage;
   const CellRender = renderValue || CellValue;

   return (
@@ -202,7 +205,7 @@ export function DataTable<Data extends Record<string, any> = Record<string, any>
                  perPage={perPage}
                  page={page}
                  items={data?.length || 0}
-                  total={total}
+                  total={hasTotal ? total : undefined}
               />
            </div>
            <div className="flex flex-row gap-2 md:gap-10 items-center">
@@ -222,11 +225,17 @@ export function DataTable<Data extends Record<string, any> = Record<string, any>
                  </div>
               )}
               <div className="text-primary/40">
-                  Page {page} of {pages}
+                  Page {page}
+                  {hasTotal ? <> of {pages}</> : ""}
               </div>
               {onClickPage && (
                  <div className="flex flex-row gap-1.5">
-                     <TableNav current={page} total={pages} onClick={onClickPage} />
+                     <TableNav
+                        current={page}
+                        total={hasTotal ? pages : page + (hasNext ? 1 : 0)}
+                        onClick={onClickPage}
+                        hasLast={hasTotal}
+                     />
                  </div>
               )}
            </div>
@@ -268,17 +277,23 @@ const SortIndicator = ({
 };

 const TableDisplay = ({ perPage, page, items, total }) => {
-   if (total === 0) {
+   if (items === 0 && page === 1) {
      return <>No rows to show</>;
   }

-   if (total === 1) {
-      return <>Showing 1 row</>;
+   const start = Math.max(perPage * (page - 1), 1);
+
+   if (!total) {
+      return (
+         <>
+            Showing {start}-{perPage * (page - 1) + items}
+         </>
+      );
   }

   return (
      <>
-         Showing {perPage * (page - 1) + 1}-{perPage * (page - 1) + items} of {total} rows
+         Showing {start}-{perPage * (page - 1) + items} of {total} rows
      </>
   );
 };
@@ -287,30 +302,44 @@ type TableNavProps = {
   current: number;
   total: number;
   onClick?: (page: number) => void;
+   hasLast?: boolean;
 };

-const TableNav: React.FC<TableNavProps> = ({ current, total, onClick }: TableNavProps) => {
+const TableNav: React.FC<TableNavProps> = ({
+   current,
+   total,
+   onClick,
+   hasLast = true,
+}: TableNavProps) => {
   const navMap = [
-      { value: 1, Icon: TbChevronsLeft, disabled: current === 1 },
-      { value: current - 1, Icon: TbChevronLeft, disabled: current === 1 },
-      { value: current + 1, Icon: TbChevronRight, disabled: current === total },
-      { value: total, Icon: TbChevronsRight, disabled: current === total },
+      { enabled: true, value: 1, Icon: TbChevronsLeft, disabled: current === 1 },
+      { enabled: true, value: current - 1, Icon: TbChevronLeft, disabled: current === 1 },
+      {
+         enabled: true,
+         value: current + 1,
+         Icon: TbChevronRight,
+         disabled: current === total,
+      },
+      { enabled: hasLast, value: total, Icon: TbChevronsRight, disabled: current === total },
   ] as const;

-   return navMap.map((nav, key) => (
-      <button
-         role="button"
-         type="button"
-         key={key}
-         disabled={nav.disabled}
-         className="px-2 py-2 border-muted border rounded-md enabled:link text-lg enabled:hover:bg-primary/5 text-primary/90 disabled:opacity-50 disabled:cursor-not-allowed"
-         onClick={() => {
-            const page = nav.value;
-            const safePage = page < 1 ? 1 : page > total ? total : page;
-            onClick?.(safePage);
-         }}
-      >
-         <nav.Icon />
-      </button>
-   ));
+   return navMap.map(
+      (nav, key) =>
+         nav.enabled && (
+            <button
+               role="button"
+               type="button"
+               key={key}
+               disabled={nav.disabled}
+               className="px-2 py-2 border-muted border rounded-md enabled:link text-lg enabled:hover:bg-primary/5 text-primary/90 disabled:opacity-50 cursor-pointer disabled:cursor-not-allowed"
+               onClick={() => {
+                  const page = nav.value;
+                  const safePage = page < 1 ? 1 : page > total ? total : page;
+                  onClick?.(safePage);
+               }}
+            >
+               <nav.Icon />
+            </button>
+         ),
+   );
 };
--- a/app/src/ui/elements/media/DropzoneContainer.tsx
+++ b/app/src/ui/elements/media/DropzoneContainer.tsx
@@ -77,7 +77,9 @@ export function DropzoneContainer({
           });

   const $q = infinite
-      ? useApiInfiniteQuery(selectApi, {})
+      ? useApiInfiniteQuery(selectApi, {
+           pageSize,
+        })
      : useApiQuery(selectApi, {
           enabled: initialItems !== false && !initialItems,
           revalidateOnFocus: false,
@@ -108,31 +110,48 @@ export function DropzoneContainer({
      []) as MediaFieldSchema[];
   const _initialItems = mediaItemsToFileStates(actualItems, { baseUrl });

-   const key = id + JSON.stringify(_initialItems);
+   const key = id + JSON.stringify(initialItems);
+
+   // check if endpoint reeturns a total, then reaching end is easy
+   const total = "_data" in $q ? $q._data?.[0]?.body.meta.count : undefined;
+   let placeholderLength = 0;
+   if (infinite && "setSize" in $q) {
+      placeholderLength =
+         typeof total === "number"
+            ? total
+            : $q.endReached
+              ? _initialItems.length
+              : _initialItems.length + pageSize;
+
+      // in case there is no total, we overfetch but SWR don't reflect an empty result
+      // therefore we check if it stopped loading, but has a bigger page size than the total.
+      // if that's the case, we assume we reached the end.
+      if (!total && !$q.isValidating && pageSize * $q.size >= placeholderLength) {
+         placeholderLength = _initialItems.length;
+      }
+   }
+
   return (
-      <Dropzone
-         key={id + key}
-         getUploadInfo={getUploadInfo}
-         handleDelete={handleDelete}
-         /* onUploaded={refresh}
-         onDeleted={refresh} */
-         autoUpload
-         initialItems={_initialItems}
-         footer={
-            infinite &&
-            "setSize" in $q && (
-               <Footer
-                  items={_initialItems.length}
-                  length={Math.min(
-                     $q._data?.[0]?.body.meta.count ?? 0,
-                     _initialItems.length + pageSize,
-                  )}
-                  onFirstVisible={() => $q.setSize($q.size + 1)}
-               />
-            )
-         }
-         {...props}
-      />
+      <>
+         <Dropzone
+            key={key}
+            getUploadInfo={getUploadInfo}
+            handleDelete={handleDelete}
+            autoUpload
+            initialItems={_initialItems}
+            footer={
+               infinite &&
+               "setSize" in $q && (
+                  <Footer
+                     items={_initialItems.length}
+                     length={placeholderLength}
+                     onFirstVisible={() => $q.setSize($q.size + 1)}
+                  />
+               )
+            }
+            {...props}
+         />
+      </>
   );
 }

--- a/app/src/ui/hooks/use-search.ts
+++ b/app/src/ui/hooks/use-search.ts
@@ -14,10 +14,6 @@ export function useSearch<Schema extends s.Schema = s.Schema>(
 ) {
   const searchString = useWouterSearch();
   const [location, navigate] = useLocation();
-   const [value, setValue] = useState<s.StaticCoerced<Schema>>(
-      options?.defaultValue ?? ({} as any),
-   );
-
   const defaults = useMemo(() => {
      return mergeObject(
         // @ts-ignore
@@ -25,6 +21,7 @@ export function useSearch<Schema extends s.Schema = s.Schema>(
         options?.defaultValue ?? {},
      );
   }, [JSON.stringify({ schema, dflt: options?.defaultValue })]);
+   const [value, setValue] = useState<s.StaticCoerced<Schema>>(defaults);

   useEffect(() => {
      const initial =
--- a/app/src/ui/modules/data/components/EntityForm.tsx
+++ b/app/src/ui/modules/data/components/EntityForm.tsx
@@ -301,19 +301,9 @@ function EntityJsonFormField({
               onChange={handleUpdate}
               onBlur={fieldApi.handleBlur}
               minHeight="100"
-               /*required={field.isRequired()}*/
               {...props}
            />
         </Suspense>
-         {/*<Formy.Textarea
-            name={fieldApi.name}
-            id={fieldApi.name}
-            value={fieldApi.state.value}
-            onBlur={fieldApi.handleBlur}
-            onChange={handleUpdate}
-            required={field.isRequired()}
-            {...props}
-         />*/}
      </Formy.Group>
   );
 }
@@ -340,8 +330,8 @@ function EntityEnumFormField({
            {...props}
         >
            {!field.isRequired() && <option value="">- Select -</option>}
-            {field.getOptions().map((option) => (
-               <option key={option.value} value={option.value}>
+            {field.getOptions().map((option, i) => (
+               <option key={`${option.value}-${i}`} value={option.value}>
                  {option.label}
               </option>
            ))}
--- a/app/src/ui/modules/data/components/fields/EntityRelationalFormField.tsx
+++ b/app/src/ui/modules/data/components/fields/EntityRelationalFormField.tsx
@@ -44,7 +44,7 @@ export function EntityRelationalFormField({
   const ref = useRef<any>(null);
   const $q = useEntityQuery(field.target(), undefined, {
      select: query.select,
-      limit: query.limit,
+      limit: query.limit + 1 /* overfetch for softscan=false */,
      offset: (query.page - 1) * query.limit,
   });
   const [_value, _setValue] = useState<{ id: number | undefined; [key: string]: any }>();
--- a/app/src/ui/routes/data/data.$entity.index.tsx
+++ b/app/src/ui/routes/data/data.$entity.index.tsx
@@ -61,7 +61,7 @@ function DataEntityListImpl({ params }) {
      (api) =>
         api.data.readMany(entity?.name as any, {
            select: search.value.select,
-            limit: search.value.perPage,
+            limit: search.value.perPage + 1 /* overfetch for softscan=false */,
            offset: (search.value.page - 1) * search.value.perPage,
            sort: `${search.value.sort.dir === "asc" ? "" : "-"}${search.value.sort.by}`,
         }),
--- a/app/vite.dev.ts
+++ b/app/vite.dev.ts
@@ -1,4 +1,4 @@
-import { readFile } from "node:fs/promises";
+import { readFile, writeFile } from "node:fs/promises";
 import { serveStatic } from "@hono/node-server/serve-static";
 import { showRoutes } from "hono/dev";
 import { App, registries, type CreateAppConfig } from "./src";
@@ -9,6 +9,7 @@ import { $console } from "core/utils/console";
 import { createClient } from "@libsql/client";
 import util from "node:util";
 import { d1Sqlite } from "adapter/cloudflare/connection/D1Connection";
+import { slugify } from "./src/core/utils/strings";

 util.inspect.defaultOptions.depth = 5;
 registries.media.register("local", StorageLocalAdapter);
@@ -21,16 +22,19 @@ $console.debug("Using db type", dbType);
 let dbUrl = import.meta.env.VITE_DB_URL ?? ":memory:";

 const example = import.meta.env.VITE_EXAMPLE;
-if (example) {
-   const configPath = `.configs/${example}.json`;
-   $console.debug("Loading config from", configPath);
-   const exampleConfig = JSON.parse(await readFile(configPath, "utf-8"));
-   config.config = exampleConfig;
-   dbUrl = `file:.configs/${example}.db`;
+async function loadExampleConfig() {
+   if (example) {
+      const configPath = `.configs/${example}.json`;
+      $console.debug("Loading config from", configPath);
+      const exampleConfig = JSON.parse(await readFile(configPath, "utf-8"));
+      config.config = exampleConfig;
+      dbUrl = `file:.configs/${example}.db`;
+   }
 }

 switch (dbType) {
   case "libsql": {
+      await loadExampleConfig();
      $console.debug("Using libsql connection", dbUrl);
      const authToken = import.meta.env.VITE_DB_LIBSQL_TOKEN;
      config.connection = libsql(
@@ -43,15 +47,48 @@ switch (dbType) {
   }
   case "d1": {
      $console.debug("Using d1 connection");
+      const wranglerConfig = {
+         name: "vite-dev",
+         main: "src/index.ts",
+         compatibility_date: "2025-08-03",
+         compatibility_flags: ["nodejs_compat"],
+         d1_databases: [
+            {
+               binding: "DB",
+               database_name: "vite-dev",
+               database_id: "00000000-0000-0000-0000-000000000000",
+            },
+         ],
+         r2_buckets: [
+            {
+               binding: "BUCKET",
+               bucket_name: "vite-dev",
+            },
+         ],
+      };
+      let configPath = ".configs/vite.wrangler.json";
+      if (example) {
+         const name = slugify(example);
+         configPath = `.configs/${slugify(example)}.wrangler.json`;
+         const exists = await readFile(configPath, "utf-8");
+         if (!exists) {
+            wranglerConfig.name = name;
+            wranglerConfig.d1_databases[0]!.database_name = name;
+            wranglerConfig.d1_databases[0]!.database_id = crypto.randomUUID();
+            wranglerConfig.r2_buckets[0]!.bucket_name = name;
+            await writeFile(configPath, JSON.stringify(wranglerConfig, null, 2));
+         }
+      }

      const { getPlatformProxy } = await import("wrangler");
      const platformProxy = await getPlatformProxy({
-         configPath: "./vite.wrangler.json",
+         configPath,
      });
      config.connection = d1Sqlite({ binding: platformProxy.env.DB as any });
      break;
   }
   default: {
+      await loadExampleConfig();
      $console.debug("Using node-sqlite connection", dbUrl);
      config.connection = nodeSqlite({ url: dbUrl });
      break;