fix pagination if endpoint's total is not available

when using a connection that has softscans disabled (e.g. D1) pagination failed. Fixing it by overfetching and slicing

app/src/ui/client/api/use-api.ts

@@ -35,7 +35,7 @@ export const useApiInfiniteQuery = <
    RefineFn extends (data: ResponseObject<Data>) => unknown = (data: ResponseObject<Data>) => Data,
 >(
    fn: (api: Api, page: number) => FetchPromise<Data>,
-   options?: SWRConfiguration & { refine?: RefineFn },
+   options?: SWRConfiguration & { refine?: RefineFn; pageSize?: number },
 ) => {
    const [endReached, setEndReached] = useState(false);
    const api = useApi();
@@ -47,14 +47,14 @@ export const useApiInfiniteQuery = <
    // @ts-ignore
    const swr = useSWRInfinite<RefinedData>(
       (index, previousPageData: any) => {
-         if (previousPageData && !previousPageData.length) {
+         if (index > 0 && previousPageData && previousPageData.length < (options?.pageSize ?? 0)) {
             setEndReached(true);
             return null; // reached the end
          }
          return promise(index).request.url;
       },
       (url: string) => {
-         return new FetchPromise(new Request(url), { fetcher: api.fetcher }, refine).execute();
+         return new FetchPromise(new Request(url), { fetcher: api.fetcher }).execute();
       },
       {
          revalidateFirstPage: false,

app/src/ui/components/table/DataTable.tsx

@@ -53,7 +53,7 @@ export type DataTableProps<Data> = {
 };
 
 export function DataTable<Data extends Record<string, any> = Record<string, any>>({
-   data = [],
+   data: _data = [],
    columns,
    checkable,
    onClickRow,
@@ -71,11 +71,14 @@ export function DataTable<Data extends Record<string, any> = Record<string, any>
    renderValue,
    onClickNew,
 }: DataTableProps<Data>) {
+   const hasTotal = !!total;
+   const data = Array.isArray(_data) ? _data.slice(0, perPage) : _data;
    total = total || data?.length || 0;
    page = page || 1;
 
    const select = columns && columns.length > 0 ? columns : Object.keys(data?.[0] || {});
    const pages = Math.max(Math.ceil(total / perPage), 1);
+   const hasNext = hasTotal ? pages > page : (_data?.length || 0) > perPage;
    const CellRender = renderValue || CellValue;
 
    return (
@@ -202,7 +205,7 @@ export function DataTable<Data extends Record<string, any> = Record<string, any>
                   perPage={perPage}
                   page={page}
                   items={data?.length || 0}
-                  total={total}
+                  total={hasTotal ? total : undefined}
                />
             </div>
             <div className="flex flex-row gap-2 md:gap-10 items-center">
@@ -222,11 +225,17 @@ export function DataTable<Data extends Record<string, any> = Record<string, any>
                   </div>
                )}
                <div className="text-primary/40">
-                  Page {page} of {pages}
+                  Page {page}
+                  {hasTotal ? <> of {pages}</> : ""}
                </div>
                {onClickPage && (
                   <div className="flex flex-row gap-1.5">
-                     <TableNav current={page} total={pages} onClick={onClickPage} />
+                     <TableNav
+                        current={page}
+                        total={hasTotal ? pages : page + (hasNext ? 1 : 0)}
+                        onClick={onClickPage}
+                        hasLast={hasTotal}
+                     />
                   </div>
                )}
             </div>
@@ -268,17 +277,23 @@ const SortIndicator = ({
 };
 
 const TableDisplay = ({ perPage, page, items, total }) => {
-   if (total === 0) {
+   if (items === 0 && page === 1) {
       return <>No rows to show</>;
    }
 
-   if (total === 1) {
+   const start = Math.max(perPage * (page - 1), 1);
-      return <>Showing 1 row</>;
+
+   if (!total) {
+      return (
+         <>
+            Showing {start}-{perPage * (page - 1) + items}
+         </>
+      );
    }
 
    return (
       <>
-         Showing {perPage * (page - 1) + 1}-{perPage * (page - 1) + items} of {total} rows
+         Showing {start}-{perPage * (page - 1) + items} of {total} rows
       </>
    );
 };
@@ -287,23 +302,36 @@ type TableNavProps = {
    current: number;
    total: number;
    onClick?: (page: number) => void;
+   hasLast?: boolean;
 };
 
-const TableNav: React.FC<TableNavProps> = ({ current, total, onClick }: TableNavProps) => {
+const TableNav: React.FC<TableNavProps> = ({
+   current,
+   total,
+   onClick,
+   hasLast = true,
+}: TableNavProps) => {
    const navMap = [
-      { value: 1, Icon: TbChevronsLeft, disabled: current === 1 },
+      { enabled: true, value: 1, Icon: TbChevronsLeft, disabled: current === 1 },
-      { value: current - 1, Icon: TbChevronLeft, disabled: current === 1 },
+      { enabled: true, value: current - 1, Icon: TbChevronLeft, disabled: current === 1 },
-      { value: current + 1, Icon: TbChevronRight, disabled: current === total },
+      {
-      { value: total, Icon: TbChevronsRight, disabled: current === total },
+         enabled: true,
+         value: current + 1,
+         Icon: TbChevronRight,
+         disabled: current === total,
+      },
+      { enabled: hasLast, value: total, Icon: TbChevronsRight, disabled: current === total },
    ] as const;
 
-   return navMap.map((nav, key) => (
+   return navMap.map(
+      (nav, key) =>
+         nav.enabled && (
             <button
                role="button"
                type="button"
                key={key}
                disabled={nav.disabled}
-         className="px-2 py-2 border-muted border rounded-md enabled:link text-lg enabled:hover:bg-primary/5 text-primary/90 disabled:opacity-50 disabled:cursor-not-allowed"
+               className="px-2 py-2 border-muted border rounded-md enabled:link text-lg enabled:hover:bg-primary/5 text-primary/90 disabled:opacity-50 cursor-pointer disabled:cursor-not-allowed"
                onClick={() => {
                   const page = nav.value;
                   const safePage = page < 1 ? 1 : page > total ? total : page;
@@ -312,5 +340,6 @@ const TableNav: React.FC<TableNavProps> = ({ current, total, onClick }: TableNav
             >
                <nav.Icon />
             </button>
-   ));
+         ),
+   );
 };

app/src/ui/elements/media/DropzoneContainer.tsx

@@ -77,7 +77,9 @@ export function DropzoneContainer({
            });
 
    const $q = infinite
-      ? useApiInfiniteQuery(selectApi, {})
+      ? useApiInfiniteQuery(selectApi, {
+           pageSize,
+        })
       : useApiQuery(selectApi, {
            enabled: initialItems !== false && !initialItems,
            revalidateOnFocus: false,
@@ -108,14 +110,33 @@ export function DropzoneContainer({
       []) as MediaFieldSchema[];
    const _initialItems = mediaItemsToFileStates(actualItems, { baseUrl });
 
-   const key = id + JSON.stringify(_initialItems);
+   const key = id + JSON.stringify(initialItems);
+
+   // check if endpoint reeturns a total, then reaching end is easy
+   const total = "_data" in $q ? $q._data?.[0]?.body.meta.count : undefined;
+   let placeholderLength = 0;
+   if (infinite && "setSize" in $q) {
+      placeholderLength =
+         typeof total === "number"
+            ? total
+            : $q.endReached
+              ? _initialItems.length
+              : _initialItems.length + pageSize;
+
+      // in case there is no total, we overfetch but SWR don't reflect an empty result
+      // therefore we check if it stopped loading, but has a bigger page size than the total.
+      // if that's the case, we assume we reached the end.
+      if (!total && !$q.isValidating && pageSize * $q.size >= placeholderLength) {
+         placeholderLength = _initialItems.length;
+      }
+   }
+
    return (
+      <>
          <Dropzone
-         key={id + key}
+            key={key}
             getUploadInfo={getUploadInfo}
             handleDelete={handleDelete}
-         /* onUploaded={refresh}
-         onDeleted={refresh} */
             autoUpload
             initialItems={_initialItems}
             footer={
@@ -123,16 +144,14 @@ export function DropzoneContainer({
                "setSize" in $q && (
                   <Footer
                      items={_initialItems.length}
-                  length={Math.min(
+                     length={placeholderLength}
-                     $q._data?.[0]?.body.meta.count ?? 0,
-                     _initialItems.length + pageSize,
-                  )}
                      onFirstVisible={() => $q.setSize($q.size + 1)}
                   />
                )
             }
             {...props}
          />
+      </>
    );
 }
 

app/src/ui/hooks/use-search.ts

@@ -14,10 +14,6 @@ export function useSearch<Schema extends s.Schema = s.Schema>(
 ) {
    const searchString = useWouterSearch();
    const [location, navigate] = useLocation();
-   const [value, setValue] = useState<s.StaticCoerced<Schema>>(
-      options?.defaultValue ?? ({} as any),
-   );
-
    const defaults = useMemo(() => {
       return mergeObject(
          // @ts-ignore
@@ -25,6 +21,7 @@ export function useSearch<Schema extends s.Schema = s.Schema>(
          options?.defaultValue ?? {},
       );
    }, [JSON.stringify({ schema, dflt: options?.defaultValue })]);
+   const [value, setValue] = useState<s.StaticCoerced<Schema>>(defaults);
 
    useEffect(() => {
       const initial =

app/src/ui/modules/data/components/EntityForm.tsx

@@ -301,19 +301,9 @@ function EntityJsonFormField({
                onChange={handleUpdate}
                onBlur={fieldApi.handleBlur}
                minHeight="100"
-               /*required={field.isRequired()}*/
                {...props}
             />
          </Suspense>
-         {/*<Formy.Textarea
-            name={fieldApi.name}
-            id={fieldApi.name}
-            value={fieldApi.state.value}
-            onBlur={fieldApi.handleBlur}
-            onChange={handleUpdate}
-            required={field.isRequired()}
-            {...props}
-         />*/}
       </Formy.Group>
    );
 }
@@ -340,8 +330,8 @@ function EntityEnumFormField({
             {...props}
          >
             {!field.isRequired() && <option value="">- Select -</option>}
-            {field.getOptions().map((option) => (
+            {field.getOptions().map((option, i) => (
-               <option key={option.value} value={option.value}>
+               <option key={`${option.value}-${i}`} value={option.value}>
                   {option.label}
                </option>
             ))}

app/src/ui/modules/data/components/fields/EntityRelationalFormField.tsx

@@ -44,7 +44,7 @@ export function EntityRelationalFormField({
    const ref = useRef<any>(null);
    const $q = useEntityQuery(field.target(), undefined, {
       select: query.select,
-      limit: query.limit,
+      limit: query.limit + 1 /* overfetch for softscan=false */,
       offset: (query.page - 1) * query.limit,
    });
    const [_value, _setValue] = useState<{ id: number | undefined; [key: string]: any }>();

app/src/ui/routes/data/data.$entity.index.tsx

@@ -61,7 +61,7 @@ function DataEntityListImpl({ params }) {
       (api) =>
          api.data.readMany(entity?.name as any, {
             select: search.value.select,
-            limit: search.value.perPage,
+            limit: search.value.perPage + 1 /* overfetch for softscan=false */,
             offset: (search.value.page - 1) * search.value.perPage,
             sort: `${search.value.sort.dir === "asc" ? "" : "-"}${search.value.sort.by}`,
          }),