Merge pull request #73 from bknd-io/feat/cf-d1-and-r2

adding d1 and r2 for cloudflare environments

app/build.ts

@@ -49,7 +49,8 @@ if (types && !watch) {
 /**
  * Building backend and general API
  */
-await tsup.build({
+async function buildApi() {
+   await tsup.build({
       minify,
       sourcemap,
       watch,
@@ -67,12 +68,14 @@ await tsup.build({
       onSuccess: async () => {
          delayTypes();
       }
-});
+   });
+}
 
 /**
  * Building UI for direct imports
  */
-await tsup.build({
+async function buildUi() {
+   await tsup.build({
       minify,
       sourcemap,
       watch,
@@ -104,14 +107,16 @@ await tsup.build({
       onSuccess: async () => {
          delayTypes();
       }
-});
+   });
+}
 
 /**
  * Building UI Elements
  * - tailwind-merge is mocked, no exclude
  * - ui/client is external, and after built replaced with "bknd/client"
  */
-await tsup.build({
+async function buildUiElements() {
+   await tsup.build({
       minify,
       sourcemap,
       watch,
@@ -148,12 +153,13 @@ await tsup.build({
 
          delayTypes();
       }
-});
+   });
+}
 
 /**
  * Building adapters
  */
-function baseConfig(adapter: string): tsup.Options {
+function baseConfig(adapter: string, overrides: Partial<tsup.Options> = {}): tsup.Options {
    return {
       minify,
       sourcemap,
@@ -162,47 +168,61 @@ function baseConfig(adapter: string): tsup.Options {
       format: ["esm"],
       platform: "neutral",
       outDir: `dist/adapter/${adapter}`,
+      metafile: true,
+      splitting: false,
+      onSuccess: async () => {
+         delayTypes();
+      },
+      ...overrides,
       define: {
-         __isDev: "0"
+         __isDev: "0",
+         ...overrides.define
       },
       external: [
          /^cloudflare*/,
          /^@?(hono|libsql).*?/,
          /^(bknd|react|next|node).*?/,
-         /.*\.(html)$/
+         /.*\.(html)$/,
-      ],
+         ...(Array.isArray(overrides.external) ? overrides.external : [])
-      metafile: true,
+      ]
-      splitting: false,
-      onSuccess: async () => {
-         delayTypes();
-      }
    };
 }
 
-// base adapter handles
+async function buildAdapters() {
-await tsup.build({
+   // base adapter handles
+   await tsup.build({
       ...baseConfig(""),
       entry: ["src/adapter/index.ts"],
       outDir: "dist/adapter"
-});
+   });
 
-// specific adatpers
+   // specific adatpers
-await tsup.build(baseConfig("remix"));
+   await tsup.build(baseConfig("remix"));
-await tsup.build(baseConfig("bun"));
+   await tsup.build(baseConfig("bun"));
-await tsup.build(baseConfig("astro"));
+   await tsup.build(baseConfig("astro"));
-await tsup.build(baseConfig("cloudflare"));
+   await tsup.build(
+      baseConfig("cloudflare", {
+         external: [/^kysely/]
+      })
+   );
 
-await tsup.build({
+   await tsup.build({
       ...baseConfig("vite"),
       platform: "node"
-});
+   });
 
-await tsup.build({
+   await tsup.build({
       ...baseConfig("nextjs"),
       platform: "node"
-});
+   });
 
-await tsup.build({
+   await tsup.build({
       ...baseConfig("node"),
       platform: "node"
-});
+   });
+}
+
+await buildApi();
+await buildUi();
+await buildUiElements();
+await buildAdapters();

app/package.json

@@ -76,6 +76,7 @@
     "clsx": "^2.1.1",
     "esbuild-postcss": "^0.0.4",
     "jotai": "^2.10.1",
+    "kysely-d1": "^0.3.0",
     "open": "^10.1.0",
     "openapi-types": "^12.1.3",
     "postcss": "^8.4.47",

app/src/adapter/cloudflare/D1Connection.ts

@@ -0,0 +1,63 @@
+/// <reference types="@cloudflare/workers-types" />
+
+import { KyselyPluginRunner, SqliteConnection, SqliteIntrospector } from "bknd/data";
+import type { QB } from "data/connection/Connection";
+import { type DatabaseIntrospector, Kysely, ParseJSONResultsPlugin } from "kysely";
+import { D1Dialect } from "kysely-d1";
+
+export type D1ConnectionConfig = {
+   binding: D1Database;
+};
+
+class CustomD1Dialect extends D1Dialect {
+   override createIntrospector(db: Kysely<any>): DatabaseIntrospector {
+      return new SqliteIntrospector(db, {
+         excludeTables: ["_cf_KV"]
+      });
+   }
+}
+
+export class D1Connection extends SqliteConnection {
+   constructor(private config: D1ConnectionConfig) {
+      const plugins = [new ParseJSONResultsPlugin()];
+
+      const kysely = new Kysely({
+         dialect: new CustomD1Dialect({ database: config.binding }),
+         plugins
+      });
+      super(kysely, {}, plugins);
+   }
+
+   override supportsBatching(): boolean {
+      return true;
+   }
+
+   override supportsIndices(): boolean {
+      return true;
+   }
+
+   protected override async batch<Queries extends QB[]>(
+      queries: [...Queries]
+   ): Promise<{
+      [K in keyof Queries]: Awaited<ReturnType<Queries[K]["execute"]>>;
+   }> {
+      const db = this.config.binding;
+
+      const res = await db.batch(
+         queries.map((q) => {
+            const { sql, parameters } = q.compile();
+            return db.prepare(sql).bind(...parameters);
+         })
+      );
+
+      // let it run through plugins
+      const kyselyPlugins = new KyselyPluginRunner(this.plugins);
+      const data: any = [];
+      for (const r of res) {
+         const rows = await kyselyPlugins.transformResultRows(r.results);
+         data.push(rows);
+      }
+
+      return data;
+   }
+}

app/src/adapter/cloudflare/StorageR2Adapter.ts

@@ -1,6 +1,47 @@
-import { isDebug } from "core";
+import { registries } from "bknd";
-import type { FileBody, StorageAdapter } from "../Storage";
+import { isDebug } from "bknd/core";
-import { guessMimeType } from "../mime-types";
+import { StringEnum, Type } from "bknd/utils";
+import type { FileBody, StorageAdapter } from "media/storage/Storage";
+import { guess } from "media/storage/mime-types-tiny";
+import { getBindings } from "./bindings";
+
+export function makeSchema(bindings: string[] = []) {
+   return Type.Object(
+      {
+         binding: bindings.length > 0 ? StringEnum(bindings) : Type.Optional(Type.String())
+      },
+      { title: "R2", description: "Cloudflare R2 storage" }
+   );
+}
+
+export function registerMedia(env: Record<string, any>) {
+   const r2_bindings = getBindings(env, "R2Bucket");
+
+   registries.media.register(
+      "r2",
+      class extends StorageR2Adapter {
+         constructor(private config: any) {
+            const binding = r2_bindings.find((b) => b.key === config.binding);
+            if (!binding) {
+               throw new Error(`No R2Bucket found with key ${config.binding}`);
+            }
+
+            super(binding?.value);
+         }
+
+         override getSchema() {
+            return makeSchema(r2_bindings.map((b) => b.key));
+         }
+
+         override toJSON() {
+            return {
+               ...super.toJSON(),
+               config: this.config
+            };
+         }
+      }
+   );
+}
 
 /**
  * Adapter for R2 storage
@@ -14,7 +55,7 @@ export class StorageR2Adapter implements StorageAdapter {
    }
 
    getSchema() {
-      return undefined;
+      return makeSchema();
    }
 
    async putObject(key: string, body: FileBody) {
@@ -47,7 +88,8 @@ export class StorageR2Adapter implements StorageAdapter {
    async getObject(key: string, headers: Headers): Promise<Response> {
       let object: R2ObjectBody | null;
       const responseHeaders = new Headers({
-         "Accept-Ranges": "bytes"
+         "Accept-Ranges": "bytes",
+         "Content-Type": guess(key)
       });
 
       //console.log("getObject:headers", headersToObject(headers));
@@ -97,10 +139,9 @@ export class StorageR2Adapter implements StorageAdapter {
       if (!metadata || Object.keys(metadata).length === 0) {
          // guessing is especially required for dev environment (miniflare)
          metadata = {
-            contentType: guessMimeType(object.key)
+            contentType: guess(object.key)
          };
       }
-      //console.log("writeHttpMetadata", object.httpMetadata, metadata);
 
       for (const [key, value] of Object.entries(metadata)) {
          const camelToDash = key.replace(/([A-Z])/g, "-$1").toLowerCase();
@@ -115,7 +156,7 @@ export class StorageR2Adapter implements StorageAdapter {
       }
 
       return {
-         type: String(head.httpMetadata?.contentType ?? "application/octet-stream"),
+         type: String(head.httpMetadata?.contentType ?? guess(key)),
          size: head.size
       };
    }

app/src/adapter/cloudflare/bindings.ts

@@ -0,0 +1,32 @@
+export type BindingTypeMap = {
+   D1Database: D1Database;
+   KVNamespace: KVNamespace;
+   DurableObjectNamespace: DurableObjectNamespace;
+   R2Bucket: R2Bucket;
+};
+
+export type GetBindingType = keyof BindingTypeMap;
+export type BindingMap<T extends GetBindingType> = { key: string; value: BindingTypeMap[T] };
+
+export function getBindings<T extends GetBindingType>(env: any, type: T): BindingMap<T>[] {
+   const bindings: BindingMap<T>[] = [];
+   for (const key in env) {
+      try {
+         if (env[key] && (env[key] as any).constructor.name === type) {
+            bindings.push({
+               key,
+               value: env[key] as BindingTypeMap[T]
+            });
+         }
+      } catch (e) {}
+   }
+   return bindings;
+}
+
+export function getBinding<T extends GetBindingType>(env: any, type: T): BindingMap<T> {
+   const bindings = getBindings(env, type);
+   if (bindings.length === 0) {
+      throw new Error(`No ${type} found in bindings`);
+   }
+   return bindings[0] as BindingMap<T>;
+}

app/src/adapter/cloudflare/cloudflare-workers.adapter.ts

@@ -1,6 +1,11 @@
-import type { FrameworkBkndConfig } from "bknd/adapter";
+/// <reference types="@cloudflare/workers-types" />
+
+import { type FrameworkBkndConfig, makeConfig } from "bknd/adapter";
 import { Hono } from "hono";
 import { serveStatic } from "hono/cloudflare-workers";
+import { D1Connection } from "./D1Connection";
+import { registerMedia } from "./StorageR2Adapter";
+import { getBinding } from "./bindings";
 import { getCached } from "./modes/cached";
 import { getDurable } from "./modes/durable";
 import { getFresh, getWarm } from "./modes/fresh";
@@ -10,6 +15,7 @@ export type CloudflareBkndConfig<Env = any> = FrameworkBkndConfig<Context<Env>>
    bindings?: (args: Context<Env>) => {
       kv?: KVNamespace;
       dobj?: DurableObjectNamespace;
+      db?: D1Database;
    };
    static?: "kv" | "assets";
    key?: string;
@@ -26,7 +32,39 @@ export type Context<Env = any> = {
    ctx: ExecutionContext;
 };
 
-export function serve<Env = any>(config: CloudflareBkndConfig<Env>) {
+let media_registered: boolean = false;
+export function makeCfConfig(config: CloudflareBkndConfig, context: Context) {
+   if (!media_registered) {
+      registerMedia(context.env as any);
+      media_registered = true;
+   }
+
+   const appConfig = makeConfig(config, context);
+   const bindings = config.bindings?.(context);
+   if (!appConfig.connection) {
+      let db: D1Database | undefined;
+      if (bindings?.db) {
+         console.log("Using database from bindings");
+         db = bindings.db;
+      } else if (Object.keys(context.env ?? {}).length > 0) {
+         const binding = getBinding(context.env, "D1Database");
+         if (binding) {
+            console.log(`Using database from env "${binding.key}"`);
+            db = binding.value;
+         }
+      }
+
+      if (db) {
+         appConfig.connection = new D1Connection({ binding: db });
+      } else {
+         throw new Error("No database connection given");
+      }
+   }
+
+   return appConfig;
+}
+
+export function serve<Env = any>(config: CloudflareBkndConfig<Env> = {}) {
    return {
       async fetch(request: Request, env: Env, ctx: ExecutionContext) {
          const url = new URL(request.url);
@@ -61,8 +99,6 @@ export function serve<Env = any>(config: CloudflareBkndConfig<Env>) {
             }
          }
 
-         config.setAdminHtml = config.setAdminHtml && !!config.manifest;
-
          const context = { request, env, ctx } as Context;
          const mode = config.mode ?? "warm";
 

app/src/adapter/cloudflare/index.ts

@@ -1,4 +1,18 @@
+import { D1Connection, type D1ConnectionConfig } from "./D1Connection";
+
 export * from "./cloudflare-workers.adapter";
 export { makeApp, getFresh, getWarm } from "./modes/fresh";
 export { getCached } from "./modes/cached";
 export { DurableBkndApp, getDurable } from "./modes/durable";
+export { D1Connection, type D1ConnectionConfig };
+export {
+   getBinding,
+   getBindings,
+   type BindingTypeMap,
+   type GetBindingType,
+   type BindingMap
+} from "./bindings";
+
+export function d1(config: D1ConnectionConfig) {
+   return new D1Connection(config);
+}

app/src/adapter/cloudflare/modes/cached.ts

@@ -1,6 +1,6 @@
 import { App } from "bknd";
 import { createRuntimeApp } from "bknd/adapter";
-import type { CloudflareBkndConfig, Context } from "../index";
+import { type CloudflareBkndConfig, type Context, makeCfConfig } from "../index";
 
 export async function getCached(config: CloudflareBkndConfig, { env, ctx, ...args }: Context) {
    const { kv } = config.bindings?.(env)!;
@@ -16,7 +16,7 @@ export async function getCached(config: CloudflareBkndConfig, { env, ctx, ...arg
 
    const app = await createRuntimeApp(
       {
-         ...config,
+         ...makeCfConfig(config, { env, ctx, ...args }),
          initialConfig,
          onBuilt: async (app) => {
             app.module.server.client.get("/__bknd/cache", async (c) => {

app/src/adapter/cloudflare/modes/fresh.ts

@@ -1,11 +1,11 @@
 import type { App } from "bknd";
 import { createRuntimeApp } from "bknd/adapter";
-import type { CloudflareBkndConfig, Context } from "../index";
+import { type CloudflareBkndConfig, type Context, makeCfConfig } from "../index";
 
 export async function makeApp(config: CloudflareBkndConfig, ctx: Context) {
    return await createRuntimeApp(
       {
-         ...config,
+         ...makeCfConfig(config, ctx),
          adminOptions: config.html ? { html: config.html } : undefined
       },
       ctx

app/src/data/connection/LibsqlConnection.ts

@@ -74,7 +74,6 @@ export class LibsqlConnection extends SqliteConnection {
    }> {
       const stms: InStatement[] = queries.map((q) => {
          const compiled = q.compile();
-         //console.log("compiled", compiled.sql, compiled.parameters);
          return {
             sql: compiled.sql,
             args: compiled.parameters as any[]

app/src/data/connection/SqliteIntrospector.ts

@@ -5,7 +5,7 @@ import type {
    ExpressionBuilder,
    Kysely,
    SchemaMetadata,
-   TableMetadata,
+   TableMetadata
 } from "kysely";
 import { DEFAULT_MIGRATION_LOCK_TABLE, DEFAULT_MIGRATION_TABLE, sql } from "kysely";
 import type { ConnectionIntrospector, IndexMetadata } from "./Connection";
@@ -62,7 +62,7 @@ export class SqliteIntrospector implements DatabaseIntrospector, ConnectionIntro
                seqno: number;
                cid: number;
                name: string;
-            }>`pragma_index_info(${index})`.as("index_info"),
+            }>`pragma_index_info(${index})`.as("index_info")
          )
          .select(["seqno", "cid", "name"])
          .orderBy("cid")
@@ -74,8 +74,8 @@ export class SqliteIntrospector implements DatabaseIntrospector, ConnectionIntro
          isUnique: isUnique,
          columns: columns.map((col) => ({
             name: col.name,
-            order: col.seqno,
+            order: col.seqno
-         })),
+         }))
       };
    }
 
@@ -87,7 +87,7 @@ export class SqliteIntrospector implements DatabaseIntrospector, ConnectionIntro
    }
 
    async getTables(
-      options: DatabaseMetadataOptions = { withInternalKyselyTables: false },
+      options: DatabaseMetadataOptions = { withInternalKyselyTables: false }
    ): Promise<TableMetadata[]> {
       let query = this.#db
          .selectFrom("sqlite_master")
@@ -99,7 +99,7 @@ export class SqliteIntrospector implements DatabaseIntrospector, ConnectionIntro
 
       if (!options.withInternalKyselyTables) {
          query = query.where(
-            this.excludeTables([DEFAULT_MIGRATION_TABLE, DEFAULT_MIGRATION_LOCK_TABLE]),
+            this.excludeTables([DEFAULT_MIGRATION_TABLE, DEFAULT_MIGRATION_LOCK_TABLE])
          );
       }
       if (this._excludeTables.length > 0) {
@@ -107,17 +107,19 @@ export class SqliteIntrospector implements DatabaseIntrospector, ConnectionIntro
       }
 
       const tables = await query.execute();
+      console.log("tables", tables);
       return Promise.all(tables.map(({ name }) => this.#getTableMetadata(name)));
    }
 
    async getMetadata(options?: DatabaseMetadataOptions): Promise<DatabaseMetadata> {
       return {
-         tables: await this.getTables(options),
+         tables: await this.getTables(options)
       };
    }
 
    async #getTableMetadata(table: string): Promise<TableMetadata> {
       const db = this.#db;
+      console.log("get table metadata", table);
 
       // Get the SQL that was used to create the table.
       const tableDefinition = await db
@@ -142,7 +144,7 @@ export class SqliteIntrospector implements DatabaseIntrospector, ConnectionIntro
                type: string;
                notnull: 0 | 1;
                dflt_value: any;
-            }>`pragma_table_info(${table})`.as("table_info"),
+            }>`pragma_table_info(${table})`.as("table_info")
          )
          .select(["name", "type", "notnull", "dflt_value"])
          .orderBy("cid")
@@ -157,8 +159,8 @@ export class SqliteIntrospector implements DatabaseIntrospector, ConnectionIntro
             isNullable: !col.notnull,
             isAutoIncrementing: col.name === autoIncrementCol,
             hasDefaultValue: col.dflt_value != null,
-            comment: undefined,
+            comment: undefined
-         })),
+         }))
       };
    }
 }

app/src/data/index.ts

@@ -18,6 +18,8 @@ export { Connection } from "./connection/Connection";
 export { LibsqlConnection, type LibSqlCredentials } from "./connection/LibsqlConnection";
 export { SqliteConnection } from "./connection/SqliteConnection";
 export { SqliteLocalConnection } from "./connection/SqliteLocalConnection";
+export { SqliteIntrospector } from "./connection/SqliteIntrospector";
+export { KyselyPluginRunner } from "./plugins/KyselyPluginRunner";
 
 export { constructEntity, constructRelation } from "./schema/constructor";
 

app/src/media/AppMedia.ts

@@ -40,7 +40,8 @@ export class AppMedia extends Module<typeof mediaConfigSchema> {
       let adapter: StorageAdapter;
       try {
          const { type, config } = this.config.adapter;
-         adapter = new (registry.get(type as any).cls)(config as any);
+         const cls = registry.get(type as any).cls;
+         adapter = new cls(config as any);
 
          this._storage = new Storage(adapter, this.config.storage, this.ctx.emgr);
          this.setBuilt();
@@ -53,8 +54,6 @@ export class AppMedia extends Module<typeof mediaConfigSchema> {
                index(media).on(["path"], true).on(["reference"]);
             })
          );
-
-         this.setBuilt();
       } catch (e) {
          console.error(e);
          throw new Error(

app/src/media/media-schema.ts

@@ -16,8 +16,8 @@ export function buildMediaSchema() {
             config: adapter.schema
          },
          {
-            title: adapter.schema.title ?? name,
+            title: adapter.schema?.title ?? name,
-            description: adapter.schema.description,
+            description: adapter.schema?.description,
             additionalProperties: false
          }
       );

app/src/ui/modals/index.tsx

@@ -22,11 +22,7 @@ declare module "@mantine/modals" {
 }
 
 export function BkndModalsProvider({ children }) {
-   return (
+   return <ModalsProvider modals={modals}>{children}</ModalsProvider>;
-      <ModalsProvider modals={modals} modalProps={{ className: "bknd-admin" }}>
-         {children}
-      </ModalsProvider>
-   );
 }
 
 function open<Modal extends keyof typeof modals>(

app/src/ui/routes/media/media.settings.tsx

@@ -1,4 +1,4 @@
-import { IconBrandAws, IconCloud, IconServer } from "@tabler/icons-react";
+import { IconBrandAws, IconBrandCloudflare, IconCloud, IconServer } from "@tabler/icons-react";
 import { isDebug } from "core";
 import { autoFormatString } from "core/utils";
 import { twMerge } from "tailwind-merge";
@@ -113,10 +113,15 @@ const RootFormError = () => {
    );
 };
 
-const Icons = [IconBrandAws, IconCloud, IconServer];
+const Icons = {
+   s3: IconBrandAws,
+   cloudinary: IconCloud,
+   local: IconServer,
+   r2: IconBrandCloudflare
+};
 
-const AdapterIcon = ({ index }: { index: number }) => {
+const AdapterIcon = ({ type }: { type: string }) => {
-   const Icon = Icons[index];
+   const Icon = Icons[type];
    if (!Icon) return null;
    return <Icon />;
 };
@@ -142,7 +147,7 @@ function Adapters() {
                   )}
                >
                   <div>
-                     <AdapterIcon index={i} />
+                     <AdapterIcon type={schema.properties.type.const} />
                   </div>
                   <div className="flex flex-col items-start justify-center">
                      <span>{autoFormatString(schema.title)}</span>

examples/cloudflare-worker/package.json

@@ -11,7 +11,8 @@
     "cf-typegen": "wrangler types"
   },
   "dependencies": {
-    "bknd": "workspace:*"
+    "bknd": "workspace:*",
+    "kysely-d1": "^0.3.0"
   },
   "devDependencies": {
     "@cloudflare/workers-types": "^4.20241106.0",

examples/cloudflare-worker/src/index.ts

@@ -1,14 +1,9 @@
+/// <reference types="@cloudflare/workers-types" />
+
 import { serve } from "bknd/adapter/cloudflare";
 
 export default serve({
-   app: (args) => ({
+   mode: "fresh",
-      connection: {
-         type: "libsql",
-         config: {
-            url: "http://localhost:8080"
-         }
-      }
-   }),
    onBuilt: async (app) => {
       app.modules.server.get("/custom", (c) => c.json({ hello: "world" }));
    }

examples/cloudflare-worker/wrangler.toml

@@ -1,7 +1,7 @@
 #:schema node_modules/wrangler/config-schema.json
 name = "bknd-cf-worker-example"
 main = "src/index.ts"
-compatibility_date = "2024-11-06"
+compatibility_date = "2025-02-04"
 compatibility_flags = ["nodejs_compat"]
 workers_dev = true
 minify = true
@@ -10,5 +10,11 @@ assets = { directory = "../../app/dist/static" }
 [observability]
 enabled = true
 
-#[site]
+[[d1_databases]]
-#bucket = "../../app/dist/static"
+binding = "DB"
+database_name = "bknd-cf-example"
+database_id = "7ad67953-2bbf-47fc-8696-f4517dbfe674"
+
+[[r2_buckets]]
+binding = "BUCKET"
+bucket_name = "bknd-cf-example"