shishantbiswas/bknd
1
0
Fork 0
mirror of https://github.com/shishantbiswas/bknd.git synced 2026-03-17 21:06:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

reorganized storage adapter and added test suites for adapter and fields (#124)

Browse Source 
* reorganized storage adapter and added test suites for adapter and fields

* added build command in ci pipeline

* updated workflow to also run node tests

* updated workflow: try with separate tasks

* updated workflow: try with separate tasks

* updated workflow: added tsx as dev dependency

* updated workflow: try with find instead of glob
This commit is contained in:
dswbx
2025-03-27 20:41:42 +01:00
committed by GitHub
parent 40c9ef9d90
commit 9e3c081e50
45 changed files with 605 additions and 940 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/__test__/app/repro.spec.ts
View File

@@ -1,7 +1,7 @@
import { describe, expect, test } from "bun:test";
import { createApp, registries } from "../../src";
import * as proto from "../../src/data/prototype";
import { StorageLocalAdapter } from "../../src/media/storage/adapters/StorageLocalAdapter";
import { StorageLocalAdapter } from "adapter/node/storage/StorageLocalAdapter";
describe("repros", async () => {
/**

7
app/__test__/auth/strategies/OAuthStrategy.spec.ts
View File

@@ -3,8 +3,10 @@ import { OAuthStrategy } from "../../../src/auth/authenticate/strategies";
const ALL_TESTS = !!process.env.ALL_TESTS;
// @todo: add mock response
describe("OAuthStrategy", async () => {
const strategy = new OAuthStrategy({
return;
/*const strategy = new OAuthStrategy({
type: "oidc",
client: {
client_id: process.env.OAUTH_CLIENT_ID!,
@@ -21,6 +23,7 @@ describe("OAuthStrategy", async () => {
const server = Bun.serve({
fetch: async (req) => {
console.log("req", req.method, req.url);
const url = new URL(req.url);
if (url.pathname === "/auth/google/callback") {
console.log("req", req);
@@ -42,5 +45,5 @@ describe("OAuthStrategy", async () => {
console.log("request", request);
await new Promise((resolve) => setTimeout(resolve, 100000));
});
});*/
});

57
app/__test__/core/cache/CloudflareKvCache.native-spec.ts vendored
View File

@@ -1,57 +0,0 @@
import * as assert from "node:assert/strict";
import { createWriteStream } from "node:fs";
import { after, beforeEach, describe, test } from "node:test";
import { Miniflare } from "miniflare";
import {
CloudflareKVCacheItem,
CloudflareKVCachePool,
} from "../../../src/core/cache/adapters/CloudflareKvCache";
import { runTests } from "./cache-test-suite";
// https://github.com/nodejs/node/issues/44372#issuecomment-1736530480
console.log = async (message: any) => {
const tty = createWriteStream("/dev/tty");
const msg = typeof message === "string" ? message : JSON.stringify(message, null, 2);
return tty.write(`${msg}\n`);
};
describe("CloudflareKv", async () => {
let mf: Miniflare;
runTests({
createCache: async () => {
if (mf) {
await mf.dispose();
}
mf = new Miniflare({
modules: true,
script: "export default { async fetch() { return new Response(null); } }",
kvNamespaces: ["TEST"],
});
const kv = await mf.getKVNamespace("TEST");
return new CloudflareKVCachePool(kv as any);
},
createItem: (key, value) => new CloudflareKVCacheItem(key, value),
tester: {
test,
beforeEach,
expect: (actual?: any) => {
return {
toBe(expected: any) {
assert.equal(actual, expected);
},
toEqual(expected: any) {
assert.deepEqual(actual, expected);
},
toBeUndefined() {
assert.equal(actual, undefined);
},
};
},
},
});
after(async () => {
await mf?.dispose();
});
});

15
app/__test__/core/cache/MemoryCache.spec.ts vendored
View File

@@ -1,15 +0,0 @@
import { beforeEach, describe, expect, test } from "bun:test";
import { MemoryCache, MemoryCacheItem } from "../../../src/core/cache/adapters/MemoryCache";
import { runTests } from "./cache-test-suite";
describe("MemoryCache", () => {
runTests({
createCache: async () => new MemoryCache(),
createItem: (key, value) => new MemoryCacheItem(key, value),
tester: {
test,
beforeEach,
expect,
},
});
});

84
app/__test__/core/cache/cache-test-suite.ts vendored
View File

@@ -1,84 +0,0 @@
//import { beforeEach as bunBeforeEach, expect as bunExpect, test as bunTest } from "bun:test";
import type { ICacheItem, ICachePool } from "../../../src/core/cache/cache-interface";
export type TestOptions = {
createCache: () => Promise<ICachePool>;
createItem: (key: string, value: any) => ICacheItem;
tester: {
test: (name: string, fn: () => Promise<void>) => void;
beforeEach: (fn: () => Promise<void>) => void;
expect: (actual?: any) => {
toBe(expected: any): void;
toEqual(expected: any): void;
toBeUndefined(): void;
};
};
};
export function runTests({ createCache, createItem, tester }: TestOptions) {
let cache: ICachePool<string>;
const { test, beforeEach, expect } = tester;
beforeEach(async () => {
cache = await createCache();
});
test("getItem returns correct item", async () => {
const item = createItem("key1", "value1");
await cache.save(item);
const retrievedItem = await cache.get("key1");
expect(retrievedItem.value()).toEqual(item.value());
});
test("getItem returns new item when key does not exist", async () => {
const retrievedItem = await cache.get("key1");
expect(retrievedItem.key()).toEqual("key1");
expect(retrievedItem.value()).toBeUndefined();
});
test("getItems returns correct items", async () => {
const item1 = createItem("key1", "value1");
const item2 = createItem("key2", "value2");
await cache.save(item1);
await cache.save(item2);
const retrievedItems = await cache.getMany(["key1", "key2"]);
expect(retrievedItems.get("key1")?.value()).toEqual(item1.value());
expect(retrievedItems.get("key2")?.value()).toEqual(item2.value());
});
test("hasItem returns true when item exists and is a hit", async () => {
const item = createItem("key1", "value1");
await cache.save(item);
expect(await cache.has("key1")).toBe(true);
});
test("clear and deleteItem correctly clear the cache and delete items", async () => {
const item = createItem("key1", "value1");
await cache.save(item);
if (cache.supports().clear) {
await cache.clear();
} else {
await cache.delete("key1");
}
expect(await cache.has("key1")).toBe(false);
});
test("save correctly saves items to the cache", async () => {
const item = createItem("key1", "value1");
await cache.save(item);
expect(await cache.has("key1")).toBe(true);
});
test("putItem correctly puts items in the cache ", async () => {
await cache.put("key1", "value1", { ttl: 60 });
const item = await cache.get("key1");
expect(item.value()).toEqual("value1");
expect(item.hit()).toBe(true);
});
/*test("commit returns true", async () => {
expect(await cache.commit()).toBe(true);
});*/
}

4
app/__test__/data/specs/fields/BooleanField.spec.ts
View File

@@ -1,9 +1,9 @@
import { describe, expect, test } from "bun:test";
import { BooleanField } from "../../../../src/data";
import { runBaseFieldTests, transformPersist } from "./inc";
import { fieldTestSuite, transformPersist } from "data/fields/field-test-suite";
describe("[data] BooleanField", async () => {
runBaseFieldTests(BooleanField, { defaultValue: true, schemaType: "boolean" });
fieldTestSuite({ expect, test }, BooleanField, { defaultValue: true, schemaType: "boolean" });
test("transformRetrieve", async () => {
const field = new BooleanField("test");

4
app/__test__/data/specs/fields/DateField.spec.ts
View File

@@ -1,9 +1,9 @@
import { describe, expect, test } from "bun:test";
import { DateField } from "../../../../src/data";
import { runBaseFieldTests } from "./inc";
import { fieldTestSuite } from "data/fields/field-test-suite";
describe("[data] DateField", async () => {
runBaseFieldTests(DateField, { defaultValue: new Date(), schemaType: "date" });
fieldTestSuite({ expect, test }, DateField, { defaultValue: new Date(), schemaType: "date" });
// @todo: add datefield tests
test("week", async () => {

9
app/__test__/data/specs/fields/EnumField.spec.ts
View File

@@ -1,13 +1,15 @@
import { describe, expect, test } from "bun:test";
import { EnumField } from "../../../../src/data";
import { runBaseFieldTests, transformPersist } from "./inc";
import { fieldTestSuite, transformPersist } from "data/fields/field-test-suite";
function options(strings: string[]) {
return { type: "strings", values: strings };
}
describe("[data] EnumField", async () => {
runBaseFieldTests(
fieldTestSuite(
{ expect, test },
// @ts-ignore
EnumField,
{ defaultValue: "a", schemaType: "text" },
{ options: options(["a", "b", "c"]) },
@@ -15,11 +17,13 @@ describe("[data] EnumField", async () => {
test("yields if default value is not a valid option", async () => {
expect(
// @ts-ignore
() => new EnumField("test", { options: options(["a", "b"]), default_value: "c" }),
).toThrow();
});
test("transformPersist (config)", async () => {
// @ts-ignore
const field = new EnumField("test", { options: options(["a", "b", "c"]) });
expect(transformPersist(field, null)).resolves.toBeUndefined();
@@ -29,6 +33,7 @@ describe("[data] EnumField", async () => {
test("transformRetrieve", async () => {
const field = new EnumField("test", {
// @ts-ignore
options: options(["a", "b", "c"]),
default_value: "a",
required: true,

4
app/__test__/data/specs/fields/Field.spec.ts
View File

@@ -1,7 +1,7 @@
import { describe, expect, test } from "bun:test";
import { Default, stripMark } from "../../../../src/core/utils";
import { baseFieldConfigSchema, Field } from "../../../../src/data/fields/Field";
import { runBaseFieldTests } from "./inc";
import { fieldTestSuite } from "data/fields/field-test-suite";
describe("[data] Field", async () => {
class FieldSpec extends Field {
@@ -19,7 +19,7 @@ describe("[data] Field", async () => {
});
});
runBaseFieldTests(FieldSpec, { defaultValue: "test", schemaType: "text" });
fieldTestSuite({ expect, test }, FieldSpec, { defaultValue: "test", schemaType: "text" });
test("default config", async () => {
const config = Default(baseFieldConfigSchema, {});

10
app/__test__/data/specs/fields/FieldIndex.spec.ts
View File

@@ -1,19 +1,13 @@
import { describe, expect, test } from "bun:test";
import { Type } from "../../../../src/core/utils";
import {
Entity,
EntityIndex,
type EntityManager,
Field,
type SchemaResponse,
} from "../../../../src/data";
import { Entity, EntityIndex, Field } from "../../../../src/data";
class TestField extends Field {
protected getSchema(): any {
return Type.Any();
}
schema(em: EntityManager<any>): SchemaResponse {
override schema() {
return undefined as any;
}
}

4
app/__test__/data/specs/fields/JsonField.spec.ts
View File

@@ -1,10 +1,10 @@
import { describe, expect, test } from "bun:test";
import { JsonField } from "../../../../src/data";
import { runBaseFieldTests, transformPersist } from "./inc";
import { fieldTestSuite, transformPersist } from "data/fields/field-test-suite";
describe("[data] JsonField", async () => {
const field = new JsonField("test");
runBaseFieldTests(JsonField, {
fieldTestSuite({ expect, test }, JsonField, {
defaultValue: { a: 1 },
sampleValues: ["string", { test: 1 }, 1],
schemaType: "text",

5
app/__test__/data/specs/fields/JsonSchemaField.spec.ts
View File

@@ -1,9 +1,10 @@
import { describe, expect, test } from "bun:test";
import { JsonSchemaField } from "../../../../src/data";
import { runBaseFieldTests } from "./inc";
import { fieldTestSuite } from "data/fields/field-test-suite";
describe("[data] JsonSchemaField", async () => {
runBaseFieldTests(JsonSchemaField, { defaultValue: {}, schemaType: "text" });
// @ts-ignore
fieldTestSuite({ expect, test }, JsonSchemaField, { defaultValue: {}, schemaType: "text" });
// @todo: add JsonSchemaField tests
});

4
app/__test__/data/specs/fields/NumberField.spec.ts
View File

@@ -1,6 +1,6 @@
import { describe, expect, test } from "bun:test";
import { NumberField } from "../../../../src/data";
import { runBaseFieldTests, transformPersist } from "./inc";
import { fieldTestSuite, transformPersist } from "data/fields/field-test-suite";
describe("[data] NumberField", async () => {
test("transformPersist (config)", async () => {
@@ -15,5 +15,5 @@ describe("[data] NumberField", async () => {
expect(transformPersist(field2, 10000)).resolves.toBe(10000);
});
runBaseFieldTests(NumberField, { defaultValue: 12, schemaType: "integer" });
fieldTestSuite({ expect, test }, NumberField, { defaultValue: 12, schemaType: "integer" });
});

4
app/__test__/data/specs/fields/TextField.spec.ts
View File

@@ -1,6 +1,6 @@
import { describe, expect, test } from "bun:test";
import { TextField } from "../../../../src/data";
import { runBaseFieldTests, transformPersist } from "./inc";
import { fieldTestSuite, transformPersist } from "data/fields/field-test-suite";
describe("[data] TextField", async () => {
test("transformPersist (config)", async () => {
@@ -11,5 +11,5 @@ describe("[data] TextField", async () => {
expect(transformPersist(field, "abc")).resolves.toBe("abc");
});
runBaseFieldTests(TextField, { defaultValue: "abc", schemaType: "text" });
fieldTestSuite({ expect, test }, TextField, { defaultValue: "abc", schemaType: "text" });
});

2
app/__test__/media/MediaController.spec.ts
View File

@@ -4,7 +4,7 @@ import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { createApp, registries } from "../../src";
import { mergeObject, randomString } from "../../src/core/utils";
import type { TAppMediaConfig } from "../../src/media/media-schema";
import { StorageLocalAdapter } from "../../src/media/storage/adapters/StorageLocalAdapter";
import { StorageLocalAdapter } from "adapter/node/storage/StorageLocalAdapter";
import { assetsPath, assetsTmpPath, disableConsoleLog, enableConsoleLog } from "../helper";
beforeAll(() => {

26
app/__test__/media/StorageR2Adapter.native-spec.ts
View File

@@ -1,7 +1,10 @@
import * as assert from "node:assert/strict";
import { createWriteStream } from "node:fs";
import { createWriteStream, readFileSync } from "node:fs";
import { test } from "node:test";
import { Miniflare } from "miniflare";
import { StorageR2Adapter } from "adapter/cloudflare/StorageR2Adapter";
import { adapterTestSuite } from "media";
import { nodeTestRunner } from "adapter/node";
import path from "node:path";
// https://github.com/nodejs/node/issues/44372#issuecomment-1736530480
console.log = async (message: any) => {
@@ -10,25 +13,20 @@ console.log = async (message: any) => {
return tty.write(`${msg}\n`);
};
test("what", async () => {
test("StorageR2Adapter", async () => {
const mf = new Miniflare({
modules: true,
script: "export default { async fetch() { return new Response(null); } }",
r2Buckets: ["BUCKET"],
});
const bucket = await mf.getR2Bucket("BUCKET");
console.log(await bucket.put("count", "1"));
const bucket = (await mf.getR2Bucket("BUCKET")) as unknown as R2Bucket;
const adapter = new StorageR2Adapter(bucket);
const object = await bucket.get("count");
if (object) {
/*const headers = new Headers();
object.writeHttpMetadata(headers);
headers.set("etag", object.httpEtag);*/
console.log("yo -->", await object.text());
assert.strictEqual(await object.text(), "1");
}
const basePath = path.resolve(import.meta.dirname, "../_assets");
const buffer = readFileSync(path.join(basePath, "image.png"));
const file = new File([buffer], "image.png", { type: "image/png" });
await adapterTestSuite(nodeTestRunner, adapter, file);
await mf.dispose();
});

63
app/__test__/media/adapters/StorageCloudinaryAdapter.spec.ts
View File

@@ -1,63 +0,0 @@
import { describe, expect, test } from "bun:test";
import { randomString } from "../../../src/core/utils";
import { StorageCloudinaryAdapter } from "../../../src/media";
import { config } from "dotenv";
const dotenvOutput = config({ path: `${import.meta.dir}/../../../.env` });
const {
CLOUDINARY_CLOUD_NAME,
CLOUDINARY_API_KEY,
CLOUDINARY_API_SECRET,
CLOUDINARY_UPLOAD_PRESET,
} = dotenvOutput.parsed!;
const ALL_TESTS = !!process.env.ALL_TESTS;
describe.skipIf(ALL_TESTS)("StorageCloudinaryAdapter", () => {
if (ALL_TESTS) return;
const adapter = new StorageCloudinaryAdapter({
cloud_name: CLOUDINARY_CLOUD_NAME as string,
api_key: CLOUDINARY_API_KEY as string,
api_secret: CLOUDINARY_API_SECRET as string,
upload_preset: CLOUDINARY_UPLOAD_PRESET as string,
});
const file = Bun.file(`${import.meta.dir}/icon.png`);
const _filename = randomString(10);
const filename = `${_filename}.png`;
test("object exists", async () => {
expect(await adapter.objectExists("7fCTBi6L8c.png")).toBeTrue();
process.exit();
});
test("puts object", async () => {
expect(await adapter.objectExists(filename)).toBeFalse();
const result = await adapter.putObject(filename, file);
console.log("result", result);
expect(result).toBeDefined();
expect(result?.name).toBe(filename);
});
test("object exists", async () => {
await Bun.sleep(10000);
const one = await adapter.objectExists(_filename);
const two = await adapter.objectExists(filename);
expect(await adapter.objectExists(filename)).toBeTrue();
});
test("object meta", async () => {
const result = await adapter.getObjectMeta(filename);
console.log("objectMeta:result", result);
expect(result).toBeDefined();
expect(result.type).toBe("image/png");
expect(result.size).toBeGreaterThan(0);
});
test("list objects", async () => {
const result = await adapter.listObjects();
console.log("listObjects:result", result);
});
});

47
app/__test__/media/adapters/StorageLocalAdapter.spec.ts
View File

@@ -1,47 +0,0 @@
import { describe, expect, test } from "bun:test";
import { randomString } from "../../../src/core/utils";
import { StorageLocalAdapter } from "../../../src/media/storage/adapters/StorageLocalAdapter";
import { assetsPath, assetsTmpPath } from "../../helper";
describe("StorageLocalAdapter", () => {
const adapter = new StorageLocalAdapter({
path: assetsTmpPath,
});
const file = Bun.file(`${assetsPath}/image.png`);
const _filename = randomString(10);
const filename = `${_filename}.png`;
let objects = 0;
test("puts an object", async () => {
objects = (await adapter.listObjects()).length;
expect(await adapter.putObject(filename, file as unknown as File)).toBeString();
});
test("lists objects", async () => {
expect((await adapter.listObjects()).length).toBe(objects + 1);
});
test("file exists", async () => {
expect(await adapter.objectExists(filename)).toBeTrue();
});
test("gets an object", async () => {
const res = await adapter.getObject(filename, new Headers());
expect(res.ok).toBeTrue();
// @todo: check the content
});
test("gets object meta", async () => {
expect(await adapter.getObjectMeta(filename)).toEqual({
type: file.type, // image/png
size: file.size,
});
});
test("deletes an object", async () => {
expect(await adapter.deleteObject(filename)).toBeUndefined();
expect(await adapter.objectExists(filename)).toBeFalse();
});
});

109
app/__test__/media/adapters/StorageS3Adapter.spec.ts
View File

@@ -1,109 +0,0 @@
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { randomString } from "../../../src/core/utils";
import { StorageS3Adapter } from "../../../src/media";
import { config } from "dotenv";
//import { enableFetchLogging } from "../../helper";
const dotenvOutput = config({ path: `${import.meta.dir}/../../../.env` });
const { R2_ACCESS_KEY, R2_SECRET_ACCESS_KEY, R2_URL, AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_S3_URL } =
dotenvOutput.parsed!;
// @todo: mock r2/s3 responses for faster tests
const ALL_TESTS = !!process.env.ALL_TESTS;
console.log("ALL_TESTS?", ALL_TESTS);
/*
// @todo: preparation to mock s3 calls + replace fast-xml-parser
let cleanup: () => void;
beforeAll(async () => {
cleanup = await enableFetchLogging();
});
afterAll(() => {
cleanup();
}); */
describe.skipIf(ALL_TESTS)("StorageS3Adapter", async () => {
if (ALL_TESTS) return;
const versions = [
[
"r2",
new StorageS3Adapter({
access_key: R2_ACCESS_KEY as string,
secret_access_key: R2_SECRET_ACCESS_KEY as string,
url: R2_URL as string,
}),
],
[
"s3",
new StorageS3Adapter({
access_key: AWS_ACCESS_KEY as string,
secret_access_key: AWS_SECRET_KEY as string,
url: AWS_S3_URL as string,
}),
],
] as const;
const _conf = {
adapters: ["r2", "s3"],
tests: [
"listObjects",
"putObject",
"objectExists",
"getObject",
"deleteObject",
"getObjectMeta",
],
};
const file = Bun.file(`${import.meta.dir}/icon.png`);
const filename = `${randomString(10)}.png`;
// single (dev)
//_conf = { adapters: [/*"r2",*/ "s3"], tests: [/*"putObject",*/ "listObjects"] };
function disabled(test: (typeof _conf.tests)[number]) {
return !_conf.tests.includes(test);
}
// @todo: add mocked fetch for faster tests
describe.each(versions)("StorageS3Adapter for %s", async (name, adapter) => {
if (!_conf.adapters.includes(name) || ALL_TESTS) {
console.log("Skipping", name);
return;
}
let objects = 0;
test.skipIf(disabled("putObject"))("puts an object", async () => {
objects = (await adapter.listObjects()).length;
expect(await adapter.putObject(filename, file as any)).toBeString();
});
test.skipIf(disabled("listObjects"))("lists objects", async () => {
expect((await adapter.listObjects()).length).toBe(objects + 1);
});
test.skipIf(disabled("objectExists"))("file exists", async () => {
expect(await adapter.objectExists(filename)).toBeTrue();
});
test.skipIf(disabled("getObject"))("gets an object", async () => {
const res = await adapter.getObject(filename, new Headers());
expect(res.ok).toBeTrue();
// @todo: check the content
});
test.skipIf(disabled("getObjectMeta"))("gets object meta", async () => {
expect(await adapter.getObjectMeta(filename)).toEqual({
type: file.type, // image/png
size: file.size,
});
});
test.skipIf(disabled("deleteObject"))("deletes an object", async () => {
expect(await adapter.deleteObject(filename)).toBeUndefined();
expect(await adapter.objectExists(filename)).toBeFalse();
});
});
});

2
app/__test__/modules/AppMedia.spec.ts
View File

@@ -1,7 +1,7 @@
import { describe, expect, test } from "bun:test";
import { createApp, registries } from "../../src";
import { em, entity, text } from "../../src/data";
import { StorageLocalAdapter } from "../../src/media/storage/adapters/StorageLocalAdapter";
import { StorageLocalAdapter } from "adapter/node/storage/StorageLocalAdapter";
import { AppMedia } from "../../src/modules";
import { moduleTestSuite } from "./module-test-suite";

10
app/build.ts
View File

@@ -54,7 +54,7 @@ function banner(title: string) {
}
// collection of always-external packages
const external = ["bun:test", "@libsql/client"] as const;
const external = ["bun:test", "node:test", "node:assert/strict", "@libsql/client"] as const;
/**
* Building backend and general API
@@ -65,7 +65,13 @@ async function buildApi() {
minify,
sourcemap,
watch,
entry: ["src/index.ts", "src/data/index.ts", "src/core/index.ts", "src/core/utils/index.ts"],
entry: [
"src/index.ts",
"src/core/index.ts",
"src/core/utils/index.ts",
"src/data/index.ts",
"src/media/index.ts",
],
outDir: "dist",
external: [...external],
metafile: true,

10
app/package.json
View File

@@ -16,9 +16,13 @@
"scripts": {
"dev": "vite",
"test": "ALL_TESTS=1 bun test --bail",
"test:all": "bun run test && bun run test:node",
"test:bun": "ALL_TESTS=1 bun test --bail",
"test:node": "tsx --test $(find . -type f -name '*.native-spec.ts')",
"test:coverage": "ALL_TESTS=1 bun test --bail --coverage",
"build": "NODE_ENV=production bun run build.ts --minify --types",
"build:all": "rm -rf dist && bun run build:static && NODE_ENV=production bun run build.ts --minify --types --clean && bun run build:cli",
"build:ci": "mkdir -p dist/static/.vite && echo '{}' > dist/static/.vite/manifest.json && NODE_ENV=production bun run build.ts",
"build:cli": "bun build src/cli/index.ts --target node --outdir dist/cli --env PUBLIC_* --minify",
"build:static": "vite build",
"watch": "bun run build.ts --types --watch",
@@ -101,6 +105,7 @@
"tailwindcss-animate": "^1.0.7",
"tsc-alias": "^1.8.11",
"tsup": "^8.4.0",
"tsx": "^4.19.3",
"vite": "^6.2.1",
"vite-tsconfig-paths": "^5.1.4",
"wouter": "^3.6.0"
@@ -156,6 +161,11 @@
"import": "./dist/cli/index.js",
"require": "./dist/cli/index.cjs"
},
"./media": {
"types": "./dist/types/media/index.d.ts",
"import": "./dist/media/index.js",
"require": "./dist/media/index.cjs"
},
"./adapter/cloudflare": {
"types": "./dist/types/adapter/cloudflare/index.d.ts",
"import": "./dist/adapter/cloudflare/index.js",

9
app/src/adapter/cloudflare/StorageR2Adapter.ts
View File

@@ -1,7 +1,8 @@
import { registries } from "bknd";
import { isDebug } from "bknd/core";
import { StringEnum, Type } from "bknd/utils";
import type { FileBody, StorageAdapter } from "media/storage/Storage";
import type { FileBody } from "media/storage/Storage";
import { StorageAdapter } from "media/storage/StorageAdapter";
import { guess } from "media/storage/mime-types-tiny";
import { getBindings } from "./bindings";
@@ -47,8 +48,10 @@ export function registerMedia(env: Record<string, any>) {
* Adapter for R2 storage
* @todo: add tests (bun tests won't work, need node native tests)
*/
export class StorageR2Adapter implements StorageAdapter {
constructor(private readonly bucket: R2Bucket) {}
export class StorageR2Adapter extends StorageAdapter {
constructor(private readonly bucket: R2Bucket) {
super();
}
getName(): string {
return "r2";

6
app/src/adapter/node/index.ts
View File

@@ -1,11 +1,9 @@
import { registries } from "bknd";
import {
type LocalAdapterConfig,
StorageLocalAdapter,
} from "../../media/storage/adapters/StorageLocalAdapter";
import { type LocalAdapterConfig, StorageLocalAdapter } from "./storage/StorageLocalAdapter";
export * from "./node.adapter";
export { StorageLocalAdapter, type LocalAdapterConfig };
export { nodeTestRunner } from "./test";
export function registerLocalMediaAdapter() {
registries.media.register("local", StorageLocalAdapter);

17
app/src/adapter/node/storage/StorageLocalAdapter.native-spec.ts Normal file
View File

@@ -0,0 +1,17 @@
import { describe } from "node:test";
import { StorageLocalAdapter, nodeTestRunner } from "adapter/node";
import { adapterTestSuite } from "media/storage/adapters/adapter-test-suite";
import { readFileSync } from "node:fs";
import path from "node:path";
describe("StorageLocalAdapter (node)", async () => {
const basePath = path.resolve(import.meta.dirname, "../../../../__test__/_assets");
const buffer = readFileSync(path.join(basePath, "image.png"));
const file = new File([buffer], "image.png", { type: "image/png" });
const adapter = new StorageLocalAdapter({
path: path.join(basePath, "tmp"),
});
await adapterTestSuite(nodeTestRunner, adapter, file);
});

14
app/src/adapter/node/storage/StorageLocalAdapter.spec.ts Normal file
View File

@@ -0,0 +1,14 @@
import { describe, test, expect } from "bun:test";
import { StorageLocalAdapter } from "./StorageLocalAdapter";
// @ts-ignore
import { assetsPath, assetsTmpPath } from "../../../../__test__/helper";
import { adapterTestSuite } from "media/storage/adapters/adapter-test-suite";
describe("StorageLocalAdapter (bun)", async () => {
const adapter = new StorageLocalAdapter({
path: assetsTmpPath,
});
const file = Bun.file(`${assetsPath}/image.png`);
await adapterTestSuite({ test, expect }, adapter, file);
});

15
app/src/media/storage/adapters/StorageLocalAdapter/StorageLocalAdapter.ts → app/src/adapter/node/storage/StorageLocalAdapter.ts
View File

@@ -1,13 +1,7 @@
import { readFile, readdir, stat, unlink, writeFile } from "node:fs/promises";
import { type Static, Type, isFile, parse } from "core/utils";
import type {
FileBody,
FileListObject,
FileMeta,
FileUploadPayload,
StorageAdapter,
} from "../../Storage";
import { guess } from "../../mime-types-tiny";
import { type Static, Type, isFile, parse } from "bknd/utils";
import type { FileBody, FileListObject, FileMeta, FileUploadPayload } from "bknd/media";
import { StorageAdapter, guessMimeType as guess } from "bknd/media";
export const localAdapterConfig = Type.Object(
{
@@ -17,10 +11,11 @@ export const localAdapterConfig = Type.Object(
);
export type LocalAdapterConfig = Static<typeof localAdapterConfig>;
export class StorageLocalAdapter implements StorageAdapter {
export class StorageLocalAdapter extends StorageAdapter {
private config: LocalAdapterConfig;
constructor(config: any) {
super();
this.config = parse(localAdapterConfig, config);
}

75
app/src/adapter/node/test.ts Normal file
View File

@@ -0,0 +1,75 @@
import nodeAssert from "node:assert/strict";
import { test } from "node:test";
import type { Matcher, Test, TestFn, TestRunner } from "core/test";
const nodeTestMatcher = <T = unknown>(actual: T, parentFailMsg?: string) =>
({
toEqual: (expected: T, failMsg = parentFailMsg) => {
nodeAssert.deepEqual(actual, expected, failMsg);
},
toBe: (expected: T, failMsg = parentFailMsg) => {
nodeAssert.strictEqual(actual, expected, failMsg);
},
toBeString: (failMsg = parentFailMsg) => {
nodeAssert.strictEqual(typeof actual, "string", failMsg);
},
toBeUndefined: (failMsg = parentFailMsg) => {
nodeAssert.strictEqual(actual, undefined, failMsg);
},
toBeDefined: (failMsg = parentFailMsg) => {
nodeAssert.notStrictEqual(actual, undefined, failMsg);
},
toBeOneOf: (expected: T | Array<T> | Iterable<T>, failMsg = parentFailMsg) => {
const e = Array.isArray(expected) ? expected : [expected];
nodeAssert.ok(e.includes(actual), failMsg);
},
}) satisfies Matcher<T>;
const nodeTestResolverProxy = <T = unknown>(
actual: Promise<T>,
handler: { resolve?: any; reject?: any },
) => {
return new Proxy(
{},
{
get: (_, prop) => {
if (prop === "then") {
return actual.then(handler.resolve, handler.reject);
}
return actual;
},
},
) as Matcher<Awaited<T>>;
};
function nodeTest(label: string, fn: TestFn, options?: any) {
return test(label, fn as any);
}
nodeTest.if = (condition: boolean): Test => {
if (condition) {
return nodeTest;
}
return (() => {}) as any;
};
nodeTest.skip = (label: string, fn: TestFn) => {
return test.skip(label, fn as any);
};
nodeTest.skipIf = (condition: boolean): Test => {
if (condition) {
return (() => {}) as any;
}
return nodeTest;
};
export const nodeTestRunner: TestRunner = {
test: nodeTest,
expect: <T = unknown>(actual?: T, failMsg?: string) => ({
...nodeTestMatcher(actual, failMsg),
resolves: nodeTestResolverProxy(actual as Promise<T>, {
resolve: (r) => nodeTestMatcher(r, failMsg),
}),
rejects: nodeTestResolverProxy(actual as Promise<T>, {
reject: (r) => nodeTestMatcher(r, failMsg),
}),
}),
};

127
app/src/core/cache/adapters/CloudflareKvCache.ts vendored
View File

@@ -1,127 +0,0 @@
import type { ICacheItem, ICachePool } from "../cache-interface";
export class CloudflareKVCachePool<Data = any> implements ICachePool<Data> {
constructor(private namespace: KVNamespace) {}
supports = () => ({
metadata: true,
clear: false,
});
async get(key: string): Promise<ICacheItem<Data>> {
const result = await this.namespace.getWithMetadata<any>(key);
const hit = result.value !== null && typeof result.value !== "undefined";
// Assuming metadata is not supported directly;
// you may adjust if Cloudflare KV supports it in future.
return new CloudflareKVCacheItem(key, result.value ?? undefined, hit, result.metadata) as any;
}
async getMany(keys: string[] = []): Promise<Map<string, ICacheItem<Data>>> {
const items = new Map<string, ICacheItem<Data>>();
await Promise.all(
keys.map(async (key) => {
const item = await this.get(key);
items.set(key, item);
}),
);
return items;
}
async has(key: string): Promise<boolean> {
const data = await this.namespace.get(key);
return data !== null;
}
async clear(): Promise<boolean> {
// Cloudflare KV does not support clearing all keys in one operation
return false;
}
async delete(key: string): Promise<boolean> {
await this.namespace.delete(key);
return true;
}
async deleteMany(keys: string[]): Promise<boolean> {
const results = await Promise.all(keys.map((key) => this.delete(key)));
return results.every((result) => result);
}
async save(item: CloudflareKVCacheItem<Data>): Promise<boolean> {
await this.namespace.put(item.key(), (await item.value()) as string, {
expirationTtl: item._expirationTtl,
metadata: item.metadata(),
});
return true;
}
async put(
key: string,
value: any,
options?: { ttl?: number; expiresAt?: Date; metadata?: Record<string, string> },
): Promise<boolean> {
const item = new CloudflareKVCacheItem(key, value, true, options?.metadata);
if (options?.expiresAt) item.expiresAt(options.expiresAt);
if (options?.ttl) item.expiresAfter(options.ttl);
return await this.save(item);
}
}
export class CloudflareKVCacheItem<Data = any> implements ICacheItem<Data> {
_expirationTtl: number | undefined;
constructor(
private _key: string,
private data: Data | undefined,
private _hit: boolean = false,
private _metadata: Record<string, string> = {},
) {}
key(): string {
return this._key;
}
value(): Data | undefined {
if (this.data) {
try {
return JSON.parse(this.data as string);
} catch (e) {}
}
return this.data ?? undefined;
}
metadata(): Record<string, string> {
return this._metadata;
}
hit(): boolean {
return this._hit;
}
set(value: Data, metadata: Record<string, string> = {}): this {
this.data = value;
this._metadata = metadata;
return this;
}
expiresAt(expiration: Date | null): this {
// Cloudflare KV does not support specific date expiration; calculate ttl instead.
if (expiration) {
const now = new Date();
const ttl = (expiration.getTime() - now.getTime()) / 1000;
return this.expiresAfter(Math.max(0, Math.floor(ttl)));
}
return this.expiresAfter(null);
}
expiresAfter(time: number | null): this {
// Dummy implementation as Cloudflare KV requires setting expiration during PUT operation.
// This method will be effectively implemented in the Cache Pool save methods.
this._expirationTtl = time ?? undefined;
return this;
}
}

139
app/src/core/cache/adapters/MemoryCache.ts vendored
View File

@@ -1,139 +0,0 @@
import type { ICacheItem, ICachePool } from "../cache-interface";
export class MemoryCache<Data = any> implements ICachePool<Data> {
private cache: Map<string, MemoryCacheItem<Data>> = new Map();
private maxSize?: number;
constructor(options?: { maxSize?: number }) {
this.maxSize = options?.maxSize;
}
supports = () => ({
metadata: true,
clear: true,
});
async get(key: string): Promise<MemoryCacheItem<Data>> {
if (!this.cache.has(key)) {
// use undefined to denote a miss initially
return new MemoryCacheItem<Data>(key, undefined!);
}
return this.cache.get(key)!;
}
async getMany(keys: string[] = []): Promise<Map<string, MemoryCacheItem<Data>>> {
const items = new Map<string, MemoryCacheItem<Data>>();
for (const key of keys) {
items.set(key, await this.get(key));
}
return items;
}
async has(key: string): Promise<boolean> {
return this.cache.has(key) && this.cache.get(key)!.hit();
}
async clear(): Promise<boolean> {
this.cache.clear();
return true;
}
async delete(key: string): Promise<boolean> {
return this.cache.delete(key);
}
async deleteMany(keys: string[]): Promise<boolean> {
let success = true;
for (const key of keys) {
if (!this.delete(key)) {
success = false;
}
}
return success;
}
async save(item: MemoryCacheItem<Data>): Promise<boolean> {
this.checkSizeAndPurge();
this.cache.set(item.key(), item);
return true;
}
async put(
key: string,
value: Data,
options: { expiresAt?: Date; ttl?: number; metadata?: Record<string, string> } = {},
): Promise<boolean> {
const item = await this.get(key);
item.set(value, options.metadata || {});
if (options.expiresAt) {
item.expiresAt(options.expiresAt);
} else if (typeof options.ttl === "number") {
item.expiresAfter(options.ttl);
}
return this.save(item);
}
private checkSizeAndPurge(): void {
if (!this.maxSize) return;
if (this.cache.size >= this.maxSize) {
// Implement logic to purge items, e.g., LRU (Least Recently Used)
// For simplicity, clear the oldest item inserted
const keyToDelete = this.cache.keys().next().value;
this.cache.delete(keyToDelete!);
}
}
}
export class MemoryCacheItem<Data = any> implements ICacheItem<Data> {
private _key: string;
private _value: Data | undefined;
private expiration: Date | null = null;
private _metadata: Record<string, string> = {};
constructor(key: string, value: Data, metadata: Record<string, string> = {}) {
this._key = key;
this.set(value, metadata);
}
key(): string {
return this._key;
}
metadata(): Record<string, string> {
return this._metadata;
}
value(): Data | undefined {
return this._value;
}
hit(): boolean {
if (this.expiration !== null && new Date() > this.expiration) {
return false;
}
return this.value() !== undefined;
}
set(value: Data, metadata: Record<string, string> = {}): this {
this._value = value;
this._metadata = metadata;
return this;
}
expiresAt(expiration: Date | null): this {
this.expiration = expiration;
return this;
}
expiresAfter(time: number | null): this {
if (typeof time === "number") {
const expirationDate = new Date();
expirationDate.setSeconds(expirationDate.getSeconds() + time);
this.expiration = expirationDate;
} else {
this.expiration = null;
}
return this;
}
}

178
app/src/core/cache/cache-interface.ts vendored
View File

@@ -1,178 +0,0 @@
/**
* CacheItem defines an interface for interacting with objects inside a cache.
* based on https://www.php-fig.org/psr/psr-6/
*/
export interface ICacheItem<Data = any> {
/**
* Returns the key for the current cache item.
*
* The key is loaded by the Implementing Library, but should be available to
* the higher level callers when needed.
*
* @returns The key string for this cache item.
*/
key(): string;
/**
* Retrieves the value of the item from the cache associated with this object's key.
*
* The value returned must be identical to the value originally stored by set().
*
* If isHit() returns false, this method MUST return null. Note that null
* is a legitimate cached value, so the isHit() method SHOULD be used to
* differentiate between "null value was found" and "no value was found."
*
* @returns The value corresponding to this cache item's key, or undefined if not found.
*/
value(): Data | undefined;
/**
* Retrieves the metadata of the item from the cache associated with this object's key.
*/
metadata(): Record<string, string>;
/**
* Confirms if the cache item lookup resulted in a cache hit.
*
* Note: This method MUST NOT have a race condition between calling isHit()
* and calling get().
*
* @returns True if the request resulted in a cache hit. False otherwise.
*/
hit(): boolean;
/**
* Sets the value represented by this cache item.
*
* The value argument may be any item that can be serialized by PHP,
* although the method of serialization is left up to the Implementing
* Library.
*
* @param value The serializable value to be stored.
* @param metadata The metadata to be associated with the item.
* @returns The invoked object.
*/
set(value: Data, metadata?: Record<string, string>): this;
/**
* Sets the expiration time for this cache item.
*
* @param expiration The point in time after which the item MUST be considered expired.
* If null is passed explicitly, a default value MAY be used. If none is set,
* the value should be stored permanently or for as long as the
* implementation allows.
* @returns The called object.
*/
expiresAt(expiration: Date | null): this;
/**
* Sets the expiration time for this cache item.
*
* @param time The period of time from the present after which the item MUST be considered
* expired. An integer parameter is understood to be the time in seconds until
* expiration. If null is passed explicitly, a default value MAY be used.
* If none is set, the value should be stored permanently or for as long as the
* implementation allows.
* @returns The called object.
*/
expiresAfter(time: number | null): this;
}
/**
* CachePool generates CacheItem objects.
* based on https://www.php-fig.org/psr/psr-6/
*/
export interface ICachePool<Data = any> {
supports(): {
metadata: boolean;
clear: boolean;
};
/**
* Returns a Cache Item representing the specified key.
* This method must always return a CacheItemInterface object, even in case of
* a cache miss. It MUST NOT return null.
*
* @param key The key for which to return the corresponding Cache Item.
* @throws Error If the key string is not a legal value an Error MUST be thrown.
* @returns The corresponding Cache Item.
*/
get(key: string): Promise<ICacheItem<Data>>;
/**
* Returns a traversable set of cache items.
*
* @param keys An indexed array of keys of items to retrieve.
* @throws Error If any of the keys in keys are not a legal value an Error MUST be thrown.
* @returns A traversable collection of Cache Items keyed by the cache keys of
* each item. A Cache item will be returned for each key, even if that
* key is not found. However, if no keys are specified then an empty
* traversable MUST be returned instead.
*/
getMany(keys?: string[]): Promise<Map<string, ICacheItem<Data>>>;
/**
* Confirms if the cache contains specified cache item.
*
* Note: This method MAY avoid retrieving the cached value for performance reasons.
* This could result in a race condition with CacheItemInterface.get(). To avoid
* such situation use CacheItemInterface.isHit() instead.
*
* @param key The key for which to check existence.
* @throws Error If the key string is not a legal value an Error MUST be thrown.
* @returns True if item exists in the cache, false otherwise.
*/
has(key: string): Promise<boolean>;
/**
* Deletes all items in the pool.
* @returns True if the pool was successfully cleared. False if there was an error.
*/
clear(): Promise<boolean>;
/**
* Removes the item from the pool.
*
* @param key The key to delete.
* @throws Error If the key string is not a legal value an Error MUST be thrown.
* @returns True if the item was successfully removed. False if there was an error.
*/
delete(key: string): Promise<boolean>;
/**
* Removes multiple items from the pool.
*
* @param keys An array of keys that should be removed from the pool.
* @throws Error If any of the keys in keys are not a legal value an Error MUST be thrown.
* @returns True if the items were successfully removed. False if there was an error.
*/
deleteMany(keys: string[]): Promise<boolean>;
/**
* Persists a cache item immediately.
*
* @param item The cache item to save.
* @returns True if the item was successfully persisted. False if there was an error.
*/
save(item: ICacheItem<Data>): Promise<boolean>;
/**
* Persists any deferred cache items.
* @returns True if all not-yet-saved items were successfully saved or there were none. False otherwise.
*/
put(
key: string,
value: any,
options?: { expiresAt?: Date; metadata?: Record<string, string> },
): Promise<boolean>;
put(
key: string,
value: any,
options?: { ttl?: number; metadata?: Record<string, string> },
): Promise<boolean>;
put(
key: string,
value: any,
options?: ({ ttl?: number } | { expiresAt?: Date }) & { metadata?: Record<string, string> },
): Promise<boolean>;
}

48
app/src/core/test/index.ts Normal file
View File

@@ -0,0 +1,48 @@
export type Matcher<T = unknown> = {
toEqual: (expected: T, failMsg?: string) => void;
toBe: (expected: T, failMsg?: string) => void;
toBeUndefined: (failMsg?: string) => void;
toBeString: (failMsg?: string) => void;
toBeOneOf: (expected: T | Array<T> | Iterable<T>, failMsg?: string) => void;
toBeDefined: (failMsg?: string) => void;
};
export type TestFn = (() => void | Promise<unknown>) | ((done: (err?: unknown) => void) => void);
export interface Test {
(label: string, fn: TestFn, options?: any): void;
if: (condition: boolean) => (label: string, fn: TestFn, options?: any) => void;
skip: (label: string, fn: () => void) => void;
skipIf: (condition: boolean) => (label: string, fn: TestFn) => void;
}
export type TestRunner = {
test: Test;
expect: <T = unknown>(
actual?: T,
failMsg?: string,
) => Matcher<T> & {
resolves: Matcher<Awaited<T>>;
rejects: Matcher<Awaited<T>>;
};
};
export async function retry<T>(
fn: () => Promise<T>,
condition: (result: T) => boolean,
retries: number,
delay: number,
): Promise<T> {
let lastError: Error | null = null;
for (let i = 0; i < retries; i++) {
try {
const result = await fn();
if (condition(result)) {
return result;
} else {
await new Promise((resolve) => setTimeout(resolve, delay));
}
} catch (error) {
lastError = error as Error;
await new Promise((resolve) => setTimeout(resolve, delay));
}
}
throw lastError;
}

12
app/__test__/data/specs/fields/inc.ts → app/src/data/fields/field-test-suite.ts
View File

@@ -1,7 +1,7 @@
import { expect, test } from "bun:test";
import type { BaseFieldConfig, Field, TActionContext } from "data";
import type { ColumnDataType } from "kysely";
import { omit } from "lodash-es";
import type { BaseFieldConfig, Field, TActionContext } from "../../../../src/data";
import type { TestRunner } from "core/test";
type ConstructableField = new (name: string, config?: Partial<BaseFieldConfig>) => Field;
@@ -15,11 +15,13 @@ export function transformPersist(field: Field, value: any, context?: TActionCont
return field.transformPersist(value, undefined as any, context as any);
}
export function runBaseFieldTests(
export function fieldTestSuite(
testRunner: TestRunner,
fieldClass: ConstructableField,
config: FieldTestConfig,
_requiredConfig: any = {},
) {
const { test, expect } = testRunner;
const noConfigField = new fieldClass("no_config", _requiredConfig);
const fillable = new fieldClass("fillable", { ..._requiredConfig, fillable: true });
const required = new fieldClass("required", { ..._requiredConfig, required: true });
@@ -76,9 +78,9 @@ export function runBaseFieldTests(
const isPrimitive = (v) => ["string", "number"].includes(typeof v);
for (const value of config.sampleValues!) {
// "form"
expect(isPrimitive(noConfigField.getValue(value, "form"))).toBeTrue();
expect(isPrimitive(noConfigField.getValue(value, "form"))).toBe(true);
// "table"
expect(isPrimitive(noConfigField.getValue(value, "table"))).toBeTrue();
expect(isPrimitive(noConfigField.getValue(value, "table"))).toBe(true);
// "read"
// "submit"
}

2
app/src/data/fields/index.ts
View File

@@ -53,3 +53,5 @@ export const FieldClassMap = {
json: { schema: jsonFieldConfigSchema, field: JsonField },
jsonschema: { schema: jsonSchemaFieldConfigSchema, field: JsonSchemaField },
} as const;
export { fieldTestSuite } from "./field-test-suite";

14
app/src/media/index.ts
View File

@@ -1,22 +1,24 @@
import type { TObject, TString } from "@sinclair/typebox";
import type { TObject } from "@sinclair/typebox";
import { type Constructor, Registry } from "core";
//export { MIME_TYPES } from "./storage/mime-types";
export { guess as guessMimeType } from "./storage/mime-types-tiny";
export {
Storage,
type StorageAdapter,
type FileMeta,
type FileListObject,
type StorageConfig,
type FileBody,
type FileUploadPayload,
} from "./storage/Storage";
import type { StorageAdapter } from "./storage/Storage";
import { StorageAdapter } from "./storage/StorageAdapter";
import {
type CloudinaryConfig,
StorageCloudinaryAdapter,
} from "./storage/adapters/StorageCloudinaryAdapter";
import { type S3AdapterConfig, StorageS3Adapter } from "./storage/adapters/StorageS3Adapter";
} from "./storage/adapters/cloudinary/StorageCloudinaryAdapter";
import { type S3AdapterConfig, StorageS3Adapter } from "./storage/adapters/s3/StorageS3Adapter";
export { StorageAdapter };
export { StorageS3Adapter, type S3AdapterConfig, StorageCloudinaryAdapter, type CloudinaryConfig };
export * as StorageEvents from "./storage/events";
@@ -45,3 +47,5 @@ export const Adapters = {
schema: StorageCloudinaryAdapter.prototype.getSchema(),
},
} as const;
export { adapterTestSuite } from "./storage/adapters/adapter-test-suite";

21
app/src/media/storage/Storage.ts
View File

@@ -1,9 +1,10 @@
import { type EmitsEvents, EventManager } from "core/events";
import { type TSchema, isFile, detectImageDimensions } from "core/utils";
import { isFile, detectImageDimensions } from "core/utils";
import { isMimeType } from "media/storage/mime-types-tiny";
import * as StorageEvents from "./events";
import type { FileUploadedEventData } from "./events";
import { $console } from "core";
import type { StorageAdapter } from "./StorageAdapter";
export type FileListObject = {
key: string;
@@ -19,24 +20,6 @@ export type FileUploadPayload = {
etag: string;
};
export interface StorageAdapter {
/**
* The unique name of the storage adapter
*/
getName(): string;
// @todo: method requires limit/offset parameters
listObjects(prefix?: string): Promise<FileListObject[]>;
putObject(key: string, body: FileBody): Promise<string | FileUploadPayload | undefined>;
deleteObject(key: string): Promise<void>;
objectExists(key: string): Promise<boolean>;
getObject(key: string, headers: Headers): Promise<Response>;
getObjectUrl(key: string): string;
getObjectMeta(key: string): Promise<FileMeta>;
getSchema(): TSchema | undefined;
toJSON(secrets?: boolean): any;
}
export type StorageConfig = {
body_max_size?: number;
};

37
app/src/media/storage/StorageAdapter.ts Normal file
View File

@@ -0,0 +1,37 @@
import type { FileListObject, FileMeta } from "media";
import type { FileBody, FileUploadPayload } from "media/storage/Storage";
import type { TSchema } from "@sinclair/typebox";
const SYMBOL = Symbol.for("bknd:storage");
export abstract class StorageAdapter {
constructor() {
this[SYMBOL] = true;
}
/**
* This is a helper function to manage Connection classes
* coming from different places
* @param conn
*/
static isAdapter(conn: unknown): conn is StorageAdapter {
if (!conn) return false;
return conn[SYMBOL] === true;
}
/**
* The unique name of the storage adapter
*/
abstract getName(): string;
// @todo: method requires limit/offset parameters
abstract listObjects(prefix?: string): Promise<FileListObject[]>;
abstract putObject(key: string, body: FileBody): Promise<string | FileUploadPayload | undefined>;
abstract deleteObject(key: string): Promise<void>;
abstract objectExists(key: string): Promise<boolean>;
abstract getObject(key: string, headers: Headers): Promise<Response>;
abstract getObjectUrl(key: string): string;
abstract getObjectMeta(key: string): Promise<FileMeta>;
abstract getSchema(): TSchema | undefined;
abstract toJSON(secrets?: boolean): any;
}

5
app/src/media/storage/adapters/StorageLocalAdapter/index.ts
View File

@@ -1,5 +0,0 @@
export {
StorageLocalAdapter,
type LocalAdapterConfig,
localAdapterConfig,
} from "./StorageLocalAdapter";

79
app/src/media/storage/adapters/adapter-test-suite.ts Normal file
View File

@@ -0,0 +1,79 @@
import { retry, type TestRunner } from "core/test";
import type { StorageAdapter } from "media";
import { randomString } from "core/utils";
import type { BunFile } from "bun";
export async function adapterTestSuite(
testRunner: TestRunner,
adapter: StorageAdapter,
file: File | BunFile,
opts?: {
retries?: number;
retryTimeout?: number;
skipExistsAfterDelete?: boolean;
},
) {
const { test, expect } = testRunner;
const options = {
retries: opts?.retries ?? 1,
retryTimeout: opts?.retryTimeout ?? 1000,
};
let objects = 0;
const _filename = randomString(10);
const filename = `${_filename}.png`;
await test("puts an object", async () => {
objects = (await adapter.listObjects()).length;
const result = await adapter.putObject(filename, file as unknown as File);
expect(result).toBeDefined();
const type = typeof result;
expect(type).toBeOneOf(["string", "object"]);
if (typeof result === "object") {
expect(Object.keys(result).sort()).toEqual(["etag", "meta", "name"]);
expect(result.meta.type).toBe(file.type);
}
});
await test("lists objects", async () => {
const length = await retry(
() => adapter.listObjects().then((res) => res.length),
(length) => length > objects,
options.retries,
options.retryTimeout,
);
expect(length).toBe(objects + 1);
});
await test("file exists", async () => {
expect(await adapter.objectExists(filename)).toBe(true);
});
await test("gets an object", async () => {
const res = await adapter.getObject(filename, new Headers());
expect(res.ok).toBe(true);
// @todo: check the content
});
await test("gets object meta", async () => {
expect(await adapter.getObjectMeta(filename)).toEqual({
type: file.type, // image/png
size: file.size,
});
});
await test("deletes an object", async () => {
expect(await adapter.deleteObject(filename)).toBeUndefined();
if (opts?.skipExistsAfterDelete !== true) {
const exists = await retry(
() => adapter.objectExists(filename),
(res) => res === false,
options.retries,
options.retryTimeout,
);
expect(exists).toBe(false);
}
});
}

53
app/src/media/storage/adapters/cloudinary/StorageCloudinaryAdapter.spec.ts Normal file
View File

@@ -0,0 +1,53 @@
import { describe, expect, test } from "bun:test";
import { StorageCloudinaryAdapter } from "./StorageCloudinaryAdapter";
import { config } from "dotenv";
// @ts-ignore
import { assetsPath, assetsTmpPath } from "../../../../../__test__/helper";
import { adapterTestSuite } from "media/storage/adapters/adapter-test-suite";
const dotenvOutput = config({ path: `${import.meta.dir}/.env` });
const {
CLOUDINARY_CLOUD_NAME,
CLOUDINARY_API_KEY,
CLOUDINARY_API_SECRET,
CLOUDINARY_UPLOAD_PRESET,
} = dotenvOutput.parsed!;
const ALL_TESTS = !!process.env.ALL_TESTS;
describe.skipIf(ALL_TESTS)("StorageCloudinaryAdapter", async () => {
if (ALL_TESTS) return;
const adapter = new StorageCloudinaryAdapter({
cloud_name: CLOUDINARY_CLOUD_NAME as string,
api_key: CLOUDINARY_API_KEY as string,
api_secret: CLOUDINARY_API_SECRET as string,
upload_preset: CLOUDINARY_UPLOAD_PRESET as string,
});
const file = Bun.file(`${assetsPath}/image.png`) as unknown as File;
test("hash", async () => {
expect(
await adapter.generateSignature(
{
eager: "w_400,h_300,c_pad|w_260,h_200,c_crop",
public_id: "sample_image",
timestamp: 1315060510,
},
"abcd",
),
).toEqual({
signature: "bfd09f95f331f558cbd1320e67aa8d488770583e",
timestamp: 1315060510,
});
});
await adapterTestSuite({ test, expect }, adapter, file, {
// eventual consistency
retries: 20,
retryTimeout: 1000,
// result is cached from cloudinary
skipExistsAfterDelete: true,
});
});

63
app/src/media/storage/adapters/StorageCloudinaryAdapter.ts → app/src/media/storage/adapters/cloudinary/StorageCloudinaryAdapter.ts
View File

@@ -1,6 +1,7 @@
import { pickHeaders } from "core/utils";
import { hash, pickHeaders } from "core/utils";
import { type Static, Type, parse } from "core/utils";
import type { FileBody, FileListObject, FileMeta, StorageAdapter } from "../Storage";
import type { FileBody, FileListObject, FileMeta } from "../../Storage";
import { StorageAdapter } from "../../StorageAdapter";
export const cloudinaryAdapterConfig = Type.Object(
{
@@ -53,10 +54,11 @@ type CloudinaryListObjectsResponse = {
};
// @todo: add signed uploads
export class StorageCloudinaryAdapter implements StorageAdapter {
export class StorageCloudinaryAdapter extends StorageAdapter {
private config: CloudinaryConfig;
constructor(config: CloudinaryConfig) {
super();
this.config = parse(cloudinaryAdapterConfig, config);
}
@@ -126,6 +128,11 @@ export class StorageCloudinaryAdapter implements StorageAdapter {
};
}
/**
* https://cloudinary.com/documentation/admin_api#search_for_resources
* Cloudinary implements eventual consistency: Search results reflect any changes made to assets within a few seconds after the change
* @param prefix
*/
async listObjects(prefix?: string): Promise<FileListObject[]> {
const result = await fetch(
`https://api.cloudinary.com/v1_1/${this.config.cloud_name}/resources/search`,
@@ -133,6 +140,7 @@ export class StorageCloudinaryAdapter implements StorageAdapter {
method: "GET",
headers: {
Accept: "application/json",
"Cache-Control": "no-cache",
...this.getAuthorizationHeader(),
},
},
@@ -143,18 +151,22 @@ export class StorageCloudinaryAdapter implements StorageAdapter {
}
const data = (await result.json()) as CloudinaryListObjectsResponse;
return data.resources.map((item) => ({
const items = data.resources.map((item) => ({
key: item.public_id,
last_modified: new Date(item.uploaded_at),
size: item.bytes,
}));
return items;
}
private async headObject(key: string) {
const url = this.getObjectUrl(key);
return await fetch(url, {
method: "GET",
method: "HEAD",
headers: {
"Cache-Control": "no-cache, no-store, must-revalidate",
Pragma: "no-cache",
Expires: "0",
Range: "bytes=0-1",
},
});
@@ -196,6 +208,22 @@ export class StorageCloudinaryAdapter implements StorageAdapter {
return objectUrl;
}
async generateSignature(params: Record<string, string | number>, secret?: string) {
const timestamp = params.timestamp ?? Math.floor(Date.now() / 1000);
const content = Object.entries({ ...params, timestamp })
.sort(([keyA], [keyB]) => keyA.localeCompare(keyB))
.map(([key, value]) => `${key}=${value}`)
.join("&");
const signature = await hash.sha1(content + (secret ?? this.config.api_secret));
return { signature, timestamp };
}
// get public_id as everything before the last "."
filenameToPublicId(key: string): string {
return key.split(".").slice(0, -1).join(".");
}
async getObject(key: string, headers: Headers): Promise<Response> {
const res = await fetch(this.getObjectUrl(key), {
method: "GET",
@@ -211,13 +239,30 @@ export class StorageCloudinaryAdapter implements StorageAdapter {
async deleteObject(key: string): Promise<void> {
const type = this.guessType(key) ?? "image";
const formData = new FormData();
formData.append("public_ids[]", key);
const public_id = this.filenameToPublicId(key);
const { timestamp, signature } = await this.generateSignature({
public_id,
});
await fetch(`https://res.cloudinary.com/${this.config.cloud_name}/${type}/upload/`, {
method: "DELETE",
const formData = new FormData();
formData.append("public_id", public_id);
formData.append("timestamp", String(timestamp));
formData.append("signature", signature);
formData.append("api_key", this.config.api_key);
const url = `https://api.cloudinary.com/v1_1/${this.config.cloud_name}/${type}/destroy`;
const res = await fetch(url, {
headers: {
Accept: "application/json",
"Cache-Control": "no-cache",
...this.getAuthorizationHeader(),
},
method: "POST",
body: formData,
});
if (!res.ok) {
throw new Error(`Failed to delete object: ${res.status} ${res.statusText}`);
}
}
toJSON(secrets?: boolean) {

50
app/src/media/storage/adapters/s3/StorageS3Adapter.spec.ts Normal file
View File

@@ -0,0 +1,50 @@
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { StorageS3Adapter } from "./StorageS3Adapter";
import { config } from "dotenv";
import { adapterTestSuite } from "media";
import { assetsPath } from "../../../../../__test__/helper";
//import { enableFetchLogging } from "../../helper";
const dotenvOutput = config({ path: `${import.meta.dir}/.env` });
const { R2_ACCESS_KEY, R2_SECRET_ACCESS_KEY, R2_URL, AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_S3_URL } =
dotenvOutput.parsed!;
const ALL_TESTS = !!process.env.ALL_TESTS;
/*
// @todo: preparation to mock s3 calls + replace fast-xml-parser
let cleanup: () => void;
beforeAll(async () => {
cleanup = await enableFetchLogging();
});
afterAll(() => {
cleanup();
}); */
describe.skipIf(ALL_TESTS)("StorageS3Adapter", async () => {
if (ALL_TESTS) return;
const versions = [
[
"r2",
new StorageS3Adapter({
access_key: R2_ACCESS_KEY as string,
secret_access_key: R2_SECRET_ACCESS_KEY as string,
url: R2_URL as string,
}),
],
[
"s3",
new StorageS3Adapter({
access_key: AWS_ACCESS_KEY as string,
secret_access_key: AWS_SECRET_KEY as string,
url: AWS_S3_URL as string,
}),
],
] as const;
const file = Bun.file(`${assetsPath}/image.png`) as unknown as File;
describe.each(versions)("%s", async (_name, adapter) => {
await adapterTestSuite({ test, expect }, adapter, file);
});
});

23
app/src/media/storage/adapters/StorageS3Adapter.ts → app/src/media/storage/adapters/s3/StorageS3Adapter.ts
View File

@@ -9,7 +9,8 @@ import type {
import { AwsClient, isDebug } from "core";
import { type Static, Type, isFile, parse, pickHeaders2 } from "core/utils";
import { transform } from "lodash-es";
import type { FileBody, FileListObject, StorageAdapter } from "../Storage";
import type { FileBody, FileListObject } from "../../Storage";
import { StorageAdapter } from "../../StorageAdapter";
export const s3AdapterConfig = Type.Object(
{
@@ -32,11 +33,13 @@ export const s3AdapterConfig = Type.Object(
export type S3AdapterConfig = Static<typeof s3AdapterConfig>;
export class StorageS3Adapter extends AwsClient implements StorageAdapter {
export class StorageS3Adapter extends StorageAdapter {
readonly #config: S3AdapterConfig;
readonly client: AwsClient;
constructor(config: S3AdapterConfig) {
super(
super();
this.client = new AwsClient(
{
accessKeyId: config.access_key,
secretAccessKey: config.secret_access_key,
@@ -58,10 +61,10 @@ export class StorageS3Adapter extends AwsClient implements StorageAdapter {
return s3AdapterConfig;
}
override getUrl(path: string = "", searchParamsObj: Record<string, any> = {}): string {
getUrl(path: string = "", searchParamsObj: Record<string, any> = {}): string {
let url = this.getObjectUrl("").slice(0, -1);
if (path.length > 0) url += `/${path}`;
return super.getUrl(url, searchParamsObj);
return this.client.getUrl(url, searchParamsObj);
}
/**
@@ -82,7 +85,7 @@ export class StorageS3Adapter extends AwsClient implements StorageAdapter {
};
const url = this.getUrl("", params);
const res = await this.fetchJson<{ ListBucketResult: ListObjectsV2Output }>(url, {
const res = await this.client.fetchJson<{ ListBucketResult: ListObjectsV2Output }>(url, {
method: "GET",
});
@@ -115,7 +118,7 @@ export class StorageS3Adapter extends AwsClient implements StorageAdapter {
params: Omit<PutObjectRequest, "Bucket" | "Key"> = {},
) {
const url = this.getUrl(key, {});
const res = await this.fetch(url, {
const res = await this.client.fetch(url, {
method: "PUT",
body,
headers: isFile(body)
@@ -139,7 +142,7 @@ export class StorageS3Adapter extends AwsClient implements StorageAdapter {
params: Pick<HeadObjectRequest, "PartNumber" | "VersionId"> = {},
) {
const url = this.getUrl(key, {});
return await this.fetch(url, {
return await this.client.fetch(url, {
method: "HEAD",
headers: {
Range: "bytes=0-1",
@@ -175,7 +178,7 @@ export class StorageS3Adapter extends AwsClient implements StorageAdapter {
*/
async getObject(key: string, headers: Headers): Promise<Response> {
const url = this.getUrl(key);
const res = await this.fetch(url, {
const res = await this.client.fetch(url, {
method: "GET",
headers: pickHeaders2(headers, [
"if-none-match",
@@ -201,7 +204,7 @@ export class StorageS3Adapter extends AwsClient implements StorageAdapter {
params: Omit<DeleteObjectRequest, "Bucket" | "Key"> = {},
): Promise<void> {
const url = this.getUrl(key, params);
const res = await this.fetch(url, {
const res = await this.client.fetch(url, {
method: "DELETE",
});
}