shishantbiswas/bknd
1
0
Fork 0
mirror of https://github.com/shishantbiswas/bknd.git synced 2026-03-17 04:46:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

reorganized storage adapter and added test suites for adapter and fields (#124)

Browse Source 
* reorganized storage adapter and added test suites for adapter and fields

* added build command in ci pipeline

* updated workflow to also run node tests

* updated workflow: try with separate tasks

* updated workflow: try with separate tasks

* updated workflow: added tsx as dev dependency

* updated workflow: try with find instead of glob
This commit is contained in:
dswbx
2025-03-27 20:41:42 +01:00
committed by GitHub
parent 40c9ef9d90
commit 9e3c081e50
45 changed files with 605 additions and 940 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
app/src/adapter/cloudflare/StorageR2Adapter.ts
View File

@@ -1,7 +1,8 @@
import { registries } from "bknd";
import { isDebug } from "bknd/core";
import { StringEnum, Type } from "bknd/utils";
import type { FileBody, StorageAdapter } from "media/storage/Storage";
import type { FileBody } from "media/storage/Storage";
import { StorageAdapter } from "media/storage/StorageAdapter";
import { guess } from "media/storage/mime-types-tiny";
import { getBindings } from "./bindings";
@@ -47,8 +48,10 @@ export function registerMedia(env: Record<string, any>) {
* Adapter for R2 storage
* @todo: add tests (bun tests won't work, need node native tests)
*/
export class StorageR2Adapter implements StorageAdapter {
constructor(private readonly bucket: R2Bucket) {}
export class StorageR2Adapter extends StorageAdapter {
constructor(private readonly bucket: R2Bucket) {
super();
}
getName(): string {
return "r2";

6
app/src/adapter/node/index.ts
View File

@@ -1,11 +1,9 @@
import { registries } from "bknd";
import {
type LocalAdapterConfig,
StorageLocalAdapter,
} from "../../media/storage/adapters/StorageLocalAdapter";
import { type LocalAdapterConfig, StorageLocalAdapter } from "./storage/StorageLocalAdapter";
export * from "./node.adapter";
export { StorageLocalAdapter, type LocalAdapterConfig };
export { nodeTestRunner } from "./test";
export function registerLocalMediaAdapter() {
registries.media.register("local", StorageLocalAdapter);

17
app/src/adapter/node/storage/StorageLocalAdapter.native-spec.ts Normal file
View File

@@ -0,0 +1,17 @@
import { describe } from "node:test";
import { StorageLocalAdapter, nodeTestRunner } from "adapter/node";
import { adapterTestSuite } from "media/storage/adapters/adapter-test-suite";
import { readFileSync } from "node:fs";
import path from "node:path";
describe("StorageLocalAdapter (node)", async () => {
const basePath = path.resolve(import.meta.dirname, "../../../../__test__/_assets");
const buffer = readFileSync(path.join(basePath, "image.png"));
const file = new File([buffer], "image.png", { type: "image/png" });
const adapter = new StorageLocalAdapter({
path: path.join(basePath, "tmp"),
});
await adapterTestSuite(nodeTestRunner, adapter, file);
});

14
app/src/adapter/node/storage/StorageLocalAdapter.spec.ts Normal file
View File

@@ -0,0 +1,14 @@
import { describe, test, expect } from "bun:test";
import { StorageLocalAdapter } from "./StorageLocalAdapter";
// @ts-ignore
import { assetsPath, assetsTmpPath } from "../../../../__test__/helper";
import { adapterTestSuite } from "media/storage/adapters/adapter-test-suite";
describe("StorageLocalAdapter (bun)", async () => {
const adapter = new StorageLocalAdapter({
path: assetsTmpPath,
});
const file = Bun.file(`${assetsPath}/image.png`);
await adapterTestSuite({ test, expect }, adapter, file);
});

15
app/src/media/storage/adapters/StorageLocalAdapter/StorageLocalAdapter.ts → app/src/adapter/node/storage/StorageLocalAdapter.ts
View File

@@ -1,13 +1,7 @@
import { readFile, readdir, stat, unlink, writeFile } from "node:fs/promises";
import { type Static, Type, isFile, parse } from "core/utils";
import type {
FileBody,
FileListObject,
FileMeta,
FileUploadPayload,
StorageAdapter,
} from "../../Storage";
import { guess } from "../../mime-types-tiny";
import { type Static, Type, isFile, parse } from "bknd/utils";
import type { FileBody, FileListObject, FileMeta, FileUploadPayload } from "bknd/media";
import { StorageAdapter, guessMimeType as guess } from "bknd/media";
export const localAdapterConfig = Type.Object(
{
@@ -17,10 +11,11 @@ export const localAdapterConfig = Type.Object(
);
export type LocalAdapterConfig = Static<typeof localAdapterConfig>;
export class StorageLocalAdapter implements StorageAdapter {
export class StorageLocalAdapter extends StorageAdapter {
private config: LocalAdapterConfig;
constructor(config: any) {
super();
this.config = parse(localAdapterConfig, config);
}

75
app/src/adapter/node/test.ts Normal file
View File

@@ -0,0 +1,75 @@
import nodeAssert from "node:assert/strict";
import { test } from "node:test";
import type { Matcher, Test, TestFn, TestRunner } from "core/test";
const nodeTestMatcher = <T = unknown>(actual: T, parentFailMsg?: string) =>
({
toEqual: (expected: T, failMsg = parentFailMsg) => {
nodeAssert.deepEqual(actual, expected, failMsg);
},
toBe: (expected: T, failMsg = parentFailMsg) => {
nodeAssert.strictEqual(actual, expected, failMsg);
},
toBeString: (failMsg = parentFailMsg) => {
nodeAssert.strictEqual(typeof actual, "string", failMsg);
},
toBeUndefined: (failMsg = parentFailMsg) => {
nodeAssert.strictEqual(actual, undefined, failMsg);
},
toBeDefined: (failMsg = parentFailMsg) => {
nodeAssert.notStrictEqual(actual, undefined, failMsg);
},
toBeOneOf: (expected: T | Array<T> | Iterable<T>, failMsg = parentFailMsg) => {
const e = Array.isArray(expected) ? expected : [expected];
nodeAssert.ok(e.includes(actual), failMsg);
},
}) satisfies Matcher<T>;
const nodeTestResolverProxy = <T = unknown>(
actual: Promise<T>,
handler: { resolve?: any; reject?: any },
) => {
return new Proxy(
{},
{
get: (_, prop) => {
if (prop === "then") {
return actual.then(handler.resolve, handler.reject);
}
return actual;
},
},
) as Matcher<Awaited<T>>;
};
function nodeTest(label: string, fn: TestFn, options?: any) {
return test(label, fn as any);
}
nodeTest.if = (condition: boolean): Test => {
if (condition) {
return nodeTest;
}
return (() => {}) as any;
};
nodeTest.skip = (label: string, fn: TestFn) => {
return test.skip(label, fn as any);
};
nodeTest.skipIf = (condition: boolean): Test => {
if (condition) {
return (() => {}) as any;
}
return nodeTest;
};
export const nodeTestRunner: TestRunner = {
test: nodeTest,
expect: <T = unknown>(actual?: T, failMsg?: string) => ({
...nodeTestMatcher(actual, failMsg),
resolves: nodeTestResolverProxy(actual as Promise<T>, {
resolve: (r) => nodeTestMatcher(r, failMsg),
}),
rejects: nodeTestResolverProxy(actual as Promise<T>, {
reject: (r) => nodeTestMatcher(r, failMsg),
}),
}),
};

127
app/src/core/cache/adapters/CloudflareKvCache.ts vendored
View File

@@ -1,127 +0,0 @@
import type { ICacheItem, ICachePool } from "../cache-interface";
export class CloudflareKVCachePool<Data = any> implements ICachePool<Data> {
constructor(private namespace: KVNamespace) {}
supports = () => ({
metadata: true,
clear: false,
});
async get(key: string): Promise<ICacheItem<Data>> {
const result = await this.namespace.getWithMetadata<any>(key);
const hit = result.value !== null && typeof result.value !== "undefined";
// Assuming metadata is not supported directly;
// you may adjust if Cloudflare KV supports it in future.
return new CloudflareKVCacheItem(key, result.value ?? undefined, hit, result.metadata) as any;
}
async getMany(keys: string[] = []): Promise<Map<string, ICacheItem<Data>>> {
const items = new Map<string, ICacheItem<Data>>();
await Promise.all(
keys.map(async (key) => {
const item = await this.get(key);
items.set(key, item);
}),
);
return items;
}
async has(key: string): Promise<boolean> {
const data = await this.namespace.get(key);
return data !== null;
}
async clear(): Promise<boolean> {
// Cloudflare KV does not support clearing all keys in one operation
return false;
}
async delete(key: string): Promise<boolean> {
await this.namespace.delete(key);
return true;
}
async deleteMany(keys: string[]): Promise<boolean> {
const results = await Promise.all(keys.map((key) => this.delete(key)));
return results.every((result) => result);
}
async save(item: CloudflareKVCacheItem<Data>): Promise<boolean> {
await this.namespace.put(item.key(), (await item.value()) as string, {
expirationTtl: item._expirationTtl,
metadata: item.metadata(),
});
return true;
}
async put(
key: string,
value: any,
options?: { ttl?: number; expiresAt?: Date; metadata?: Record<string, string> },
): Promise<boolean> {
const item = new CloudflareKVCacheItem(key, value, true, options?.metadata);
if (options?.expiresAt) item.expiresAt(options.expiresAt);
if (options?.ttl) item.expiresAfter(options.ttl);
return await this.save(item);
}
}
export class CloudflareKVCacheItem<Data = any> implements ICacheItem<Data> {
_expirationTtl: number | undefined;
constructor(
private _key: string,
private data: Data | undefined,
private _hit: boolean = false,
private _metadata: Record<string, string> = {},
) {}
key(): string {
return this._key;
}
value(): Data | undefined {
if (this.data) {
try {
return JSON.parse(this.data as string);
} catch (e) {}
}
return this.data ?? undefined;
}
metadata(): Record<string, string> {
return this._metadata;
}
hit(): boolean {
return this._hit;
}
set(value: Data, metadata: Record<string, string> = {}): this {
this.data = value;
this._metadata = metadata;
return this;
}
expiresAt(expiration: Date | null): this {
// Cloudflare KV does not support specific date expiration; calculate ttl instead.
if (expiration) {
const now = new Date();
const ttl = (expiration.getTime() - now.getTime()) / 1000;
return this.expiresAfter(Math.max(0, Math.floor(ttl)));
}
return this.expiresAfter(null);
}
expiresAfter(time: number | null): this {
// Dummy implementation as Cloudflare KV requires setting expiration during PUT operation.
// This method will be effectively implemented in the Cache Pool save methods.
this._expirationTtl = time ?? undefined;
return this;
}
}

139
app/src/core/cache/adapters/MemoryCache.ts vendored
View File

@@ -1,139 +0,0 @@
import type { ICacheItem, ICachePool } from "../cache-interface";
export class MemoryCache<Data = any> implements ICachePool<Data> {
private cache: Map<string, MemoryCacheItem<Data>> = new Map();
private maxSize?: number;
constructor(options?: { maxSize?: number }) {
this.maxSize = options?.maxSize;
}
supports = () => ({
metadata: true,
clear: true,
});
async get(key: string): Promise<MemoryCacheItem<Data>> {
if (!this.cache.has(key)) {
// use undefined to denote a miss initially
return new MemoryCacheItem<Data>(key, undefined!);
}
return this.cache.get(key)!;
}
async getMany(keys: string[] = []): Promise<Map<string, MemoryCacheItem<Data>>> {
const items = new Map<string, MemoryCacheItem<Data>>();
for (const key of keys) {
items.set(key, await this.get(key));
}
return items;
}
async has(key: string): Promise<boolean> {
return this.cache.has(key) && this.cache.get(key)!.hit();
}
async clear(): Promise<boolean> {
this.cache.clear();
return true;
}
async delete(key: string): Promise<boolean> {
return this.cache.delete(key);
}
async deleteMany(keys: string[]): Promise<boolean> {
let success = true;
for (const key of keys) {
if (!this.delete(key)) {
success = false;
}
}
return success;
}
async save(item: MemoryCacheItem<Data>): Promise<boolean> {
this.checkSizeAndPurge();
this.cache.set(item.key(), item);
return true;
}
async put(
key: string,
value: Data,
options: { expiresAt?: Date; ttl?: number; metadata?: Record<string, string> } = {},
): Promise<boolean> {
const item = await this.get(key);
item.set(value, options.metadata || {});
if (options.expiresAt) {
item.expiresAt(options.expiresAt);
} else if (typeof options.ttl === "number") {
item.expiresAfter(options.ttl);
}
return this.save(item);
}
private checkSizeAndPurge(): void {
if (!this.maxSize) return;
if (this.cache.size >= this.maxSize) {
// Implement logic to purge items, e.g., LRU (Least Recently Used)
// For simplicity, clear the oldest item inserted
const keyToDelete = this.cache.keys().next().value;
this.cache.delete(keyToDelete!);
}
}
}
export class MemoryCacheItem<Data = any> implements ICacheItem<Data> {
private _key: string;
private _value: Data | undefined;
private expiration: Date | null = null;
private _metadata: Record<string, string> = {};
constructor(key: string, value: Data, metadata: Record<string, string> = {}) {
this._key = key;
this.set(value, metadata);
}
key(): string {
return this._key;
}
metadata(): Record<string, string> {
return this._metadata;
}
value(): Data | undefined {
return this._value;
}
hit(): boolean {
if (this.expiration !== null && new Date() > this.expiration) {
return false;
}
return this.value() !== undefined;
}
set(value: Data, metadata: Record<string, string> = {}): this {
this._value = value;
this._metadata = metadata;
return this;
}
expiresAt(expiration: Date | null): this {
this.expiration = expiration;
return this;
}
expiresAfter(time: number | null): this {
if (typeof time === "number") {
const expirationDate = new Date();
expirationDate.setSeconds(expirationDate.getSeconds() + time);
this.expiration = expirationDate;
} else {
this.expiration = null;
}
return this;
}
}

178
app/src/core/cache/cache-interface.ts vendored
View File

@@ -1,178 +0,0 @@
/**
* CacheItem defines an interface for interacting with objects inside a cache.
* based on https://www.php-fig.org/psr/psr-6/
*/
export interface ICacheItem<Data = any> {
/**
* Returns the key for the current cache item.
*
* The key is loaded by the Implementing Library, but should be available to
* the higher level callers when needed.
*
* @returns The key string for this cache item.
*/
key(): string;
/**
* Retrieves the value of the item from the cache associated with this object's key.
*
* The value returned must be identical to the value originally stored by set().
*
* If isHit() returns false, this method MUST return null. Note that null
* is a legitimate cached value, so the isHit() method SHOULD be used to
* differentiate between "null value was found" and "no value was found."
*
* @returns The value corresponding to this cache item's key, or undefined if not found.
*/
value(): Data | undefined;
/**
* Retrieves the metadata of the item from the cache associated with this object's key.
*/
metadata(): Record<string, string>;
/**
* Confirms if the cache item lookup resulted in a cache hit.
*
* Note: This method MUST NOT have a race condition between calling isHit()
* and calling get().
*
* @returns True if the request resulted in a cache hit. False otherwise.
*/
hit(): boolean;
/**
* Sets the value represented by this cache item.
*
* The value argument may be any item that can be serialized by PHP,
* although the method of serialization is left up to the Implementing
* Library.
*
* @param value The serializable value to be stored.
* @param metadata The metadata to be associated with the item.
* @returns The invoked object.
*/
set(value: Data, metadata?: Record<string, string>): this;
/**
* Sets the expiration time for this cache item.
*
* @param expiration The point in time after which the item MUST be considered expired.
* If null is passed explicitly, a default value MAY be used. If none is set,
* the value should be stored permanently or for as long as the
* implementation allows.
* @returns The called object.
*/
expiresAt(expiration: Date | null): this;
/**
* Sets the expiration time for this cache item.
*
* @param time The period of time from the present after which the item MUST be considered
* expired. An integer parameter is understood to be the time in seconds until
* expiration. If null is passed explicitly, a default value MAY be used.
* If none is set, the value should be stored permanently or for as long as the
* implementation allows.
* @returns The called object.
*/
expiresAfter(time: number | null): this;
}
/**
* CachePool generates CacheItem objects.
* based on https://www.php-fig.org/psr/psr-6/
*/
export interface ICachePool<Data = any> {
supports(): {
metadata: boolean;
clear: boolean;
};
/**
* Returns a Cache Item representing the specified key.
* This method must always return a CacheItemInterface object, even in case of
* a cache miss. It MUST NOT return null.
*
* @param key The key for which to return the corresponding Cache Item.
* @throws Error If the key string is not a legal value an Error MUST be thrown.
* @returns The corresponding Cache Item.
*/
get(key: string): Promise<ICacheItem<Data>>;
/**
* Returns a traversable set of cache items.
*
* @param keys An indexed array of keys of items to retrieve.
* @throws Error If any of the keys in keys are not a legal value an Error MUST be thrown.
* @returns A traversable collection of Cache Items keyed by the cache keys of
* each item. A Cache item will be returned for each key, even if that
* key is not found. However, if no keys are specified then an empty
* traversable MUST be returned instead.
*/
getMany(keys?: string[]): Promise<Map<string, ICacheItem<Data>>>;
/**
* Confirms if the cache contains specified cache item.
*
* Note: This method MAY avoid retrieving the cached value for performance reasons.
* This could result in a race condition with CacheItemInterface.get(). To avoid
* such situation use CacheItemInterface.isHit() instead.
*
* @param key The key for which to check existence.
* @throws Error If the key string is not a legal value an Error MUST be thrown.
* @returns True if item exists in the cache, false otherwise.
*/
has(key: string): Promise<boolean>;
/**
* Deletes all items in the pool.
* @returns True if the pool was successfully cleared. False if there was an error.
*/
clear(): Promise<boolean>;
/**
* Removes the item from the pool.
*
* @param key The key to delete.
* @throws Error If the key string is not a legal value an Error MUST be thrown.
* @returns True if the item was successfully removed. False if there was an error.
*/
delete(key: string): Promise<boolean>;
/**
* Removes multiple items from the pool.
*
* @param keys An array of keys that should be removed from the pool.
* @throws Error If any of the keys in keys are not a legal value an Error MUST be thrown.
* @returns True if the items were successfully removed. False if there was an error.
*/
deleteMany(keys: string[]): Promise<boolean>;
/**
* Persists a cache item immediately.
*
* @param item The cache item to save.
* @returns True if the item was successfully persisted. False if there was an error.
*/
save(item: ICacheItem<Data>): Promise<boolean>;
/**
* Persists any deferred cache items.
* @returns True if all not-yet-saved items were successfully saved or there were none. False otherwise.
*/
put(
key: string,
value: any,
options?: { expiresAt?: Date; metadata?: Record<string, string> },
): Promise<boolean>;
put(
key: string,
value: any,
options?: { ttl?: number; metadata?: Record<string, string> },
): Promise<boolean>;
put(
key: string,
value: any,
options?: ({ ttl?: number } | { expiresAt?: Date }) & { metadata?: Record<string, string> },
): Promise<boolean>;
}

48
app/src/core/test/index.ts Normal file
View File

@@ -0,0 +1,48 @@
export type Matcher<T = unknown> = {
toEqual: (expected: T, failMsg?: string) => void;
toBe: (expected: T, failMsg?: string) => void;
toBeUndefined: (failMsg?: string) => void;
toBeString: (failMsg?: string) => void;
toBeOneOf: (expected: T | Array<T> | Iterable<T>, failMsg?: string) => void;
toBeDefined: (failMsg?: string) => void;
};
export type TestFn = (() => void | Promise<unknown>) | ((done: (err?: unknown) => void) => void);
export interface Test {
(label: string, fn: TestFn, options?: any): void;
if: (condition: boolean) => (label: string, fn: TestFn, options?: any) => void;
skip: (label: string, fn: () => void) => void;
skipIf: (condition: boolean) => (label: string, fn: TestFn) => void;
}
export type TestRunner = {
test: Test;
expect: <T = unknown>(
actual?: T,
failMsg?: string,
) => Matcher<T> & {
resolves: Matcher<Awaited<T>>;
rejects: Matcher<Awaited<T>>;
};
};
export async function retry<T>(
fn: () => Promise<T>,
condition: (result: T) => boolean,
retries: number,
delay: number,
): Promise<T> {
let lastError: Error | null = null;
for (let i = 0; i < retries; i++) {
try {
const result = await fn();
if (condition(result)) {
return result;
} else {
await new Promise((resolve) => setTimeout(resolve, delay));
}
} catch (error) {
lastError = error as Error;
await new Promise((resolve) => setTimeout(resolve, delay));
}
}
throw lastError;
}

167
app/src/data/fields/field-test-suite.ts Normal file
View File

@@ -0,0 +1,167 @@
import type { BaseFieldConfig, Field, TActionContext } from "data";
import type { ColumnDataType } from "kysely";
import { omit } from "lodash-es";
import type { TestRunner } from "core/test";
type ConstructableField = new (name: string, config?: Partial<BaseFieldConfig>) => Field;
type FieldTestConfig = {
defaultValue: any;
sampleValues?: any[];
schemaType: ColumnDataType;
};
export function transformPersist(field: Field, value: any, context?: TActionContext) {
return field.transformPersist(value, undefined as any, context as any);
}
export function fieldTestSuite(
testRunner: TestRunner,
fieldClass: ConstructableField,
config: FieldTestConfig,
_requiredConfig: any = {},
) {
const { test, expect } = testRunner;
const noConfigField = new fieldClass("no_config", _requiredConfig);
const fillable = new fieldClass("fillable", { ..._requiredConfig, fillable: true });
const required = new fieldClass("required", { ..._requiredConfig, required: true });
const hidden = new fieldClass("hidden", { ..._requiredConfig, hidden: true });
const dflt = new fieldClass("dflt", { ..._requiredConfig, default_value: config.defaultValue });
const requiredAndDefault = new fieldClass("full", {
..._requiredConfig,
fillable: true,
required: true,
default_value: config.defaultValue,
});
test("schema", () => {
expect(noConfigField.name).toBe("no_config");
const { type, name, nullable, dflt } = noConfigField.schema()!;
expect({ type, name, nullable, dflt }).toEqual({
type: config.schemaType as any,
name: "no_config",
nullable: true, // always true
dflt: undefined, // never using default value
});
});
test("hasDefault", async () => {
expect(noConfigField.hasDefault()).toBe(false);
expect(noConfigField.getDefault()).toBeUndefined();
expect(dflt.hasDefault()).toBe(true);
expect(dflt.getDefault()).toBe(config.defaultValue);
});
test("isFillable", async () => {
expect(noConfigField.isFillable()).toBe(true);
expect(fillable.isFillable()).toBe(true);
expect(hidden.isFillable()).toBe(true);
expect(required.isFillable()).toBe(true);
});
test("isHidden", async () => {
expect(noConfigField.isHidden()).toBe(false);
expect(hidden.isHidden()).toBe(true);
expect(fillable.isHidden()).toBe(false);
expect(required.isHidden()).toBe(false);
});
test("isRequired", async () => {
expect(noConfigField.isRequired()).toBe(false);
expect(required.isRequired()).toBe(true);
expect(hidden.isRequired()).toBe(false);
expect(fillable.isRequired()).toBe(false);
});
test.if(Array.isArray(config.sampleValues))("getValue (RenderContext)", async () => {
const isPrimitive = (v) => ["string", "number"].includes(typeof v);
for (const value of config.sampleValues!) {
// "form"
expect(isPrimitive(noConfigField.getValue(value, "form"))).toBe(true);
// "table"
expect(isPrimitive(noConfigField.getValue(value, "table"))).toBe(true);
// "read"
// "submit"
}
});
test("transformPersist", async () => {
const persist = await transformPersist(noConfigField, config.defaultValue);
expect(config.defaultValue).toEqual(noConfigField.transformRetrieve(config.defaultValue));
expect(transformPersist(noConfigField, null)).resolves.toBeUndefined();
expect(transformPersist(noConfigField, undefined)).resolves.toBeUndefined();
expect(transformPersist(requiredAndDefault, null)).resolves.toBe(persist);
expect(transformPersist(dflt, null)).resolves.toBe(persist);
});
test("toJSON", async () => {
const _config = {
..._requiredConfig,
//order: 1,
fillable: true,
required: false,
hidden: false,
//virtual: false,
//default_value: undefined
};
function fieldJson(field: Field) {
const json = field.toJSON();
return {
...json,
config: omit(json.config, ["html"]),
};
}
expect(fieldJson(noConfigField)).toEqual({
//name: "no_config",
type: noConfigField.type,
config: _config,
});
expect(fieldJson(fillable)).toEqual({
//name: "fillable",
type: noConfigField.type,
config: _config,
});
expect(fieldJson(required)).toEqual({
//name: "required",
type: required.type,
config: {
..._config,
required: true,
},
});
expect(fieldJson(hidden)).toEqual({
//name: "hidden",
type: required.type,
config: {
..._config,
hidden: true,
},
});
expect(fieldJson(dflt)).toEqual({
//name: "dflt",
type: dflt.type,
config: {
..._config,
default_value: config.defaultValue,
},
});
expect(fieldJson(requiredAndDefault)).toEqual({
//name: "full",
type: requiredAndDefault.type,
config: {
..._config,
fillable: true,
required: true,
default_value: config.defaultValue,
},
});
});
}

2
app/src/data/fields/index.ts
View File

@@ -53,3 +53,5 @@ export const FieldClassMap = {
json: { schema: jsonFieldConfigSchema, field: JsonField },
jsonschema: { schema: jsonSchemaFieldConfigSchema, field: JsonSchemaField },
} as const;
export { fieldTestSuite } from "./field-test-suite";

14
app/src/media/index.ts
View File

@@ -1,22 +1,24 @@
import type { TObject, TString } from "@sinclair/typebox";
import type { TObject } from "@sinclair/typebox";
import { type Constructor, Registry } from "core";
//export { MIME_TYPES } from "./storage/mime-types";
export { guess as guessMimeType } from "./storage/mime-types-tiny";
export {
Storage,
type StorageAdapter,
type FileMeta,
type FileListObject,
type StorageConfig,
type FileBody,
type FileUploadPayload,
} from "./storage/Storage";
import type { StorageAdapter } from "./storage/Storage";
import { StorageAdapter } from "./storage/StorageAdapter";
import {
type CloudinaryConfig,
StorageCloudinaryAdapter,
} from "./storage/adapters/StorageCloudinaryAdapter";
import { type S3AdapterConfig, StorageS3Adapter } from "./storage/adapters/StorageS3Adapter";
} from "./storage/adapters/cloudinary/StorageCloudinaryAdapter";
import { type S3AdapterConfig, StorageS3Adapter } from "./storage/adapters/s3/StorageS3Adapter";
export { StorageAdapter };
export { StorageS3Adapter, type S3AdapterConfig, StorageCloudinaryAdapter, type CloudinaryConfig };
export * as StorageEvents from "./storage/events";
@@ -45,3 +47,5 @@ export const Adapters = {
schema: StorageCloudinaryAdapter.prototype.getSchema(),
},
} as const;
export { adapterTestSuite } from "./storage/adapters/adapter-test-suite";

21
app/src/media/storage/Storage.ts
View File

@@ -1,9 +1,10 @@
import { type EmitsEvents, EventManager } from "core/events";
import { type TSchema, isFile, detectImageDimensions } from "core/utils";
import { isFile, detectImageDimensions } from "core/utils";
import { isMimeType } from "media/storage/mime-types-tiny";
import * as StorageEvents from "./events";
import type { FileUploadedEventData } from "./events";
import { $console } from "core";
import type { StorageAdapter } from "./StorageAdapter";
export type FileListObject = {
key: string;
@@ -19,24 +20,6 @@ export type FileUploadPayload = {
etag: string;
};
export interface StorageAdapter {
/**
* The unique name of the storage adapter
*/
getName(): string;
// @todo: method requires limit/offset parameters
listObjects(prefix?: string): Promise<FileListObject[]>;
putObject(key: string, body: FileBody): Promise<string | FileUploadPayload | undefined>;
deleteObject(key: string): Promise<void>;
objectExists(key: string): Promise<boolean>;
getObject(key: string, headers: Headers): Promise<Response>;
getObjectUrl(key: string): string;
getObjectMeta(key: string): Promise<FileMeta>;
getSchema(): TSchema | undefined;
toJSON(secrets?: boolean): any;
}
export type StorageConfig = {
body_max_size?: number;
};

37
app/src/media/storage/StorageAdapter.ts Normal file
View File

@@ -0,0 +1,37 @@
import type { FileListObject, FileMeta } from "media";
import type { FileBody, FileUploadPayload } from "media/storage/Storage";
import type { TSchema } from "@sinclair/typebox";
const SYMBOL = Symbol.for("bknd:storage");
export abstract class StorageAdapter {
constructor() {
this[SYMBOL] = true;
}
/**
* This is a helper function to manage Connection classes
* coming from different places
* @param conn
*/
static isAdapter(conn: unknown): conn is StorageAdapter {
if (!conn) return false;
return conn[SYMBOL] === true;
}
/**
* The unique name of the storage adapter
*/
abstract getName(): string;
// @todo: method requires limit/offset parameters
abstract listObjects(prefix?: string): Promise<FileListObject[]>;
abstract putObject(key: string, body: FileBody): Promise<string | FileUploadPayload | undefined>;
abstract deleteObject(key: string): Promise<void>;
abstract objectExists(key: string): Promise<boolean>;
abstract getObject(key: string, headers: Headers): Promise<Response>;
abstract getObjectUrl(key: string): string;
abstract getObjectMeta(key: string): Promise<FileMeta>;
abstract getSchema(): TSchema | undefined;
abstract toJSON(secrets?: boolean): any;
}

5
app/src/media/storage/adapters/StorageLocalAdapter/index.ts
View File

@@ -1,5 +0,0 @@
export {
StorageLocalAdapter,
type LocalAdapterConfig,
localAdapterConfig,
} from "./StorageLocalAdapter";

79
app/src/media/storage/adapters/adapter-test-suite.ts Normal file
View File

@@ -0,0 +1,79 @@
import { retry, type TestRunner } from "core/test";
import type { StorageAdapter } from "media";
import { randomString } from "core/utils";
import type { BunFile } from "bun";
export async function adapterTestSuite(
testRunner: TestRunner,
adapter: StorageAdapter,
file: File | BunFile,
opts?: {
retries?: number;
retryTimeout?: number;
skipExistsAfterDelete?: boolean;
},
) {
const { test, expect } = testRunner;
const options = {
retries: opts?.retries ?? 1,
retryTimeout: opts?.retryTimeout ?? 1000,
};
let objects = 0;
const _filename = randomString(10);
const filename = `${_filename}.png`;
await test("puts an object", async () => {
objects = (await adapter.listObjects()).length;
const result = await adapter.putObject(filename, file as unknown as File);
expect(result).toBeDefined();
const type = typeof result;
expect(type).toBeOneOf(["string", "object"]);
if (typeof result === "object") {
expect(Object.keys(result).sort()).toEqual(["etag", "meta", "name"]);
expect(result.meta.type).toBe(file.type);
}
});
await test("lists objects", async () => {
const length = await retry(
() => adapter.listObjects().then((res) => res.length),
(length) => length > objects,
options.retries,
options.retryTimeout,
);
expect(length).toBe(objects + 1);
});
await test("file exists", async () => {
expect(await adapter.objectExists(filename)).toBe(true);
});
await test("gets an object", async () => {
const res = await adapter.getObject(filename, new Headers());
expect(res.ok).toBe(true);
// @todo: check the content
});
await test("gets object meta", async () => {
expect(await adapter.getObjectMeta(filename)).toEqual({
type: file.type, // image/png
size: file.size,
});
});
await test("deletes an object", async () => {
expect(await adapter.deleteObject(filename)).toBeUndefined();
if (opts?.skipExistsAfterDelete !== true) {
const exists = await retry(
() => adapter.objectExists(filename),
(res) => res === false,
options.retries,
options.retryTimeout,
);
expect(exists).toBe(false);
}
});
}

53
app/src/media/storage/adapters/cloudinary/StorageCloudinaryAdapter.spec.ts Normal file
View File

@@ -0,0 +1,53 @@
import { describe, expect, test } from "bun:test";
import { StorageCloudinaryAdapter } from "./StorageCloudinaryAdapter";
import { config } from "dotenv";
// @ts-ignore
import { assetsPath, assetsTmpPath } from "../../../../../__test__/helper";
import { adapterTestSuite } from "media/storage/adapters/adapter-test-suite";
const dotenvOutput = config({ path: `${import.meta.dir}/.env` });
const {
CLOUDINARY_CLOUD_NAME,
CLOUDINARY_API_KEY,
CLOUDINARY_API_SECRET,
CLOUDINARY_UPLOAD_PRESET,
} = dotenvOutput.parsed!;
const ALL_TESTS = !!process.env.ALL_TESTS;
describe.skipIf(ALL_TESTS)("StorageCloudinaryAdapter", async () => {
if (ALL_TESTS) return;
const adapter = new StorageCloudinaryAdapter({
cloud_name: CLOUDINARY_CLOUD_NAME as string,
api_key: CLOUDINARY_API_KEY as string,
api_secret: CLOUDINARY_API_SECRET as string,
upload_preset: CLOUDINARY_UPLOAD_PRESET as string,
});
const file = Bun.file(`${assetsPath}/image.png`) as unknown as File;
test("hash", async () => {
expect(
await adapter.generateSignature(
{
eager: "w_400,h_300,c_pad|w_260,h_200,c_crop",
public_id: "sample_image",
timestamp: 1315060510,
},
"abcd",
),
).toEqual({
signature: "bfd09f95f331f558cbd1320e67aa8d488770583e",
timestamp: 1315060510,
});
});
await adapterTestSuite({ test, expect }, adapter, file, {
// eventual consistency
retries: 20,
retryTimeout: 1000,
// result is cached from cloudinary
skipExistsAfterDelete: true,
});
});

63
app/src/media/storage/adapters/StorageCloudinaryAdapter.ts → app/src/media/storage/adapters/cloudinary/StorageCloudinaryAdapter.ts
View File

@@ -1,6 +1,7 @@
import { pickHeaders } from "core/utils";
import { hash, pickHeaders } from "core/utils";
import { type Static, Type, parse } from "core/utils";
import type { FileBody, FileListObject, FileMeta, StorageAdapter } from "../Storage";
import type { FileBody, FileListObject, FileMeta } from "../../Storage";
import { StorageAdapter } from "../../StorageAdapter";
export const cloudinaryAdapterConfig = Type.Object(
{
@@ -53,10 +54,11 @@ type CloudinaryListObjectsResponse = {
};
// @todo: add signed uploads
export class StorageCloudinaryAdapter implements StorageAdapter {
export class StorageCloudinaryAdapter extends StorageAdapter {
private config: CloudinaryConfig;
constructor(config: CloudinaryConfig) {
super();
this.config = parse(cloudinaryAdapterConfig, config);
}
@@ -126,6 +128,11 @@ export class StorageCloudinaryAdapter implements StorageAdapter {
};
}
/**
* https://cloudinary.com/documentation/admin_api#search_for_resources
* Cloudinary implements eventual consistency: Search results reflect any changes made to assets within a few seconds after the change
* @param prefix
*/
async listObjects(prefix?: string): Promise<FileListObject[]> {
const result = await fetch(
`https://api.cloudinary.com/v1_1/${this.config.cloud_name}/resources/search`,
@@ -133,6 +140,7 @@ export class StorageCloudinaryAdapter implements StorageAdapter {
method: "GET",
headers: {
Accept: "application/json",
"Cache-Control": "no-cache",
...this.getAuthorizationHeader(),
},
},
@@ -143,18 +151,22 @@ export class StorageCloudinaryAdapter implements StorageAdapter {
}
const data = (await result.json()) as CloudinaryListObjectsResponse;
return data.resources.map((item) => ({
const items = data.resources.map((item) => ({
key: item.public_id,
last_modified: new Date(item.uploaded_at),
size: item.bytes,
}));
return items;
}
private async headObject(key: string) {
const url = this.getObjectUrl(key);
return await fetch(url, {
method: "GET",
method: "HEAD",
headers: {
"Cache-Control": "no-cache, no-store, must-revalidate",
Pragma: "no-cache",
Expires: "0",
Range: "bytes=0-1",
},
});
@@ -196,6 +208,22 @@ export class StorageCloudinaryAdapter implements StorageAdapter {
return objectUrl;
}
async generateSignature(params: Record<string, string | number>, secret?: string) {
const timestamp = params.timestamp ?? Math.floor(Date.now() / 1000);
const content = Object.entries({ ...params, timestamp })
.sort(([keyA], [keyB]) => keyA.localeCompare(keyB))
.map(([key, value]) => `${key}=${value}`)
.join("&");
const signature = await hash.sha1(content + (secret ?? this.config.api_secret));
return { signature, timestamp };
}
// get public_id as everything before the last "."
filenameToPublicId(key: string): string {
return key.split(".").slice(0, -1).join(".");
}
async getObject(key: string, headers: Headers): Promise<Response> {
const res = await fetch(this.getObjectUrl(key), {
method: "GET",
@@ -211,13 +239,30 @@ export class StorageCloudinaryAdapter implements StorageAdapter {
async deleteObject(key: string): Promise<void> {
const type = this.guessType(key) ?? "image";
const formData = new FormData();
formData.append("public_ids[]", key);
const public_id = this.filenameToPublicId(key);
const { timestamp, signature } = await this.generateSignature({
public_id,
});
await fetch(`https://res.cloudinary.com/${this.config.cloud_name}/${type}/upload/`, {
method: "DELETE",
const formData = new FormData();
formData.append("public_id", public_id);
formData.append("timestamp", String(timestamp));
formData.append("signature", signature);
formData.append("api_key", this.config.api_key);
const url = `https://api.cloudinary.com/v1_1/${this.config.cloud_name}/${type}/destroy`;
const res = await fetch(url, {
headers: {
Accept: "application/json",
"Cache-Control": "no-cache",
...this.getAuthorizationHeader(),
},
method: "POST",
body: formData,
});
if (!res.ok) {
throw new Error(`Failed to delete object: ${res.status} ${res.statusText}`);
}
}
toJSON(secrets?: boolean) {

50
app/src/media/storage/adapters/s3/StorageS3Adapter.spec.ts Normal file
View File

@@ -0,0 +1,50 @@
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { StorageS3Adapter } from "./StorageS3Adapter";
import { config } from "dotenv";
import { adapterTestSuite } from "media";
import { assetsPath } from "../../../../../__test__/helper";
//import { enableFetchLogging } from "../../helper";
const dotenvOutput = config({ path: `${import.meta.dir}/.env` });
const { R2_ACCESS_KEY, R2_SECRET_ACCESS_KEY, R2_URL, AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_S3_URL } =
dotenvOutput.parsed!;
const ALL_TESTS = !!process.env.ALL_TESTS;
/*
// @todo: preparation to mock s3 calls + replace fast-xml-parser
let cleanup: () => void;
beforeAll(async () => {
cleanup = await enableFetchLogging();
});
afterAll(() => {
cleanup();
}); */
describe.skipIf(ALL_TESTS)("StorageS3Adapter", async () => {
if (ALL_TESTS) return;
const versions = [
[
"r2",
new StorageS3Adapter({
access_key: R2_ACCESS_KEY as string,
secret_access_key: R2_SECRET_ACCESS_KEY as string,
url: R2_URL as string,
}),
],
[
"s3",
new StorageS3Adapter({
access_key: AWS_ACCESS_KEY as string,
secret_access_key: AWS_SECRET_KEY as string,
url: AWS_S3_URL as string,
}),
],
] as const;
const file = Bun.file(`${assetsPath}/image.png`) as unknown as File;
describe.each(versions)("%s", async (_name, adapter) => {
await adapterTestSuite({ test, expect }, adapter, file);
});
});

23
app/src/media/storage/adapters/StorageS3Adapter.ts → app/src/media/storage/adapters/s3/StorageS3Adapter.ts
View File

@@ -9,7 +9,8 @@ import type {
import { AwsClient, isDebug } from "core";
import { type Static, Type, isFile, parse, pickHeaders2 } from "core/utils";
import { transform } from "lodash-es";
import type { FileBody, FileListObject, StorageAdapter } from "../Storage";
import type { FileBody, FileListObject } from "../../Storage";
import { StorageAdapter } from "../../StorageAdapter";
export const s3AdapterConfig = Type.Object(
{
@@ -32,11 +33,13 @@ export const s3AdapterConfig = Type.Object(
export type S3AdapterConfig = Static<typeof s3AdapterConfig>;
export class StorageS3Adapter extends AwsClient implements StorageAdapter {
export class StorageS3Adapter extends StorageAdapter {
readonly #config: S3AdapterConfig;
readonly client: AwsClient;
constructor(config: S3AdapterConfig) {
super(
super();
this.client = new AwsClient(
{
accessKeyId: config.access_key,
secretAccessKey: config.secret_access_key,
@@ -58,10 +61,10 @@ export class StorageS3Adapter extends AwsClient implements StorageAdapter {
return s3AdapterConfig;
}
override getUrl(path: string = "", searchParamsObj: Record<string, any> = {}): string {
getUrl(path: string = "", searchParamsObj: Record<string, any> = {}): string {
let url = this.getObjectUrl("").slice(0, -1);
if (path.length > 0) url += `/${path}`;
return super.getUrl(url, searchParamsObj);
return this.client.getUrl(url, searchParamsObj);
}
/**
@@ -82,7 +85,7 @@ export class StorageS3Adapter extends AwsClient implements StorageAdapter {
};
const url = this.getUrl("", params);
const res = await this.fetchJson<{ ListBucketResult: ListObjectsV2Output }>(url, {
const res = await this.client.fetchJson<{ ListBucketResult: ListObjectsV2Output }>(url, {
method: "GET",
});
@@ -115,7 +118,7 @@ export class StorageS3Adapter extends AwsClient implements StorageAdapter {
params: Omit<PutObjectRequest, "Bucket" | "Key"> = {},
) {
const url = this.getUrl(key, {});
const res = await this.fetch(url, {
const res = await this.client.fetch(url, {
method: "PUT",
body,
headers: isFile(body)
@@ -139,7 +142,7 @@ export class StorageS3Adapter extends AwsClient implements StorageAdapter {
params: Pick<HeadObjectRequest, "PartNumber" | "VersionId"> = {},
) {
const url = this.getUrl(key, {});
return await this.fetch(url, {
return await this.client.fetch(url, {
method: "HEAD",
headers: {
Range: "bytes=0-1",
@@ -175,7 +178,7 @@ export class StorageS3Adapter extends AwsClient implements StorageAdapter {
*/
async getObject(key: string, headers: Headers): Promise<Response> {
const url = this.getUrl(key);
const res = await this.fetch(url, {
const res = await this.client.fetch(url, {
method: "GET",
headers: pickHeaders2(headers, [
"if-none-match",
@@ -201,7 +204,7 @@ export class StorageS3Adapter extends AwsClient implements StorageAdapter {
params: Omit<DeleteObjectRequest, "Bucket" | "Key"> = {},
): Promise<void> {
const url = this.getUrl(key, params);
const res = await this.fetch(url, {
const res = await this.client.fetch(url, {
method: "DELETE",
});
}