reorganized storage adapter and added test suites for adapter and fields (#124)

* reorganized storage adapter and added test suites for adapter and fields

* added build command in ci pipeline

* updated workflow to also run node tests

* updated workflow: try with separate tasks

* updated workflow: try with separate tasks

* updated workflow: added tsx as dev dependency

* updated workflow: try with find instead of glob

app/src/media/index.ts

@@ -1,22 +1,24 @@
-import type { TObject, TString } from "@sinclair/typebox";
+import type { TObject } from "@sinclair/typebox";
 import { type Constructor, Registry } from "core";
 
 //export { MIME_TYPES } from "./storage/mime-types";
 export { guess as guessMimeType } from "./storage/mime-types-tiny";
 export {
    Storage,
-   type StorageAdapter,
    type FileMeta,
    type FileListObject,
    type StorageConfig,
+   type FileBody,
+   type FileUploadPayload,
 } from "./storage/Storage";
-import type { StorageAdapter } from "./storage/Storage";
+import { StorageAdapter } from "./storage/StorageAdapter";
 import {
    type CloudinaryConfig,
    StorageCloudinaryAdapter,
-} from "./storage/adapters/StorageCloudinaryAdapter";
-import { type S3AdapterConfig, StorageS3Adapter } from "./storage/adapters/StorageS3Adapter";
+} from "./storage/adapters/cloudinary/StorageCloudinaryAdapter";
+import { type S3AdapterConfig, StorageS3Adapter } from "./storage/adapters/s3/StorageS3Adapter";
 
+export { StorageAdapter };
 export { StorageS3Adapter, type S3AdapterConfig, StorageCloudinaryAdapter, type CloudinaryConfig };
 
 export * as StorageEvents from "./storage/events";
@@ -45,3 +47,5 @@ export const Adapters = {
       schema: StorageCloudinaryAdapter.prototype.getSchema(),
    },
 } as const;
+
+export { adapterTestSuite } from "./storage/adapters/adapter-test-suite";

app/src/media/storage/Storage.ts

@@ -1,9 +1,10 @@
 import { type EmitsEvents, EventManager } from "core/events";
-import { type TSchema, isFile, detectImageDimensions } from "core/utils";
+import { isFile, detectImageDimensions } from "core/utils";
 import { isMimeType } from "media/storage/mime-types-tiny";
 import * as StorageEvents from "./events";
 import type { FileUploadedEventData } from "./events";
 import { $console } from "core";
+import type { StorageAdapter } from "./StorageAdapter";
 
 export type FileListObject = {
    key: string;
@@ -19,24 +20,6 @@ export type FileUploadPayload = {
    etag: string;
 };
 
-export interface StorageAdapter {
-   /**
-    * The unique name of the storage adapter
-    */
-   getName(): string;
-
-   // @todo: method requires limit/offset parameters
-   listObjects(prefix?: string): Promise<FileListObject[]>;
-   putObject(key: string, body: FileBody): Promise<string | FileUploadPayload | undefined>;
-   deleteObject(key: string): Promise<void>;
-   objectExists(key: string): Promise<boolean>;
-   getObject(key: string, headers: Headers): Promise<Response>;
-   getObjectUrl(key: string): string;
-   getObjectMeta(key: string): Promise<FileMeta>;
-   getSchema(): TSchema | undefined;
-   toJSON(secrets?: boolean): any;
-}
-
 export type StorageConfig = {
    body_max_size?: number;
 };

app/src/media/storage/StorageAdapter.ts

@@ -0,0 +1,37 @@
+import type { FileListObject, FileMeta } from "media";
+import type { FileBody, FileUploadPayload } from "media/storage/Storage";
+import type { TSchema } from "@sinclair/typebox";
+
+const SYMBOL = Symbol.for("bknd:storage");
+
+export abstract class StorageAdapter {
+   constructor() {
+      this[SYMBOL] = true;
+   }
+
+   /**
+    * This is a helper function to manage Connection classes
+    * coming from different places
+    * @param conn
+    */
+   static isAdapter(conn: unknown): conn is StorageAdapter {
+      if (!conn) return false;
+      return conn[SYMBOL] === true;
+   }
+
+   /**
+    * The unique name of the storage adapter
+    */
+   abstract getName(): string;
+
+   // @todo: method requires limit/offset parameters
+   abstract listObjects(prefix?: string): Promise<FileListObject[]>;
+   abstract putObject(key: string, body: FileBody): Promise<string | FileUploadPayload | undefined>;
+   abstract deleteObject(key: string): Promise<void>;
+   abstract objectExists(key: string): Promise<boolean>;
+   abstract getObject(key: string, headers: Headers): Promise<Response>;
+   abstract getObjectUrl(key: string): string;
+   abstract getObjectMeta(key: string): Promise<FileMeta>;
+   abstract getSchema(): TSchema | undefined;
+   abstract toJSON(secrets?: boolean): any;
+}

app/src/media/storage/adapters/StorageLocalAdapter/StorageLocalAdapter.ts

@@ -1,125 +0,0 @@
-import { readFile, readdir, stat, unlink, writeFile } from "node:fs/promises";
-import { type Static, Type, isFile, parse } from "core/utils";
-import type {
-   FileBody,
-   FileListObject,
-   FileMeta,
-   FileUploadPayload,
-   StorageAdapter,
-} from "../../Storage";
-import { guess } from "../../mime-types-tiny";
-
-export const localAdapterConfig = Type.Object(
-   {
-      path: Type.String({ default: "./" }),
-   },
-   { title: "Local", description: "Local file system storage" },
-);
-export type LocalAdapterConfig = Static<typeof localAdapterConfig>;
-
-export class StorageLocalAdapter implements StorageAdapter {
-   private config: LocalAdapterConfig;
-
-   constructor(config: any) {
-      this.config = parse(localAdapterConfig, config);
-   }
-
-   getSchema() {
-      return localAdapterConfig;
-   }
-
-   getName(): string {
-      return "local";
-   }
-
-   async listObjects(prefix?: string): Promise<FileListObject[]> {
-      const files = await readdir(this.config.path);
-      const fileStats = await Promise.all(
-         files
-            .filter((file) => !prefix || file.startsWith(prefix))
-            .map(async (file) => {
-               const stats = await stat(`${this.config.path}/${file}`);
-               return {
-                  key: file,
-                  last_modified: stats.mtime,
-                  size: stats.size,
-               };
-            }),
-      );
-      return fileStats;
-   }
-
-   private async computeEtag(body: FileBody): Promise<string> {
-      const content = isFile(body) ? body : new Response(body);
-      const hashBuffer = await crypto.subtle.digest("SHA-256", await content.arrayBuffer());
-      const hashArray = Array.from(new Uint8Array(hashBuffer));
-      const hashHex = hashArray.map((byte) => byte.toString(16).padStart(2, "0")).join("");
-
-      // Wrap the hex string in quotes for ETag format
-      return `"${hashHex}"`;
-   }
-
-   async putObject(key: string, body: FileBody): Promise<string | FileUploadPayload> {
-      if (body === null) {
-         throw new Error("Body is empty");
-      }
-
-      const filePath = `${this.config.path}/${key}`;
-      const is_file = isFile(body);
-      await writeFile(filePath, is_file ? body.stream() : body);
-
-      return await this.computeEtag(body);
-   }
-
-   async deleteObject(key: string): Promise<void> {
-      try {
-         await unlink(`${this.config.path}/${key}`);
-      } catch (e) {}
-   }
-
-   async objectExists(key: string): Promise<boolean> {
-      try {
-         const stats = await stat(`${this.config.path}/${key}`);
-         return stats.isFile();
-      } catch (error) {
-         return false;
-      }
-   }
-
-   async getObject(key: string, headers: Headers): Promise<Response> {
-      try {
-         const content = await readFile(`${this.config.path}/${key}`);
-         const mimeType = guess(key);
-
-         return new Response(content, {
-            status: 200,
-            headers: {
-               "Content-Type": mimeType || "application/octet-stream",
-               "Content-Length": content.length.toString(),
-            },
-         });
-      } catch (error) {
-         // Handle file reading errors
-         return new Response("", { status: 404 });
-      }
-   }
-
-   getObjectUrl(key: string): string {
-      throw new Error("Method not implemented.");
-   }
-
-   async getObjectMeta(key: string): Promise<FileMeta> {
-      const stats = await stat(`${this.config.path}/${key}`);
-      return {
-         type: guess(key) || "application/octet-stream",
-         size: stats.size,
-      };
-   }
-
-   toJSON(secrets?: boolean) {
-      return {
-         type: this.getName(),
-         config: this.config,
-      };
-   }
-}

app/src/media/storage/adapters/StorageLocalAdapter/index.ts

@@ -1,5 +0,0 @@
-export {
-   StorageLocalAdapter,
-   type LocalAdapterConfig,
-   localAdapterConfig,
-} from "./StorageLocalAdapter";

app/src/media/storage/adapters/adapter-test-suite.ts

@@ -0,0 +1,79 @@
+import { retry, type TestRunner } from "core/test";
+import type { StorageAdapter } from "media";
+import { randomString } from "core/utils";
+import type { BunFile } from "bun";
+
+export async function adapterTestSuite(
+   testRunner: TestRunner,
+   adapter: StorageAdapter,
+   file: File | BunFile,
+   opts?: {
+      retries?: number;
+      retryTimeout?: number;
+      skipExistsAfterDelete?: boolean;
+   },
+) {
+   const { test, expect } = testRunner;
+   const options = {
+      retries: opts?.retries ?? 1,
+      retryTimeout: opts?.retryTimeout ?? 1000,
+   };
+
+   let objects = 0;
+   const _filename = randomString(10);
+   const filename = `${_filename}.png`;
+
+   await test("puts an object", async () => {
+      objects = (await adapter.listObjects()).length;
+      const result = await adapter.putObject(filename, file as unknown as File);
+      expect(result).toBeDefined();
+      const type = typeof result;
+      expect(type).toBeOneOf(["string", "object"]);
+      if (typeof result === "object") {
+         expect(Object.keys(result).sort()).toEqual(["etag", "meta", "name"]);
+         expect(result.meta.type).toBe(file.type);
+      }
+   });
+
+   await test("lists objects", async () => {
+      const length = await retry(
+         () => adapter.listObjects().then((res) => res.length),
+         (length) => length > objects,
+         options.retries,
+         options.retryTimeout,
+      );
+
+      expect(length).toBe(objects + 1);
+   });
+
+   await test("file exists", async () => {
+      expect(await adapter.objectExists(filename)).toBe(true);
+   });
+
+   await test("gets an object", async () => {
+      const res = await adapter.getObject(filename, new Headers());
+      expect(res.ok).toBe(true);
+      // @todo: check the content
+   });
+
+   await test("gets object meta", async () => {
+      expect(await adapter.getObjectMeta(filename)).toEqual({
+         type: file.type, // image/png
+         size: file.size,
+      });
+   });
+
+   await test("deletes an object", async () => {
+      expect(await adapter.deleteObject(filename)).toBeUndefined();
+
+      if (opts?.skipExistsAfterDelete !== true) {
+         const exists = await retry(
+            () => adapter.objectExists(filename),
+            (res) => res === false,
+            options.retries,
+            options.retryTimeout,
+         );
+         expect(exists).toBe(false);
+      }
+   });
+}

app/src/media/storage/adapters/cloudinary/StorageCloudinaryAdapter.spec.ts

@@ -0,0 +1,53 @@
+import { describe, expect, test } from "bun:test";
+import { StorageCloudinaryAdapter } from "./StorageCloudinaryAdapter";
+import { config } from "dotenv";
+// @ts-ignore
+import { assetsPath, assetsTmpPath } from "../../../../../__test__/helper";
+import { adapterTestSuite } from "media/storage/adapters/adapter-test-suite";
+
+const dotenvOutput = config({ path: `${import.meta.dir}/.env` });
+const {
+   CLOUDINARY_CLOUD_NAME,
+   CLOUDINARY_API_KEY,
+   CLOUDINARY_API_SECRET,
+   CLOUDINARY_UPLOAD_PRESET,
+} = dotenvOutput.parsed!;
+
+const ALL_TESTS = !!process.env.ALL_TESTS;
+
+describe.skipIf(ALL_TESTS)("StorageCloudinaryAdapter", async () => {
+   if (ALL_TESTS) return;
+
+   const adapter = new StorageCloudinaryAdapter({
+      cloud_name: CLOUDINARY_CLOUD_NAME as string,
+      api_key: CLOUDINARY_API_KEY as string,
+      api_secret: CLOUDINARY_API_SECRET as string,
+      upload_preset: CLOUDINARY_UPLOAD_PRESET as string,
+   });
+
+   const file = Bun.file(`${assetsPath}/image.png`) as unknown as File;
+
+   test("hash", async () => {
+      expect(
+         await adapter.generateSignature(
+            {
+               eager: "w_400,h_300,c_pad|w_260,h_200,c_crop",
+               public_id: "sample_image",
+               timestamp: 1315060510,
+            },
+            "abcd",
+         ),
+      ).toEqual({
+         signature: "bfd09f95f331f558cbd1320e67aa8d488770583e",
+         timestamp: 1315060510,
+      });
+   });
+
+   await adapterTestSuite({ test, expect }, adapter, file, {
+      // eventual consistency
+      retries: 20,
+      retryTimeout: 1000,
+      // result is cached from cloudinary
+      skipExistsAfterDelete: true,
+   });
+});

app/src/media/storage/adapters/cloudinary/StorageCloudinaryAdapter.ts

@@ -1,6 +1,7 @@
-import { pickHeaders } from "core/utils";
+import { hash, pickHeaders } from "core/utils";
 import { type Static, Type, parse } from "core/utils";
-import type { FileBody, FileListObject, FileMeta, StorageAdapter } from "../Storage";
+import type { FileBody, FileListObject, FileMeta } from "../../Storage";
+import { StorageAdapter } from "../../StorageAdapter";
 
 export const cloudinaryAdapterConfig = Type.Object(
    {
@@ -53,10 +54,11 @@ type CloudinaryListObjectsResponse = {
 };
 
 // @todo: add signed uploads
-export class StorageCloudinaryAdapter implements StorageAdapter {
+export class StorageCloudinaryAdapter extends StorageAdapter {
    private config: CloudinaryConfig;
 
    constructor(config: CloudinaryConfig) {
+      super();
       this.config = parse(cloudinaryAdapterConfig, config);
    }
 
@@ -126,6 +128,11 @@ export class StorageCloudinaryAdapter implements StorageAdapter {
       };
    }
 
+   /**
+    * https://cloudinary.com/documentation/admin_api#search_for_resources
+    * Cloudinary implements eventual consistency: Search results reflect any changes made to assets within a few seconds after the change
+    * @param prefix
+    */
    async listObjects(prefix?: string): Promise<FileListObject[]> {
       const result = await fetch(
          `https://api.cloudinary.com/v1_1/${this.config.cloud_name}/resources/search`,
@@ -133,6 +140,7 @@ export class StorageCloudinaryAdapter implements StorageAdapter {
             method: "GET",
             headers: {
                Accept: "application/json",
+               "Cache-Control": "no-cache",
                ...this.getAuthorizationHeader(),
             },
          },
@@ -143,18 +151,22 @@ export class StorageCloudinaryAdapter implements StorageAdapter {
       }
 
       const data = (await result.json()) as CloudinaryListObjectsResponse;
-      return data.resources.map((item) => ({
+      const items = data.resources.map((item) => ({
          key: item.public_id,
          last_modified: new Date(item.uploaded_at),
          size: item.bytes,
       }));
+      return items;
    }
 
    private async headObject(key: string) {
       const url = this.getObjectUrl(key);
       return await fetch(url, {
-         method: "GET",
+         method: "HEAD",
          headers: {
+            "Cache-Control": "no-cache, no-store, must-revalidate",
+            Pragma: "no-cache",
+            Expires: "0",
             Range: "bytes=0-1",
          },
       });
@@ -196,6 +208,22 @@ export class StorageCloudinaryAdapter implements StorageAdapter {
       return objectUrl;
    }
 
+   async generateSignature(params: Record<string, string | number>, secret?: string) {
+      const timestamp = params.timestamp ?? Math.floor(Date.now() / 1000);
+      const content = Object.entries({ ...params, timestamp })
+         .sort(([keyA], [keyB]) => keyA.localeCompare(keyB))
+         .map(([key, value]) => `${key}=${value}`)
+         .join("&");
+
+      const signature = await hash.sha1(content + (secret ?? this.config.api_secret));
+      return { signature, timestamp };
+   }
+
+   // get public_id as everything before the last "."
+   filenameToPublicId(key: string): string {
+      return key.split(".").slice(0, -1).join(".");
+   }
+
    async getObject(key: string, headers: Headers): Promise<Response> {
       const res = await fetch(this.getObjectUrl(key), {
          method: "GET",
@@ -211,13 +239,30 @@ export class StorageCloudinaryAdapter implements StorageAdapter {
 
    async deleteObject(key: string): Promise<void> {
       const type = this.guessType(key) ?? "image";
-      const formData = new FormData();
-      formData.append("public_ids[]", key);
+      const public_id = this.filenameToPublicId(key);
+      const { timestamp, signature } = await this.generateSignature({
+         public_id,
+      });
 
-      await fetch(`https://res.cloudinary.com/${this.config.cloud_name}/${type}/upload/`, {
-         method: "DELETE",
+      const formData = new FormData();
+      formData.append("public_id", public_id);
+      formData.append("timestamp", String(timestamp));
+      formData.append("signature", signature);
+      formData.append("api_key", this.config.api_key);
+
+      const url = `https://api.cloudinary.com/v1_1/${this.config.cloud_name}/${type}/destroy`;
+      const res = await fetch(url, {
+         headers: {
+            Accept: "application/json",
+            "Cache-Control": "no-cache",
+            ...this.getAuthorizationHeader(),
+         },
+         method: "POST",
          body: formData,
       });
+      if (!res.ok) {
+         throw new Error(`Failed to delete object: ${res.status} ${res.statusText}`);
+      }
    }
 
    toJSON(secrets?: boolean) {

app/src/media/storage/adapters/s3/StorageS3Adapter.spec.ts

@@ -0,0 +1,50 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { StorageS3Adapter } from "./StorageS3Adapter";
+
+import { config } from "dotenv";
+import { adapterTestSuite } from "media";
+import { assetsPath } from "../../../../../__test__/helper";
+//import { enableFetchLogging } from "../../helper";
+const dotenvOutput = config({ path: `${import.meta.dir}/.env` });
+const { R2_ACCESS_KEY, R2_SECRET_ACCESS_KEY, R2_URL, AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_S3_URL } =
+   dotenvOutput.parsed!;
+
+const ALL_TESTS = !!process.env.ALL_TESTS;
+
+/* 
+// @todo: preparation to mock s3 calls + replace fast-xml-parser
+let cleanup: () => void;
+beforeAll(async () => {
+   cleanup = await enableFetchLogging();
+});
+afterAll(() => {
+   cleanup();
+}); */
+
+describe.skipIf(ALL_TESTS)("StorageS3Adapter", async () => {
+   if (ALL_TESTS) return;
+
+   const versions = [
+      [
+         "r2",
+         new StorageS3Adapter({
+            access_key: R2_ACCESS_KEY as string,
+            secret_access_key: R2_SECRET_ACCESS_KEY as string,
+            url: R2_URL as string,
+         }),
+      ],
+      [
+         "s3",
+         new StorageS3Adapter({
+            access_key: AWS_ACCESS_KEY as string,
+            secret_access_key: AWS_SECRET_KEY as string,
+            url: AWS_S3_URL as string,
+         }),
+      ],
+   ] as const;
+   const file = Bun.file(`${assetsPath}/image.png`) as unknown as File;
+
+   describe.each(versions)("%s", async (_name, adapter) => {
+      await adapterTestSuite({ test, expect }, adapter, file);
+   });
+});

app/src/media/storage/adapters/s3/StorageS3Adapter.ts

@@ -9,7 +9,8 @@ import type {
 import { AwsClient, isDebug } from "core";
 import { type Static, Type, isFile, parse, pickHeaders2 } from "core/utils";
 import { transform } from "lodash-es";
-import type { FileBody, FileListObject, StorageAdapter } from "../Storage";
+import type { FileBody, FileListObject } from "../../Storage";
+import { StorageAdapter } from "../../StorageAdapter";
 
 export const s3AdapterConfig = Type.Object(
    {
@@ -32,11 +33,13 @@ export const s3AdapterConfig = Type.Object(
 
 export type S3AdapterConfig = Static<typeof s3AdapterConfig>;
 
-export class StorageS3Adapter extends AwsClient implements StorageAdapter {
+export class StorageS3Adapter extends StorageAdapter {
    readonly #config: S3AdapterConfig;
+   readonly client: AwsClient;
 
    constructor(config: S3AdapterConfig) {
-      super(
+      super();
+      this.client = new AwsClient(
          {
             accessKeyId: config.access_key,
             secretAccessKey: config.secret_access_key,
@@ -58,10 +61,10 @@ export class StorageS3Adapter extends AwsClient implements StorageAdapter {
       return s3AdapterConfig;
    }
 
-   override getUrl(path: string = "", searchParamsObj: Record<string, any> = {}): string {
+   getUrl(path: string = "", searchParamsObj: Record<string, any> = {}): string {
       let url = this.getObjectUrl("").slice(0, -1);
       if (path.length > 0) url += `/${path}`;
-      return super.getUrl(url, searchParamsObj);
+      return this.client.getUrl(url, searchParamsObj);
    }
 
    /**
@@ -82,7 +85,7 @@ export class StorageS3Adapter extends AwsClient implements StorageAdapter {
       };
 
       const url = this.getUrl("", params);
-      const res = await this.fetchJson<{ ListBucketResult: ListObjectsV2Output }>(url, {
+      const res = await this.client.fetchJson<{ ListBucketResult: ListObjectsV2Output }>(url, {
          method: "GET",
       });
 
@@ -115,7 +118,7 @@ export class StorageS3Adapter extends AwsClient implements StorageAdapter {
       params: Omit<PutObjectRequest, "Bucket" | "Key"> = {},
    ) {
       const url = this.getUrl(key, {});
-      const res = await this.fetch(url, {
+      const res = await this.client.fetch(url, {
          method: "PUT",
          body,
          headers: isFile(body)
@@ -139,7 +142,7 @@ export class StorageS3Adapter extends AwsClient implements StorageAdapter {
       params: Pick<HeadObjectRequest, "PartNumber" | "VersionId"> = {},
    ) {
       const url = this.getUrl(key, {});
-      return await this.fetch(url, {
+      return await this.client.fetch(url, {
          method: "HEAD",
          headers: {
             Range: "bytes=0-1",
@@ -175,7 +178,7 @@ export class StorageS3Adapter extends AwsClient implements StorageAdapter {
     */
    async getObject(key: string, headers: Headers): Promise<Response> {
       const url = this.getUrl(key);
-      const res = await this.fetch(url, {
+      const res = await this.client.fetch(url, {
          method: "GET",
          headers: pickHeaders2(headers, [
             "if-none-match",
@@ -201,7 +204,7 @@ export class StorageS3Adapter extends AwsClient implements StorageAdapter {
       params: Omit<DeleteObjectRequest, "Bucket" | "Key"> = {},
    ): Promise<void> {
       const url = this.getUrl(key, params);
-      const res = await this.fetch(url, {
+      const res = await this.client.fetch(url, {
          method: "DELETE",
       });
    }