public commit

app/src/media/AppMedia.ts

@@ -0,0 +1,171 @@
+import { EntityIndex, type EntityManager } from "data";
+import { type FileUploadedEventData, Storage, type StorageAdapter } from "media";
+import { Module } from "modules/Module";
+import {
+   type FieldSchema,
+   type InferFields,
+   type Schema,
+   boolean,
+   datetime,
+   entity,
+   json,
+   number,
+   text
+} from "../data/prototype";
+import { MediaController } from "./api/MediaController";
+import { ADAPTERS, buildMediaSchema, type mediaConfigSchema, registry } from "./media-schema";
+
+export type MediaFieldSchema = FieldSchema<typeof AppMedia.mediaFields>;
+declare global {
+   interface DB {
+      media: MediaFieldSchema;
+   }
+}
+
+export class AppMedia extends Module<typeof mediaConfigSchema> {
+   private _storage?: Storage;
+
+   override async build() {
+      if (!this.config.enabled) {
+         this.setBuilt();
+         return;
+      }
+
+      if (!this.config.adapter) {
+         console.info("No storage adapter provided, skip building media.");
+         return;
+      }
+
+      // build adapter
+      let adapter: StorageAdapter;
+      try {
+         const { type, config } = this.config.adapter;
+         adapter = new (registry.get(type as any).cls)(config as any);
+
+         this._storage = new Storage(adapter, this.config.storage, this.ctx.emgr);
+         this.setBuilt();
+         this.setupListeners();
+         this.ctx.server.route(this.basepath, new MediaController(this).getController());
+
+         // @todo: add check for media entity
+         const mediaEntity = this.getMediaEntity();
+         if (!this.ctx.em.hasEntity(mediaEntity)) {
+            this.ctx.em.addEntity(mediaEntity);
+         }
+
+         const pathIndex = new EntityIndex(mediaEntity, [mediaEntity.field("path")!], true);
+         if (!this.ctx.em.hasIndex(pathIndex)) {
+            this.ctx.em.addIndex(pathIndex);
+         }
+
+         // @todo: check indices
+      } catch (e) {
+         console.error(e);
+         throw new Error(
+            `Could not build adapter with config ${JSON.stringify(this.config.adapter)}`
+         );
+      }
+   }
+
+   getSchema() {
+      return buildMediaSchema();
+   }
+
+   get basepath() {
+      return this.config.basepath;
+   }
+
+   get storage(): Storage {
+      this.throwIfNotBuilt();
+      return this._storage!;
+   }
+
+   uploadedEventDataToMediaPayload(info: FileUploadedEventData) {
+      return {
+         path: info.name,
+         mime_type: info.meta.type,
+         size: info.meta.size,
+         etag: info.etag,
+         modified_at: new Date()
+      };
+   }
+
+   static mediaFields = {
+      path: text().required(),
+      folder: boolean({ default_value: false, hidden: true, fillable: ["create"] }),
+      mime_type: text(),
+      size: number(),
+      scope: text({ hidden: true, fillable: ["create"] }),
+      etag: text(),
+      modified_at: datetime(),
+      reference: text(),
+      entity_id: number(),
+      metadata: json()
+   };
+
+   getMediaEntity() {
+      const entity_name = this.config.entity_name;
+      if (!this.em.hasEntity(entity_name)) {
+         return entity(entity_name, AppMedia.mediaFields, undefined, "system");
+      }
+
+      return this.em.entity(entity_name);
+   }
+
+   get em(): EntityManager<DB> {
+      return this.ctx.em;
+   }
+
+   private setupListeners() {
+      //const media = this._entity;
+      const { emgr, em } = this.ctx;
+      const media = this.getMediaEntity();
+
+      // when file is uploaded, sync with media entity
+      // @todo: need a way for singleton events!
+      emgr.onEvent(
+         Storage.Events.FileUploadedEvent,
+         async (e) => {
+            const mutator = em.mutator(media);
+            mutator.__unstable_toggleSystemEntityCreation(false);
+            await mutator.insertOne(this.uploadedEventDataToMediaPayload(e.params));
+            mutator.__unstable_toggleSystemEntityCreation(true);
+            console.log("App:storage:file uploaded", e);
+         },
+         "sync"
+      );
+
+      // when file is deleted, sync with media entity
+      emgr.onEvent(
+         Storage.Events.FileDeletedEvent,
+         async (e) => {
+            // simple file deletion sync
+            const item = await em.repo(media).findOne({ path: e.params.name });
+            if (item.data) {
+               console.log("item.data", item.data);
+               await em.mutator(media).deleteOne(item.data.id);
+            }
+
+            console.log("App:storage:file deleted", e);
+         },
+         "sync"
+      );
+   }
+
+   override getOverwritePaths() {
+      // if using 'set' or mocked 'set' (patch), then "." is prepended
+      return [/^\.?adapter$/];
+   }
+
+   // @todo: add unit tests for toJSON!
+   override toJSON(secrets?: boolean) {
+      if (!this.isBuilt() || !this.config.enabled) {
+         return this.configDefault;
+      }
+
+      return {
+         ...this.config,
+         adapter: this.storage.getAdapter().toJSON(secrets)
+      };
+   }
+}

app/src/media/MediaField.ts

@@ -0,0 +1,71 @@
+import { type Static, Type } from "core/utils";
+import { Field, baseFieldConfigSchema } from "data";
+
+export const mediaFieldConfigSchema = Type.Composite([
+   Type.Object({
+      entity: Type.String(), // @todo: is this really required?
+      min_items: Type.Optional(Type.Number()),
+      max_items: Type.Optional(Type.Number()),
+      mime_types: Type.Optional(Type.Array(Type.String()))
+   }),
+   baseFieldConfigSchema
+]);
+
+export type MediaFieldConfig = Static<typeof mediaFieldConfigSchema>;
+
+export type MediaItem = {
+   id: number;
+   path: string;
+   mime_type: string;
+   size: number;
+   scope: number;
+   etag: string;
+   modified_at: Date;
+   folder: boolean;
+};
+
+export class MediaField<
+   Required extends true | false = false,
+   TypeOverride = MediaItem[]
+> extends Field<MediaFieldConfig, TypeOverride, Required> {
+   override readonly type = "media";
+
+   constructor(name: string, config: Partial<MediaFieldConfig>) {
+      // field must be virtual, as it doesn't store a reference to the entity
+      super(name, { ...config, fillable: ["update"], virtual: true });
+   }
+
+   protected getSchema() {
+      return mediaFieldConfigSchema;
+   }
+
+   getMaxItems(): number | undefined {
+      return this.config.max_items;
+   }
+
+   getMinItems(): number | undefined {
+      return this.config.min_items;
+   }
+
+   schema() {
+      return undefined;
+   }
+
+   override toJsonSchema() {
+      // @todo: should be a variable, since media could be a diff entity
+      const $ref = "../schema.json#/properties/media";
+      const minItems = this.config?.min_items;
+      const maxItems = this.config?.max_items;
+
+      if (maxItems === 1) {
+         return { $ref } as any;
+      } else {
+         return {
+            type: "array",
+            items: { $ref },
+            minItems,
+            maxItems
+         };
+      }
+   }
+}

app/src/media/api/MediaApi.ts

@@ -0,0 +1,44 @@
+import { type BaseModuleApiOptions, ModuleApi, type PrimaryFieldType } from "modules/ModuleApi";
+import type { FileWithPath } from "ui/modules/media/components/dropzone/file-selector";
+
+export type MediaApiOptions = BaseModuleApiOptions & {};
+
+export class MediaApi extends ModuleApi<MediaApiOptions> {
+   protected override getDefaultOptions(): Partial<MediaApiOptions> {
+      return {
+         basepath: "/api/media"
+      };
+   }
+
+   async getFiles() {
+      return this.get(["files"]);
+   }
+
+   async getFile(filename: string) {
+      return this.get(["file", filename]);
+   }
+
+   getFileUploadUrl(file: FileWithPath): string {
+      return this.getUrl(`/upload/${file.path}`);
+   }
+
+   getEntityUploadUrl(entity: string, id: PrimaryFieldType, field: string) {
+      return this.getUrl(`/entity/${entity}/${id}/${field}`);
+   }
+
+   getUploadHeaders(): Headers {
+      return new Headers({
+         Authorization: `Bearer ${this.options.token}`
+      });
+   }
+
+   async uploadFile(file: File) {
+      const formData = new FormData();
+      formData.append("file", file);
+      return this.post(["upload"], formData);
+   }
+
+   async deleteFile(filename: string) {
+      return this.delete(["file", filename]);
+   }
+}

app/src/media/api/MediaController.ts

@@ -0,0 +1,193 @@
+import { type ClassController, tbValidator as tb } from "core";
+import { Type } from "core/utils";
+import { Hono } from "hono";
+import { bodyLimit } from "hono/body-limit";
+import type { StorageAdapter } from "media";
+import { StorageEvents } from "media";
+import { getRandomizedFilename } from "media";
+import type { AppMedia } from "../AppMedia";
+import { MediaField } from "../MediaField";
+
+const booleanLike = Type.Transform(Type.String())
+   .Decode((v) => v === "1")
+   .Encode((v) => (v ? "1" : "0"));
+
+export class MediaController implements ClassController {
+   constructor(private readonly media: AppMedia) {}
+
+   private getStorageAdapter(): StorageAdapter {
+      return this.getStorage().getAdapter();
+   }
+
+   private getStorage() {
+      return this.media.storage;
+   }
+
+   getController(): Hono<any> {
+      // @todo: multiple providers?
+      // @todo: implement range requests
+
+      const hono = new Hono();
+
+      // get files list (temporary)
+      hono.get("/files", async (c) => {
+         const files = await this.getStorageAdapter().listObjects();
+         return c.json(files);
+      });
+
+      // get file by name
+      hono.get("/file/:filename", async (c) => {
+         const { filename } = c.req.param();
+         if (!filename) {
+            throw new Error("No file name provided");
+         }
+         //console.log("getting file", filename, headersToObject(c.req.raw.headers));
+
+         await this.getStorage().emgr.emit(new StorageEvents.FileAccessEvent({ name: filename }));
+         return await this.getStorageAdapter().getObject(filename, c.req.raw.headers);
+      });
+
+      // delete a file by name
+      hono.delete("/file/:filename", async (c) => {
+         const { filename } = c.req.param();
+         if (!filename) {
+            throw new Error("No file name provided");
+         }
+         await this.getStorage().deleteFile(filename);
+
+         return c.json({ message: "File deleted" });
+      });
+
+      const uploadSizeMiddleware = bodyLimit({
+         maxSize: this.getStorage().getConfig().body_max_size,
+         onError: (c: any) => {
+            return c.text(`Payload exceeds ${this.getStorage().getConfig().body_max_size}`, 413);
+         }
+      });
+
+      // upload file
+      // @todo: add required type for "upload endpoints"
+      hono.post("/upload/:filename", uploadSizeMiddleware, async (c) => {
+         const { filename } = c.req.param();
+         if (!filename) {
+            throw new Error("No file name provided");
+         }
+
+         const file = await this.getStorage().getFileFromRequest(c);
+         console.log("----file", file);
+         return c.json(await this.getStorage().uploadFile(file, filename));
+      });
+
+      // add upload file to entity
+      // @todo: add required type for "upload endpoints"
+      hono.post(
+         "/entity/:entity/:id/:field",
+         tb(
+            "query",
+            Type.Object({
+               overwrite: Type.Optional(booleanLike)
+            })
+         ),
+         uploadSizeMiddleware,
+         async (c) => {
+            const entity_name = c.req.param("entity");
+            const field_name = c.req.param("field");
+            const entity_id = Number.parseInt(c.req.param("id"));
+            console.log("params", { entity_name, field_name, entity_id });
+
+            // check if entity exists
+            const entity = this.media.em.entity(entity_name);
+            if (!entity) {
+               return c.json({ error: `Entity "${entity_name}" not found` }, 404);
+            }
+
+            // check if field exists and is of type MediaField
+            const field = entity.field(field_name);
+            if (!field || !(field instanceof MediaField)) {
+               return c.json({ error: `Invalid field "${field_name}"` }, 400);
+            }
+
+            const mediaEntity = this.media.getMediaEntity();
+            const reference = `${entity_name}.${field_name}`;
+            const mediaRef = {
+               scope: field_name,
+               reference,
+               entity_id: entity_id
+            };
+
+            // check max items
+            const max_items = field.getMaxItems();
+            const ids_to_delete: number[] = [];
+            const id_field = mediaEntity.getPrimaryField().name;
+            if (max_items) {
+               const { overwrite } = c.req.valid("query");
+               const { count } = await this.media.em.repository(mediaEntity).count(mediaRef);
+
+               // if there are more than or equal to max items
+               if (count >= max_items) {
+                  // if overwrite not set, abort early
+                  if (!overwrite) {
+                     return c.json({ error: `Max items (${max_items}) reached` }, 400);
+                  }
+
+                  // if already more in database than allowed, abort early
+                  // because we don't know if we can delete multiple items
+                  if (count > max_items) {
+                     return c.json(
+                        { error: `Max items (${max_items}) exceeded already with ${count} items.` },
+                        400
+                     );
+                  }
+
+                  // collect items to delete
+                  const deleteRes = await this.media.em.repo(mediaEntity).findMany({
+                     select: [id_field],
+                     where: mediaRef,
+                     sort: {
+                        by: id_field,
+                        dir: "asc"
+                     },
+                     limit: count - max_items + 1
+                  });
+
+                  if (deleteRes.data && deleteRes.data.length > 0) {
+                     deleteRes.data.map((item) => ids_to_delete.push(item[id_field]));
+                  }
+               }
+            }
+
+            // check if entity exists in database
+            const { exists } = await this.media.em.repository(entity).exists({ id: entity_id });
+            if (!exists) {
+               return c.json(
+                  { error: `Entity "${entity_name}" with ID "${entity_id}" doesn't exist found` },
+                  404
+               );
+            }
+
+            const file = await this.getStorage().getFileFromRequest(c);
+            const file_name = getRandomizedFilename(file as File);
+            const info = await this.getStorage().uploadFile(file, file_name, true);
+
+            const mutator = this.media.em.mutator(mediaEntity);
+            mutator.__unstable_toggleSystemEntityCreation(false);
+            const result = await mutator.insertOne({
+               ...this.media.uploadedEventDataToMediaPayload(info),
+               ...mediaRef
+            });
+            mutator.__unstable_toggleSystemEntityCreation(true);
+
+            // delete items if needed
+            if (ids_to_delete.length > 0) {
+               await this.media.em
+                  .mutator(mediaEntity)
+                  .deleteMany({ [id_field]: { $in: ids_to_delete } });
+            }
+
+            return c.json({ ok: true, result: result.data, ...info });
+         }
+      );
+
+      return hono;
+   }
+}

app/src/media/index.ts

@@ -0,0 +1,54 @@
+import type { TObject, TString } from "@sinclair/typebox";
+import { type Constructor, Registry } from "core";
+
+export { MIME_TYPES } from "./storage/mime-types";
+export {
+   Storage,
+   type StorageAdapter,
+   type FileMeta,
+   type FileListObject,
+   type StorageConfig
+} from "./storage/Storage";
+import type { StorageAdapter } from "./storage/Storage";
+import {
+   type CloudinaryConfig,
+   StorageCloudinaryAdapter
+} from "./storage/adapters/StorageCloudinaryAdapter";
+import { type S3AdapterConfig, StorageS3Adapter } from "./storage/adapters/StorageS3Adapter";
+
+export { StorageS3Adapter, type S3AdapterConfig, StorageCloudinaryAdapter, type CloudinaryConfig };
+/*export {
+   StorageLocalAdapter,
+   type LocalAdapterConfig
+} from "./storage/adapters/StorageLocalAdapter";*/
+
+export * as StorageEvents from "./storage/events";
+export { type FileUploadedEventData } from "./storage/events";
+export * from "./utils";
+
+type ClassThatImplements<T> = Constructor<T> & { prototype: T };
+
+export const MediaAdapterRegistry = new Registry<{
+   cls: ClassThatImplements<StorageAdapter>;
+   schema: TObject;
+}>().set({
+   s3: {
+      cls: StorageS3Adapter,
+      schema: StorageS3Adapter.prototype.getSchema()
+   },
+   cloudinary: {
+      cls: StorageCloudinaryAdapter,
+      schema: StorageCloudinaryAdapter.prototype.getSchema()
+   }
+});
+
+export const Adapters = {
+   s3: {
+      cls: StorageS3Adapter,
+      schema: StorageS3Adapter.prototype.getSchema()
+   },
+   cloudinary: {
+      cls: StorageCloudinaryAdapter,
+      schema: StorageCloudinaryAdapter.prototype.getSchema()
+   }
+} as const;

app/src/media/media-schema.ts

@@ -0,0 +1,49 @@
+import { Const, Type, objectTransform } from "core/utils";
+import { Adapters } from "media";
+import { registries } from "modules/registries";
+
+export const ADAPTERS = {
+   ...Adapters
+} as const;
+
+export const registry = registries.media;
+
+export function buildMediaSchema() {
+   const adapterSchemaObject = objectTransform(registry.all(), (adapter, name) => {
+      return Type.Object(
+         {
+            type: Const(name),
+            config: adapter.schema
+         },
+         {
+            title: name,
+            additionalProperties: false
+         }
+      );
+   });
+   const adapterSchema = Type.Union(Object.values(adapterSchemaObject));
+
+   return Type.Object(
+      {
+         enabled: Type.Boolean({ default: false }),
+         basepath: Type.String({ default: "/api/media" }),
+         entity_name: Type.String({ default: "media" }),
+         storage: Type.Object(
+            {
+               body_max_size: Type.Optional(
+                  Type.Number({
+                     description: "Max size of the body in bytes. Leave blank for unlimited."
+                  })
+               )
+            },
+            { default: {} }
+         ),
+         adapter: Type.Optional(adapterSchema)
+      },
+      {
+         additionalProperties: false
+      }
+   );
+}
+
+export const mediaConfigSchema = buildMediaSchema();

app/src/media/storage/Storage.ts

@@ -0,0 +1,228 @@
+import { type EmitsEvents, EventManager } from "core/events";
+import type { TSchema } from "core/utils";
+import { type Context, Hono } from "hono";
+import { bodyLimit } from "hono/body-limit";
+import * as StorageEvents from "./events";
+import type { FileUploadedEventData } from "./events";
+
+export type FileListObject = {
+   key: string;
+   last_modified: Date;
+   size: number;
+};
+
+export type FileMeta = { type: string; size: number };
+export type FileBody = ReadableStream | ArrayBuffer | ArrayBufferView | string | null | Blob | File;
+export type FileUploadPayload = {
+   name: string;
+   meta: FileMeta;
+   etag: string;
+};
+
+export interface StorageAdapter {
+   /**
+    * The unique name of the storage adapter
+    */
+   getName(): string;
+
+   // @todo: method requires limit/offset parameters
+   listObjects(prefix?: string): Promise<FileListObject[]>;
+   putObject(key: string, body: FileBody): Promise<string | FileUploadPayload | undefined>;
+   deleteObject(key: string): Promise<void>;
+   objectExists(key: string): Promise<boolean>;
+   getObject(key: string, headers: Headers): Promise<Response>;
+   getObjectUrl(key: string): string;
+   getObjectMeta(key: string): Promise<FileMeta>;
+   getSchema(): TSchema | undefined;
+   toJSON(secrets?: boolean): any;
+}
+
+export type StorageConfig = {
+   body_max_size: number;
+};
+
+export class Storage implements EmitsEvents {
+   readonly #adapter: StorageAdapter;
+   static readonly Events = StorageEvents;
+   readonly emgr: EventManager<typeof Storage.Events>;
+   readonly config: StorageConfig;
+
+   constructor(
+      adapter: StorageAdapter,
+      config: Partial<StorageConfig> = {},
+      emgr?: EventManager<any>
+   ) {
+      this.#adapter = adapter;
+      this.config = {
+         ...config,
+         body_max_size: config.body_max_size ?? 20 * 1024 * 1024
+      };
+
+      this.emgr = emgr ?? new EventManager();
+      this.emgr.registerEvents(Storage.Events);
+   }
+
+   getAdapter(): StorageAdapter {
+      return this.#adapter;
+   }
+
+   async objectMetadata(key: string): Promise<FileMeta> {
+      return await this.#adapter.getObjectMeta(key);
+   }
+
+   //randomizeFilename(filename: string): string {}
+
+   getConfig(): StorageConfig {
+      return this.config;
+   }
+
+   async uploadFile(
+      file: FileBody,
+      name: string,
+      noEmit?: boolean
+   ): Promise<FileUploadedEventData> {
+      const result = await this.#adapter.putObject(name, file);
+      console.log("result", result);
+
+      let info: FileUploadPayload;
+
+      switch (typeof result) {
+         case "undefined":
+            throw new Error("Failed to upload file");
+         case "string": {
+            // get object meta
+            const meta = await this.#adapter.getObjectMeta(name);
+            if (!meta) {
+               throw new Error("Failed to get object meta");
+            }
+
+            info = { name, meta, etag: result };
+            break;
+         }
+         case "object":
+            info = result;
+            break;
+      }
+
+      const eventData = {
+         file,
+         ...info,
+         state: {
+            name: info.name,
+            path: info.name
+         }
+      };
+      if (!noEmit) {
+         await this.emgr.emit(new StorageEvents.FileUploadedEvent(eventData));
+      }
+
+      return eventData;
+   }
+
+   async deleteFile(name: string): Promise<void> {
+      await this.#adapter.deleteObject(name);
+      await this.emgr.emit(new StorageEvents.FileDeletedEvent({ name }));
+   }
+
+   async fileExists(name: string) {
+      return await this.#adapter.objectExists(name);
+   }
+
+   getController(): any {
+      // @todo: multiple providers?
+      // @todo: implement range requests
+
+      const hono = new Hono();
+
+      // get files list (temporary)
+      hono.get("/files", async (c) => {
+         const files = await this.#adapter.listObjects();
+         return c.json(files);
+      });
+
+      // get file by name
+      hono.get("/file/:filename", async (c) => {
+         const { filename } = c.req.param();
+         if (!filename) {
+            throw new Error("No file name provided");
+         }
+         //console.log("getting file", filename, headersToObject(c.req.raw.headers));
+
+         await this.emgr.emit(new StorageEvents.FileAccessEvent({ name: filename }));
+         return await this.#adapter.getObject(filename, c.req.raw.headers);
+      });
+
+      // delete a file by name
+      hono.delete("/file/:filename", async (c) => {
+         const { filename } = c.req.param();
+         if (!filename) {
+            throw new Error("No file name provided");
+         }
+         await this.deleteFile(filename);
+
+         return c.json({ message: "File deleted" });
+      });
+
+      // upload file
+      hono.post(
+         "/upload/:filename",
+         bodyLimit({
+            maxSize: this.config.body_max_size,
+            onError: (c: any) => {
+               return c.text(`Payload exceeds ${this.config.body_max_size}`, 413);
+            }
+         }),
+         async (c) => {
+            const { filename } = c.req.param();
+            if (!filename) {
+               throw new Error("No file name provided");
+            }
+
+            const file = await this.getFileFromRequest(c);
+            return c.json(await this.uploadFile(file, filename));
+         }
+      );
+
+      return hono;
+   }
+
+   /**
+    * If uploaded through HttpPie -> ReadableStream
+    * If uploaded in tests -> file == ReadableStream
+    * If uploaded in FE -> content_type:body multipart/form-data; boundary=----WebKitFormBoundary7euoBFF12B0AHWLn
+    * file File {
+    *   size: 223052,
+    *   type: 'image/png',
+    *   name: 'noise_white.png',
+    *   lastModified: 1731743671176
+    * }
+    * @param c
+    */
+   async getFileFromRequest(c: Context): Promise<FileBody> {
+      const content_type = c.req.header("Content-Type") ?? "application/octet-stream";
+      console.log("content_type:body", content_type);
+      const body = c.req.raw.body;
+      if (!body) {
+         throw new Error("No body");
+      }
+
+      let file: FileBody | undefined;
+      if (content_type?.startsWith("multipart/form-data")) {
+         file = (await c.req.formData()).get("file") as File;
+         // @todo: check nextjs, it's not *that* [File] type (but it's uploadable)
+         if (typeof file === "undefined") {
+            throw new Error("No file given at form data 'file'");
+         }
+         /*console.log("file", file);
+         if (!(file instanceof File)) {
+            throw new Error("No file given at form data 'file'");
+         }*/
+      } else if (content_type?.startsWith("application/octet-stream")) {
+         file = body;
+      } else {
+         throw new Error(`Unsupported content type: ${content_type}`);
+      }
+
+      return file;
+   }
+}

app/src/media/storage/adapters/StorageCloudinaryAdapter.ts

@@ -0,0 +1,256 @@
+import { pickHeaders } from "core/utils";
+import { type Static, Type, parse } from "core/utils";
+import type { FileBody, FileListObject, FileMeta, StorageAdapter } from "../Storage";
+
+export const cloudinaryAdapterConfig = Type.Object(
+   {
+      cloud_name: Type.String(),
+      api_key: Type.String(),
+      api_secret: Type.String(),
+      upload_preset: Type.Optional(Type.String())
+   },
+   { title: "Cloudinary" }
+);
+
+export type CloudinaryConfig = Static<typeof cloudinaryAdapterConfig>;
+/*export type CloudinaryConfig = {
+   cloud_name: string;
+   api_key: string;
+   api_secret: string;
+   upload_preset?: string;
+};*/
+
+type CloudinaryObject = {
+   asset_id: string;
+   public_id: string;
+   version: number;
+   version_id: string;
+   signature: string;
+   width?: number;
+   height?: number;
+   format: string;
+   resource_type: string;
+   created_at: string; // date format
+   tags: string[];
+   bytes: number;
+   type: string; // "upload" ?
+   etag: string;
+   placeholder: boolean;
+   url: string;
+   secure_url: string;
+   folder: string;
+   existing: boolean;
+   original_filename: string;
+};
+
+type CloudinaryPutObjectResponse = CloudinaryObject;
+type CloudinaryListObjectsResponse = {
+   total_count: number;
+   time: number;
+   next_cursor: string;
+   resources: (CloudinaryObject & {
+      uploaded_at: string; // date format
+      backup_bytes: number;
+      aspect_ratio?: number;
+      pixels?: number;
+      status: string;
+      access_mode: string;
+   })[];
+};
+
+// @todo: add signed uploads
+export class StorageCloudinaryAdapter implements StorageAdapter {
+   private config: CloudinaryConfig;
+
+   constructor(config: CloudinaryConfig) {
+      this.config = parse(cloudinaryAdapterConfig, config);
+   }
+
+   getSchema() {
+      return cloudinaryAdapterConfig;
+   }
+
+   private getMimeType(object: CloudinaryObject): string {
+      switch (true) {
+         case object.format === "jpeg" || object.format === "jpg":
+            return "image/jpeg";
+      }
+
+      return `${object.resource_type}/${object.format}`;
+   }
+
+   getName(): string {
+      return "cloudinary";
+   }
+
+   private getAuthorizationHeader() {
+      const credentials = btoa(`${this.config.api_key}:${this.config.api_secret}`);
+      return {
+         Authorization: `Basic ${credentials}`
+      };
+   }
+
+   async putObject(_key: string, body: FileBody) {
+      //console.log("_key", _key);
+      // remove extension, as it is added by cloudinary
+      const key = _key.replace(/\.[a-z0-9]{2,5}$/, "");
+      //console.log("key", key);
+
+      const formData = new FormData();
+      formData.append("file", body as any);
+      formData.append("public_id", key);
+      formData.append("api_key", this.config.api_key);
+
+      if (this.config.upload_preset) {
+         formData.append("upload_preset", this.config.upload_preset);
+      }
+
+      const result = await fetch(
+         `https://api.cloudinary.com/v1_1/${this.config.cloud_name}/auto/upload`,
+         {
+            method: "POST",
+            headers: {
+               Accept: "application/json"
+               // content type must be undefined to use correct boundaries
+               //"Content-Type": "multipart/form-data",
+            },
+            body: formData
+         }
+      );
+      //console.log("putObject:cloudinary", formData);
+
+      if (!result.ok) {
+         /*console.log(
+            "failed to upload using cloudinary",
+            Object.fromEntries(formData.entries()),
+            result
+         );*/
+         return undefined;
+      }
+
+      //console.log("putObject:result", result);
+
+      const data = (await result.json()) as CloudinaryPutObjectResponse;
+      //console.log("putObject:result:json", data);
+
+      return {
+         name: data.public_id + "." + data.format,
+         etag: data.etag,
+         meta: {
+            type: this.getMimeType(data),
+            size: data.bytes
+         }
+      };
+   }
+
+   async listObjects(prefix?: string): Promise<FileListObject[]> {
+      const result = await fetch(
+         `https://api.cloudinary.com/v1_1/${this.config.cloud_name}/resources/search`,
+         {
+            method: "GET",
+            headers: {
+               Accept: "application/json",
+               ...this.getAuthorizationHeader()
+            }
+         }
+      );
+      //console.log("result", result);
+
+      if (!result.ok) {
+         throw new Error("Failed to list objects");
+      }
+
+      const data = (await result.json()) as CloudinaryListObjectsResponse;
+      return data.resources.map((item) => ({
+         key: item.public_id,
+         last_modified: new Date(item.uploaded_at),
+         size: item.bytes
+      }));
+   }
+
+   private async headObject(key: string) {
+      const url = this.getObjectUrl(key);
+      return await fetch(url, {
+         method: "GET",
+         headers: {
+            Range: "bytes=0-1"
+         }
+      });
+   }
+
+   async objectExists(key: string): Promise<boolean> {
+      //console.log("--object exists?", key);
+      const result = await this.headObject(key);
+      //console.log("object exists", result);
+
+      return result.ok;
+   }
+
+   async getObjectMeta(key: string): Promise<FileMeta> {
+      const result = await this.headObject(key);
+      if (result.ok) {
+         const type = result.headers.get("content-type");
+         const size = Number(result.headers.get("content-range")?.split("/")[1]);
+         return {
+            type: type as string,
+            size: size
+         };
+      }
+
+      throw new Error("Cannot get object meta");
+   }
+
+   private guessType(key: string): string | undefined {
+      const extensions = {
+         image: ["jpg", "jpeg", "png", "gif", "webp", "svg"],
+         video: ["mp4", "webm", "ogg"]
+      };
+
+      const ext = key.split(".").pop();
+      return Object.keys(extensions).find((type) => extensions[type].includes(ext));
+   }
+
+   getObjectUrl(key: string): string {
+      const type = this.guessType(key) ?? "image";
+
+      const objectUrl = `https://res.cloudinary.com/${this.config.cloud_name}/${type}/upload/${key}`;
+      //console.log("objectUrl", objectUrl);
+      return objectUrl;
+   }
+
+   async getObject(key: string, headers: Headers): Promise<Response> {
+      //console.log("url", this.getObjectUrl(key));
+      const res = await fetch(this.getObjectUrl(key), {
+         method: "GET",
+         headers: pickHeaders(headers, ["range"])
+      });
+
+      return new Response(res.body, {
+         status: res.status,
+         statusText: res.statusText,
+         headers: res.headers
+      });
+   }
+
+   async deleteObject(key: string): Promise<void> {
+      const type = this.guessType(key) ?? "image";
+      const formData = new FormData();
+      formData.append("public_ids[]", key);
+
+      const result = await fetch(
+         `https://res.cloudinary.com/${this.config.cloud_name}/${type}/upload/`,
+         {
+            method: "DELETE",
+            body: formData
+         }
+      );
+      //console.log("deleteObject:result", result);
+   }
+
+   toJSON(secrets?: boolean) {
+      return {
+         type: "cloudinary",
+         config: secrets ? this.config : { cloud_name: this.config.cloud_name }
+      };
+   }
+}

app/src/media/storage/adapters/StorageLocalAdapter/StorageLocalAdapter.ts

@@ -0,0 +1,125 @@
+import { readFile, readdir, stat, unlink, writeFile } from "node:fs/promises";
+import { type Static, Type, parse } from "core/utils";
+import type {
+   FileBody,
+   FileListObject,
+   FileMeta,
+   FileUploadPayload,
+   StorageAdapter
+} from "../../Storage";
+import { guessMimeType } from "../../mime-types";
+
+export const localAdapterConfig = Type.Object(
+   {
+      path: Type.String()
+   },
+   { title: "Local" }
+);
+export type LocalAdapterConfig = Static<typeof localAdapterConfig>;
+
+export class StorageLocalAdapter implements StorageAdapter {
+   private config: LocalAdapterConfig;
+
+   constructor(config: any) {
+      this.config = parse(localAdapterConfig, config);
+   }
+
+   getSchema() {
+      return localAdapterConfig;
+   }
+
+   getName(): string {
+      return "local";
+   }
+
+   async listObjects(prefix?: string): Promise<FileListObject[]> {
+      const files = await readdir(this.config.path);
+      const fileStats = await Promise.all(
+         files
+            .filter((file) => !prefix || file.startsWith(prefix))
+            .map(async (file) => {
+               const stats = await stat(`${this.config.path}/${file}`);
+               return {
+                  key: file,
+                  last_modified: stats.mtime,
+                  size: stats.size
+               };
+            })
+      );
+      return fileStats;
+   }
+
+   private async computeEtag(content: BufferSource): Promise<string> {
+      const hashBuffer = await crypto.subtle.digest("SHA-256", content);
+      const hashArray = Array.from(new Uint8Array(hashBuffer));
+      const hashHex = hashArray.map((byte) => byte.toString(16).padStart(2, "0")).join("");
+
+      // Wrap the hex string in quotes for ETag format
+      return `"${hashHex}"`;
+   }
+
+   async putObject(key: string, body: FileBody): Promise<string> {
+      if (body === null) {
+         throw new Error("Body is empty");
+      }
+
+      // @todo: this is too hacky
+      const file = body as File;
+
+      const filePath = `${this.config.path}/${key}`;
+      await writeFile(filePath, file.stream());
+      return await this.computeEtag(await file.arrayBuffer());
+   }
+
+   async deleteObject(key: string): Promise<void> {
+      try {
+         await unlink(`${this.config.path}/${key}`);
+      } catch (e) {}
+   }
+
+   async objectExists(key: string): Promise<boolean> {
+      try {
+         const stats = await stat(`${this.config.path}/${key}`);
+         return stats.isFile();
+      } catch (error) {
+         return false;
+      }
+   }
+
+   async getObject(key: string, headers: Headers): Promise<Response> {
+      try {
+         const content = await readFile(`${this.config.path}/${key}`);
+         const mimeType = guessMimeType(key);
+
+         return new Response(content, {
+            status: 200,
+            headers: {
+               "Content-Type": mimeType || "application/octet-stream",
+               "Content-Length": content.length.toString()
+            }
+         });
+      } catch (error) {
+         // Handle file reading errors
+         return new Response("", { status: 404 });
+      }
+   }
+
+   getObjectUrl(key: string): string {
+      throw new Error("Method not implemented.");
+   }
+
+   async getObjectMeta(key: string): Promise<FileMeta> {
+      const stats = await stat(`${this.config.path}/${key}`);
+      return {
+         type: guessMimeType(key) || "application/octet-stream",
+         size: stats.size
+      };
+   }
+
+   toJSON(secrets?: boolean) {
+      return {
+         type: this.getName(),
+         config: this.config
+      };
+   }
+}

app/src/media/storage/adapters/StorageLocalAdapter/index.ts

@@ -0,0 +1,5 @@
+export {
+   StorageLocalAdapter,
+   type LocalAdapterConfig,
+   localAdapterConfig
+} from "./StorageLocalAdapter";

app/src/media/storage/adapters/StorageR2Adapter.ts

@@ -0,0 +1,137 @@
+import { isDebug } from "core";
+import type { FileBody, StorageAdapter } from "../Storage";
+import { guessMimeType } from "../mime-types";
+
+/**
+ * Adapter for R2 storage
+ * @todo: add tests (bun tests won't work, need node native tests)
+ */
+export class StorageR2Adapter implements StorageAdapter {
+   constructor(private readonly bucket: R2Bucket) {}
+
+   getName(): string {
+      return "r2";
+   }
+
+   getSchema() {
+      return undefined;
+   }
+
+   async putObject(key: string, body: FileBody) {
+      try {
+         const res = await this.bucket.put(key, body);
+         return res?.etag;
+      } catch (e) {
+         return undefined;
+      }
+   }
+   async listObjects(
+      prefix?: string
+   ): Promise<{ key: string; last_modified: Date; size: number }[]> {
+      const list = await this.bucket.list({ limit: 50 });
+      return list.objects.map((item) => ({
+         key: item.key,
+         size: item.size,
+         last_modified: item.uploaded
+      }));
+   }
+
+   private async headObject(key: string): Promise<R2Object | null> {
+      return await this.bucket.head(key);
+   }
+
+   async objectExists(key: string): Promise<boolean> {
+      return (await this.headObject(key)) !== null;
+   }
+
+   async getObject(key: string, headers: Headers): Promise<Response> {
+      let object: R2ObjectBody | null;
+      const responseHeaders = new Headers({
+         "Accept-Ranges": "bytes"
+      });
+
+      //console.log("getObject:headers", headersToObject(headers));
+      if (headers.has("range")) {
+         const options = isDebug()
+            ? {} // miniflare doesn't support range requests
+            : {
+                 range: headers,
+                 onlyIf: headers
+              };
+         object = (await this.bucket.get(key, options)) as R2ObjectBody;
+
+         if (!object) {
+            return new Response(null, { status: 404 });
+         }
+
+         if (object.range) {
+            const offset = "offset" in object.range ? object.range.offset : 0;
+            const end = "end" in object.range ? object.range.end : object.size - 1;
+            responseHeaders.set("Content-Range", `bytes ${offset}-${end}/${object.size}`);
+            responseHeaders.set("Connection", "keep-alive");
+            responseHeaders.set("Vary", "Accept-Encoding");
+         }
+      } else {
+         object = (await this.bucket.get(key)) as R2ObjectBody;
+
+         if (object === null) {
+            return new Response(null, { status: 404 });
+         }
+      }
+
+      //console.log("response headers:before", headersToObject(responseHeaders));
+      this.writeHttpMetadata(responseHeaders, object);
+      responseHeaders.set("etag", object.httpEtag);
+      responseHeaders.set("Content-Length", String(object.size));
+      responseHeaders.set("Last-Modified", object.uploaded.toUTCString());
+      //console.log("response headers:after", headersToObject(responseHeaders));
+
+      return new Response(object.body, {
+         status: object.range ? 206 : 200,
+         headers: responseHeaders
+      });
+   }
+
+   private writeHttpMetadata(headers: Headers, object: R2Object | R2ObjectBody): void {
+      let metadata = object.httpMetadata;
+      if (!metadata || Object.keys(metadata).length === 0) {
+         // guessing is especially required for dev environment (miniflare)
+         metadata = {
+            contentType: guessMimeType(object.key)
+         };
+      }
+      //console.log("writeHttpMetadata", object.httpMetadata, metadata);
+
+      for (const [key, value] of Object.entries(metadata)) {
+         const camelToDash = key.replace(/([A-Z])/g, "-$1").toLowerCase();
+         headers.set(camelToDash, value);
+      }
+   }
+
+   async getObjectMeta(key: string): Promise<{ type: string; size: number }> {
+      const head = await this.headObject(key);
+      if (!head) {
+         throw new Error("Object not found");
+      }
+
+      return {
+         type: String(head.httpMetadata?.contentType ?? "application/octet-stream"),
+         size: head.size
+      };
+   }
+
+   async deleteObject(key: string): Promise<void> {
+      await this.bucket.delete(key);
+   }
+
+   getObjectUrl(key: string): string {
+      throw new Error("Method getObjectUrl not implemented.");
+   }
+
+   toJSON(secrets?: boolean) {
+      return {
+         type: this.getName(),
+         config: {}
+      };
+   }
+}

app/src/media/storage/adapters/StorageS3Adapter.ts

@@ -0,0 +1,213 @@
+import type {
+   DeleteObjectRequest,
+   GetObjectRequest,
+   HeadObjectRequest,
+   ListObjectsV2Output,
+   ListObjectsV2Request,
+   PutObjectRequest
+} from "@aws-sdk/client-s3";
+import { AwsClient, isDebug } from "core";
+import { type Static, Type, parse, pickHeaders } from "core/utils";
+import { transform } from "lodash-es";
+import type { FileBody, FileListObject, StorageAdapter } from "../Storage";
+
+export const s3AdapterConfig = Type.Object(
+   {
+      access_key: Type.String(),
+      secret_access_key: Type.String(),
+      url: Type.String({
+         pattern: "^https?://(?:.*)?[^/.]+$",
+         description: "URL to S3 compatible endpoint without trailing slash",
+         examples: [
+            "https://{account_id}.r2.cloudflarestorage.com/{bucket}",
+            "https://{bucket}.s3.{region}.amazonaws.com"
+         ]
+      })
+   },
+   {
+      title: "S3"
+   }
+);
+
+export type S3AdapterConfig = Static<typeof s3AdapterConfig>;
+
+export class StorageS3Adapter extends AwsClient implements StorageAdapter {
+   readonly #config: S3AdapterConfig;
+
+   constructor(config: S3AdapterConfig) {
+      super(
+         {
+            accessKeyId: config.access_key,
+            secretAccessKey: config.secret_access_key,
+            retries: isDebug() ? 0 : 10
+         },
+         {
+            convertParams: "pascalToKebab",
+            responseType: "xml"
+         }
+      );
+      this.#config = parse(s3AdapterConfig, config);
+   }
+
+   getName(): string {
+      return "s3";
+   }
+
+   getSchema() {
+      return s3AdapterConfig;
+   }
+
+   override getUrl(path: string = "", searchParamsObj: Record<string, any> = {}): string {
+      let url = this.getObjectUrl("").slice(0, -1);
+      if (path.length > 0) url += `/${path}`;
+      return super.getUrl(url, searchParamsObj);
+   }
+
+   /**
+    * Returns the URL of an object
+    * @param key the key of the object
+    */
+   getObjectUrl(key: string): string {
+      return `${this.#config.url}/${key}`;
+   }
+
+   /**
+    * https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
+    */
+   async listObjects(key: string = ""): Promise<FileListObject[]> {
+      const params: Omit<ListObjectsV2Request, "Bucket"> & { ListType: number } = {
+         ListType: 2,
+         Prefix: key
+      };
+
+      const url = this.getUrl("", params);
+      //console.log("url", url);
+      const res = await this.fetchJson<{ ListBucketResult: ListObjectsV2Output }>(url, {
+         method: "GET"
+      });
+      //console.log("res", res);
+
+      // absolutely weird, but if only one object is there, it's an object, not an array
+      const { Contents } = res.ListBucketResult;
+      const objects = !Contents ? [] : Array.isArray(Contents) ? Contents : [Contents];
+
+      //console.log(JSON.stringify(res.ListBucketResult, null, 2), objects);
+      const transformed = transform(
+         objects,
+         (acc, obj) => {
+            // s3 contains folders, but Size is 0, which is filtered here
+            if (obj.Key && obj.LastModified && obj.Size) {
+               acc.push({
+                  key: obj.Key,
+                  last_modified: obj.LastModified,
+                  size: obj.Size
+               });
+            }
+         },
+         [] as FileListObject[]
+      );
+      //console.log(transformed);
+
+      return transformed;
+   }
+
+   async putObject(
+      key: string,
+      body: FileBody | null,
+      // @todo: params must be added as headers, skipping for now
+      params: Omit<PutObjectRequest, "Bucket" | "Key"> = {}
+   ) {
+      const url = this.getUrl(key, {});
+      //console.log("url", url);
+      const res = await this.fetch(url, {
+         method: "PUT",
+         body
+      });
+      /*console.log("putObject:raw:res", {
+         ok: res.ok,
+         status: res.status,
+         statusText: res.statusText,
+      });*/
+
+      if (res.ok) {
+         // "df20fcb574dba1446cf5ec997940492b"
+         return String(res.headers.get("etag"));
+      }
+
+      return undefined;
+   }
+
+   private async headObject(
+      key: string,
+      params: Pick<HeadObjectRequest, "PartNumber" | "VersionId"> = {}
+   ) {
+      const url = this.getUrl(key, {});
+      return await this.fetch(url, {
+         method: "HEAD",
+         headers: {
+            Range: "bytes=0-1"
+         }
+      });
+   }
+
+   async getObjectMeta(key: string) {
+      const res = await this.headObject(key);
+      const type = String(res.headers.get("content-type"));
+      const size = Number(String(res.headers.get("content-range")?.split("/")[1]));
+
+      return {
+         type,
+         size
+      };
+   }
+
+   /**
+    * Check if an object exists by fetching the first byte of the object
+    * @param key
+    * @param params
+    */
+   async objectExists(
+      key: string,
+      params: Pick<HeadObjectRequest, "PartNumber" | "VersionId"> = {}
+   ) {
+      return (await this.headObject(key)).ok;
+   }
+
+   /**
+    * Simply returns the Response of the object to download body as needed
+    */
+   async getObject(key: string, headers: Headers): Promise<Response> {
+      const url = this.getUrl(key);
+      const res = await this.fetch(url, {
+         method: "GET",
+         headers: pickHeaders(headers, ["range"])
+      });
+
+      // Response has to be copied, because of middlewares that might set headers
+      return new Response(res.body, {
+         status: res.status,
+         statusText: res.statusText,
+         headers: res.headers
+      });
+   }
+
+   /**
+    * Deletes a single object. Method is void, because it doesn't return anything
+    */
+   async deleteObject(
+      key: string,
+      params: Omit<DeleteObjectRequest, "Bucket" | "Key"> = {}
+   ): Promise<void> {
+      const url = this.getUrl(key, params);
+      const res = await this.fetch(url, {
+         method: "DELETE"
+      });
+   }
+
+   toJSON(secrets?: boolean) {
+      return {
+         type: this.getName(),
+         config: secrets ? this.#config : undefined
+      };
+   }
+}

app/src/media/storage/events/index.ts

@@ -0,0 +1,17 @@
+import { Event } from "core/events";
+import type { FileBody, FileUploadPayload } from "../Storage";
+
+export type FileUploadedEventData = FileUploadPayload & {
+   file: FileBody;
+};
+export class FileUploadedEvent extends Event<FileUploadedEventData> {
+   static override slug = "file-uploaded";
+}
+
+export class FileDeletedEvent extends Event<{ name: string }> {
+   static override slug = "file-deleted";
+}
+
+export class FileAccessEvent extends Event<{ name: string }> {
+   static override slug = "file-access";
+}

app/src/media/utils/index.ts

@@ -0,0 +1,21 @@
+import { randomString } from "core/utils";
+
+export function getExtension(filename: string): string | undefined {
+   if (!filename.includes(".")) return;
+
+   const parts = filename.split(".");
+   return parts[parts.length - 1];
+}
+
+export function getRandomizedFilename(file: File, length?: number): string;
+export function getRandomizedFilename(file: string, length?: number): string;
+export function getRandomizedFilename(file: File | string, length = 16): string {
+   const filename = file instanceof File ? file.name : file;
+
+   if (typeof filename !== "string") {
+      console.error("Couldn't extract filename from", file);
+      throw new Error("Invalid file name");
+   }
+
+   return [randomString(length), getExtension(filename)].filter(Boolean).join(".");
+}