shishantbiswas/bknd
1
0
Fork 0
mirror of https://github.com/shishantbiswas/bknd.git synced 2026-03-16 12:37:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

updated media api and added tests, fixed body limit

Browse Source
This commit is contained in:
dswbx
2025-02-14 10:42:36 +01:00
parent cc938db4b8
commit c4e505582b
18 changed files with 769 additions and 251 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
app/src/media/api/MediaApi.ts
View File

@@ -14,8 +14,28 @@ export class MediaApi extends ModuleApi<MediaApiOptions> {
return this.get(["files"]);
}
getFile(filename: string) {
return this.get(["file", filename]);
getFileResponse(filename: string) {
return this.get(["file", filename], undefined, {
headers: {
Accept: "*/*"
}
});
}
async getFile(filename: string): Promise<Blob> {
const { res } = await this.getFileResponse(filename);
if (!res.ok || !res.body) {
throw new Error("Failed to fetch file");
}
return await res.blob();
}
async getFileStream(filename: string): Promise<ReadableStream<Uint8Array>> {
const { res } = await this.getFileResponse(filename);
if (!res.ok || !res.body) {
throw new Error("Failed to fetch file");
}
return res.body;
}
getFileUploadUrl(file: FileWithPath): string {
@@ -32,10 +52,46 @@ export class MediaApi extends ModuleApi<MediaApiOptions> {
});
}
uploadFile(file: File) {
const formData = new FormData();
formData.append("file", file);
return this.post(["upload"], formData);
uploadFile(body: File | ReadableStream, filename?: string) {
let type: string = "application/octet-stream";
let name: string = filename || "";
try {
type = (body as File).type;
if (!filename) {
name = (body as File).name;
}
} catch (e) {}
if (name && name.length > 0 && name.includes("/")) {
name = name.split("/").pop() || "";
}
if (!name || name.length === 0) {
throw new Error("Invalid filename");
}
return this.post(["upload", name], body, {
headers: {
"Content-Type": type
}
});
}
async upload(item: Request | Response | string | File | ReadableStream, filename?: string) {
if (item instanceof Request || typeof item === "string") {
const res = await this.fetcher(item);
if (!res.ok || !res.body) {
throw new Error("Failed to fetch file");
}
return this.uploadFile(res.body, filename);
} else if (item instanceof Response) {
if (!item.body) {
throw new Error("Invalid response");
}
return this.uploadFile(item.body, filename);
}
return this.uploadFile(item, filename);
}
deleteFile(filename: string) {

70
app/src/media/api/MediaController.ts
View File

@@ -1,6 +1,5 @@
import { tbValidator as tb } from "core";
import { Type } from "core/utils";
import { bodyLimit } from "hono/body-limit";
import { isDebug, tbValidator as tb } from "core";
import { HttpStatus, Type, getFileFromContext } from "core/utils";
import type { StorageAdapter } from "media";
import { StorageEvents, getRandomizedFilename } from "media";
import { Controller } from "modules/Controller";
@@ -42,7 +41,6 @@ export class MediaController extends Controller {
if (!filename) {
throw new Error("No file name provided");
}
//console.log("getting file", filename, headersToObject(c.req.raw.headers));
await this.getStorage().emgr.emit(new StorageEvents.FileAccessEvent({ name: filename }));
return await this.getStorageAdapter().getObject(filename, c.req.raw.headers);
@@ -59,24 +57,39 @@ export class MediaController extends Controller {
return c.json({ message: "File deleted" });
});
const uploadSizeMiddleware = bodyLimit({
maxSize: this.getStorage().getConfig().body_max_size,
onError: (c: any) => {
return c.text(`Payload exceeds ${this.getStorage().getConfig().body_max_size}`, 413);
}
});
const maxSize = this.getStorage().getConfig().body_max_size ?? Number.POSITIVE_INFINITY;
if (isDebug()) {
hono.post("/inspect", async (c) => {
const file = await getFileFromContext(c);
return c.json({
type: file?.type,
name: file?.name,
size: file?.size
});
});
}
// upload file
// @todo: add required type for "upload endpoints"
hono.post("/upload/:filename", uploadSizeMiddleware, async (c) => {
hono.post("/upload/:filename", async (c) => {
const { filename } = c.req.param();
if (!filename) {
throw new Error("No file name provided");
}
const file = await this.getStorage().getFileFromRequest(c);
console.log("----file", file);
return c.json(await this.getStorage().uploadFile(file, filename));
const body = await getFileFromContext(c);
if (!body) {
return c.json({ error: "No file provided" }, HttpStatus.BAD_REQUEST);
}
if (body.size > maxSize) {
return c.json(
{ error: `Max size (${maxSize} bytes) exceeded` },
HttpStatus.PAYLOAD_TOO_LARGE
);
}
return c.json(await this.getStorage().uploadFile(body, filename), HttpStatus.CREATED);
});
// add upload file to entity
@@ -89,23 +102,21 @@ export class MediaController extends Controller {
overwrite: Type.Optional(booleanLike)
})
),
uploadSizeMiddleware,
async (c) => {
const entity_name = c.req.param("entity");
const field_name = c.req.param("field");
const entity_id = Number.parseInt(c.req.param("id"));
console.log("params", { entity_name, field_name, entity_id });
// check if entity exists
const entity = this.media.em.entity(entity_name);
if (!entity) {
return c.json({ error: `Entity "${entity_name}" not found` }, 404);
return c.json({ error: `Entity "${entity_name}" not found` }, HttpStatus.NOT_FOUND);
}
// check if field exists and is of type MediaField
const field = entity.field(field_name);
if (!field || !(field instanceof MediaField)) {
return c.json({ error: `Invalid field "${field_name}"` }, 400);
return c.json({ error: `Invalid field "${field_name}"` }, HttpStatus.BAD_REQUEST);
}
const media_entity = this.media.getMediaEntity().name as "media";
@@ -127,7 +138,10 @@ export class MediaController extends Controller {
if (count >= max_items) {
// if overwrite not set, abort early
if (!overwrite) {
return c.json({ error: `Max items (${max_items}) reached` }, 400);
return c.json(
{ error: `Max items (${max_items}) reached` },
HttpStatus.BAD_REQUEST
);
}
// if already more in database than allowed, abort early
@@ -135,7 +149,7 @@ export class MediaController extends Controller {
if (count > max_items) {
return c.json(
{ error: `Max items (${max_items}) exceeded already with ${count} items.` },
400
HttpStatus.UNPROCESSABLE_ENTITY
);
}
@@ -161,11 +175,21 @@ export class MediaController extends Controller {
if (!exists) {
return c.json(
{ error: `Entity "${entity_name}" with ID "${entity_id}" doesn't exist found` },
404
HttpStatus.NOT_FOUND
);
}
const file = await getFileFromContext(c);
if (!file) {
return c.json({ error: "No file provided" }, HttpStatus.BAD_REQUEST);
}
if (file.size > maxSize) {
return c.json(
{ error: `Max size (${maxSize} bytes) exceeded` },
HttpStatus.PAYLOAD_TOO_LARGE
);
}
const file = await this.getStorage().getFileFromRequest(c);
const file_name = getRandomizedFilename(file as File);
const info = await this.getStorage().uploadFile(file, file_name, true);
@@ -185,7 +209,7 @@ export class MediaController extends Controller {
}
}
return c.json({ ok: true, result: result.data, ...info });
return c.json({ ok: true, result: result.data, ...info }, HttpStatus.CREATED);
}
);

132
app/src/media/storage/Storage.ts
View File

@@ -1,7 +1,5 @@
import { type EmitsEvents, EventManager } from "core/events";
import type { TSchema } from "core/utils";
import { type Context, Hono } from "hono";
import { bodyLimit } from "hono/body-limit";
import { type TSchema, isFile } from "core/utils";
import * as StorageEvents from "./events";
import type { FileUploadedEventData } from "./events";
@@ -12,7 +10,7 @@ export type FileListObject = {
};
export type FileMeta = { type: string; size: number };
export type FileBody = ReadableStream | ArrayBuffer | ArrayBufferView | string | null | Blob | File;
export type FileBody = ReadableStream | File;
export type FileUploadPayload = {
name: string;
meta: FileMeta;
@@ -38,7 +36,7 @@ export interface StorageAdapter {
}
export type StorageConfig = {
body_max_size: number;
body_max_size?: number;
};
export class Storage implements EmitsEvents {
@@ -55,7 +53,7 @@ export class Storage implements EmitsEvents {
this.#adapter = adapter;
this.config = {
...config,
body_max_size: config.body_max_size ?? 20 * 1024 * 1024
body_max_size: config.body_max_size
};
this.emgr = emgr ?? new EventManager();
@@ -90,13 +88,25 @@ export class Storage implements EmitsEvents {
case "undefined":
throw new Error("Failed to upload file");
case "string": {
// get object meta
const meta = await this.#adapter.getObjectMeta(name);
if (!meta) {
throw new Error("Failed to get object meta");
}
if (isFile(file)) {
info = {
name,
meta: {
size: file.size,
type: file.type
},
etag: result
};
break;
} else {
// get object meta
const meta = await this.#adapter.getObjectMeta(name);
if (!meta) {
throw new Error("Failed to get object meta");
}
info = { name, meta, etag: result };
info = { name, meta, etag: result };
}
break;
}
case "object":
@@ -127,102 +137,4 @@ export class Storage implements EmitsEvents {
async fileExists(name: string) {
return await this.#adapter.objectExists(name);
}
getController(): any {
// @todo: multiple providers?
// @todo: implement range requests
const hono = new Hono();
// get files list (temporary)
hono.get("/files", async (c) => {
const files = await this.#adapter.listObjects();
return c.json(files);
});
// get file by name
hono.get("/file/:filename", async (c) => {
const { filename } = c.req.param();
if (!filename) {
throw new Error("No file name provided");
}
//console.log("getting file", filename, headersToObject(c.req.raw.headers));
await this.emgr.emit(new StorageEvents.FileAccessEvent({ name: filename }));
return await this.#adapter.getObject(filename, c.req.raw.headers);
});
// delete a file by name
hono.delete("/file/:filename", async (c) => {
const { filename } = c.req.param();
if (!filename) {
throw new Error("No file name provided");
}
await this.deleteFile(filename);
return c.json({ message: "File deleted" });
});
// upload file
hono.post(
"/upload/:filename",
bodyLimit({
maxSize: this.config.body_max_size,
onError: (c: any) => {
return c.text(`Payload exceeds ${this.config.body_max_size}`, 413);
}
}),
async (c) => {
const { filename } = c.req.param();
if (!filename) {
throw new Error("No file name provided");
}
const file = await this.getFileFromRequest(c);
return c.json(await this.uploadFile(file, filename));
}
);
return hono;
}
/**
* If uploaded through HttpPie -> ReadableStream
* If uploaded in tests -> file == ReadableStream
* If uploaded in FE -> content_type:body multipart/form-data; boundary=----WebKitFormBoundary7euoBFF12B0AHWLn
* file File {
* size: 223052,
* type: 'image/png',
* name: 'noise_white.png',
* lastModified: 1731743671176
* }
* @param c
*/
async getFileFromRequest(c: Context): Promise<FileBody> {
const content_type = c.req.header("Content-Type") ?? "application/octet-stream";
console.log("content_type:body", content_type);
const body = c.req.raw.body;
if (!body) {
throw new Error("No body");
}
let file: FileBody | undefined;
if (content_type?.startsWith("multipart/form-data")) {
file = (await c.req.formData()).get("file") as File;
// @todo: check nextjs, it's not *that* [File] type (but it's uploadable)
if (typeof file === "undefined") {
throw new Error("No file given at form data 'file'");
}
/*console.log("file", file);
if (!(file instanceof File)) {
throw new Error("No file given at form data 'file'");
}*/
} else if (content_type?.startsWith("application/octet-stream")) {
file = body;
} else {
throw new Error(`Unsupported content type: ${content_type}`);
}
return file;
}
}

35
app/src/media/storage/adapters/StorageCloudinaryAdapter.ts
View File

@@ -13,12 +13,6 @@ export const cloudinaryAdapterConfig = Type.Object(
);
export type CloudinaryConfig = Static<typeof cloudinaryAdapterConfig>;
/*export type CloudinaryConfig = {
cloud_name: string;
api_key: string;
api_secret: string;
upload_preset?: string;
};*/
type CloudinaryObject = {
asset_id: string;
@@ -91,10 +85,8 @@ export class StorageCloudinaryAdapter implements StorageAdapter {
}
async putObject(_key: string, body: FileBody) {
//console.log("_key", _key);
// remove extension, as it is added by cloudinary
const key = _key.replace(/\.[a-z0-9]{2,5}$/, "");
//console.log("key", key);
const formData = new FormData();
formData.append("file", body as any);
@@ -117,21 +109,12 @@ export class StorageCloudinaryAdapter implements StorageAdapter {
body: formData
}
);
//console.log("putObject:cloudinary", formData);
if (!result.ok) {
/*console.log(
"failed to upload using cloudinary",
Object.fromEntries(formData.entries()),
result
);*/
return undefined;
}
//console.log("putObject:result", result);
const data = (await result.json()) as CloudinaryPutObjectResponse;
//console.log("putObject:result:json", data);
return {
name: data.public_id + "." + data.format,
@@ -154,7 +137,6 @@ export class StorageCloudinaryAdapter implements StorageAdapter {
}
}
);
//console.log("result", result);
if (!result.ok) {
throw new Error("Failed to list objects");
@@ -179,10 +161,7 @@ export class StorageCloudinaryAdapter implements StorageAdapter {
}
async objectExists(key: string): Promise<boolean> {
//console.log("--object exists?", key);
const result = await this.headObject(key);
//console.log("object exists", result);
return result.ok;
}
@@ -214,12 +193,10 @@ export class StorageCloudinaryAdapter implements StorageAdapter {
const type = this.guessType(key) ?? "image";
const objectUrl = `https://res.cloudinary.com/${this.config.cloud_name}/${type}/upload/${key}`;
//console.log("objectUrl", objectUrl);
return objectUrl;
}
async getObject(key: string, headers: Headers): Promise<Response> {
//console.log("url", this.getObjectUrl(key));
const res = await fetch(this.getObjectUrl(key), {
method: "GET",
headers: pickHeaders(headers, ["range"])
@@ -237,14 +214,10 @@ export class StorageCloudinaryAdapter implements StorageAdapter {
const formData = new FormData();
formData.append("public_ids[]", key);
const result = await fetch(
`https://res.cloudinary.com/${this.config.cloud_name}/${type}/upload/`,
{
method: "DELETE",
body: formData
}
);
//console.log("deleteObject:result", result);
await fetch(`https://res.cloudinary.com/${this.config.cloud_name}/${type}/upload/`, {
method: "DELETE",
body: formData
});
}
toJSON(secrets?: boolean) {

26
app/src/media/storage/adapters/StorageLocalAdapter/StorageLocalAdapter.ts
View File

@@ -1,6 +1,12 @@
import { readFile, readdir, stat, unlink, writeFile } from "node:fs/promises";
import { type Static, Type, parse } from "core/utils";
import type { FileBody, FileListObject, FileMeta, StorageAdapter } from "../../Storage";
import { type Static, Type, isFile, parse } from "core/utils";
import type {
FileBody,
FileListObject,
FileMeta,
FileUploadPayload,
StorageAdapter
} from "../../Storage";
import { guess } from "../../mime-types-tiny";
export const localAdapterConfig = Type.Object(
@@ -43,8 +49,9 @@ export class StorageLocalAdapter implements StorageAdapter {
return fileStats;
}
private async computeEtag(content: BufferSource): Promise<string> {
const hashBuffer = await crypto.subtle.digest("SHA-256", content);
private async computeEtag(body: FileBody): Promise<string> {
const content = isFile(body) ? body : new Response(body);
const hashBuffer = await crypto.subtle.digest("SHA-256", await content.arrayBuffer());
const hashArray = Array.from(new Uint8Array(hashBuffer));
const hashHex = hashArray.map((byte) => byte.toString(16).padStart(2, "0")).join("");
@@ -52,17 +59,16 @@ export class StorageLocalAdapter implements StorageAdapter {
return `"${hashHex}"`;
}
async putObject(key: string, body: FileBody): Promise<string> {
async putObject(key: string, body: FileBody): Promise<string | FileUploadPayload> {
if (body === null) {
throw new Error("Body is empty");
}
// @todo: this is too hacky
const file = body as File;
const filePath = `${this.config.path}/${key}`;
await writeFile(filePath, file.stream());
return await this.computeEtag(await file.arrayBuffer());
const is_file = isFile(body);
await writeFile(filePath, is_file ? body.stream() : body);
return await this.computeEtag(body);
}
async deleteObject(key: string): Promise<void> {

27
app/src/media/storage/adapters/StorageS3Adapter.ts
View File

@@ -7,7 +7,7 @@ import type {
PutObjectRequest
} from "@aws-sdk/client-s3";
import { AwsClient, isDebug } from "core";
import { type Static, Type, parse, pickHeaders } from "core/utils";
import { type Static, Type, isFile, parse, pickHeaders } from "core/utils";
import { transform } from "lodash-es";
import type { FileBody, FileListObject, StorageAdapter } from "../Storage";
@@ -82,17 +82,14 @@ export class StorageS3Adapter extends AwsClient implements StorageAdapter {
};
const url = this.getUrl("", params);
//console.log("url", url);
const res = await this.fetchJson<{ ListBucketResult: ListObjectsV2Output }>(url, {
method: "GET"
});
//console.log("res", res);
// absolutely weird, but if only one object is there, it's an object, not an array
const { Contents } = res.ListBucketResult;
const objects = !Contents ? [] : Array.isArray(Contents) ? Contents : [Contents];
//console.log(JSON.stringify(res.ListBucketResult, null, 2), objects);
const transformed = transform(
objects,
(acc, obj) => {
@@ -107,32 +104,36 @@ export class StorageS3Adapter extends AwsClient implements StorageAdapter {
},
[] as FileListObject[]
);
//console.log(transformed);
return transformed;
}
async putObject(
key: string,
body: FileBody | null,
body: FileBody,
// @todo: params must be added as headers, skipping for now
params: Omit<PutObjectRequest, "Bucket" | "Key"> = {}
) {
const url = this.getUrl(key, {});
//console.log("url", url);
const res = await this.fetch(url, {
method: "PUT",
body
});
/*console.log("putObject:raw:res", {
ok: res.ok,
status: res.status,
statusText: res.statusText,
});*/
if (res.ok) {
// "df20fcb574dba1446cf5ec997940492b"
return String(res.headers.get("etag"));
const etag = String(res.headers.get("etag"));
if (isFile(body)) {
return {
etag,
name: body.name,
meta: {
size: body.size,
type: body.type
}
};
}
return etag;
}
return undefined;

18
app/src/media/storage/mime-types-tiny.ts
View File

@@ -75,3 +75,21 @@ export function guess(f: string): string {
return c.a();
}
}
export function isMimeType(mime: any, exclude: string[] = []) {
for (const [k, v] of M.entries()) {
if (v === mime && !exclude.includes(k)) {
return true;
}
}
return false;
}
export function extension(mime: string) {
for (const [k, v] of M.entries()) {
if (v === mime) {
return k;
}
}
return "";
}