mirror of
https://github.com/shishantbiswas/bknd.git
synced 2026-03-17 12:56:05 +00:00
public commit
This commit is contained in:
228
app/src/media/storage/Storage.ts
Normal file
228
app/src/media/storage/Storage.ts
Normal file
@@ -0,0 +1,228 @@
|
||||
import { type EmitsEvents, EventManager } from "core/events";
|
||||
import type { TSchema } from "core/utils";
|
||||
import { type Context, Hono } from "hono";
|
||||
import { bodyLimit } from "hono/body-limit";
|
||||
import * as StorageEvents from "./events";
|
||||
import type { FileUploadedEventData } from "./events";
|
||||
|
||||
export type FileListObject = {
|
||||
key: string;
|
||||
last_modified: Date;
|
||||
size: number;
|
||||
};
|
||||
|
||||
export type FileMeta = { type: string; size: number };
|
||||
export type FileBody = ReadableStream | ArrayBuffer | ArrayBufferView | string | null | Blob | File;
|
||||
export type FileUploadPayload = {
|
||||
name: string;
|
||||
meta: FileMeta;
|
||||
etag: string;
|
||||
};
|
||||
|
||||
export interface StorageAdapter {
|
||||
/**
|
||||
* The unique name of the storage adapter
|
||||
*/
|
||||
getName(): string;
|
||||
|
||||
// @todo: method requires limit/offset parameters
|
||||
listObjects(prefix?: string): Promise<FileListObject[]>;
|
||||
putObject(key: string, body: FileBody): Promise<string | FileUploadPayload | undefined>;
|
||||
deleteObject(key: string): Promise<void>;
|
||||
objectExists(key: string): Promise<boolean>;
|
||||
getObject(key: string, headers: Headers): Promise<Response>;
|
||||
getObjectUrl(key: string): string;
|
||||
getObjectMeta(key: string): Promise<FileMeta>;
|
||||
getSchema(): TSchema | undefined;
|
||||
toJSON(secrets?: boolean): any;
|
||||
}
|
||||
|
||||
export type StorageConfig = {
|
||||
body_max_size: number;
|
||||
};
|
||||
|
||||
export class Storage implements EmitsEvents {
|
||||
readonly #adapter: StorageAdapter;
|
||||
static readonly Events = StorageEvents;
|
||||
readonly emgr: EventManager<typeof Storage.Events>;
|
||||
readonly config: StorageConfig;
|
||||
|
||||
constructor(
|
||||
adapter: StorageAdapter,
|
||||
config: Partial<StorageConfig> = {},
|
||||
emgr?: EventManager<any>
|
||||
) {
|
||||
this.#adapter = adapter;
|
||||
this.config = {
|
||||
...config,
|
||||
body_max_size: config.body_max_size ?? 20 * 1024 * 1024
|
||||
};
|
||||
|
||||
this.emgr = emgr ?? new EventManager();
|
||||
this.emgr.registerEvents(Storage.Events);
|
||||
}
|
||||
|
||||
getAdapter(): StorageAdapter {
|
||||
return this.#adapter;
|
||||
}
|
||||
|
||||
async objectMetadata(key: string): Promise<FileMeta> {
|
||||
return await this.#adapter.getObjectMeta(key);
|
||||
}
|
||||
|
||||
//randomizeFilename(filename: string): string {}
|
||||
|
||||
getConfig(): StorageConfig {
|
||||
return this.config;
|
||||
}
|
||||
|
||||
async uploadFile(
|
||||
file: FileBody,
|
||||
name: string,
|
||||
noEmit?: boolean
|
||||
): Promise<FileUploadedEventData> {
|
||||
const result = await this.#adapter.putObject(name, file);
|
||||
console.log("result", result);
|
||||
|
||||
let info: FileUploadPayload;
|
||||
|
||||
switch (typeof result) {
|
||||
case "undefined":
|
||||
throw new Error("Failed to upload file");
|
||||
case "string": {
|
||||
// get object meta
|
||||
const meta = await this.#adapter.getObjectMeta(name);
|
||||
if (!meta) {
|
||||
throw new Error("Failed to get object meta");
|
||||
}
|
||||
|
||||
info = { name, meta, etag: result };
|
||||
break;
|
||||
}
|
||||
case "object":
|
||||
info = result;
|
||||
break;
|
||||
}
|
||||
|
||||
const eventData = {
|
||||
file,
|
||||
...info,
|
||||
state: {
|
||||
name: info.name,
|
||||
path: info.name
|
||||
}
|
||||
};
|
||||
if (!noEmit) {
|
||||
await this.emgr.emit(new StorageEvents.FileUploadedEvent(eventData));
|
||||
}
|
||||
|
||||
return eventData;
|
||||
}
|
||||
|
||||
async deleteFile(name: string): Promise<void> {
|
||||
await this.#adapter.deleteObject(name);
|
||||
await this.emgr.emit(new StorageEvents.FileDeletedEvent({ name }));
|
||||
}
|
||||
|
||||
async fileExists(name: string) {
|
||||
return await this.#adapter.objectExists(name);
|
||||
}
|
||||
|
||||
getController(): any {
|
||||
// @todo: multiple providers?
|
||||
// @todo: implement range requests
|
||||
|
||||
const hono = new Hono();
|
||||
|
||||
// get files list (temporary)
|
||||
hono.get("/files", async (c) => {
|
||||
const files = await this.#adapter.listObjects();
|
||||
return c.json(files);
|
||||
});
|
||||
|
||||
// get file by name
|
||||
hono.get("/file/:filename", async (c) => {
|
||||
const { filename } = c.req.param();
|
||||
if (!filename) {
|
||||
throw new Error("No file name provided");
|
||||
}
|
||||
//console.log("getting file", filename, headersToObject(c.req.raw.headers));
|
||||
|
||||
await this.emgr.emit(new StorageEvents.FileAccessEvent({ name: filename }));
|
||||
return await this.#adapter.getObject(filename, c.req.raw.headers);
|
||||
});
|
||||
|
||||
// delete a file by name
|
||||
hono.delete("/file/:filename", async (c) => {
|
||||
const { filename } = c.req.param();
|
||||
if (!filename) {
|
||||
throw new Error("No file name provided");
|
||||
}
|
||||
await this.deleteFile(filename);
|
||||
|
||||
return c.json({ message: "File deleted" });
|
||||
});
|
||||
|
||||
// upload file
|
||||
hono.post(
|
||||
"/upload/:filename",
|
||||
bodyLimit({
|
||||
maxSize: this.config.body_max_size,
|
||||
onError: (c: any) => {
|
||||
return c.text(`Payload exceeds ${this.config.body_max_size}`, 413);
|
||||
}
|
||||
}),
|
||||
async (c) => {
|
||||
const { filename } = c.req.param();
|
||||
if (!filename) {
|
||||
throw new Error("No file name provided");
|
||||
}
|
||||
|
||||
const file = await this.getFileFromRequest(c);
|
||||
return c.json(await this.uploadFile(file, filename));
|
||||
}
|
||||
);
|
||||
|
||||
return hono;
|
||||
}
|
||||
|
||||
/**
|
||||
* If uploaded through HttpPie -> ReadableStream
|
||||
* If uploaded in tests -> file == ReadableStream
|
||||
* If uploaded in FE -> content_type:body multipart/form-data; boundary=----WebKitFormBoundary7euoBFF12B0AHWLn
|
||||
* file File {
|
||||
* size: 223052,
|
||||
* type: 'image/png',
|
||||
* name: 'noise_white.png',
|
||||
* lastModified: 1731743671176
|
||||
* }
|
||||
* @param c
|
||||
*/
|
||||
async getFileFromRequest(c: Context): Promise<FileBody> {
|
||||
const content_type = c.req.header("Content-Type") ?? "application/octet-stream";
|
||||
console.log("content_type:body", content_type);
|
||||
const body = c.req.raw.body;
|
||||
if (!body) {
|
||||
throw new Error("No body");
|
||||
}
|
||||
|
||||
let file: FileBody | undefined;
|
||||
if (content_type?.startsWith("multipart/form-data")) {
|
||||
file = (await c.req.formData()).get("file") as File;
|
||||
// @todo: check nextjs, it's not *that* [File] type (but it's uploadable)
|
||||
if (typeof file === "undefined") {
|
||||
throw new Error("No file given at form data 'file'");
|
||||
}
|
||||
/*console.log("file", file);
|
||||
if (!(file instanceof File)) {
|
||||
throw new Error("No file given at form data 'file'");
|
||||
}*/
|
||||
} else if (content_type?.startsWith("application/octet-stream")) {
|
||||
file = body;
|
||||
} else {
|
||||
throw new Error(`Unsupported content type: ${content_type}`);
|
||||
}
|
||||
|
||||
return file;
|
||||
}
|
||||
}
|
||||
256
app/src/media/storage/adapters/StorageCloudinaryAdapter.ts
Normal file
256
app/src/media/storage/adapters/StorageCloudinaryAdapter.ts
Normal file
@@ -0,0 +1,256 @@
|
||||
import { pickHeaders } from "core/utils";
|
||||
import { type Static, Type, parse } from "core/utils";
|
||||
import type { FileBody, FileListObject, FileMeta, StorageAdapter } from "../Storage";
|
||||
|
||||
export const cloudinaryAdapterConfig = Type.Object(
|
||||
{
|
||||
cloud_name: Type.String(),
|
||||
api_key: Type.String(),
|
||||
api_secret: Type.String(),
|
||||
upload_preset: Type.Optional(Type.String())
|
||||
},
|
||||
{ title: "Cloudinary" }
|
||||
);
|
||||
|
||||
export type CloudinaryConfig = Static<typeof cloudinaryAdapterConfig>;
|
||||
/*export type CloudinaryConfig = {
|
||||
cloud_name: string;
|
||||
api_key: string;
|
||||
api_secret: string;
|
||||
upload_preset?: string;
|
||||
};*/
|
||||
|
||||
type CloudinaryObject = {
|
||||
asset_id: string;
|
||||
public_id: string;
|
||||
version: number;
|
||||
version_id: string;
|
||||
signature: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
format: string;
|
||||
resource_type: string;
|
||||
created_at: string; // date format
|
||||
tags: string[];
|
||||
bytes: number;
|
||||
type: string; // "upload" ?
|
||||
etag: string;
|
||||
placeholder: boolean;
|
||||
url: string;
|
||||
secure_url: string;
|
||||
folder: string;
|
||||
existing: boolean;
|
||||
original_filename: string;
|
||||
};
|
||||
|
||||
type CloudinaryPutObjectResponse = CloudinaryObject;
|
||||
type CloudinaryListObjectsResponse = {
|
||||
total_count: number;
|
||||
time: number;
|
||||
next_cursor: string;
|
||||
resources: (CloudinaryObject & {
|
||||
uploaded_at: string; // date format
|
||||
backup_bytes: number;
|
||||
aspect_ratio?: number;
|
||||
pixels?: number;
|
||||
status: string;
|
||||
access_mode: string;
|
||||
})[];
|
||||
};
|
||||
|
||||
// @todo: add signed uploads
|
||||
export class StorageCloudinaryAdapter implements StorageAdapter {
|
||||
private config: CloudinaryConfig;
|
||||
|
||||
constructor(config: CloudinaryConfig) {
|
||||
this.config = parse(cloudinaryAdapterConfig, config);
|
||||
}
|
||||
|
||||
getSchema() {
|
||||
return cloudinaryAdapterConfig;
|
||||
}
|
||||
|
||||
private getMimeType(object: CloudinaryObject): string {
|
||||
switch (true) {
|
||||
case object.format === "jpeg" || object.format === "jpg":
|
||||
return "image/jpeg";
|
||||
}
|
||||
|
||||
return `${object.resource_type}/${object.format}`;
|
||||
}
|
||||
|
||||
getName(): string {
|
||||
return "cloudinary";
|
||||
}
|
||||
|
||||
private getAuthorizationHeader() {
|
||||
const credentials = btoa(`${this.config.api_key}:${this.config.api_secret}`);
|
||||
return {
|
||||
Authorization: `Basic ${credentials}`
|
||||
};
|
||||
}
|
||||
|
||||
async putObject(_key: string, body: FileBody) {
|
||||
//console.log("_key", _key);
|
||||
// remove extension, as it is added by cloudinary
|
||||
const key = _key.replace(/\.[a-z0-9]{2,5}$/, "");
|
||||
//console.log("key", key);
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append("file", body as any);
|
||||
formData.append("public_id", key);
|
||||
formData.append("api_key", this.config.api_key);
|
||||
|
||||
if (this.config.upload_preset) {
|
||||
formData.append("upload_preset", this.config.upload_preset);
|
||||
}
|
||||
|
||||
const result = await fetch(
|
||||
`https://api.cloudinary.com/v1_1/${this.config.cloud_name}/auto/upload`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
Accept: "application/json"
|
||||
// content type must be undefined to use correct boundaries
|
||||
//"Content-Type": "multipart/form-data",
|
||||
},
|
||||
body: formData
|
||||
}
|
||||
);
|
||||
//console.log("putObject:cloudinary", formData);
|
||||
|
||||
if (!result.ok) {
|
||||
/*console.log(
|
||||
"failed to upload using cloudinary",
|
||||
Object.fromEntries(formData.entries()),
|
||||
result
|
||||
);*/
|
||||
return undefined;
|
||||
}
|
||||
|
||||
//console.log("putObject:result", result);
|
||||
|
||||
const data = (await result.json()) as CloudinaryPutObjectResponse;
|
||||
//console.log("putObject:result:json", data);
|
||||
|
||||
return {
|
||||
name: data.public_id + "." + data.format,
|
||||
etag: data.etag,
|
||||
meta: {
|
||||
type: this.getMimeType(data),
|
||||
size: data.bytes
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async listObjects(prefix?: string): Promise<FileListObject[]> {
|
||||
const result = await fetch(
|
||||
`https://api.cloudinary.com/v1_1/${this.config.cloud_name}/resources/search`,
|
||||
{
|
||||
method: "GET",
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
...this.getAuthorizationHeader()
|
||||
}
|
||||
}
|
||||
);
|
||||
//console.log("result", result);
|
||||
|
||||
if (!result.ok) {
|
||||
throw new Error("Failed to list objects");
|
||||
}
|
||||
|
||||
const data = (await result.json()) as CloudinaryListObjectsResponse;
|
||||
return data.resources.map((item) => ({
|
||||
key: item.public_id,
|
||||
last_modified: new Date(item.uploaded_at),
|
||||
size: item.bytes
|
||||
}));
|
||||
}
|
||||
|
||||
private async headObject(key: string) {
|
||||
const url = this.getObjectUrl(key);
|
||||
return await fetch(url, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
Range: "bytes=0-1"
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async objectExists(key: string): Promise<boolean> {
|
||||
//console.log("--object exists?", key);
|
||||
const result = await this.headObject(key);
|
||||
//console.log("object exists", result);
|
||||
|
||||
return result.ok;
|
||||
}
|
||||
|
||||
async getObjectMeta(key: string): Promise<FileMeta> {
|
||||
const result = await this.headObject(key);
|
||||
if (result.ok) {
|
||||
const type = result.headers.get("content-type");
|
||||
const size = Number(result.headers.get("content-range")?.split("/")[1]);
|
||||
return {
|
||||
type: type as string,
|
||||
size: size
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error("Cannot get object meta");
|
||||
}
|
||||
|
||||
private guessType(key: string): string | undefined {
|
||||
const extensions = {
|
||||
image: ["jpg", "jpeg", "png", "gif", "webp", "svg"],
|
||||
video: ["mp4", "webm", "ogg"]
|
||||
};
|
||||
|
||||
const ext = key.split(".").pop();
|
||||
return Object.keys(extensions).find((type) => extensions[type].includes(ext));
|
||||
}
|
||||
|
||||
getObjectUrl(key: string): string {
|
||||
const type = this.guessType(key) ?? "image";
|
||||
|
||||
const objectUrl = `https://res.cloudinary.com/${this.config.cloud_name}/${type}/upload/${key}`;
|
||||
//console.log("objectUrl", objectUrl);
|
||||
return objectUrl;
|
||||
}
|
||||
|
||||
async getObject(key: string, headers: Headers): Promise<Response> {
|
||||
//console.log("url", this.getObjectUrl(key));
|
||||
const res = await fetch(this.getObjectUrl(key), {
|
||||
method: "GET",
|
||||
headers: pickHeaders(headers, ["range"])
|
||||
});
|
||||
|
||||
return new Response(res.body, {
|
||||
status: res.status,
|
||||
statusText: res.statusText,
|
||||
headers: res.headers
|
||||
});
|
||||
}
|
||||
|
||||
async deleteObject(key: string): Promise<void> {
|
||||
const type = this.guessType(key) ?? "image";
|
||||
const formData = new FormData();
|
||||
formData.append("public_ids[]", key);
|
||||
|
||||
const result = await fetch(
|
||||
`https://res.cloudinary.com/${this.config.cloud_name}/${type}/upload/`,
|
||||
{
|
||||
method: "DELETE",
|
||||
body: formData
|
||||
}
|
||||
);
|
||||
//console.log("deleteObject:result", result);
|
||||
}
|
||||
|
||||
toJSON(secrets?: boolean) {
|
||||
return {
|
||||
type: "cloudinary",
|
||||
config: secrets ? this.config : { cloud_name: this.config.cloud_name }
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
import { readFile, readdir, stat, unlink, writeFile } from "node:fs/promises";
|
||||
import { type Static, Type, parse } from "core/utils";
|
||||
import type {
|
||||
FileBody,
|
||||
FileListObject,
|
||||
FileMeta,
|
||||
FileUploadPayload,
|
||||
StorageAdapter
|
||||
} from "../../Storage";
|
||||
import { guessMimeType } from "../../mime-types";
|
||||
|
||||
export const localAdapterConfig = Type.Object(
|
||||
{
|
||||
path: Type.String()
|
||||
},
|
||||
{ title: "Local" }
|
||||
);
|
||||
export type LocalAdapterConfig = Static<typeof localAdapterConfig>;
|
||||
|
||||
export class StorageLocalAdapter implements StorageAdapter {
|
||||
private config: LocalAdapterConfig;
|
||||
|
||||
constructor(config: any) {
|
||||
this.config = parse(localAdapterConfig, config);
|
||||
}
|
||||
|
||||
getSchema() {
|
||||
return localAdapterConfig;
|
||||
}
|
||||
|
||||
getName(): string {
|
||||
return "local";
|
||||
}
|
||||
|
||||
async listObjects(prefix?: string): Promise<FileListObject[]> {
|
||||
const files = await readdir(this.config.path);
|
||||
const fileStats = await Promise.all(
|
||||
files
|
||||
.filter((file) => !prefix || file.startsWith(prefix))
|
||||
.map(async (file) => {
|
||||
const stats = await stat(`${this.config.path}/${file}`);
|
||||
return {
|
||||
key: file,
|
||||
last_modified: stats.mtime,
|
||||
size: stats.size
|
||||
};
|
||||
})
|
||||
);
|
||||
return fileStats;
|
||||
}
|
||||
|
||||
private async computeEtag(content: BufferSource): Promise<string> {
|
||||
const hashBuffer = await crypto.subtle.digest("SHA-256", content);
|
||||
const hashArray = Array.from(new Uint8Array(hashBuffer));
|
||||
const hashHex = hashArray.map((byte) => byte.toString(16).padStart(2, "0")).join("");
|
||||
|
||||
// Wrap the hex string in quotes for ETag format
|
||||
return `"${hashHex}"`;
|
||||
}
|
||||
|
||||
async putObject(key: string, body: FileBody): Promise<string> {
|
||||
if (body === null) {
|
||||
throw new Error("Body is empty");
|
||||
}
|
||||
|
||||
// @todo: this is too hacky
|
||||
const file = body as File;
|
||||
|
||||
const filePath = `${this.config.path}/${key}`;
|
||||
await writeFile(filePath, file.stream());
|
||||
return await this.computeEtag(await file.arrayBuffer());
|
||||
}
|
||||
|
||||
async deleteObject(key: string): Promise<void> {
|
||||
try {
|
||||
await unlink(`${this.config.path}/${key}`);
|
||||
} catch (e) {}
|
||||
}
|
||||
|
||||
async objectExists(key: string): Promise<boolean> {
|
||||
try {
|
||||
const stats = await stat(`${this.config.path}/${key}`);
|
||||
return stats.isFile();
|
||||
} catch (error) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async getObject(key: string, headers: Headers): Promise<Response> {
|
||||
try {
|
||||
const content = await readFile(`${this.config.path}/${key}`);
|
||||
const mimeType = guessMimeType(key);
|
||||
|
||||
return new Response(content, {
|
||||
status: 200,
|
||||
headers: {
|
||||
"Content-Type": mimeType || "application/octet-stream",
|
||||
"Content-Length": content.length.toString()
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
// Handle file reading errors
|
||||
return new Response("", { status: 404 });
|
||||
}
|
||||
}
|
||||
|
||||
getObjectUrl(key: string): string {
|
||||
throw new Error("Method not implemented.");
|
||||
}
|
||||
|
||||
async getObjectMeta(key: string): Promise<FileMeta> {
|
||||
const stats = await stat(`${this.config.path}/${key}`);
|
||||
return {
|
||||
type: guessMimeType(key) || "application/octet-stream",
|
||||
size: stats.size
|
||||
};
|
||||
}
|
||||
|
||||
toJSON(secrets?: boolean) {
|
||||
return {
|
||||
type: this.getName(),
|
||||
config: this.config
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
export {
|
||||
StorageLocalAdapter,
|
||||
type LocalAdapterConfig,
|
||||
localAdapterConfig
|
||||
} from "./StorageLocalAdapter";
|
||||
137
app/src/media/storage/adapters/StorageR2Adapter.ts
Normal file
137
app/src/media/storage/adapters/StorageR2Adapter.ts
Normal file
@@ -0,0 +1,137 @@
|
||||
import { isDebug } from "core";
|
||||
import type { FileBody, StorageAdapter } from "../Storage";
|
||||
import { guessMimeType } from "../mime-types";
|
||||
|
||||
/**
|
||||
* Adapter for R2 storage
|
||||
* @todo: add tests (bun tests won't work, need node native tests)
|
||||
*/
|
||||
export class StorageR2Adapter implements StorageAdapter {
|
||||
constructor(private readonly bucket: R2Bucket) {}
|
||||
|
||||
getName(): string {
|
||||
return "r2";
|
||||
}
|
||||
|
||||
getSchema() {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async putObject(key: string, body: FileBody) {
|
||||
try {
|
||||
const res = await this.bucket.put(key, body);
|
||||
return res?.etag;
|
||||
} catch (e) {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
async listObjects(
|
||||
prefix?: string
|
||||
): Promise<{ key: string; last_modified: Date; size: number }[]> {
|
||||
const list = await this.bucket.list({ limit: 50 });
|
||||
return list.objects.map((item) => ({
|
||||
key: item.key,
|
||||
size: item.size,
|
||||
last_modified: item.uploaded
|
||||
}));
|
||||
}
|
||||
|
||||
private async headObject(key: string): Promise<R2Object | null> {
|
||||
return await this.bucket.head(key);
|
||||
}
|
||||
|
||||
async objectExists(key: string): Promise<boolean> {
|
||||
return (await this.headObject(key)) !== null;
|
||||
}
|
||||
|
||||
async getObject(key: string, headers: Headers): Promise<Response> {
|
||||
let object: R2ObjectBody | null;
|
||||
const responseHeaders = new Headers({
|
||||
"Accept-Ranges": "bytes"
|
||||
});
|
||||
|
||||
//console.log("getObject:headers", headersToObject(headers));
|
||||
if (headers.has("range")) {
|
||||
const options = isDebug()
|
||||
? {} // miniflare doesn't support range requests
|
||||
: {
|
||||
range: headers,
|
||||
onlyIf: headers
|
||||
};
|
||||
object = (await this.bucket.get(key, options)) as R2ObjectBody;
|
||||
|
||||
if (!object) {
|
||||
return new Response(null, { status: 404 });
|
||||
}
|
||||
|
||||
if (object.range) {
|
||||
const offset = "offset" in object.range ? object.range.offset : 0;
|
||||
const end = "end" in object.range ? object.range.end : object.size - 1;
|
||||
responseHeaders.set("Content-Range", `bytes ${offset}-${end}/${object.size}`);
|
||||
responseHeaders.set("Connection", "keep-alive");
|
||||
responseHeaders.set("Vary", "Accept-Encoding");
|
||||
}
|
||||
} else {
|
||||
object = (await this.bucket.get(key)) as R2ObjectBody;
|
||||
|
||||
if (object === null) {
|
||||
return new Response(null, { status: 404 });
|
||||
}
|
||||
}
|
||||
|
||||
//console.log("response headers:before", headersToObject(responseHeaders));
|
||||
this.writeHttpMetadata(responseHeaders, object);
|
||||
responseHeaders.set("etag", object.httpEtag);
|
||||
responseHeaders.set("Content-Length", String(object.size));
|
||||
responseHeaders.set("Last-Modified", object.uploaded.toUTCString());
|
||||
//console.log("response headers:after", headersToObject(responseHeaders));
|
||||
|
||||
return new Response(object.body, {
|
||||
status: object.range ? 206 : 200,
|
||||
headers: responseHeaders
|
||||
});
|
||||
}
|
||||
|
||||
private writeHttpMetadata(headers: Headers, object: R2Object | R2ObjectBody): void {
|
||||
let metadata = object.httpMetadata;
|
||||
if (!metadata || Object.keys(metadata).length === 0) {
|
||||
// guessing is especially required for dev environment (miniflare)
|
||||
metadata = {
|
||||
contentType: guessMimeType(object.key)
|
||||
};
|
||||
}
|
||||
//console.log("writeHttpMetadata", object.httpMetadata, metadata);
|
||||
|
||||
for (const [key, value] of Object.entries(metadata)) {
|
||||
const camelToDash = key.replace(/([A-Z])/g, "-$1").toLowerCase();
|
||||
headers.set(camelToDash, value);
|
||||
}
|
||||
}
|
||||
|
||||
async getObjectMeta(key: string): Promise<{ type: string; size: number }> {
|
||||
const head = await this.headObject(key);
|
||||
if (!head) {
|
||||
throw new Error("Object not found");
|
||||
}
|
||||
|
||||
return {
|
||||
type: String(head.httpMetadata?.contentType ?? "application/octet-stream"),
|
||||
size: head.size
|
||||
};
|
||||
}
|
||||
|
||||
async deleteObject(key: string): Promise<void> {
|
||||
await this.bucket.delete(key);
|
||||
}
|
||||
|
||||
getObjectUrl(key: string): string {
|
||||
throw new Error("Method getObjectUrl not implemented.");
|
||||
}
|
||||
|
||||
toJSON(secrets?: boolean) {
|
||||
return {
|
||||
type: this.getName(),
|
||||
config: {}
|
||||
};
|
||||
}
|
||||
}
|
||||
213
app/src/media/storage/adapters/StorageS3Adapter.ts
Normal file
213
app/src/media/storage/adapters/StorageS3Adapter.ts
Normal file
@@ -0,0 +1,213 @@
|
||||
import type {
|
||||
DeleteObjectRequest,
|
||||
GetObjectRequest,
|
||||
HeadObjectRequest,
|
||||
ListObjectsV2Output,
|
||||
ListObjectsV2Request,
|
||||
PutObjectRequest
|
||||
} from "@aws-sdk/client-s3";
|
||||
import { AwsClient, isDebug } from "core";
|
||||
import { type Static, Type, parse, pickHeaders } from "core/utils";
|
||||
import { transform } from "lodash-es";
|
||||
import type { FileBody, FileListObject, StorageAdapter } from "../Storage";
|
||||
|
||||
export const s3AdapterConfig = Type.Object(
|
||||
{
|
||||
access_key: Type.String(),
|
||||
secret_access_key: Type.String(),
|
||||
url: Type.String({
|
||||
pattern: "^https?://(?:.*)?[^/.]+$",
|
||||
description: "URL to S3 compatible endpoint without trailing slash",
|
||||
examples: [
|
||||
"https://{account_id}.r2.cloudflarestorage.com/{bucket}",
|
||||
"https://{bucket}.s3.{region}.amazonaws.com"
|
||||
]
|
||||
})
|
||||
},
|
||||
{
|
||||
title: "S3"
|
||||
}
|
||||
);
|
||||
|
||||
export type S3AdapterConfig = Static<typeof s3AdapterConfig>;
|
||||
|
||||
export class StorageS3Adapter extends AwsClient implements StorageAdapter {
|
||||
readonly #config: S3AdapterConfig;
|
||||
|
||||
constructor(config: S3AdapterConfig) {
|
||||
super(
|
||||
{
|
||||
accessKeyId: config.access_key,
|
||||
secretAccessKey: config.secret_access_key,
|
||||
retries: isDebug() ? 0 : 10
|
||||
},
|
||||
{
|
||||
convertParams: "pascalToKebab",
|
||||
responseType: "xml"
|
||||
}
|
||||
);
|
||||
this.#config = parse(s3AdapterConfig, config);
|
||||
}
|
||||
|
||||
getName(): string {
|
||||
return "s3";
|
||||
}
|
||||
|
||||
getSchema() {
|
||||
return s3AdapterConfig;
|
||||
}
|
||||
|
||||
override getUrl(path: string = "", searchParamsObj: Record<string, any> = {}): string {
|
||||
let url = this.getObjectUrl("").slice(0, -1);
|
||||
if (path.length > 0) url += `/${path}`;
|
||||
return super.getUrl(url, searchParamsObj);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the URL of an object
|
||||
* @param key the key of the object
|
||||
*/
|
||||
getObjectUrl(key: string): string {
|
||||
return `${this.#config.url}/${key}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
|
||||
*/
|
||||
async listObjects(key: string = ""): Promise<FileListObject[]> {
|
||||
const params: Omit<ListObjectsV2Request, "Bucket"> & { ListType: number } = {
|
||||
ListType: 2,
|
||||
Prefix: key
|
||||
};
|
||||
|
||||
const url = this.getUrl("", params);
|
||||
//console.log("url", url);
|
||||
const res = await this.fetchJson<{ ListBucketResult: ListObjectsV2Output }>(url, {
|
||||
method: "GET"
|
||||
});
|
||||
//console.log("res", res);
|
||||
|
||||
// absolutely weird, but if only one object is there, it's an object, not an array
|
||||
const { Contents } = res.ListBucketResult;
|
||||
const objects = !Contents ? [] : Array.isArray(Contents) ? Contents : [Contents];
|
||||
|
||||
//console.log(JSON.stringify(res.ListBucketResult, null, 2), objects);
|
||||
const transformed = transform(
|
||||
objects,
|
||||
(acc, obj) => {
|
||||
// s3 contains folders, but Size is 0, which is filtered here
|
||||
if (obj.Key && obj.LastModified && obj.Size) {
|
||||
acc.push({
|
||||
key: obj.Key,
|
||||
last_modified: obj.LastModified,
|
||||
size: obj.Size
|
||||
});
|
||||
}
|
||||
},
|
||||
[] as FileListObject[]
|
||||
);
|
||||
//console.log(transformed);
|
||||
|
||||
return transformed;
|
||||
}
|
||||
|
||||
async putObject(
|
||||
key: string,
|
||||
body: FileBody | null,
|
||||
// @todo: params must be added as headers, skipping for now
|
||||
params: Omit<PutObjectRequest, "Bucket" | "Key"> = {}
|
||||
) {
|
||||
const url = this.getUrl(key, {});
|
||||
//console.log("url", url);
|
||||
const res = await this.fetch(url, {
|
||||
method: "PUT",
|
||||
body
|
||||
});
|
||||
/*console.log("putObject:raw:res", {
|
||||
ok: res.ok,
|
||||
status: res.status,
|
||||
statusText: res.statusText,
|
||||
});*/
|
||||
|
||||
if (res.ok) {
|
||||
// "df20fcb574dba1446cf5ec997940492b"
|
||||
return String(res.headers.get("etag"));
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
private async headObject(
|
||||
key: string,
|
||||
params: Pick<HeadObjectRequest, "PartNumber" | "VersionId"> = {}
|
||||
) {
|
||||
const url = this.getUrl(key, {});
|
||||
return await this.fetch(url, {
|
||||
method: "HEAD",
|
||||
headers: {
|
||||
Range: "bytes=0-1"
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async getObjectMeta(key: string) {
|
||||
const res = await this.headObject(key);
|
||||
const type = String(res.headers.get("content-type"));
|
||||
const size = Number(String(res.headers.get("content-range")?.split("/")[1]));
|
||||
|
||||
return {
|
||||
type,
|
||||
size
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if an object exists by fetching the first byte of the object
|
||||
* @param key
|
||||
* @param params
|
||||
*/
|
||||
async objectExists(
|
||||
key: string,
|
||||
params: Pick<HeadObjectRequest, "PartNumber" | "VersionId"> = {}
|
||||
) {
|
||||
return (await this.headObject(key)).ok;
|
||||
}
|
||||
|
||||
/**
|
||||
* Simply returns the Response of the object to download body as needed
|
||||
*/
|
||||
async getObject(key: string, headers: Headers): Promise<Response> {
|
||||
const url = this.getUrl(key);
|
||||
const res = await this.fetch(url, {
|
||||
method: "GET",
|
||||
headers: pickHeaders(headers, ["range"])
|
||||
});
|
||||
|
||||
// Response has to be copied, because of middlewares that might set headers
|
||||
return new Response(res.body, {
|
||||
status: res.status,
|
||||
statusText: res.statusText,
|
||||
headers: res.headers
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes a single object. Method is void, because it doesn't return anything
|
||||
*/
|
||||
async deleteObject(
|
||||
key: string,
|
||||
params: Omit<DeleteObjectRequest, "Bucket" | "Key"> = {}
|
||||
): Promise<void> {
|
||||
const url = this.getUrl(key, params);
|
||||
const res = await this.fetch(url, {
|
||||
method: "DELETE"
|
||||
});
|
||||
}
|
||||
|
||||
toJSON(secrets?: boolean) {
|
||||
return {
|
||||
type: this.getName(),
|
||||
config: secrets ? this.#config : undefined
|
||||
};
|
||||
}
|
||||
}
|
||||
17
app/src/media/storage/events/index.ts
Normal file
17
app/src/media/storage/events/index.ts
Normal file
@@ -0,0 +1,17 @@
|
||||
import { Event } from "core/events";
|
||||
import type { FileBody, FileUploadPayload } from "../Storage";
|
||||
|
||||
export type FileUploadedEventData = FileUploadPayload & {
|
||||
file: FileBody;
|
||||
};
|
||||
export class FileUploadedEvent extends Event<FileUploadedEventData> {
|
||||
static override slug = "file-uploaded";
|
||||
}
|
||||
|
||||
export class FileDeletedEvent extends Event<{ name: string }> {
|
||||
static override slug = "file-deleted";
|
||||
}
|
||||
|
||||
export class FileAccessEvent extends Event<{ name: string }> {
|
||||
static override slug = "file-access";
|
||||
}
|
||||
1214
app/src/media/storage/mime-types.ts
Normal file
1214
app/src/media/storage/mime-types.ts
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user