public commit

This commit is contained in:
dswbx
2024-11-16 12:01:47 +01:00
commit 90f80c4280
582 changed files with 49291 additions and 0 deletions

View File

@@ -0,0 +1,228 @@
import { type EmitsEvents, EventManager } from "core/events";
import type { TSchema } from "core/utils";
import { type Context, Hono } from "hono";
import { bodyLimit } from "hono/body-limit";
import * as StorageEvents from "./events";
import type { FileUploadedEventData } from "./events";
export type FileListObject = {
key: string;
last_modified: Date;
size: number;
};
export type FileMeta = { type: string; size: number };
export type FileBody = ReadableStream | ArrayBuffer | ArrayBufferView | string | null | Blob | File;
export type FileUploadPayload = {
name: string;
meta: FileMeta;
etag: string;
};
export interface StorageAdapter {
/**
* The unique name of the storage adapter
*/
getName(): string;
// @todo: method requires limit/offset parameters
listObjects(prefix?: string): Promise<FileListObject[]>;
putObject(key: string, body: FileBody): Promise<string | FileUploadPayload | undefined>;
deleteObject(key: string): Promise<void>;
objectExists(key: string): Promise<boolean>;
getObject(key: string, headers: Headers): Promise<Response>;
getObjectUrl(key: string): string;
getObjectMeta(key: string): Promise<FileMeta>;
getSchema(): TSchema | undefined;
toJSON(secrets?: boolean): any;
}
export type StorageConfig = {
body_max_size: number;
};
export class Storage implements EmitsEvents {
readonly #adapter: StorageAdapter;
static readonly Events = StorageEvents;
readonly emgr: EventManager<typeof Storage.Events>;
readonly config: StorageConfig;
constructor(
adapter: StorageAdapter,
config: Partial<StorageConfig> = {},
emgr?: EventManager<any>
) {
this.#adapter = adapter;
this.config = {
...config,
body_max_size: config.body_max_size ?? 20 * 1024 * 1024
};
this.emgr = emgr ?? new EventManager();
this.emgr.registerEvents(Storage.Events);
}
getAdapter(): StorageAdapter {
return this.#adapter;
}
async objectMetadata(key: string): Promise<FileMeta> {
return await this.#adapter.getObjectMeta(key);
}
//randomizeFilename(filename: string): string {}
getConfig(): StorageConfig {
return this.config;
}
async uploadFile(
file: FileBody,
name: string,
noEmit?: boolean
): Promise<FileUploadedEventData> {
const result = await this.#adapter.putObject(name, file);
console.log("result", result);
let info: FileUploadPayload;
switch (typeof result) {
case "undefined":
throw new Error("Failed to upload file");
case "string": {
// get object meta
const meta = await this.#adapter.getObjectMeta(name);
if (!meta) {
throw new Error("Failed to get object meta");
}
info = { name, meta, etag: result };
break;
}
case "object":
info = result;
break;
}
const eventData = {
file,
...info,
state: {
name: info.name,
path: info.name
}
};
if (!noEmit) {
await this.emgr.emit(new StorageEvents.FileUploadedEvent(eventData));
}
return eventData;
}
async deleteFile(name: string): Promise<void> {
await this.#adapter.deleteObject(name);
await this.emgr.emit(new StorageEvents.FileDeletedEvent({ name }));
}
async fileExists(name: string) {
return await this.#adapter.objectExists(name);
}
getController(): any {
// @todo: multiple providers?
// @todo: implement range requests
const hono = new Hono();
// get files list (temporary)
hono.get("/files", async (c) => {
const files = await this.#adapter.listObjects();
return c.json(files);
});
// get file by name
hono.get("/file/:filename", async (c) => {
const { filename } = c.req.param();
if (!filename) {
throw new Error("No file name provided");
}
//console.log("getting file", filename, headersToObject(c.req.raw.headers));
await this.emgr.emit(new StorageEvents.FileAccessEvent({ name: filename }));
return await this.#adapter.getObject(filename, c.req.raw.headers);
});
// delete a file by name
hono.delete("/file/:filename", async (c) => {
const { filename } = c.req.param();
if (!filename) {
throw new Error("No file name provided");
}
await this.deleteFile(filename);
return c.json({ message: "File deleted" });
});
// upload file
hono.post(
"/upload/:filename",
bodyLimit({
maxSize: this.config.body_max_size,
onError: (c: any) => {
return c.text(`Payload exceeds ${this.config.body_max_size}`, 413);
}
}),
async (c) => {
const { filename } = c.req.param();
if (!filename) {
throw new Error("No file name provided");
}
const file = await this.getFileFromRequest(c);
return c.json(await this.uploadFile(file, filename));
}
);
return hono;
}
/**
* If uploaded through HttpPie -> ReadableStream
* If uploaded in tests -> file == ReadableStream
* If uploaded in FE -> content_type:body multipart/form-data; boundary=----WebKitFormBoundary7euoBFF12B0AHWLn
* file File {
* size: 223052,
* type: 'image/png',
* name: 'noise_white.png',
* lastModified: 1731743671176
* }
* @param c
*/
async getFileFromRequest(c: Context): Promise<FileBody> {
const content_type = c.req.header("Content-Type") ?? "application/octet-stream";
console.log("content_type:body", content_type);
const body = c.req.raw.body;
if (!body) {
throw new Error("No body");
}
let file: FileBody | undefined;
if (content_type?.startsWith("multipart/form-data")) {
file = (await c.req.formData()).get("file") as File;
// @todo: check nextjs, it's not *that* [File] type (but it's uploadable)
if (typeof file === "undefined") {
throw new Error("No file given at form data 'file'");
}
/*console.log("file", file);
if (!(file instanceof File)) {
throw new Error("No file given at form data 'file'");
}*/
} else if (content_type?.startsWith("application/octet-stream")) {
file = body;
} else {
throw new Error(`Unsupported content type: ${content_type}`);
}
return file;
}
}

View File

@@ -0,0 +1,256 @@
import { pickHeaders } from "core/utils";
import { type Static, Type, parse } from "core/utils";
import type { FileBody, FileListObject, FileMeta, StorageAdapter } from "../Storage";
export const cloudinaryAdapterConfig = Type.Object(
{
cloud_name: Type.String(),
api_key: Type.String(),
api_secret: Type.String(),
upload_preset: Type.Optional(Type.String())
},
{ title: "Cloudinary" }
);
export type CloudinaryConfig = Static<typeof cloudinaryAdapterConfig>;
/*export type CloudinaryConfig = {
cloud_name: string;
api_key: string;
api_secret: string;
upload_preset?: string;
};*/
type CloudinaryObject = {
asset_id: string;
public_id: string;
version: number;
version_id: string;
signature: string;
width?: number;
height?: number;
format: string;
resource_type: string;
created_at: string; // date format
tags: string[];
bytes: number;
type: string; // "upload" ?
etag: string;
placeholder: boolean;
url: string;
secure_url: string;
folder: string;
existing: boolean;
original_filename: string;
};
type CloudinaryPutObjectResponse = CloudinaryObject;
type CloudinaryListObjectsResponse = {
total_count: number;
time: number;
next_cursor: string;
resources: (CloudinaryObject & {
uploaded_at: string; // date format
backup_bytes: number;
aspect_ratio?: number;
pixels?: number;
status: string;
access_mode: string;
})[];
};
// @todo: add signed uploads
export class StorageCloudinaryAdapter implements StorageAdapter {
private config: CloudinaryConfig;
constructor(config: CloudinaryConfig) {
this.config = parse(cloudinaryAdapterConfig, config);
}
getSchema() {
return cloudinaryAdapterConfig;
}
private getMimeType(object: CloudinaryObject): string {
switch (true) {
case object.format === "jpeg" || object.format === "jpg":
return "image/jpeg";
}
return `${object.resource_type}/${object.format}`;
}
getName(): string {
return "cloudinary";
}
private getAuthorizationHeader() {
const credentials = btoa(`${this.config.api_key}:${this.config.api_secret}`);
return {
Authorization: `Basic ${credentials}`
};
}
async putObject(_key: string, body: FileBody) {
//console.log("_key", _key);
// remove extension, as it is added by cloudinary
const key = _key.replace(/\.[a-z0-9]{2,5}$/, "");
//console.log("key", key);
const formData = new FormData();
formData.append("file", body as any);
formData.append("public_id", key);
formData.append("api_key", this.config.api_key);
if (this.config.upload_preset) {
formData.append("upload_preset", this.config.upload_preset);
}
const result = await fetch(
`https://api.cloudinary.com/v1_1/${this.config.cloud_name}/auto/upload`,
{
method: "POST",
headers: {
Accept: "application/json"
// content type must be undefined to use correct boundaries
//"Content-Type": "multipart/form-data",
},
body: formData
}
);
//console.log("putObject:cloudinary", formData);
if (!result.ok) {
/*console.log(
"failed to upload using cloudinary",
Object.fromEntries(formData.entries()),
result
);*/
return undefined;
}
//console.log("putObject:result", result);
const data = (await result.json()) as CloudinaryPutObjectResponse;
//console.log("putObject:result:json", data);
return {
name: data.public_id + "." + data.format,
etag: data.etag,
meta: {
type: this.getMimeType(data),
size: data.bytes
}
};
}
async listObjects(prefix?: string): Promise<FileListObject[]> {
const result = await fetch(
`https://api.cloudinary.com/v1_1/${this.config.cloud_name}/resources/search`,
{
method: "GET",
headers: {
Accept: "application/json",
...this.getAuthorizationHeader()
}
}
);
//console.log("result", result);
if (!result.ok) {
throw new Error("Failed to list objects");
}
const data = (await result.json()) as CloudinaryListObjectsResponse;
return data.resources.map((item) => ({
key: item.public_id,
last_modified: new Date(item.uploaded_at),
size: item.bytes
}));
}
private async headObject(key: string) {
const url = this.getObjectUrl(key);
return await fetch(url, {
method: "GET",
headers: {
Range: "bytes=0-1"
}
});
}
async objectExists(key: string): Promise<boolean> {
//console.log("--object exists?", key);
const result = await this.headObject(key);
//console.log("object exists", result);
return result.ok;
}
async getObjectMeta(key: string): Promise<FileMeta> {
const result = await this.headObject(key);
if (result.ok) {
const type = result.headers.get("content-type");
const size = Number(result.headers.get("content-range")?.split("/")[1]);
return {
type: type as string,
size: size
};
}
throw new Error("Cannot get object meta");
}
private guessType(key: string): string | undefined {
const extensions = {
image: ["jpg", "jpeg", "png", "gif", "webp", "svg"],
video: ["mp4", "webm", "ogg"]
};
const ext = key.split(".").pop();
return Object.keys(extensions).find((type) => extensions[type].includes(ext));
}
getObjectUrl(key: string): string {
const type = this.guessType(key) ?? "image";
const objectUrl = `https://res.cloudinary.com/${this.config.cloud_name}/${type}/upload/${key}`;
//console.log("objectUrl", objectUrl);
return objectUrl;
}
async getObject(key: string, headers: Headers): Promise<Response> {
//console.log("url", this.getObjectUrl(key));
const res = await fetch(this.getObjectUrl(key), {
method: "GET",
headers: pickHeaders(headers, ["range"])
});
return new Response(res.body, {
status: res.status,
statusText: res.statusText,
headers: res.headers
});
}
async deleteObject(key: string): Promise<void> {
const type = this.guessType(key) ?? "image";
const formData = new FormData();
formData.append("public_ids[]", key);
const result = await fetch(
`https://res.cloudinary.com/${this.config.cloud_name}/${type}/upload/`,
{
method: "DELETE",
body: formData
}
);
//console.log("deleteObject:result", result);
}
toJSON(secrets?: boolean) {
return {
type: "cloudinary",
config: secrets ? this.config : { cloud_name: this.config.cloud_name }
};
}
}

View File

@@ -0,0 +1,125 @@
import { readFile, readdir, stat, unlink, writeFile } from "node:fs/promises";
import { type Static, Type, parse } from "core/utils";
import type {
FileBody,
FileListObject,
FileMeta,
FileUploadPayload,
StorageAdapter
} from "../../Storage";
import { guessMimeType } from "../../mime-types";
export const localAdapterConfig = Type.Object(
{
path: Type.String()
},
{ title: "Local" }
);
export type LocalAdapterConfig = Static<typeof localAdapterConfig>;
export class StorageLocalAdapter implements StorageAdapter {
private config: LocalAdapterConfig;
constructor(config: any) {
this.config = parse(localAdapterConfig, config);
}
getSchema() {
return localAdapterConfig;
}
getName(): string {
return "local";
}
async listObjects(prefix?: string): Promise<FileListObject[]> {
const files = await readdir(this.config.path);
const fileStats = await Promise.all(
files
.filter((file) => !prefix || file.startsWith(prefix))
.map(async (file) => {
const stats = await stat(`${this.config.path}/${file}`);
return {
key: file,
last_modified: stats.mtime,
size: stats.size
};
})
);
return fileStats;
}
private async computeEtag(content: BufferSource): Promise<string> {
const hashBuffer = await crypto.subtle.digest("SHA-256", content);
const hashArray = Array.from(new Uint8Array(hashBuffer));
const hashHex = hashArray.map((byte) => byte.toString(16).padStart(2, "0")).join("");
// Wrap the hex string in quotes for ETag format
return `"${hashHex}"`;
}
async putObject(key: string, body: FileBody): Promise<string> {
if (body === null) {
throw new Error("Body is empty");
}
// @todo: this is too hacky
const file = body as File;
const filePath = `${this.config.path}/${key}`;
await writeFile(filePath, file.stream());
return await this.computeEtag(await file.arrayBuffer());
}
async deleteObject(key: string): Promise<void> {
try {
await unlink(`${this.config.path}/${key}`);
} catch (e) {}
}
async objectExists(key: string): Promise<boolean> {
try {
const stats = await stat(`${this.config.path}/${key}`);
return stats.isFile();
} catch (error) {
return false;
}
}
async getObject(key: string, headers: Headers): Promise<Response> {
try {
const content = await readFile(`${this.config.path}/${key}`);
const mimeType = guessMimeType(key);
return new Response(content, {
status: 200,
headers: {
"Content-Type": mimeType || "application/octet-stream",
"Content-Length": content.length.toString()
}
});
} catch (error) {
// Handle file reading errors
return new Response("", { status: 404 });
}
}
getObjectUrl(key: string): string {
throw new Error("Method not implemented.");
}
async getObjectMeta(key: string): Promise<FileMeta> {
const stats = await stat(`${this.config.path}/${key}`);
return {
type: guessMimeType(key) || "application/octet-stream",
size: stats.size
};
}
toJSON(secrets?: boolean) {
return {
type: this.getName(),
config: this.config
};
}
}

View File

@@ -0,0 +1,5 @@
export {
StorageLocalAdapter,
type LocalAdapterConfig,
localAdapterConfig
} from "./StorageLocalAdapter";

View File

@@ -0,0 +1,137 @@
import { isDebug } from "core";
import type { FileBody, StorageAdapter } from "../Storage";
import { guessMimeType } from "../mime-types";
/**
* Adapter for R2 storage
* @todo: add tests (bun tests won't work, need node native tests)
*/
export class StorageR2Adapter implements StorageAdapter {
constructor(private readonly bucket: R2Bucket) {}
getName(): string {
return "r2";
}
getSchema() {
return undefined;
}
async putObject(key: string, body: FileBody) {
try {
const res = await this.bucket.put(key, body);
return res?.etag;
} catch (e) {
return undefined;
}
}
async listObjects(
prefix?: string
): Promise<{ key: string; last_modified: Date; size: number }[]> {
const list = await this.bucket.list({ limit: 50 });
return list.objects.map((item) => ({
key: item.key,
size: item.size,
last_modified: item.uploaded
}));
}
private async headObject(key: string): Promise<R2Object | null> {
return await this.bucket.head(key);
}
async objectExists(key: string): Promise<boolean> {
return (await this.headObject(key)) !== null;
}
async getObject(key: string, headers: Headers): Promise<Response> {
let object: R2ObjectBody | null;
const responseHeaders = new Headers({
"Accept-Ranges": "bytes"
});
//console.log("getObject:headers", headersToObject(headers));
if (headers.has("range")) {
const options = isDebug()
? {} // miniflare doesn't support range requests
: {
range: headers,
onlyIf: headers
};
object = (await this.bucket.get(key, options)) as R2ObjectBody;
if (!object) {
return new Response(null, { status: 404 });
}
if (object.range) {
const offset = "offset" in object.range ? object.range.offset : 0;
const end = "end" in object.range ? object.range.end : object.size - 1;
responseHeaders.set("Content-Range", `bytes ${offset}-${end}/${object.size}`);
responseHeaders.set("Connection", "keep-alive");
responseHeaders.set("Vary", "Accept-Encoding");
}
} else {
object = (await this.bucket.get(key)) as R2ObjectBody;
if (object === null) {
return new Response(null, { status: 404 });
}
}
//console.log("response headers:before", headersToObject(responseHeaders));
this.writeHttpMetadata(responseHeaders, object);
responseHeaders.set("etag", object.httpEtag);
responseHeaders.set("Content-Length", String(object.size));
responseHeaders.set("Last-Modified", object.uploaded.toUTCString());
//console.log("response headers:after", headersToObject(responseHeaders));
return new Response(object.body, {
status: object.range ? 206 : 200,
headers: responseHeaders
});
}
private writeHttpMetadata(headers: Headers, object: R2Object | R2ObjectBody): void {
let metadata = object.httpMetadata;
if (!metadata || Object.keys(metadata).length === 0) {
// guessing is especially required for dev environment (miniflare)
metadata = {
contentType: guessMimeType(object.key)
};
}
//console.log("writeHttpMetadata", object.httpMetadata, metadata);
for (const [key, value] of Object.entries(metadata)) {
const camelToDash = key.replace(/([A-Z])/g, "-$1").toLowerCase();
headers.set(camelToDash, value);
}
}
async getObjectMeta(key: string): Promise<{ type: string; size: number }> {
const head = await this.headObject(key);
if (!head) {
throw new Error("Object not found");
}
return {
type: String(head.httpMetadata?.contentType ?? "application/octet-stream"),
size: head.size
};
}
async deleteObject(key: string): Promise<void> {
await this.bucket.delete(key);
}
getObjectUrl(key: string): string {
throw new Error("Method getObjectUrl not implemented.");
}
toJSON(secrets?: boolean) {
return {
type: this.getName(),
config: {}
};
}
}

View File

@@ -0,0 +1,213 @@
import type {
DeleteObjectRequest,
GetObjectRequest,
HeadObjectRequest,
ListObjectsV2Output,
ListObjectsV2Request,
PutObjectRequest
} from "@aws-sdk/client-s3";
import { AwsClient, isDebug } from "core";
import { type Static, Type, parse, pickHeaders } from "core/utils";
import { transform } from "lodash-es";
import type { FileBody, FileListObject, StorageAdapter } from "../Storage";
export const s3AdapterConfig = Type.Object(
{
access_key: Type.String(),
secret_access_key: Type.String(),
url: Type.String({
pattern: "^https?://(?:.*)?[^/.]+$",
description: "URL to S3 compatible endpoint without trailing slash",
examples: [
"https://{account_id}.r2.cloudflarestorage.com/{bucket}",
"https://{bucket}.s3.{region}.amazonaws.com"
]
})
},
{
title: "S3"
}
);
export type S3AdapterConfig = Static<typeof s3AdapterConfig>;
export class StorageS3Adapter extends AwsClient implements StorageAdapter {
readonly #config: S3AdapterConfig;
constructor(config: S3AdapterConfig) {
super(
{
accessKeyId: config.access_key,
secretAccessKey: config.secret_access_key,
retries: isDebug() ? 0 : 10
},
{
convertParams: "pascalToKebab",
responseType: "xml"
}
);
this.#config = parse(s3AdapterConfig, config);
}
getName(): string {
return "s3";
}
getSchema() {
return s3AdapterConfig;
}
override getUrl(path: string = "", searchParamsObj: Record<string, any> = {}): string {
let url = this.getObjectUrl("").slice(0, -1);
if (path.length > 0) url += `/${path}`;
return super.getUrl(url, searchParamsObj);
}
/**
* Returns the URL of an object
* @param key the key of the object
*/
getObjectUrl(key: string): string {
return `${this.#config.url}/${key}`;
}
/**
* https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
*/
async listObjects(key: string = ""): Promise<FileListObject[]> {
const params: Omit<ListObjectsV2Request, "Bucket"> & { ListType: number } = {
ListType: 2,
Prefix: key
};
const url = this.getUrl("", params);
//console.log("url", url);
const res = await this.fetchJson<{ ListBucketResult: ListObjectsV2Output }>(url, {
method: "GET"
});
//console.log("res", res);
// absolutely weird, but if only one object is there, it's an object, not an array
const { Contents } = res.ListBucketResult;
const objects = !Contents ? [] : Array.isArray(Contents) ? Contents : [Contents];
//console.log(JSON.stringify(res.ListBucketResult, null, 2), objects);
const transformed = transform(
objects,
(acc, obj) => {
// s3 contains folders, but Size is 0, which is filtered here
if (obj.Key && obj.LastModified && obj.Size) {
acc.push({
key: obj.Key,
last_modified: obj.LastModified,
size: obj.Size
});
}
},
[] as FileListObject[]
);
//console.log(transformed);
return transformed;
}
async putObject(
key: string,
body: FileBody | null,
// @todo: params must be added as headers, skipping for now
params: Omit<PutObjectRequest, "Bucket" | "Key"> = {}
) {
const url = this.getUrl(key, {});
//console.log("url", url);
const res = await this.fetch(url, {
method: "PUT",
body
});
/*console.log("putObject:raw:res", {
ok: res.ok,
status: res.status,
statusText: res.statusText,
});*/
if (res.ok) {
// "df20fcb574dba1446cf5ec997940492b"
return String(res.headers.get("etag"));
}
return undefined;
}
private async headObject(
key: string,
params: Pick<HeadObjectRequest, "PartNumber" | "VersionId"> = {}
) {
const url = this.getUrl(key, {});
return await this.fetch(url, {
method: "HEAD",
headers: {
Range: "bytes=0-1"
}
});
}
async getObjectMeta(key: string) {
const res = await this.headObject(key);
const type = String(res.headers.get("content-type"));
const size = Number(String(res.headers.get("content-range")?.split("/")[1]));
return {
type,
size
};
}
/**
* Check if an object exists by fetching the first byte of the object
* @param key
* @param params
*/
async objectExists(
key: string,
params: Pick<HeadObjectRequest, "PartNumber" | "VersionId"> = {}
) {
return (await this.headObject(key)).ok;
}
/**
* Simply returns the Response of the object to download body as needed
*/
async getObject(key: string, headers: Headers): Promise<Response> {
const url = this.getUrl(key);
const res = await this.fetch(url, {
method: "GET",
headers: pickHeaders(headers, ["range"])
});
// Response has to be copied, because of middlewares that might set headers
return new Response(res.body, {
status: res.status,
statusText: res.statusText,
headers: res.headers
});
}
/**
* Deletes a single object. Method is void, because it doesn't return anything
*/
async deleteObject(
key: string,
params: Omit<DeleteObjectRequest, "Bucket" | "Key"> = {}
): Promise<void> {
const url = this.getUrl(key, params);
const res = await this.fetch(url, {
method: "DELETE"
});
}
toJSON(secrets?: boolean) {
return {
type: this.getName(),
config: secrets ? this.#config : undefined
};
}
}

View File

@@ -0,0 +1,17 @@
import { Event } from "core/events";
import type { FileBody, FileUploadPayload } from "../Storage";
export type FileUploadedEventData = FileUploadPayload & {
file: FileBody;
};
export class FileUploadedEvent extends Event<FileUploadedEventData> {
static override slug = "file-uploaded";
}
export class FileDeletedEvent extends Event<{ name: string }> {
static override slug = "file-deleted";
}
export class FileAccessEvent extends Event<{ name: string }> {
static override slug = "file-access";
}

File diff suppressed because it is too large Load Diff