diff --git a/app/__test__/_assets/test.mp3 b/app/__test__/_assets/test.mp3 new file mode 100644 index 0000000..ab94045 Binary files /dev/null and b/app/__test__/_assets/test.mp3 differ diff --git a/app/__test__/_assets/test.txt b/app/__test__/_assets/test.txt new file mode 100644 index 0000000..b6fc4c6 --- /dev/null +++ b/app/__test__/_assets/test.txt @@ -0,0 +1 @@ +hello \ No newline at end of file diff --git a/app/__test__/media/MediaController.spec.ts b/app/__test__/media/MediaController.spec.ts index 8ce6a0b..bf62599 100644 --- a/app/__test__/media/MediaController.spec.ts +++ b/app/__test__/media/MediaController.spec.ts @@ -10,7 +10,7 @@ import { assetsPath, assetsTmpPath } from "../helper"; import { disableConsoleLog, enableConsoleLog } from "core/utils/test"; beforeAll(() => { - disableConsoleLog(); + //disableConsoleLog(); registries.media.register("local", StorageLocalAdapter); }); afterAll(enableConsoleLog); @@ -94,4 +94,38 @@ describe("MediaController", () => { expect(res.status).toBe(413); expect(await Bun.file(assetsTmpPath + "/" + name).exists()).toBe(false); }); + + test("audio files", async () => { + const app = await makeApp(); + const file = Bun.file(`${assetsPath}/test.mp3`); + const name = makeName("mp3"); + const res = await app.server.request("/api/media/upload/" + name, { + method: "POST", + body: file, + }); + const result = (await res.json()) as any; + expect(result.data.mime_type).toStartWith("audio/mpeg"); + expect(result.name).toBe(name); + + const destFile = Bun.file(assetsTmpPath + "/" + name); + expect(destFile.exists()).resolves.toBe(true); + await destFile.delete(); + }); + + test("text files", async () => { + const app = await makeApp(); + const file = Bun.file(`${assetsPath}/test.txt`); + const name = makeName("txt"); + const res = await app.server.request("/api/media/upload/" + name, { + method: "POST", + body: file, + }); + const result = (await res.json()) as any; + expect(result.data.mime_type).toStartWith("text/plain"); + expect(result.name).toBe(name); + + const destFile = Bun.file(assetsTmpPath + "/" + name); + expect(destFile.exists()).resolves.toBe(true); + await destFile.delete(); + }); }); diff --git a/app/__test__/media/mime-types.spec.ts b/app/__test__/media/mime-types.spec.ts index d326aca..1435dd6 100644 --- a/app/__test__/media/mime-types.spec.ts +++ b/app/__test__/media/mime-types.spec.ts @@ -72,6 +72,7 @@ describe("media/mime-types", () => { ["text/tab-separated-values", "tsv"], ["application/zip", "zip"], ["application/pdf", "pdf"], + ["audio/mpeg", "mp3"], ] as const; for (const [mime, ext] of tests) { @@ -90,6 +91,8 @@ describe("media/mime-types", () => { ["-473Wx593H-466453554-black-MODEL.jpg", "jpg"], ["-473Wx593H-466453554-black-MODEL.avif", "avif"], ["file.pdf", "pdf"], + ["file.mp3", "mp3"], + ["robots.txt", "txt"], ] as const; for (const [filename, ext] of tests) { @@ -127,5 +130,13 @@ describe("media/mime-types", () => { ).split("."); expect(ext).toBe("what"); } + + { + // txt + const [, ext] = getRandomizedFilename( + new File([""], "file.txt", { type: "text/plain" }), + ).split("."); + expect(ext).toBe("txt"); + } }); }); diff --git a/app/src/adapter/cloudflare/storage/StorageR2Adapter.vitest.ts b/app/src/adapter/cloudflare/storage/StorageR2Adapter.vitest.ts index 8ad1d90..6ecd8df 100644 --- a/app/src/adapter/cloudflare/storage/StorageR2Adapter.vitest.ts +++ b/app/src/adapter/cloudflare/storage/StorageR2Adapter.vitest.ts @@ -28,7 +28,8 @@ describe("StorageR2Adapter", async () => { const buffer = readFileSync(path.join(basePath, "image.png")); const file = new File([buffer], "image.png", { type: "image/png" }); - await adapterTestSuite(viTestRunner, adapter, file); + // miniflare doesn't support range requests + await adapterTestSuite(viTestRunner, adapter, file, { testRange: false }); }); afterAll(async () => { diff --git a/app/src/adapter/node/storage/StorageLocalAdapter.ts b/app/src/adapter/node/storage/StorageLocalAdapter.ts index fa3e336..46db8fd 100644 --- a/app/src/adapter/node/storage/StorageLocalAdapter.ts +++ b/app/src/adapter/node/storage/StorageLocalAdapter.ts @@ -80,18 +80,79 @@ export class StorageLocalAdapter extends StorageAdapter { } } + private parseRangeHeader( + rangeHeader: string, + fileSize: number, + ): { start: number; end: number } | null { + // Parse "bytes=start-end" format + const match = rangeHeader.match(/^bytes=(\d*)-(\d*)$/); + if (!match) return null; + + const [, startStr, endStr] = match; + let start = startStr ? Number.parseInt(startStr, 10) : 0; + let end = endStr ? Number.parseInt(endStr, 10) : fileSize - 1; + + // Handle suffix-byte-range-spec (e.g., "bytes=-500") + if (!startStr && endStr) { + start = Math.max(0, fileSize - Number.parseInt(endStr, 10)); + end = fileSize - 1; + } + + // Validate range + if (start < 0 || end >= fileSize || start > end) { + return null; + } + + return { start, end }; + } + async getObject(key: string, headers: Headers): Promise { try { - const content = await readFile(`${this.config.path}/${key}`); + const filePath = `${this.config.path}/${key}`; + const stats = await stat(filePath); + const fileSize = stats.size; const mimeType = guessMimeType(key); - return new Response(content, { - status: 200, - headers: { - "Content-Type": mimeType || "application/octet-stream", - "Content-Length": content.length.toString(), - }, + const responseHeaders = new Headers({ + "Accept-Ranges": "bytes", + "Content-Type": mimeType || "application/octet-stream", }); + + const rangeHeader = headers.get("range"); + + if (rangeHeader) { + const range = this.parseRangeHeader(rangeHeader, fileSize); + + if (!range) { + // Invalid range - return 416 Range Not Satisfiable + responseHeaders.set("Content-Range", `bytes */${fileSize}`); + return new Response("", { + status: 416, + headers: responseHeaders, + }); + } + + const { start, end } = range; + const content = await readFile(filePath, { encoding: null }); + const chunk = content.slice(start, end + 1); + + responseHeaders.set("Content-Range", `bytes ${start}-${end}/${fileSize}`); + responseHeaders.set("Content-Length", chunk.length.toString()); + + return new Response(chunk, { + status: 206, // Partial Content + headers: responseHeaders, + }); + } else { + // Normal request - return entire file + const content = await readFile(filePath); + responseHeaders.set("Content-Length", content.length.toString()); + + return new Response(content, { + status: 200, + headers: responseHeaders, + }); + } } catch (error) { // Handle file reading errors return new Response("", { status: 404 }); diff --git a/app/src/media/AppMedia.ts b/app/src/media/AppMedia.ts index 0971187..2c1b6b2 100644 --- a/app/src/media/AppMedia.ts +++ b/app/src/media/AppMedia.ts @@ -8,6 +8,7 @@ import { MediaController } from "./api/MediaController"; import { buildMediaSchema, registry, type TAppMediaConfig } from "./media-schema"; import { mediaFields } from "./media-entities"; import * as MediaPermissions from "media/media-permissions"; +import * as DatabaseEvents from "data/events"; export type MediaFields = typeof AppMedia.mediaFields; export type MediaFieldSchema = FieldSchema; @@ -139,6 +140,30 @@ export class AppMedia extends Module> { }, { mode: "sync", id: "delete-data-media" }, ); + + emgr.onEvent( + DatabaseEvents.MutatorDeleteAfter, + async (e) => { + const { entity, data } = e.params; + const fields = entity.fields.filter((f) => f.type === "media"); + if (fields.length > 0) { + const references = fields.map((f) => `${entity.name}.${f.name}`); + $console.log("App:storage:file cleaning up", { + reference: { $in: references }, + entity_id: String(data.id), + }); + const { data: deleted } = await em.mutator(media).deleteWhere({ + reference: { $in: references }, + entity_id: String(data.id), + }); + for (const file of deleted) { + await this.storage.deleteFile(file.path); + } + $console.log("App:storage:file cleaned up files:", deleted.length); + } + }, + { mode: "async", id: "delete-data-media-after" }, + ); } override getOverwritePaths() { diff --git a/app/src/media/MediaField.ts b/app/src/media/MediaField.ts index 5f005bc..6ff8cd3 100644 --- a/app/src/media/MediaField.ts +++ b/app/src/media/MediaField.ts @@ -43,6 +43,10 @@ export class MediaField< return this.config.max_items; } + getAllowedMimeTypes(): string[] | undefined { + return this.config.mime_types; + } + getMinItems(): number | undefined { return this.config.min_items; } diff --git a/app/src/media/storage/Storage.ts b/app/src/media/storage/Storage.ts index 6615f8e..1d11b1d 100644 --- a/app/src/media/storage/Storage.ts +++ b/app/src/media/storage/Storage.ts @@ -71,22 +71,29 @@ export class Storage implements EmitsEvents { let info: FileUploadPayload = { name, - meta: { - size: 0, - type: "application/octet-stream", - }, + meta: isFile(file) + ? { + size: file.size, + type: file.type, + } + : { + size: 0, + type: "application/octet-stream", + }, etag: typeof result === "string" ? result : "", }; + // normally only etag is returned if (typeof result === "object") { info = result; - } else if (isFile(file)) { - info.meta.size = file.size; - info.meta.type = file.type; } // try to get better meta info - if (!isMimeType(info.meta.type, ["application/octet-stream", "application/json"])) { + if ( + !info.meta.type || + ["application/octet-stream", "application/json"].includes(info.meta.type) || + !info.meta.size + ) { const meta = await this.#adapter.getObjectMeta(name); if (!meta) { throw new Error("Failed to get object meta"); diff --git a/app/src/media/storage/adapters/adapter-test-suite.ts b/app/src/media/storage/adapters/adapter-test-suite.ts index 1a92d34..a6d1917 100644 --- a/app/src/media/storage/adapters/adapter-test-suite.ts +++ b/app/src/media/storage/adapters/adapter-test-suite.ts @@ -11,12 +11,14 @@ export async function adapterTestSuite( retries?: number; retryTimeout?: number; skipExistsAfterDelete?: boolean; + testRange?: boolean; }, ) { const { test, expect } = testRunner; const options = { retries: opts?.retries ?? 1, retryTimeout: opts?.retryTimeout ?? 1000, + testRange: opts?.testRange ?? true, }; let objects = 0; @@ -53,9 +55,34 @@ export async function adapterTestSuite( await test("gets an object", async () => { const res = await adapter.getObject(filename, new Headers()); expect(res.ok).toBe(true); + expect(res.headers.get("Accept-Ranges")).toBe("bytes"); // @todo: check the content }); + if (options.testRange) { + await test("handles range request - partial content", async () => { + const headers = new Headers({ Range: "bytes=0-99" }); + const res = await adapter.getObject(filename, headers); + expect(res.status).toBe(206); // Partial Content + expect(/^bytes 0-99\/\d+$/.test(res.headers.get("Content-Range")!)).toBe(true); + expect(res.headers.get("Accept-Ranges")).toBe("bytes"); + }); + + await test("handles range request - suffix range", async () => { + const headers = new Headers({ Range: "bytes=-100" }); + const res = await adapter.getObject(filename, headers); + expect(res.status).toBe(206); // Partial Content + expect(/^bytes \d+-\d+\/\d+$/.test(res.headers.get("Content-Range")!)).toBe(true); + }); + + await test("handles invalid range request", async () => { + const headers = new Headers({ Range: "bytes=invalid" }); + const res = await adapter.getObject(filename, headers); + expect(res.status).toBe(416); // Range Not Satisfiable + expect(/^bytes \*\/\d+$/.test(res.headers.get("Content-Range")!)).toBe(true); + }); + } + await test("gets object meta", async () => { expect(await adapter.getObjectMeta(filename)).toEqual({ type: file.type, // image/png diff --git a/app/src/media/storage/mime-types-tiny.ts b/app/src/media/storage/mime-types-tiny.ts index 6a3e241..bbc8d9d 100644 --- a/app/src/media/storage/mime-types-tiny.ts +++ b/app/src/media/storage/mime-types-tiny.ts @@ -3,7 +3,7 @@ export const Q = { audio: ["ogg"], image: ["jpeg", "png", "gif", "webp", "bmp", "tiff", "avif", "heic", "heif"], text: ["html", "css", "mdx", "yaml", "vcard", "csv", "vtt"], - application: ["zip", "xml", "toml", "json", "json5", "pdf"], + application: ["zip", "toml", "json", "json5", "pdf", "xml"], font: ["woff", "woff2", "ttf", "otf"], } as const; @@ -15,10 +15,12 @@ const c = { a: (w = "octet-stream") => `application/${w}`, i: (w) => `image/${w}`, v: (w) => `video/${w}`, + au: (w) => `audio/${w}`, } as const; export const M = new Map([ ["7z", c.z], ["7zip", c.z], + ["txt", c.t()], ["ai", c.a("postscript")], ["apk", c.a("vnd.android.package-archive")], ["doc", c.a("msword")], @@ -32,12 +34,12 @@ export const M = new Map([ ["jpg", c.i("jpeg")], ["js", c.t("javascript")], ["log", c.t()], - ["m3u", c.t()], + ["m3u", c.au("x-mpegurl")], ["m3u8", c.a("vnd.apple.mpegurl")], ["manifest", c.t("cache-manifest")], ["md", c.t("markdown")], ["mkv", c.v("x-matroska")], - ["mp3", c.a("mpeg")], + ["mp3", c.au("mpeg")], ["mobi", c.a("x-mobipocket-ebook")], ["ppt", c.a("powerpoint")], ["pptx", `${c.vnd}.presentationml.presentation`], @@ -46,11 +48,10 @@ export const M = new Map([ ["tif", c.i("tiff")], ["tsv", c.t("tab-separated-values")], ["tgz", c.a("x-tar")], - ["txt", c.t()], ["text", c.t()], ["vcd", c.a("x-cdlink")], ["vcs", c.t("x-vcalendar")], - ["wav", c.a("x-wav")], + ["wav", c.au("vnd.wav")], ["webmanifest", c.a("manifest+json")], ["xls", c.a("vnd.ms-excel")], ["xlsx", `${c.vnd}.spreadsheetml.sheet`], diff --git a/app/src/ui/elements/media/Dropzone.tsx b/app/src/ui/elements/media/Dropzone.tsx index 908c49a..ffaa5df 100644 --- a/app/src/ui/elements/media/Dropzone.tsx +++ b/app/src/ui/elements/media/Dropzone.tsx @@ -42,7 +42,10 @@ export type DropzoneRenderProps = { showPlaceholder: boolean; onClick?: (file: { path: string }) => void; footer?: ReactNode; - dropzoneProps: Pick; + dropzoneProps: Pick< + DropzoneProps, + "maxItems" | "placeholder" | "autoUpload" | "flow" | "allowedMimeTypes" + >; }; export type DropzoneProps = { @@ -151,6 +154,7 @@ export function Dropzone({ const setIsOver = useStore(store, (state) => state.setIsOver); const uploading = useStore(store, (state) => state.uploading); const setFileState = useStore(store, (state) => state.setFileState); + const overrideFile = useStore(store, (state) => state.overrideFile); const removeFile = useStore(store, (state) => state.removeFile); const inputRef = useRef(null); @@ -169,9 +173,14 @@ export function Dropzone({ return specs.every((spec) => { if (spec.kind !== "file") { + console.log("not a file", spec.kind); return false; } - return !(allowedMimeTypes && !allowedMimeTypes.includes(spec.type)); + if (allowedMimeTypes && allowedMimeTypes.length > 0) { + console.log("not allowed mimetype", spec.type); + return allowedMimeTypes.includes(spec.type); + } + return true; }); } @@ -359,7 +368,7 @@ export function Dropzone({ state: "uploaded", }; - setFileState(file.path, newState.state); + overrideFile(file.path, newState); resolve({ ...response, ...file, ...newState }); } catch (e) { setFileState(file.path, "uploaded", 1); @@ -382,7 +391,9 @@ export function Dropzone({ }; xhr.setRequestHeader("Accept", "application/json"); - xhr.send(file.body); + const formData = new FormData(); + formData.append("file", file.body); + xhr.send(formData); }); } @@ -411,7 +422,9 @@ export function Dropzone({ const openFileInput = useCallback(() => inputRef.current?.click(), [inputRef]); const showPlaceholder = useMemo( () => - Boolean(placeholder?.show === true || !maxItems || (maxItems && files.length < maxItems)), + Boolean( + placeholder?.show !== false && (!maxItems || (maxItems && files.length < maxItems)), + ), [placeholder, maxItems, files.length], ); @@ -424,6 +437,7 @@ export function Dropzone({ type: "file", multiple: !maxItems || maxItems > 1, onChange: handleFileInputChange, + accept: allowedMimeTypes?.join(","), }, showPlaceholder, actions: { @@ -437,11 +451,12 @@ export function Dropzone({ placeholder, autoUpload, flow, + allowedMimeTypes, }, onClick, footer, }), - [maxItems, flow, placeholder, autoUpload, footer], + [maxItems, files.length, flow, placeholder, autoUpload, footer, allowedMimeTypes], ) as unknown as DropzoneRenderProps; return ( diff --git a/app/src/ui/elements/media/DropzoneInner.tsx b/app/src/ui/elements/media/DropzoneInner.tsx index dfbdf8f..6c3cb87 100644 --- a/app/src/ui/elements/media/DropzoneInner.tsx +++ b/app/src/ui/elements/media/DropzoneInner.tsx @@ -1,7 +1,22 @@ import { type ComponentPropsWithoutRef, memo, type ReactNode, useCallback, useMemo } from "react"; import { twMerge } from "tailwind-merge"; import { useRenderCount } from "ui/hooks/use-render-count"; -import { TbDots, TbExternalLink, TbTrash, TbUpload } from "react-icons/tb"; +import { + TbDots, + TbExternalLink, + TbFileTypeCsv, + TbFileText, + TbJson, + TbFileTypePdf, + TbMarkdown, + TbMusic, + TbTrash, + TbUpload, + TbFileTypeTxt, + TbFileTypeXml, + TbZip, + TbFileTypeSql, +} from "react-icons/tb"; import { Dropdown, type DropdownItem } from "ui/components/overlay/Dropdown"; import { IconButton } from "ui/components/buttons/IconButton"; import { formatNumber } from "core/utils"; @@ -22,7 +37,7 @@ export const DropzoneInner = ({ inputProps, showPlaceholder, actions: { uploadFile, deleteFile, openFileInput }, - dropzoneProps: { placeholder, flow }, + dropzoneProps: { placeholder, flow, maxItems, allowedMimeTypes }, onClick, footer, }: DropzoneRenderProps) => { @@ -85,7 +100,7 @@ const UploadPlaceholder = ({ onClick, text = "Upload files" }) => { ); }; -type ReducedFile = Pick; +type ReducedFile = Omit; export type PreviewComponentProps = { file: ReducedFile; fallback?: (props: { file: ReducedFile }) => ReactNode; @@ -159,9 +174,9 @@ const Preview = memo(

{file.name}

-
+
{file.type} - {formatNumber.fileSize(file.size)} + {formatNumber.fileSize(file.size)}
@@ -271,6 +286,59 @@ const VideoPreview = ({ return