diff --git a/functions/webdav/utils.ts b/functions/webdav/utils.ts index 1b43209..bc99284 100644 --- a/functions/webdav/utils.ts +++ b/functions/webdav/utils.ts @@ -6,6 +6,19 @@ export interface RequestHandlerParams { export const WEBDAV_ENDPOINT = "/webdav/"; +export const ROOT_OBJECT = { + key: "", + uploaded: new Date(), + httpMetadata: { + contentType: "application/x-directory", + contentDisposition: undefined, + contentLanguage: undefined, + }, + customMetadata: undefined, + size: 0, + etag: undefined, +}; + export function notFound() { return new Response("Not found", { status: 404 }); } @@ -42,16 +55,3 @@ export async function* listAll( if (r2Objects.truncated) cursor = r2Objects.cursor; } while (r2Objects.truncated); } - -export const ROOT_OBJECT = { - key: "", - uploaded: new Date(), - httpMetadata: { - contentType: "application/x-directory", - contentDisposition: undefined, - contentLanguage: undefined, - }, - customMetadata: undefined, - size: 0, - etag: undefined, -}; diff --git a/src/App.tsx b/src/App.tsx index c9e1311..78c1219 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -11,6 +11,7 @@ import React, { useState } from "react"; import Header from "./Header"; import Main from "./Main"; import ProgressDialog from "./ProgressDialog"; +import { TransferQueueProvider } from "./app/transferQueue"; const globalStyles = ( @@ -29,24 +30,26 @@ function App() { {globalStyles} - -
setSearch(newSearch)} - setShowProgressDialog={setShowProgressDialog} + + +
setSearch(newSearch)} + setShowProgressDialog={setShowProgressDialog} + /> +
+ + setError(null)} /> -
- - setError(null)} - /> - setShowProgressDialog(false)} - /> + setShowProgressDialog(false)} + /> + ); } diff --git a/src/FileGrid.tsx b/src/FileGrid.tsx index a70c545..755031a 100644 --- a/src/FileGrid.tsx +++ b/src/FileGrid.tsx @@ -7,6 +7,7 @@ import { ListItemText, } from "@mui/material"; import MimeIcon from "./MimeIcon"; +import { humanReadableSize } from "./app/utils"; export interface FileItem { key: string; @@ -16,16 +17,6 @@ export interface FileItem { customMetadata?: { thumbnail?: string }; } -function humanReadableSize(size: number) { - const units = ["B", "KB", "MB", "GB", "TB"]; - let i = 0; - while (size >= 1024) { - size /= 1024; - i++; - } - return `${size.toFixed(1)} ${units[i]}`; -} - function extractFilename(key: string) { return key.split("/").pop(); } diff --git a/src/Main.tsx b/src/Main.tsx index dc3e034..b2418ee 100644 --- a/src/Main.tsx +++ b/src/Main.tsx @@ -12,12 +12,8 @@ import React, { useCallback, useEffect, useMemo, useState } from "react"; import FileGrid, { encodeKey, FileItem, isDirectory } from "./FileGrid"; import MultiSelectToolbar from "./MultiSelectToolbar"; import UploadDrawer, { UploadFab } from "./UploadDrawer"; -import { - copyPaste, - fetchPath, - processUploadQueue, - uploadQueue, -} from "./app/transfer"; +import { copyPaste, fetchPath } from "./app/transfer"; +import { useTransferQueue, useUploadEnqueue } from "./app/transferQueue"; function Centered({ children }: { children: React.ReactNode }) { return ( @@ -105,6 +101,7 @@ function DropZone({ onDrop={(event) => { event.preventDefault(); onDrop(event.dataTransfer.files); + setDragging(false); }} > {children} @@ -124,6 +121,10 @@ function Main({ const [loading, setLoading] = useState(true); const [multiSelected, setMultiSelected] = useState(null); const [showUploadDrawer, setShowUploadDrawer] = useState(false); + const [lastUploadKey, setLastUploadKey] = useState(null); + + const transferQueue = useTransferQueue(); + const uploadEnqueue = useUploadEnqueue(); const fetchFiles = useCallback(() => { setLoading(true); @@ -140,6 +141,19 @@ function Main({ fetchFiles(); }, [fetchFiles]); + useEffect(() => { + if (!transferQueue.length) return; + const lastFile = transferQueue[transferQueue.length - 1]; + if (lastFile.loaded < lastFile.total) setLastUploadKey(lastFile.remoteKey); + else if (lastUploadKey) { + fetchPath(cwd).then((files) => { + setFiles(files); + setMultiSelected(null); + }); + setLastUploadKey(null); + } + }, [cwd, fetchFiles, lastUploadKey, transferQueue]); + const filteredFiles = useMemo( () => (search @@ -173,11 +187,9 @@ function Main({ ) : ( { - uploadQueue.push( + uploadEnqueue( ...Array.from(files).map((file) => ({ file, basedir: cwd })) ); - await processUploadQueue(); - fetchFiles(); }} > void; }) { const [tab, setTab] = useState(0); + const transferQueue: TransferTask[] = useTransferQueue(); + + const tasks = useMemo(() => { + const taskType = tab === 0 ? "download" : "upload"; + return Object.values(transferQueue).filter( + (task) => task.type === taskType + ); + }, [tab, transferQueue]); return ( @@ -21,7 +47,41 @@ function ProgressDialog({ - {tab === 0 ? "Downloads" : "Uploads"} + {tasks.length === 0 ? ( + + + No tasks + + + ) : ( + + + {tasks.map((task) => ( + + + {task.error ? ( + + + + ) : task.loaded === 0 ? null : task.loaded === task.total ? ( + + ) : ( + + )} + + ))} + + + )} ); } diff --git a/src/UploadDrawer.tsx b/src/UploadDrawer.tsx index 7708e44..4e218de 100644 --- a/src/UploadDrawer.tsx +++ b/src/UploadDrawer.tsx @@ -7,7 +7,8 @@ import { Image as ImageIcon, Upload as UploadIcon, } from "@mui/icons-material"; -import { createFolder, processUploadQueue, uploadQueue } from "./app/transfer"; +import { createFolder } from "./app/transfer"; +import { useUploadEnqueue } from "./app/transferQueue"; function IconCaptionButton({ icon, @@ -64,6 +65,8 @@ function UploadDrawer({ cwd: string; onUpload: () => void; }) { + const uploadEnqueue = useUploadEnqueue(); + const handleUpload = useCallback( (action: string) => () => { const input = document.createElement("input"); @@ -84,14 +87,13 @@ function UploadDrawer({ input.onchange = async () => { if (!input.files) return; const files = Array.from(input.files); - uploadQueue.push(...files.map((file) => ({ file, basedir: cwd }))); - await processUploadQueue(); + uploadEnqueue(...files.map((file) => ({ file, basedir: cwd }))); setOpen(false); onUpload(); }; input.click(); }, - [cwd, onUpload, setOpen] + [cwd, onUpload, setOpen, uploadEnqueue] ); const takePhoto = useMemo(() => handleUpload("photo"), [handleUpload]); diff --git a/src/app/transfer.ts b/src/app/transfer.ts index 8a044f4..bb4b3c0 100644 --- a/src/app/transfer.ts +++ b/src/app/transfer.ts @@ -1,6 +1,7 @@ import pLimit from "p-limit"; import { encodeKey, FileItem } from "../FileGrid"; +import { TransferTask } from "./transferQueue"; const WEBDAV_ENDPOINT = "/webdav/"; @@ -170,6 +171,7 @@ export async function multipartUpload( const limit = pLimit(2); const parts = Array.from({ length: totalChunks }, (_, i) => i + 1); + const partsLoaded = Array.from({ length: totalChunks + 1 }, () => 0); const promises = parts.map((i) => limit(async () => { const chunk = file.slice((i - 1) * SIZE_LIMIT, i * SIZE_LIMIT); @@ -177,27 +179,44 @@ export async function multipartUpload( partNumber: i.toString(), uploadId, }); - const res = await xhrFetch(`/webdav/${encodeKey(key)}?${searchParams}`, { - method: "PUT", - headers, - body: chunk, - onUploadProgress: (progressEvent) => { - if (typeof options?.onUploadProgress !== "function") return; - options.onUploadProgress({ - loaded: (i - 1) * SIZE_LIMIT + progressEvent.loaded, - total: file.size, - }); - }, - }); - return { partNumber: i, etag: res.headers.get("etag")! }; + const uploadUrl = `/webdav/${encodeKey(key)}?${searchParams}`; + if (i === limit.concurrency) + await new Promise((resolve) => setTimeout(resolve, 1000)); + + const uploadPart = () => + xhrFetch(uploadUrl, { + method: "PUT", + headers, + body: chunk, + onUploadProgress: (progressEvent) => { + partsLoaded[i] = progressEvent.loaded; + options?.onUploadProgress?.({ + loaded: partsLoaded.reduce((a, b) => a + b), + total: file.size, + }); + }, + }); + + const retryReducer = (acc: Promise) => + acc + .then((res) => { + const retryAfter = res.headers.get("retry-after"); + if (!retryAfter) return res; + return uploadPart(); + }) + .catch(uploadPart); + const response = await [1, 2].reduce(retryReducer, uploadPart()); + return { partNumber: i, etag: response.headers.get("etag")! }; }) ); const uploadedParts = await Promise.all(promises); const completeParams = new URLSearchParams({ uploadId }); - await fetch(`/webdav/${encodeKey(key)}?${completeParams}`, { + const response = await fetch(`/webdav/${encodeKey(key)}?${completeParams}`, { method: "POST", body: JSON.stringify({ parts: uploadedParts }), }); + if (!response.ok) throw new Error(await response.text()); + return response; } export async function copyPaste(source: string, target: string, move = false) { @@ -228,17 +247,15 @@ export async function createFolder(cwd: string) { } } -export const uploadQueue: { - basedir: string; - file: File; -}[] = []; - -export async function processUploadQueue() { - if (!uploadQueue.length) { - return; - } - - const { basedir, file } = uploadQueue.shift()!; +export async function processTransferTask({ + task, + onTaskProgress, +}: { + task: TransferTask; + onTaskProgress?: (event: { loaded: number; total: number }) => void; +}) { + const { remoteKey, file } = task; + if (task.type !== "upload" || !file) return; let thumbnailDigest = null; if ( @@ -265,17 +282,20 @@ export async function processUploadQueue() { } } - try { - const headers: { "fd-thumbnail"?: string } = {}; - if (thumbnailDigest) headers["fd-thumbnail"] = thumbnailDigest; - if (file.size >= SIZE_LIMIT) { - await multipartUpload(basedir + file.name, file, { headers }); - } else { - const uploadUrl = `${WEBDAV_ENDPOINT}${encodeKey(basedir + file.name)}`; - await xhrFetch(uploadUrl, { method: "PUT", headers, body: file }); - } - } catch (error) { - console.log(`Upload ${file.name} failed`, error); + const headers: { "fd-thumbnail"?: string } = {}; + if (thumbnailDigest) headers["fd-thumbnail"] = thumbnailDigest; + if (file.size >= SIZE_LIMIT) { + return await multipartUpload(remoteKey, file, { + headers, + onUploadProgress: onTaskProgress, + }); + } else { + const uploadUrl = `${WEBDAV_ENDPOINT}${encodeKey(remoteKey)}`; + await xhrFetch(uploadUrl, { + method: "PUT", + headers, + body: file, + onUploadProgress: onTaskProgress, + }); } - setTimeout(processUploadQueue); } diff --git a/src/app/transferQueue.tsx b/src/app/transferQueue.tsx new file mode 100644 index 0000000..5a30b6a --- /dev/null +++ b/src/app/transferQueue.tsx @@ -0,0 +1,98 @@ +import React, { + createContext, + useContext, + useEffect, + useRef, + useState, +} from "react"; +import { processTransferTask } from "./transfer"; + +export interface TransferTask { + type: "upload" | "download"; + remoteKey: string; + file?: File; + name: string; + loaded: number; + total: number; + error?: any; +} + +const TransferQueueContext = createContext([]); +const SetTransferQueueContext = createContext< + React.Dispatch> +>(() => {}); + +export function useTransferQueue() { + return useContext(TransferQueueContext); +} + +export function useUploadEnqueue() { + const setTransferTasks = useContext(SetTransferQueueContext); + return (...requests: { basedir: string; file: File }[]) => { + const newTasks = requests.map( + ({ basedir, file }) => + ({ + type: "upload", + name: file.name, + file, + remoteKey: basedir + file.name, + loaded: 0, + total: file.size, + } as TransferTask) + ); + setTransferTasks((tasks) => [...tasks, ...newTasks]); + }; +} + +export function TransferQueueProvider({ + children, +}: { + children: React.ReactNode; +}) { + const [transferTasks, setTransferTasks] = useState([]); + const taskProcessing = useRef(null); + + useEffect(() => { + const taskToProcess = transferTasks.find((task) => task.loaded === 0); + if (!taskToProcess || taskProcessing.current) return; + taskProcessing.current = taskToProcess; + processTransferTask({ + task: taskToProcess, + onTaskProgress: ({ loaded }) => { + setTransferTasks((tasks) => { + const newTask: TransferTask = { ...taskProcessing.current!, loaded }; + const newTasks = tasks.map((t) => + t === taskProcessing.current ? newTask : t + ); + taskProcessing.current = newTask; + return newTasks; + }); + }, + }) + .then(() => { + taskProcessing.current = null; + setTransferTasks((tasks) => [...tasks]); + }) + .catch((error) => { + setTransferTasks((tasks) => { + const newTask: TransferTask = { + ...taskProcessing.current!, + error, + } as TransferTask; + const newTasks = tasks.map((t) => + t === taskProcessing.current ? newTask : t + ); + taskProcessing.current = newTask; + return newTasks; + }); + }); + }, [transferTasks]); + + return ( + + + {children} + + + ); +} diff --git a/src/app/utils.ts b/src/app/utils.ts new file mode 100644 index 0000000..038c3ef --- /dev/null +++ b/src/app/utils.ts @@ -0,0 +1,9 @@ +export function humanReadableSize(size: number) { + const units = ["B", "KB", "MB", "GB", "TB"]; + let i = 0; + while (size >= 1024) { + size /= 1024; + i++; + } + return `${size.toFixed(1)} ${units[i]}`; +}