From a389194db7aae44f07830cd5ebe0943937944c72 Mon Sep 17 00:00:00 2001 From: Marc Seitz Date: Thu, 28 Aug 2025 12:31:43 +0200 Subject: [PATCH] feat: improve bulk uploads --- components/upload-zone.tsx | 20 ++- lib/files/tus-redis-locker.ts | 4 +- lib/files/tus-upload.ts | 166 ++++++++++++++++++--- lib/files/viewer-tus-upload.ts | 177 +++++++++++++++++++---- pages/api/file/tus-viewer/[[...file]].ts | 1 + pages/api/file/tus/[[...file]].ts | 1 + 6 files changed, 315 insertions(+), 54 deletions(-) diff --git a/components/upload-zone.tsx b/components/upload-zone.tsx index f3e6107fd..bea4ea7b9 100644 --- a/components/upload-zone.tsx +++ b/components/upload-zone.tsx @@ -251,8 +251,26 @@ export default function UploadZone({ ), ); + // Provide more specific error messages + let errorMessage = "Error uploading file"; + if (error.message?.toLowerCase().includes("timeout")) { + errorMessage = "Upload timed out. Please try again."; + } else if (error.message?.toLowerCase().includes("network")) { + errorMessage = + "Network error. Please check your connection and try again."; + } else if ((error as any).originalResponse) { + const status = (error as any).originalResponse.getStatus(); + if (status === 413) { + errorMessage = "File too large for upload"; + } else if (status >= 500) { + errorMessage = "Server error. Please try again later."; + } else if (status === 403) { + errorMessage = "Upload not permitted"; + } + } + setRejectedFiles((prev) => [ - { fileName: file.name, message: "Error uploading file" }, + { fileName: file.name, message: errorMessage }, ...prev, ]); }, diff --git a/lib/files/tus-redis-locker.ts b/lib/files/tus-redis-locker.ts index 68b72340c..779423dca 100644 --- a/lib/files/tus-redis-locker.ts +++ b/lib/files/tus-redis-locker.ts @@ -32,7 +32,7 @@ export class RedisLocker implements Locker { redisClient: Redis; constructor(options: RedisLockerOptions) { - this.timeout = options.acquireLockTimeout ?? 1000 * 30; // default: 30 seconds + this.timeout = options.acquireLockTimeout ?? 60_000; this.redisClient = options.redisClient; } @@ -45,7 +45,7 @@ class RedisLock implements Lock { constructor( private id: string, private locker: RedisLocker, - private timeout: number = 1000 * 30, // default: 30 seconds + private timeout: number = 60_000, ) {} async lock( diff --git a/lib/files/tus-upload.ts b/lib/files/tus-upload.ts index f69bac74f..a27d00e9f 100644 --- a/lib/files/tus-upload.ts +++ b/lib/files/tus-upload.ts @@ -31,9 +31,19 @@ export function resumableUpload({ teamId, numPages, relativePath, -}: ResumableUploadParams) { - return new Promise<{ upload: tus.Upload; complete: Promise }>( - (resolve, reject) => { +}: ResumableUploadParams): Promise<{ + upload: tus.Upload; + complete: Promise; +}> { + const retryDelays = [0, 3000, 5000, 10000, 15000, 20000]; + + return new Promise((resolve, reject) => { + let attempt = 0; + let networkTimeoutId: NodeJS.Timeout | undefined; + + const createUpload = () => { + console.log(`Starting upload attempt ${attempt + 1}/6`); + let completeResolve: ( value: UploadResult | PromiseLike, ) => void; @@ -41,9 +51,11 @@ export function resumableUpload({ completeResolve = res; }); + let isTimedOut = false; + const upload = new tus.Upload(file, { endpoint: `${process.env.NEXT_PUBLIC_BASE_URL}/api/file/tus`, - retryDelays: [0, 3000, 5000, 10000], + retryDelays, uploadDataDuringCreation: true, removeFingerprintOnSuccess: true, metadata: { @@ -56,25 +68,96 @@ export function resumableUpload({ }, chunkSize: 4 * 1024 * 1024, onError: (error) => { - onError && onError(error); - console.error("Failed because: " + error); + if (networkTimeoutId) clearTimeout(networkTimeoutId); + console.error(`TUS onError called on attempt ${attempt + 1}:`, error); + + if (isTimedOut) { + console.log( + "Error was caused by our manual timeout, handling retry...", + ); + return; // Let the timeout handler deal with retries + } + + // Check if we should retry this error + const shouldRetry = attempt < retryDelays.length - 1; + const detailedError = error as tus.DetailedError; + const isRetryableError = + !detailedError.originalResponse || // Network error + error.message?.toLowerCase().includes("timeout") || + error.message?.toLowerCase().includes("network") || + error.message?.toLowerCase().includes("err_timed_out") || + [0, 502, 503, 504].includes( + detailedError.originalResponse?.getStatus() || 0, + ); + + if (shouldRetry && isRetryableError) { + attempt++; + const delay = retryDelays[attempt] || 10000; + console.log( + `Retrying upload in ${delay}ms (attempt ${attempt + 1}/${retryDelays.length})`, + ); + + setTimeout(() => { + const newUpload = createUpload(); + resolve(newUpload); + }, delay); + return; + } + + // No more retries or non-retryable error + console.error("Upload failed after all retries:", error); + onError?.(error); reject(error); }, onShouldRetry(error, retryAttempt, options) { - console.error(`Should retry upload. Attempt ${retryAttempt}`); - var status = error.originalResponse - ? error.originalResponse.getStatus() - : 0; - // Do not retry if the status is a 500. - if (status === 500 || status === 403) { - return false; - } - // For any other status code, we retry. + // We see this message, great! TUS internal retry system is working + console.error( + `TUS onShouldRetry called - Attempt ${retryAttempt}, Error:`, + error, + ); + + // Let TUS handle these retries first, then we'll handle network timeouts separately return true; }, - onProgress, + onProgress: (bytesUploaded, bytesTotal) => { + // Reset timeout on any progress + if (networkTimeoutId) clearTimeout(networkTimeoutId); + + // Set a new timeout + networkTimeoutId = setTimeout(() => { + console.error( + `Network timeout after ${60_000}ms of no progress (attempt ${attempt + 1})`, + ); + isTimedOut = true; + upload.abort(); + + // Handle retry after timeout + const shouldRetry = attempt < retryDelays.length - 1; + if (shouldRetry) { + attempt++; + const delay = retryDelays[attempt] || 10000; + console.log( + `Retrying after network timeout in ${delay}ms (attempt ${attempt + 1}/${retryDelays.length})`, + ); + + setTimeout(() => { + const newUpload = createUpload(); + resolve(newUpload); + }, delay); + } else { + console.error( + "Upload failed after network timeout with no retries left", + ); + onError?.(new Error("Network timeout after 60000ms")); + reject(new Error("Network timeout after 60000ms")); + } + }, 60_000); + + onProgress?.(bytesUploaded, bytesTotal); + }, onSuccess: () => { - console.log("Uploaded successfully"); + if (networkTimeoutId) clearTimeout(networkTimeoutId); + console.log("Upload completed successfully!"); const id = upload.url!.split("/api/file/tus/")[1]; // if id contains a slash, then we use it as it otherwise we need to convert from buffer base64URL to utf-8 const newId = id.includes("/") ? id : decodeBase64Url(id); @@ -91,23 +174,60 @@ export function resumableUpload({ }, }); + // Set initial timeout + networkTimeoutId = setTimeout(() => { + console.error( + `Initial network timeout after ${60_000}ms (attempt ${attempt + 1})`, + ); + isTimedOut = true; + upload.abort(); + + // Handle retry after timeout + const shouldRetry = attempt < retryDelays.length - 1; + if (shouldRetry) { + attempt++; + const delay = retryDelays[attempt] || 10000; + console.log( + `Retrying after initial timeout in ${delay}ms (attempt ${attempt + 1}/${retryDelays.length})`, + ); + + setTimeout(() => { + const newUpload = createUpload(); + resolve(newUpload); + }, delay); + } else { + console.error( + "Upload failed after initial timeout with no retries left", + ); + onError?.(new Error("Initial network timeout after 60000ms")); + reject(new Error("Initial network timeout after 60000ms")); + } + }, 60_000); + // Check if there are any previous uploads to continue. upload .findPreviousUploads() .then((previousUploads) => { - // Found previous uploads so we select the first one. if (previousUploads.length) { + console.log("Resuming from previous upload..."); upload.resumeFromPreviousUpload(previousUploads[0]); } - upload.start(); - resolve({ upload, complete }); }) .catch((error) => { console.error("Error finding previous uploads:", error); upload.start(); - resolve({ upload, complete }); }); - }, - ); + + return { upload, complete }; + }; + + // Start the first upload attempt + try { + const result = createUpload(); + resolve(result); + } catch (error) { + reject(error); + } + }); } diff --git a/lib/files/viewer-tus-upload.ts b/lib/files/viewer-tus-upload.ts index aa43de5e8..a5db09b69 100644 --- a/lib/files/viewer-tus-upload.ts +++ b/lib/files/viewer-tus-upload.ts @@ -21,8 +21,6 @@ type UploadResult = { fileName: string; fileType: string; numPages: number; - viewerId: string; - linkId: string; dataroomId?: string; teamId: string; }; @@ -34,9 +32,21 @@ export function viewerUpload({ viewerData, teamId, numPages, -}: ViewerUploadParams) { - return new Promise<{ upload: tus.Upload; complete: Promise }>( - (resolve, reject) => { +}: ViewerUploadParams): Promise<{ + upload: tus.Upload; + complete: Promise; +}> { + const retryDelays = [0, 3000, 5000, 10000, 15000, 20000]; + + return new Promise((resolve, reject) => { + let attempt = 0; + let networkTimeoutId: NodeJS.Timeout | undefined; + + const createUpload = () => { + console.log( + `Starting viewer upload attempt ${attempt + 1}/${retryDelays.length}`, + ); + let completeResolve: ( value: UploadResult | PromiseLike, ) => void; @@ -44,9 +54,11 @@ export function viewerUpload({ completeResolve = res; }); + let isTimedOut = false; + const upload = new tus.Upload(file, { endpoint: `${process.env.NEXT_PUBLIC_BASE_URL}/api/file/tus-viewer`, - retryDelays: [0, 3000, 5000, 10000], + retryDelays, uploadDataDuringCreation: true, removeFingerprintOnSuccess: true, metadata: { @@ -60,25 +72,99 @@ export function viewerUpload({ }, chunkSize: 4 * 1024 * 1024, onError: (error) => { - onError && onError(error); - console.error("Failed because: " + error); + if (networkTimeoutId) clearTimeout(networkTimeoutId); + console.error( + `TUS viewer onError called on attempt ${attempt + 1}:`, + error, + ); + + if (isTimedOut) { + console.log( + "Error was caused by our manual timeout, handling retry...", + ); + return; // Let the timeout handler deal with retries + } + + // Check if we should retry this error + const shouldRetry = attempt < retryDelays.length - 1; + const detailedError = error as tus.DetailedError; + const isRetryableError = + !detailedError.originalResponse || // Network error + error.message?.toLowerCase().includes("timeout") || + error.message?.toLowerCase().includes("network") || + error.message?.toLowerCase().includes("err_timed_out") || + [0, 502, 503, 504].includes( + detailedError.originalResponse?.getStatus() || 0, + ); + + if (shouldRetry && isRetryableError) { + attempt++; + const delay = retryDelays[attempt] || 10000; + console.log( + `Retrying viewer upload in ${delay}ms (attempt ${attempt + 1}/${retryDelays.length})`, + ); + + setTimeout(() => { + const newUpload = createUpload(); + resolve(newUpload); + }, delay); + return; + } + + // No more retries or non-retryable error + console.error("Viewer upload failed after all retries:", error); + onError?.(error); reject(error); }, onShouldRetry(error, retryAttempt, options) { - console.error(`Should retry upload. Attempt ${retryAttempt}`); - var status = error.originalResponse - ? error.originalResponse.getStatus() - : 0; - // Do not retry if the status is a 500 or 403. - if (status === 500 || status === 403) { - return false; - } - // For any other status code, we retry. + // We see this message, great! TUS internal retry system is working + console.error( + `TUS viewer onShouldRetry called - Attempt ${retryAttempt}, Error:`, + error, + ); + + // Let TUS handle these retries first, then we'll handle network timeouts separately return true; }, - onProgress, + onProgress: (bytesUploaded, bytesTotal) => { + // Reset timeout on any progress + if (networkTimeoutId) clearTimeout(networkTimeoutId); + + // Set a new timeout + networkTimeoutId = setTimeout(() => { + console.error( + `Viewer upload network timeout after ${60_000}ms of no progress (attempt ${attempt + 1})`, + ); + isTimedOut = true; + upload.abort(); + + // Handle retry after timeout + const shouldRetry = attempt < retryDelays.length - 1; + if (shouldRetry) { + attempt++; + const delay = retryDelays[attempt] || 10000; + console.log( + `Retrying viewer upload after network timeout in ${delay}ms (attempt ${attempt + 1}/${retryDelays.length})`, + ); + + setTimeout(() => { + const newUpload = createUpload(); + resolve(newUpload); + }, delay); + } else { + console.error( + "Viewer upload failed after network timeout with no retries left", + ); + onError?.(new Error(`Network timeout after ${60_000}ms`)); + reject(new Error(`Network timeout after ${60_000}ms`)); + } + }, 60_000); + + onProgress?.(bytesUploaded, bytesTotal); + }, onSuccess: () => { - console.log("Uploaded successfully"); + if (networkTimeoutId) clearTimeout(networkTimeoutId); + console.log("Viewer upload completed successfully!"); const id = upload.url!.split("/api/file/tus-viewer/")[1]; // if id contains a slash, then we use it as it otherwise we need to convert from buffer base64URL to utf-8 const newId = id.includes("/") ? id : decodeBase64Url(id); @@ -88,31 +174,66 @@ export function viewerUpload({ fileName: file.name, fileType: file.type, numPages, - viewerId: viewerData.id, - linkId: viewerData.linkId, dataroomId: viewerData.dataroomId, teamId, }); }, }); + // Set initial timeout + networkTimeoutId = setTimeout(() => { + console.error( + `Viewer upload initial network timeout after ${60_000}ms (attempt ${attempt + 1})`, + ); + isTimedOut = true; + upload.abort(); + + // Handle retry after timeout + const shouldRetry = attempt < retryDelays.length - 1; + if (shouldRetry) { + attempt++; + const delay = retryDelays[attempt] || 10000; + console.log( + `Retrying viewer upload after initial timeout in ${delay}ms (attempt ${attempt + 1}/${retryDelays.length})`, + ); + + setTimeout(() => { + const newUpload = createUpload(); + resolve(newUpload); + }, delay); + } else { + console.error( + "Viewer upload failed after initial timeout with no retries left", + ); + onError?.(new Error("Initial network timeout after 60000ms")); + reject(new Error("Initial network timeout after 60000ms")); + } + }, 60_000); + // Check if there are any previous uploads to continue. upload .findPreviousUploads() .then((previousUploads) => { - // Found previous uploads so we select the first one. if (previousUploads.length) { + console.log("Resuming viewer upload from previous upload..."); upload.resumeFromPreviousUpload(previousUploads[0]); } - upload.start(); - resolve({ upload, complete }); }) .catch((error) => { - console.error("Error finding previous uploads:", error); + console.error("Error finding previous viewer uploads:", error); upload.start(); - resolve({ upload, complete }); }); - }, - ); + + return { upload, complete }; + }; + + // Start the first upload attempt + try { + const result = createUpload(); + resolve(result); + } catch (error) { + reject(error); + } + }); } diff --git a/pages/api/file/tus-viewer/[[...file]].ts b/pages/api/file/tus-viewer/[[...file]].ts index 74e06679b..f5a8d6ead 100644 --- a/pages/api/file/tus-viewer/[[...file]].ts +++ b/pages/api/file/tus-viewer/[[...file]].ts @@ -15,6 +15,7 @@ import { lockerRedisClient } from "@/lib/redis"; import { log } from "@/lib/utils"; export const config = { + maxDuration: 60, api: { bodyParser: false, }, diff --git a/pages/api/file/tus/[[...file]].ts b/pages/api/file/tus/[[...file]].ts index 6db621134..ef4e19330 100644 --- a/pages/api/file/tus/[[...file]].ts +++ b/pages/api/file/tus/[[...file]].ts @@ -16,6 +16,7 @@ import { log } from "@/lib/utils"; import { authOptions } from "../../auth/[...nextauth]"; export const config = { + maxDuration: 60, api: { bodyParser: false, },