From 631de1b4ebe35574b5d652ebb8e10c51bdc61342 Mon Sep 17 00:00:00 2001 From: Aarish mansur Date: Fri, 7 Nov 2025 07:22:47 +0000 Subject: [PATCH 1/4] Feat : added retry logic for object storage fetch failures --- apps/webapp/app/v3/r2.server.ts | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/apps/webapp/app/v3/r2.server.ts b/apps/webapp/app/v3/r2.server.ts index 6717009407..97c61c6c4f 100644 --- a/apps/webapp/app/v3/r2.server.ts +++ b/apps/webapp/app/v3/r2.server.ts @@ -93,11 +93,34 @@ export async function downloadPacketFromObjectStore( logger.debug("Downloading from object store", { url: url.href }); - const response = await r2.fetch(url.toString()); + async function fetchWithRetry(url:string,retries =3,delay=500):Promise{ + for(let attempt =1; attempt<=retries;attempt++){ + try { + const response = await r2.fetch(url); + if(response.ok) return response; + + // only retry on transient server/network errors + if(response.status >= 500 && response.status<600 ){ + throw new Error(`Server Error : ${response.statusText}`); + } + + // for other non server errors + throw new Error(`non-retryable error :${response.statusText}`); + + } catch (error:any) { + if(attempt == retries) throw error; + logger.warn(`Retrying object Download (attempt: ${attempt}) out of ${retries}`,{ + url, + error:error.message, + }); + await new Promise((res)=>setTimeout(res,delay*attempt)); + } + } + + throw new Error(`Failed to fetch ${url} after ${retries} retries`); - if (!response.ok) { - throw new Error(`Failed to download input from ${url}: ${response.statusText}`); } + const response = await fetchWithRetry(url.toString()); const data = await response.text(); From 7b08bba9b3a5b4804ac1f4b3209cde74606613de Mon Sep 17 00:00:00 2001 From: Aarish mansur Date: Sun, 9 Nov 2025 13:41:10 +0000 Subject: [PATCH 2/4] handled the undefined case --- apps/webapp/app/v3/r2.server.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apps/webapp/app/v3/r2.server.ts b/apps/webapp/app/v3/r2.server.ts index 97c61c6c4f..8dd3ef3a6b 100644 --- a/apps/webapp/app/v3/r2.server.ts +++ b/apps/webapp/app/v3/r2.server.ts @@ -94,6 +94,11 @@ export async function downloadPacketFromObjectStore( logger.debug("Downloading from object store", { url: url.href }); async function fetchWithRetry(url:string,retries =3,delay=500):Promise{ + + if (!r2) { + throw new Error("Object store credentials are not set"); + } + for(let attempt =1; attempt<=retries;attempt++){ try { const response = await r2.fetch(url); From da00c3ae3361c021e35679444e0cc70d0f1aff68 Mon Sep 17 00:00:00 2001 From: Aarish mansur Date: Sun, 9 Nov 2025 14:13:17 +0000 Subject: [PATCH 3/4] Fixed the critical logic flow --- apps/webapp/app/v3/r2.server.ts | 70 +++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/apps/webapp/app/v3/r2.server.ts b/apps/webapp/app/v3/r2.server.ts index 8dd3ef3a6b..4b0ad8b4a1 100644 --- a/apps/webapp/app/v3/r2.server.ts +++ b/apps/webapp/app/v3/r2.server.ts @@ -93,39 +93,51 @@ export async function downloadPacketFromObjectStore( logger.debug("Downloading from object store", { url: url.href }); - async function fetchWithRetry(url:string,retries =3,delay=500):Promise{ - - if (!r2) { - throw new Error("Object store credentials are not set"); - } - - for(let attempt =1; attempt<=retries;attempt++){ - try { - const response = await r2.fetch(url); - if(response.ok) return response; - - // only retry on transient server/network errors - if(response.status >= 500 && response.status<600 ){ - throw new Error(`Server Error : ${response.statusText}`); - } - - // for other non server errors - throw new Error(`non-retryable error :${response.statusText}`); - - } catch (error:any) { - if(attempt == retries) throw error; - logger.warn(`Retrying object Download (attempt: ${attempt}) out of ${retries}`,{ - url, - error:error.message, - }); - await new Promise((res)=>setTimeout(res,delay*attempt)); + + async function fetchWithRetry(url: string, retries = 3, delay = 500): Promise { + for (let attempt = 1; attempt <= retries; attempt++) { + try { + const response = await r2.fetch(url); + if (response.ok) return response; + + if (response.status >= 400 && response.status < 500) { + throw new Error(`Client error (non-retryable): ${response.statusText}`); + } + + if (response.status >= 500 && response.status < 600) { + if (attempt === retries) { + throw new Error(`Server error after ${retries} attempts: ${response.statusText}`); } - } - throw new Error(`Failed to fetch ${url} after ${retries} retries`); + logger.warn(`Retrying object download (attempt ${attempt}/${retries})`, { + url, + status: response.status, + error: response.statusText, + }); + + await new Promise((res) => setTimeout(res, delay * attempt)); + continue; + } + + throw new Error(`Unexpected status ${response.status}: ${response.statusText}`); + } catch (error: unknown) { + if (attempt === retries) throw error; + + const errorMessage = error instanceof Error ? error.message : String(error); + logger.warn(`Network error, retrying (attempt ${attempt}/${retries})`, { + url, + error: errorMessage, + }); + + await new Promise((res) => setTimeout(res, delay * attempt)); } - const response = await fetchWithRetry(url.toString()); + } + + throw new Error(`Failed to fetch ${url} after ${retries} retries`); +} + + const response = await fetchWithRetry(url.toString()); const data = await response.text(); From ced1c90f1cca11d388be666989545807fe4e6900 Mon Sep 17 00:00:00 2001 From: Aarish mansur Date: Sun, 9 Nov 2025 14:26:39 +0000 Subject: [PATCH 4/4] added logic for to avoid unexpected error --- apps/webapp/app/v3/r2.server.ts | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/apps/webapp/app/v3/r2.server.ts b/apps/webapp/app/v3/r2.server.ts index 4b0ad8b4a1..02c8f8d310 100644 --- a/apps/webapp/app/v3/r2.server.ts +++ b/apps/webapp/app/v3/r2.server.ts @@ -94,14 +94,22 @@ export async function downloadPacketFromObjectStore( logger.debug("Downloading from object store", { url: url.href }); - async function fetchWithRetry(url: string, retries = 3, delay = 500): Promise { + class NonRetryableError extends Error { + constructor(message: string) { + super(message); + this.name = "NonRetryableError"; + } +} + +async function fetchWithRetry(url: string, retries = 3, delay = 500): Promise { for (let attempt = 1; attempt <= retries; attempt++) { try { const response = await r2.fetch(url); + if (response.ok) return response; if (response.status >= 400 && response.status < 500) { - throw new Error(`Client error (non-retryable): ${response.statusText}`); + throw new NonRetryableError(`Client error: ${response.statusText}`); } if (response.status >= 500 && response.status < 600) { @@ -119,8 +127,12 @@ export async function downloadPacketFromObjectStore( continue; } - throw new Error(`Unexpected status ${response.status}: ${response.statusText}`); + throw new NonRetryableError(`Unexpected status ${response.status}: ${response.statusText}`); } catch (error: unknown) { + if (error instanceof NonRetryableError) { + throw error; + } + if (attempt === retries) throw error; const errorMessage = error instanceof Error ? error.message : String(error); @@ -137,6 +149,7 @@ export async function downloadPacketFromObjectStore( throw new Error(`Failed to fetch ${url} after ${retries} retries`); } + const response = await fetchWithRetry(url.toString()); const data = await response.text();