feat: add time varying timeout

Author: VinciGit00

scrapegraph-py/scrapegraph_py/async_client.py

@@ -101,14 +101,18 @@ def __init__(
 
     async def _make_request(self, method: str, url: str, **kwargs) -> Any:
         """Make HTTP request with retry logic."""
+        batch_size = len(kwargs.get('json', {}).get('urls', [1])) if kwargs.get('json') else 1
+        timeout = ClientTimeout(total=self.timeout.total * batch_size)
+        
         for attempt in range(self.max_retries):
             try:
                 logger.info(
                     f"🚀 Making {method} request to {url} (Attempt {attempt + 1}/{self.max_retries})"
                 )
                 logger.debug(f"🔍 Request parameters: {kwargs}")
 
-                async with self.session.request(method, url, **kwargs) as response:
+                # Use the calculated timeout for this request
+                async with self.session.request(method, url, timeout=timeout, **kwargs) as response:
                     logger.debug(f"📥 Response status: {response.status}")
                     result = await handle_async_response(response)
                     logger.info(f"✅ Request completed successfully: {method} {url}")

scrapegraph-py/scrapegraph_py/client.py

@@ -115,14 +115,16 @@ def __init__(
         logger.info("✅ Client initialized successfully")
 
     def _make_request(self, method: str, url: str, **kwargs) -> Any:
-        """Make HTTP request with error handling."""
+        """Make HTTP request with retry logic."""
+        # Calculate timeout based on batch size if present in kwargs
+        batch_size = len(kwargs.get('json', {}).get('urls', [1])) if kwargs.get('json') else 1
+        timeout = self.timeout * batch_size
+
         try:
             logger.info(f"🚀 Making {method} request to {url}")
             logger.debug(f"🔍 Request parameters: {kwargs}")
 
-            response = self.session.request(method, url, timeout=self.timeout, **kwargs)
-            logger.debug(f"📥 Response status: {response.status_code}")
-
+            response = self.session.request(method, url, timeout=timeout, **kwargs)
             result = handle_sync_response(response)
             logger.info(f"✅ Request completed successfully: {method} {url}")
             return result