chore: fix _make_request not using it

Author: PeriniM

scrapegraph-py/scrapegraph_py/async_client.py

@@ -1,7 +1,6 @@
 import asyncio
 from typing import Any, Optional
 
-import aiohttp
 from aiohttp import ClientSession, ClientTimeout, TCPConnector
 from aiohttp.client_exceptions import ClientError
 from pydantic import BaseModel
@@ -101,18 +100,14 @@ def __init__(
 
     async def _make_request(self, method: str, url: str, **kwargs) -> Any:
         """Make HTTP request with retry logic."""
-        batch_size = len(kwargs.get('json', {}).get('urls', [1])) if kwargs.get('json') else 1
-        timeout = ClientTimeout(total=self.timeout.total * batch_size)
-        
         for attempt in range(self.max_retries):
             try:
                 logger.info(
                     f"🚀 Making {method} request to {url} (Attempt {attempt + 1}/{self.max_retries})"
                 )
                 logger.debug(f"🔍 Request parameters: {kwargs}")
 
-                # Use the calculated timeout for this request
-                async with self.session.request(method, url, timeout=timeout, **kwargs) as response:
+                async with self.session.request(method, url, **kwargs) as response:
                     logger.debug(f"📥 Response status: {response.status}")
                     result = await handle_async_response(response)
                     logger.info(f"✅ Request completed successfully: {method} {url}")
@@ -141,6 +136,33 @@ async def _make_request(self, method: str, url: str, **kwargs) -> Any:
                 logger.info(f"⏳ Waiting {retry_delay}s before retry {attempt + 2}")
                 await asyncio.sleep(retry_delay)
 
+    async def markdownify(self, website_url: str):
+        """Send a markdownify request"""
+        logger.info(f"🔍 Starting markdownify request for {website_url}")
+
+        request = MarkdownifyRequest(website_url=website_url)
+        logger.debug("✅ Request validation passed")
+
+        result = await self._make_request(
+            "POST", f"{API_BASE_URL}/markdownify", json=request.model_dump()
+        )
+        logger.info("✨ Markdownify request completed successfully")
+        return result
+
+    async def get_markdownify(self, request_id: str):
+        """Get the result of a previous markdownify request"""
+        logger.info(f"🔍 Fetching markdownify result for request {request_id}")
+
+        # Validate input using Pydantic model
+        GetMarkdownifyRequest(request_id=request_id)
+        logger.debug("✅ Request ID validation passed")
+
+        result = await self._make_request(
+            "GET", f"{API_BASE_URL}/markdownify/{request_id}"
+        )
+        logger.info(f"✨ Successfully retrieved result for request {request_id}")
+        return result
+
     async def smartscraper(
         self,
         website_url: str,
@@ -158,17 +180,11 @@ async def smartscraper(
         )
         logger.debug("✅ Request validation passed")
 
-        try:
-            async with self.session.post(
-                f"{API_BASE_URL}/smartscraper", json=request.model_dump()
-            ) as response:
-                response.raise_for_status()
-                result = await handle_async_response(response)
-                logger.info("✨ Smartscraper request completed successfully")
-                return result
-        except aiohttp.ClientError as e:
-            logger.error(f"❌ Smartscraper request failed: {str(e)}")
-            raise ConnectionError(f"Failed to connect to API: {str(e)}")
+        result = await self._make_request(
+            "POST", f"{API_BASE_URL}/smartscraper", json=request.model_dump()
+        )
+        logger.info("✨ Smartscraper request completed successfully")
+        return result
 
     async def get_smartscraper(self, request_id: str):
         """Get the result of a previous smartscraper request"""
@@ -178,80 +194,8 @@ async def get_smartscraper(self, request_id: str):
         GetSmartScraperRequest(request_id=request_id)
         logger.debug("✅ Request ID validation passed")
 
-        async with self.session.get(
-            f"{API_BASE_URL}/smartscraper/{request_id}"
-        ) as response:
-            result = await handle_async_response(response)
-            logger.info(f"✨ Successfully retrieved result for request {request_id}")
-            return result
-
-    async def get_credits(self):
-        """Get credits information"""
-        logger.info("💳 Fetching credits information")
-
-        async with self.session.get(
-            f"{API_BASE_URL}/credits",
-        ) as response:
-            result = await handle_async_response(response)
-            logger.info(
-                f"✨ Credits info retrieved: {result.get('remaining_credits')} credits remaining"
-            )
-            return result
-
-    async def submit_feedback(
-        self, request_id: str, rating: int, feedback_text: Optional[str] = None
-    ):
-        """Submit feedback for a request"""
-        logger.info(f"📝 Submitting feedback for request {request_id}")
-        logger.debug(f"⭐ Rating: {rating}, Feedback: {feedback_text}")
-
-        feedback = FeedbackRequest(
-            request_id=request_id, rating=rating, feedback_text=feedback_text
-        )
-        logger.debug("✅ Feedback validation passed")
-
-        async with self.session.post(
-            f"{API_BASE_URL}/feedback", json=feedback.model_dump()
-        ) as response:
-            result = await handle_async_response(response)
-            logger.info("✨ Feedback submitted successfully")
-            return result
-
-    async def close(self):
-        """Close the session to free up resources"""
-        logger.info("🔒 Closing AsyncClient session")
-        await self.session.close()
-        logger.debug("✅ Session closed successfully")
-
-    async def __aenter__(self):
-        return self
-
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
-        await self.close()
-
-    async def markdownify(self, website_url: str):
-        """Send a markdownify request"""
-        logger.info(f"🔍 Starting markdownify request for {website_url}")
-
-        request = MarkdownifyRequest(website_url=website_url)
-        logger.debug("✅ Request validation passed")
-
-        result = await self._make_request(
-            "POST", f"{API_BASE_URL}/markdownify", json=request.model_dump()
-        )
-        logger.info("✨ Markdownify request completed successfully")
-        return result
-
-    async def get_markdownify(self, request_id: str):
-        """Get the result of a previous markdownify request"""
-        logger.info(f"🔍 Fetching markdownify result for request {request_id}")
-
-        # Validate input using Pydantic model
-        GetMarkdownifyRequest(request_id=request_id)
-        logger.debug("✅ Request ID validation passed")
-
         result = await self._make_request(
-            "GET", f"{API_BASE_URL}/markdownify/{request_id}"
+            "GET", f"{API_BASE_URL}/smartscraper/{request_id}"
         )
         logger.info(f"✨ Successfully retrieved result for request {request_id}")
         return result
@@ -292,3 +236,46 @@ async def get_localscraper(self, request_id: str):
         )
         logger.info(f"✨ Successfully retrieved result for request {request_id}")
         return result
+
+    async def submit_feedback(
+        self, request_id: str, rating: int, feedback_text: Optional[str] = None
+    ):
+        """Submit feedback for a request"""
+        logger.info(f"📝 Submitting feedback for request {request_id}")
+        logger.debug(f"⭐ Rating: {rating}, Feedback: {feedback_text}")
+
+        feedback = FeedbackRequest(
+            request_id=request_id, rating=rating, feedback_text=feedback_text
+        )
+        logger.debug("✅ Feedback validation passed")
+
+        result = await self._make_request(
+            "POST", f"{API_BASE_URL}/feedback", json=feedback.model_dump()
+        )
+        logger.info("✨ Feedback submitted successfully")
+        return result
+
+    async def get_credits(self):
+        """Get credits information"""
+        logger.info("💳 Fetching credits information")
+
+        result = await self._make_request(
+            "GET",
+            f"{API_BASE_URL}/credits",
+        )
+        logger.info(
+            f"✨ Credits info retrieved: {result.get('remaining_credits')} credits remaining"
+        )
+        return result
+
+    async def close(self):
+        """Close the session to free up resources"""
+        logger.info("🔒 Closing AsyncClient session")
+        await self.session.close()
+        logger.debug("✅ Session closed successfully")
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self.close()

scrapegraph-py/scrapegraph_py/client.py

@@ -115,16 +115,14 @@ def __init__(
         logger.info("✅ Client initialized successfully")
 
     def _make_request(self, method: str, url: str, **kwargs) -> Any:
-        """Make HTTP request with retry logic."""
-        # Calculate timeout based on batch size if present in kwargs
-        batch_size = len(kwargs.get('json', {}).get('urls', [1])) if kwargs.get('json') else 1
-        timeout = self.timeout * batch_size
-
+        """Make HTTP request with error handling."""
         try:
             logger.info(f"🚀 Making {method} request to {url}")
             logger.debug(f"🔍 Request parameters: {kwargs}")
 
-            response = self.session.request(method, url, timeout=timeout, **kwargs)
+            response = self.session.request(method, url, timeout=self.timeout, **kwargs)
+            logger.debug(f"📥 Response status: {response.status_code}")
+
             result = handle_sync_response(response)
             logger.info(f"✅ Request completed successfully: {method} {url}")
             return result
@@ -150,6 +148,31 @@ def _make_request(self, method: str, url: str, **kwargs) -> Any:
             logger.error(f"🔴 Connection Error: {str(e)}")
             raise ConnectionError(f"Failed to connect to API: {str(e)}")
 
+    def markdownify(self, website_url: str):
+        """Send a markdownify request"""
+        logger.info(f"🔍 Starting markdownify request for {website_url}")
+
+        request = MarkdownifyRequest(website_url=website_url)
+        logger.debug("✅ Request validation passed")
+
+        result = self._make_request(
+            "POST", f"{API_BASE_URL}/markdownify", json=request.model_dump()
+        )
+        logger.info("✨ Markdownify request completed successfully")
+        return result
+
+    def get_markdownify(self, request_id: str):
+        """Get the result of a previous markdownify request"""
+        logger.info(f"🔍 Fetching markdownify result for request {request_id}")
+
+        # Validate input using Pydantic model
+        GetMarkdownifyRequest(request_id=request_id)
+        logger.debug("✅ Request ID validation passed")
+
+        result = self._make_request("GET", f"{API_BASE_URL}/markdownify/{request_id}")
+        logger.info(f"✨ Successfully retrieved result for request {request_id}")
+        return result
+
     def smartscraper(
         self,
         website_url: str,
@@ -185,74 +208,6 @@ def get_smartscraper(self, request_id: str):
         logger.info(f"✨ Successfully retrieved result for request {request_id}")
         return result
 
-    def get_credits(self):
-        """Get credits information"""
-        logger.info("💳 Fetching credits information")
-
-        result = self._make_request(
-            "GET",
-            f"{API_BASE_URL}/credits",
-        )
-        logger.info(
-            f"✨ Credits info retrieved: {result.get('remaining_credits')} credits remaining"
-        )
-        return result
-
-    def submit_feedback(
-        self, request_id: str, rating: int, feedback_text: Optional[str] = None
-    ):
-        """Submit feedback for a request"""
-        logger.info(f"📝 Submitting feedback for request {request_id}")
-        logger.debug(f"⭐ Rating: {rating}, Feedback: {feedback_text}")
-
-        feedback = FeedbackRequest(
-            request_id=request_id, rating=rating, feedback_text=feedback_text
-        )
-        logger.debug("✅ Feedback validation passed")
-
-        result = self._make_request(
-            "POST", f"{API_BASE_URL}/feedback", json=feedback.model_dump()
-        )
-        logger.info("✨ Feedback submitted successfully")
-        return result
-
-    def close(self):
-        """Close the session to free up resources"""
-        logger.info("🔒 Closing Client session")
-        self.session.close()
-        logger.debug("✅ Session closed successfully")
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self.close()
-
-    def markdownify(self, website_url: str):
-        """Send a markdownify request"""
-        logger.info(f"🔍 Starting markdownify request for {website_url}")
-
-        request = MarkdownifyRequest(website_url=website_url)
-        logger.debug("✅ Request validation passed")
-
-        result = self._make_request(
-            "POST", f"{API_BASE_URL}/markdownify", json=request.model_dump()
-        )
-        logger.info("✨ Markdownify request completed successfully")
-        return result
-
-    def get_markdownify(self, request_id: str):
-        """Get the result of a previous markdownify request"""
-        logger.info(f"🔍 Fetching markdownify result for request {request_id}")
-
-        # Validate input using Pydantic model
-        GetMarkdownifyRequest(request_id=request_id)
-        logger.debug("✅ Request ID validation passed")
-
-        result = self._make_request("GET", f"{API_BASE_URL}/markdownify/{request_id}")
-        logger.info(f"✨ Successfully retrieved result for request {request_id}")
-        return result
-
     def localscraper(
         self,
         user_prompt: str,
@@ -287,3 +242,46 @@ def get_localscraper(self, request_id: str):
         result = self._make_request("GET", f"{API_BASE_URL}/localscraper/{request_id}")
         logger.info(f"✨ Successfully retrieved result for request {request_id}")
         return result
+
+    def submit_feedback(
+        self, request_id: str, rating: int, feedback_text: Optional[str] = None
+    ):
+        """Submit feedback for a request"""
+        logger.info(f"📝 Submitting feedback for request {request_id}")
+        logger.debug(f"⭐ Rating: {rating}, Feedback: {feedback_text}")
+
+        feedback = FeedbackRequest(
+            request_id=request_id, rating=rating, feedback_text=feedback_text
+        )
+        logger.debug("✅ Feedback validation passed")
+
+        result = self._make_request(
+            "POST", f"{API_BASE_URL}/feedback", json=feedback.model_dump()
+        )
+        logger.info("✨ Feedback submitted successfully")
+        return result
+
+    def get_credits(self):
+        """Get credits information"""
+        logger.info("💳 Fetching credits information")
+
+        result = self._make_request(
+            "GET",
+            f"{API_BASE_URL}/credits",
+        )
+        logger.info(
+            f"✨ Credits info retrieved: {result.get('remaining_credits')} credits remaining"
+        )
+        return result
+
+    def close(self):
+        """Close the session to free up resources"""
+        logger.info("🔒 Closing Client session")
+        self.session.close()
+        logger.debug("✅ Session closed successfully")
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()