docs: refactor nodes docstrings

Author: PeriniM

scrapegraphai/nodes/base_node.py

@@ -1,5 +1,5 @@
 """ 
-Module for defining BaseNode, an abstract base class for nodes in a graph-based workflow.
+BaseNode Module
 """
 
 from abc import ABC, abstractmethod

scrapegraphai/nodes/fetch_node.py

@@ -1,5 +1,5 @@
 """ 
-Module for fetching the HTML node
+FetchNode Module
 """
 
 from typing import List, Optional
@@ -27,10 +27,6 @@ class FetchNode(BaseNode):
         output (List[str]): List of output keys to be updated in the state.
         node_config (Optional[dict]): Additional configuration for the node.
         node_name (str): The unique identifier name for the node, defaulting to "Fetch".
-
-    Methods:
-        execute(state): Fetches the HTML content for the URL specified in the state
-        and updates the state with the fetched content under the specified output key.
     """
 
     def __init__(self, input: str, output: List[str], node_config: Optional[dict], node_name: str = "Fetch"):
@@ -45,13 +41,14 @@ def execute(self, state):
         update the state with this content.
 
         Args:
-            state (dict): The current state of the graph, expected to contain a 'url' key.
+            state (dict): The current state of the graph. The input keys will be used
+                            to fetch the correct data types from the state.
 
         Returns:
-            dict: The updated state with a new 'document' key containing the fetched HTML content.
+            dict: The updated state with a new output key containing the fetched HTML content.
 
         Raises:
-            KeyError: If the 'url' key is not found in the state, indicating that the
+            KeyError: If the input key is not found in the state, indicating that the
                     necessary information to perform the operation is missing.
         """
         if self.verbose:

scrapegraphai/nodes/generate_answer_node.py

@@ -1,6 +1,7 @@
 """
-Module for generating the answer node
+GenerateAnswerNode Module
 """
+
 # Imports from standard library
 from typing import List
 from tqdm import tqdm
@@ -16,57 +17,43 @@
 
 class GenerateAnswerNode(BaseNode):
     """
-    A node that generates an answer using a language model (LLM) based on the user's input
+    A node that generates an answer using a large language model (LLM) based on the user's input
     and the content extracted from a webpage. It constructs a prompt from the user's input
     and the scraped content, feeds it to the LLM, and parses the LLM's response to produce
     an answer.
 
     Attributes:
-        llm: An instance of a language model client, configured for generating answers.
-        node_name (str): The unique identifier name for the node, defaulting 
-        to "GenerateAnswerNode".
-        node_type (str): The type of the node, set to "node" indicating a 
-        standard operational node.
+        llm_model: An instance of a language model client, configured for generating answers.
+        verbose (bool): A flag indicating whether to show print statements during execution.
 
     Args:
-        llm: An instance of the language model client (e.g., ChatOpenAI) used 
-        for generating answers.
-        node_name (str, optional): The unique identifier name for the node. 
-        Defaults to "GenerateAnswerNode".
-
-    Methods:
-        execute(state): Processes the input and document from the state to generate an answer,
-                        updating the state with the generated answer under the 'answer' key.
+        input (str): Boolean expression defining the input keys needed from the state.
+        output (List[str]): List of output keys to be updated in the state.
+        node_config (dict): Additional configuration for the node.
+        node_name (str): The unique identifier name for the node, defaulting to "GenerateAnswer".
     """
 
     def __init__(self, input: str, output: List[str], node_config: dict,
                  node_name: str = "GenerateAnswer"):
-        """
-        Initializes the GenerateAnswerNode with a language model client and a node name.
-        Args:
-            llm: An instance of the OpenAIImageToText class.
-            node_name (str): name of the node
-        """
         super().__init__(node_name, "node", input, output, 2, node_config)
+        
         self.llm_model = node_config["llm"]
         self.verbose = True if node_config is None else node_config.get("verbose", False)
 
-    def execute(self, state):
+    def execute(self, state: dict) -> dict:
         """
         Generates an answer by constructing a prompt from the user's input and the scraped
         content, querying the language model, and parsing its response.
 
-        The method updates the state with the generated answer under the 'answer' key.
-
         Args:
-            state (dict): The current state of the graph, expected to contain 'user_input',
-                          and optionally 'parsed_document' or 'relevant_chunks' within 'keys'.
+            state (dict): The current state of the graph. The input keys will be used
+                            to fetch the correct data from the state.
 
         Returns:
-            dict: The updated state with the 'answer' key containing the generated answer.
+            dict: The updated state with the output key containing the generated answer.
 
         Raises:
-            KeyError: If 'user_input' or 'document' is not found in the state, indicating
+            KeyError: If the input keys are not found in the state, indicating
                       that the necessary information for generating an answer is missing.
         """
 

scrapegraphai/nodes/generate_scraper_node.py

@@ -1,6 +1,7 @@
 """
-Module for generating the answer node
+GenerateScraperNode Module
 """
+
 # Imports from standard library
 from typing import List
 from tqdm import tqdm
@@ -16,58 +17,46 @@
 
 class GenerateScraperNode(BaseNode):
     """
-    A node that generates an answer using a language model (LLM) based on the user's input
-    and the content extracted from a webpage. It constructs a prompt from the user's input
-    and the scraped content, feeds it to the LLM, and parses the LLM's response to produce
-    an answer.
+    Generates a python script for scraping a website using the specified library.
+    It takes the user's prompt and the scraped content as input and generates a python script
+    that extracts the information requested by the user.
 
     Attributes:
-        llm: An instance of a language model client, configured for generating answers.
-        node_name (str): The unique identifier name for the node, defaulting 
-        to "GenerateScraperNode".
-        node_type (str): The type of the node, set to "node" indicating a 
-        standard operational node.
+        llm_model: An instance of a language model client, configured for generating answers.
+        library (str): The python library to use for scraping the website.
+        source (str): The website to scrape.
 
     Args:
-        llm: An instance of the language model client (e.g., ChatOpenAI) used 
-        for generating answers.
-        node_name (str, optional): The unique identifier name for the node. 
-        Defaults to "GenerateScraperNode".
-
-    Methods:
-        execute(state): Processes the input and document from the state to generate an answer,
-                        updating the state with the generated answer under the 'answer' key.
+        input (str): Boolean expression defining the input keys needed from the state.
+        output (List[str]): List of output keys to be updated in the state.
+        node_config (dict): Additional configuration for the node.
+        library (str): The python library to use for scraping the website.
+        website (str): The website to scrape.
+        node_name (str): The unique identifier name for the node, defaulting to "GenerateAnswer".
+
     """
 
     def __init__(self, input: str, output: List[str], node_config: dict,
                  library: str, website: str, node_name: str = "GenerateAnswer"):
-        """
-        Initializes the GenerateScraperNode with a language model client and a node name.
-        Args:
-            llm (OpenAIImageToText): An instance of the OpenAIImageToText class.
-            node_name (str): name of the node
-        """
         super().__init__(node_name, "node", input, output, 2, node_config)
+
         self.llm_model = node_config["llm"]
         self.library = library
         self.source = website
 
-    def execute(self, state):
+    def execute(self, state: dict) -> dict:
         """
-        Generates an answer by constructing a prompt from the user's input and the scraped
-        content, querying the language model, and parsing its response.
-
-        The method updates the state with the generated answer under the 'answer' key.
+        Generates a python script for scraping a website using the specified library.
 
         Args:
-            state (dict): The current state of the graph, expected to contain 'user_input',
-                          and optionally 'parsed_document' or 'relevant_chunks' within 'keys'.
+            state (dict): The current state of the graph. The input keys will be used
+                            to fetch the correct data from the state.
 
         Returns:
-            dict: The updated state with the 'answer' key containing the generated answer.
+            dict: The updated state with the output key containing the generated answer.
 
         Raises:
-            KeyError: If 'user_input' or 'document' is not found in the state, indicating
+            KeyError: If input keys are not found in the state, indicating
                       that the necessary information for generating an answer is missing.
         """
 

scrapegraphai/nodes/get_probable_tags_node.py

@@ -1,6 +1,7 @@
 """
-Module for proobable tags
+GetProbableTagsNode Module
 """
+
 from typing import List
 from langchain.output_parsers import CommaSeparatedListOutputParser
 from langchain.prompts import PromptTemplate
@@ -15,47 +16,36 @@ class GetProbableTagsNode(BaseNode):
     list of probable tags.
 
     Attributes:
-        llm: An instance of a language model client, configured for generating tag predictions.
-        node_name (str): The unique identifier name for the node,
-        defaulting to "GetProbableTagsNode".
-        node_type (str): The type of the node, set to "node" indicating a standard operational node.
+        llm_model: An instance of the language model client used for tag predictions.
 
     Args:
-        llm: An instance of the language model client (e.g., ChatOpenAI) used for tag predictions.
-        node_name (str, optional): The unique identifier name for the node. 
-        Defaults to "GetProbableTagsNode".
-
-    Methods:
-        execute(state): Processes the user's input and the URL from the state to generate a list of 
-                        probable HTML tags, updating the state with these tags under the 'tags' key.
+        input (str): Boolean expression defining the input keys needed from the state.
+        output (List[str]): List of output keys to be updated in the state.
+        model_config (dict): Additional configuration for the language model.
+        node_name (str): The unique identifier name for the node, defaulting to "GetProbableTags".
     """
 
     def __init__(self, input: str, output: List[str], model_config: dict,
                  node_name: str = "GetProbableTags"):
-        """
-        Initializes the GetProbableTagsNode with a language model client and a node name.
-        Args:
-            llm (OpenAIImageToText): An instance of the OpenAIImageToText class.
-            node_name (str): name of the node
-        """
         super().__init__(node_name, "node", input, output, 2, model_config)
+
         self.llm_model = model_config["llm_model"]
 
-    def execute(self, state):
+    def execute(self, state: dict) -> dict:
         """
         Generates a list of probable HTML tags based on the user's input and updates the state 
         with this list. The method constructs a prompt for the language model, submits it, and 
         parses the output to identify probable tags.
 
         Args:
-            state (dict): The current state of the graph, expected to contain 'user_input', 'url',
-                          and optionally 'document' within 'keys'.
+            state (dict): The current state of the graph. The input keys will be used to fetch the
+                            correct data types from the state.
 
         Returns:
-            dict: The updated state with the 'tags' key containing a list of probable HTML tags.
+            dict: The updated state with the input key containing a list of probable HTML tags.
 
         Raises:
-            KeyError: If 'user_input' or 'url' is not found in the state, indicating that the
+            KeyError: If input keys are not found in the state, indicating that the
                       necessary information for generating tag predictions is missing.
         """
 

scrapegraphai/nodes/image_to_text_node.py

@@ -1,45 +1,44 @@
 """
-Module for the ImageToTextNode class.
+ImageToTextNode Module
 """
+
 from typing import List
 from .base_node import BaseNode
 
 
 class ImageToTextNode(BaseNode):
     """
-    A class representing a node that processes an image and returns the text description.
+    Retrieve an image from an URL and convert it to text using an ImageToText model.
 
     Attributes:
-        llm_model (OpenAIImageToText): An instance of the OpenAIImageToText class.
-
-    Methods:
-        execute(state, url): Execute the node's logic and return the updated state.
+        llm_model: An instance of the language model client used for image-to-text conversion.
+        verbose (bool): A flag indicating whether to show print statements during execution.
+
+    Args:
+        input (str): Boolean expression defining the input keys needed from the state.
+        output (List[str]): List of output keys to be updated in the state.
+        node_config (dict): Additional configuration for the node.
+        node_name (str): The unique identifier name for the node, defaulting to "ImageToText".
     """
 
     def __init__(self, input: str, output: List[str], node_config: dict,
                  node_name: str = "ImageToText"):
-        """
-        Initializes an instance of the ImageToTextNode class.
-
-        Args:
-            input (str): The input for the node.
-            output (List[str]): The output of the node.
-            node_config (dict): Configuration for the model.
-            node_name (str): Name of the node.
-        """
         super().__init__(node_name, "node", input, output, 1, node_config)
+
         self.llm_model = node_config["llm_model"]
         self.verbose = True if node_config is None else node_config.get("verbose", False)
 
     def execute(self, state: dict) -> dict:
         """
-        Execute the node's logic and return the updated state.
+        Generate text from an image using an image-to-text model. The method retrieves the image
+        from the URL provided in the state.
 
         Args:
-            state (dict): The current state of the graph.
+            state (dict): The current state of the graph. The input keys will be used to fetch the
+                            correct data types from the state.
 
         Returns:
-            dict: The updated state after executing this node.
+            dict: The updated state with the input key containing the text extracted from the image.
         """
 
         if self.verbose: