Add get_text_separator parameter to BSHTMLLoader (langchain-ai#3551)

By default get_text doesn't separate content of different HTML tag.
Adding option for specifying separator helps with document splitting.

Author: maver1ck

langchain/document_loaders/html_bs.py

@@ -17,6 +17,7 @@ def __init__(
         file_path: str,
         open_encoding: Union[str, None] = None,
         bs_kwargs: Union[dict, None] = None,
+        get_text_separator: str = "",
     ) -> None:
         """Initialise with path, and optionally, file encoding to use, and any kwargs
         to pass to the BeautifulSoup object."""
@@ -33,6 +34,7 @@ def __init__(
         if bs_kwargs is None:
             bs_kwargs = {"features": "lxml"}
         self.bs_kwargs = bs_kwargs
+        self.get_text_separator = get_text_separator
 
     def load(self) -> List[Document]:
         from bs4 import BeautifulSoup
@@ -41,7 +43,7 @@ def load(self) -> List[Document]:
         with open(self.file_path, "r", encoding=self.open_encoding) as f:
             soup = BeautifulSoup(f, **self.bs_kwargs)
 
-        text = soup.get_text()
+        text = soup.get_text(self.get_text_separator)
 
         if soup.title:
             title = str(soup.title.string)

tests/integration_tests/document_loaders/test_bshtml.py

@@ -9,15 +9,17 @@
 def test_bs_html_loader() -> None:
     """Test unstructured loader."""
     file_path = Path(__file__).parent.parent / "examples/example.html"
-    loader = BSHTMLLoader(str(file_path))
+    loader = BSHTMLLoader(str(file_path), get_text_separator="|")
     docs = loader.load()
 
     assert len(docs) == 1
 
     metadata = docs[0].metadata
+    content = docs[0].page_content
 
     assert metadata["title"] == "Chew dad's slippers"
     assert metadata["source"] == str(file_path)
+    assert content[:2] == "\n|"
 
 
 @pytest.mark.skipif(