support is_extracted metadata for elements

Author: qued

unstructured/documents/elements.py

@@ -206,6 +206,7 @@ class ElementMetadata:
 
     # -- used for Table elements to capture rows/col structure --
     text_as_html: Optional[str]
+    is_extracted: Optional[str]
     table_as_cells: Optional[dict[str, str | int]]
     url: Optional[str]
 
@@ -498,6 +499,7 @@ def field_consolidation_strategies(cls) -> dict[str, ConsolidationStrategy]:
             "image_base64": cls.DROP,
             "image_mime_type": cls.DROP,
             "is_continuation": cls.DROP,  # -- not expected, added by chunking, not before --
+            "is_extracted": cls.DROP,
             "languages": cls.LIST_UNIQUE,
             "last_modified": cls.FIRST,
             "link_texts": cls.LIST_CONCATENATE,

unstructured/partition/common/common.py

@@ -2,6 +2,7 @@
 
 import numbers
 import subprocess
+from enum import Enum
 from io import BufferedReader, BytesIO, TextIOWrapper
 from tempfile import SpooledTemporaryFile
 from time import sleep
@@ -58,12 +59,17 @@ def normalize_layout_element(
     prob = layout_dict.get("prob")
     aux_origin = layout_dict.get("source", None)
     origin = None
+    if isinstance(layout_dict.get("is_extracted"), Enum):
+        is_extracted = layout_dict["is_extracted"].value
+    else:
+        is_extracted = None
     if aux_origin:
         origin = aux_origin.value
     if prob and isinstance(prob, (int, str, float, numbers.Number)):
         class_prob_metadata = ElementMetadata(detection_class_prob=float(prob))  # type: ignore
     else:
         class_prob_metadata = ElementMetadata()
+    class_prob_metadata.is_extracted = is_extracted
     common_kwargs = {
         "coordinates": coordinates,
         "coordinate_system": coordinate_system,