Merge pull request #697 from onekey-sec/padding-auto-id

Unknown chunks auto-identification (padding)

Author: qkaiser

tests/integration/archive/cpio/cpio_portable_ascii/__output__/sample.nofilepad.unaligned.cpio-newc_extract/0-1.unknown renamed to tests/integration/archive/cpio/cpio_portable_ascii/__output__/sample.nofilepad.unaligned.cpio-newc_extract/0-1.padding

@@ -73,6 +73,7 @@ def hello_kitty_task_results(
     extract_root: Path,
     hello_id: str,
     kitty_id: str,
+    padding_id: str,
     container_id="",
     start_depth=0,
 ):
@@ -133,12 +134,14 @@ def hello_kitty_task_results(
                     size=7,
                     entropy=None,
                 ),
-                UnknownChunkReport(
-                    id=ANY,
+                ChunkReport(
+                    id=padding_id,
                     start_offset=263,
                     end_offset=264,
                     size=1,
-                    entropy=None,
+                    handler_name="padding",
+                    is_encrypted=False,
+                    extraction_reports=[],
                 ),
                 ChunkReport(
                     id=hello_id,
@@ -286,13 +289,14 @@ def test_flat_report_structure(hello_kitty: Path, extract_root):
     task_results = get_normalized_task_results(process_result)
 
     # extract the ids from the chunks
-    hello_id, kitty_id = get_chunk_ids(task_results[0])
+    padding_id, hello_id, kitty_id = get_chunk_ids(task_results[0])
 
     assert task_results == hello_kitty_task_results(
         hello_kitty=hello_kitty,
         extract_root=extract_root,
         hello_id=hello_id,
         kitty_id=kitty_id,
+        padding_id=padding_id,
     )
 
 
@@ -416,7 +420,7 @@ def test_chunk_in_chunk_report_structure(hello_kitty_container: Path, extract_ro
     # and they should be the only differences
     [main_id] = get_chunk_ids(task_results[0])
 
-    hello_id, kitty_id = get_chunk_ids(task_results[2])
+    padding_id, hello_id, kitty_id = get_chunk_ids(task_results[2])
 
     # We test, that the container is referenced from the internal file
     # through the chunk id `main_id`
@@ -428,6 +432,7 @@ def test_chunk_in_chunk_report_structure(hello_kitty_container: Path, extract_ro
         extract_root=extract_root / "container_extract",
         hello_id=hello_id,
         kitty_id=kitty_id,
+        padding_id=padding_id,
         container_id=main_id,
         start_depth=1,
     )

tests/integration/compression/bzip2/__output__/collated.headpad.bzip2_extract/0-17.unknown renamed to tests/integration/compression/bzip2/__output__/collated.headpad.bzip2_extract/0-17.padding

@@ -2,11 +2,12 @@
 import errno
 import os
 from pathlib import Path
+from typing import Union
 
 from structlog import get_logger
 
 from .file_utils import carve, is_safe_path
-from .models import Chunk, File, TaskResult, UnknownChunk, ValidChunk
+from .models import Chunk, File, PaddingChunk, TaskResult, UnknownChunk, ValidChunk
 from .report import MaliciousSymlinkRemoved
 
 logger = get_logger()
@@ -113,8 +114,14 @@ def _fix_extracted_directory(directory: Path):
     _fix_extracted_directory(outdir)
 
 
-def carve_unknown_chunk(extract_dir: Path, file: File, chunk: UnknownChunk) -> Path:
-    filename = f"{chunk.start_offset}-{chunk.end_offset}.unknown"
+def carve_unknown_chunk(
+    extract_dir: Path, file: File, chunk: Union[UnknownChunk, PaddingChunk]
+) -> Path:
+    extension = "unknown"
+    if isinstance(chunk, PaddingChunk):
+        extension = "padding"
+
+    filename = f"{chunk.start_offset}-{chunk.end_offset}.{extension}"
     carve_path = extract_dir / filename
     logger.info("Extracting unknown chunk", path=carve_path, chunk=chunk)
     carve_chunk_to_file(carve_path, file, chunk)

tests/integration/executable/elf/elf64/__output__/kernel-initramfs-padding_extract/17-11651465.elf64_extract/initramfs_extract/366109-366113.unknown renamed to tests/integration/executable/elf/elf64/__output__/kernel-initramfs-padding_extract/17-11651465.elf64_extract/initramfs_extract/366109-366113.padding

@@ -147,6 +147,28 @@ def as_report(self, entropy: Optional[EntropyReport]) -> UnknownChunkReport:
         )
 
 
+@attr.define(repr=False)
+class PaddingChunk(Chunk):
+    r"""Gaps between valid chunks or otherwise unknown chunks.
+
+    Important for manual analysis, and analytical certanity: for example
+    entropy, other chunks inside it, metadata, etc.
+    """
+
+    def as_report(
+        self, entropy: Optional[EntropyReport]  #   noqa: ARG002
+    ) -> ChunkReport:
+        return ChunkReport(
+            id=self.id,
+            start_offset=self.start_offset,
+            end_offset=self.end_offset,
+            size=self.size,
+            is_encrypted=False,
+            handler_name="padding",
+            extraction_reports=[],
+        )
+
+
 @attrs.define
 class MultiFile(Blob):
     name: str = attr.field(kw_only=True)