feat(inspect): true source frame capture (#3137)

maxxgx · Copilot · web-flow · commit c46946f5a8db · 2025-11-24T13:52:43.000+01:00
* capture direct feed

Signed-off-by: Ma, Xiangxiang &lt;xiangxiang.ma@intel.com&gt;

* /capture -&gt; /images

Signed-off-by: Ma, Xiangxiang &lt;xiangxiang.ma@intel.com&gt;

* Update application/backend/src/api/endpoints/media_endpoints.py

Co-authored-by: Copilot &lt;175728472+Copilot@users.noreply.github.com&gt;
Signed-off-by: Max Xiang &lt;maxx.rift@gmail.com&gt;

* Update application/backend/src/api/endpoints/media_endpoints.py

Co-authored-by: Copilot &lt;175728472+Copilot@users.noreply.github.com&gt;
Signed-off-by: Max Xiang &lt;maxx.rift@gmail.com&gt;

* Update application/backend/src/services/media_service.py

Co-authored-by: Copilot &lt;175728472+Copilot@users.noreply.github.com&gt;
Signed-off-by: Max Xiang &lt;maxx.rift@gmail.com&gt;

* fix

Signed-off-by: Ma, Xiangxiang &lt;xiangxiang.ma@intel.com&gt;

* fix style and add missing test

Signed-off-by: Ma, Xiangxiang &lt;xiangxiang.ma@intel.com&gt;

* fix style

Signed-off-by: Ma, Xiangxiang &lt;xiangxiang.ma@intel.com&gt;

---------

Signed-off-by: Ma, Xiangxiang &lt;xiangxiang.ma@intel.com&gt;
Signed-off-by: Max Xiang &lt;maxx.rift@gmail.com&gt;
Co-authored-by: Copilot &lt;175728472+Copilot@users.noreply.github.com&gt;
diff --git a/application/backend/src/api/endpoints/capture_endpoints.py b/application/backend/src/api/endpoints/capture_endpoints.py
@@ -0,0 +1,61 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+import asyncio
+import queue
+from typing import Annotated
+from uuid import UUID
+
+from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, status
+
+from api.dependencies import get_media_service, get_project_id, get_scheduler
+from api.endpoints import API_PREFIX
+from core import Scheduler
+from pydantic_models import Media
+from services.media_service import MediaService
+
+capture_api_prefix_url = API_PREFIX + "/projects/{project_id}"
+router = APIRouter(
+    prefix=capture_api_prefix_url,
+    tags=["capture"],
+)
+
+
+@router.get(
+    "/capture",
+    response_model_exclude_none=True,
+    status_code=status.HTTP_201_CREATED,
+    responses={
+        status.HTTP_201_CREATED: {"description": "Image captured successfully"},
+        status.HTTP_400_BAD_REQUEST: {"description": "Invalid image file"},
+        status.HTTP_404_NOT_FOUND: {"description": "Source or project not found"},
+    },
+)
+async def capture(
+    media_service: Annotated[MediaService, Depends(get_media_service)],
+    project_id: Annotated[UUID, Depends(get_project_id)],
+    scheduler: Annotated[Scheduler, Depends(get_scheduler)],
+    background_tasks: BackgroundTasks,
+) -> Media:
+    """Endpoint to capture an image"""
+    try:
+        stream_data = await asyncio.to_thread(lambda: scheduler.frame_queue.get(timeout=1))
+    except queue.Empty as e:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="No frame available to capture. Please make sure the stream is active.",
+        ) from e
+
+    image = stream_data.frame_data
+    # Convert BGR to RGB if needed
+    if image.ndim == 3 and image.shape[2] == 3:
+        image = image[..., ::-1]
+
+    media = await media_service.upload_image(project_id=project_id, image=image, is_anomalous=False, extension=".png")
+
+    background_tasks.add_task(
+        media_service.generate_thumbnail,
+        project_id=project_id,
+        media_id=media.id,
+        image=image,
+    )
+    return media
diff --git a/application/backend/src/api/endpoints/media_endpoints.py b/application/backend/src/api/endpoints/media_endpoints.py
@@ -1,9 +1,10 @@
 # Copyright (C) 2025 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
-
+import asyncio
 from typing import Annotated
 from uuid import UUID
 
+import anyio
 from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, UploadFile, status
 from fastapi.responses import FileResponse
 
@@ -73,7 +74,17 @@ async def get_media_thumbnail(
 ) -> FileResponse:
     """Return a PNG thumbnail for the requested image."""
     try:
-        return FileResponse(await media_service.get_thumbnail_file_path(project_id=project_id, media_id=media_id))
+        thumbnail_path = await media_service.get_thumbnail_file_path(project_id=project_id, media_id=media_id)
+        # Wait for thumbnail with timeout
+        max_retries = 10  # 0.5 seconds
+        for _ in range(max_retries):
+            if await anyio.Path(thumbnail_path).exists():
+                return FileResponse(thumbnail_path)
+            await asyncio.sleep(0.05)
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Thumbnail for media with ID {media_id} not found",
+        )
     except FileNotFoundError as e:
         raise HTTPException(
             status_code=status.HTTP_404_NOT_FOUND,
@@ -102,7 +113,7 @@ async def delete_media(
 
 
 @media_router.post(
-    "/capture",
+    "/images",
     response_model_exclude_none=True,
     status_code=status.HTTP_201_CREATED,
     responses={
@@ -118,14 +129,17 @@ async def capture_image(
 ) -> Media:
     """Endpoint to capture an image"""
     image_bytes = await file.read()
+    if not file.filename or "." not in file.filename:
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Uploaded file must have an extension.")
+    extension = "." + file.filename.rsplit(".", maxsplit=1)[-1]
     media = await media_service.upload_image(
-        project_id=project_id, file=file, image_bytes=image_bytes, is_anomalous=False
+        project_id=project_id, image=image_bytes, is_anomalous=False, extension=extension, size=file.size
     )
 
     background_tasks.add_task(
         media_service.generate_thumbnail,
         project_id=project_id,
         media_id=media.id,
-        image_bytes=image_bytes,
+        image=image_bytes,
     )
     return media
diff --git a/application/backend/src/main.py b/application/backend/src/main.py
@@ -8,6 +8,7 @@
 from fastapi.middleware.cors import CORSMiddleware
 
 from api.endpoints.active_pipeline_endpoints import router as active_pipeline_router
+from api.endpoints.capture_endpoints import router as capture_router
 from api.endpoints.devices_endpoints import device_router
 from api.endpoints.job_endpoints import job_router
 from api.endpoints.media_endpoints import media_router
@@ -56,6 +57,7 @@
 app.include_router(webrtc_router)
 app.include_router(trainable_model_router)
 app.include_router(device_router)
+app.include_router(capture_router)
 
 
 if __name__ == "__main__":
diff --git a/application/backend/src/services/media_service.py b/application/backend/src/services/media_service.py
@@ -1,11 +1,10 @@
 # Copyright (C) 2025 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 import asyncio
-import os
 from io import BytesIO
 from uuid import UUID, uuid4
 
-from fastapi import UploadFile
+import numpy as np
 from loguru import logger
 from PIL import Image
 
@@ -56,14 +55,33 @@ async def get_thumbnail_file_path(cls, project_id: UUID, media_id: UUID) -> str:
         return bin_repo.get_full_path(filename=thumbnail_filename)
 
     @classmethod
-    async def upload_image(cls, project_id: UUID, file: UploadFile, image_bytes: bytes, is_anomalous: bool) -> Media:
+    async def upload_image(
+        cls,
+        project_id: UUID,
+        image: np.ndarray | bytes,
+        is_anomalous: bool,
+        extension: str | None = None,
+        size: int | None = None,
+    ) -> Media:
         # Generate unique filename and media ID
         media_id = uuid4()
 
-        if file.filename is None or file.size is None:
-            raise ValueError("File must have a filename and size")
+        if not extension or not extension.lstrip("."):
+            raise ValueError("File extension must be provided")
+
+        if isinstance(image, np.ndarray):
+
+            def _encode_image() -> bytes:
+                with BytesIO() as output:
+                    # Determine format from extension, default to PNG if unknown or generic
+                    fmt = extension.lstrip(".").upper()
+                    Image.fromarray(image).save(output, format=fmt)
+                    return output.getvalue()
+
+            image_bytes = await asyncio.to_thread(_encode_image)
+        else:
+            image_bytes = image
 
-        extension = list(os.path.splitext(file.filename)).pop().lower()
         filename = f"{media_id}{extension}"
         bin_repo = ImageBinaryRepository(project_id=project_id)
         saved_media: Media | None = None
@@ -90,7 +108,7 @@ def _get_image_size() -> tuple[int, int]:
                     id=media_id,
                     project_id=project_id,
                     filename=filename,
-                    size=file.size,
+                    size=size or len(image_bytes),
                     is_anomalous=is_anomalous,
                     width=width,
                     height=height,
@@ -137,7 +155,7 @@ async def generate_thumbnail(
         cls,
         project_id: UUID,
         media_id: UUID,
-        image_bytes: bytes,
+        image: np.ndarray | bytes,
         height_px: int = THUMBNAIL_SIZE,
         width_px: int = THUMBNAIL_SIZE,
     ) -> None:
@@ -150,7 +168,7 @@ async def generate_thumbnail(
         Args:
             project_id: Identifier of the owning project.
             media_id: Identifier of the media item the thumbnail belongs to.
-            image_bytes: Original image bytes used to create the thumbnail.
+            image: Original image (bytes or numpy array) used to create the thumbnail.
             height_px: Maximum thumbnail height in pixels. Defaults to
                 ``THUMBNAIL_SIZE``.
             width_px: Maximum thumbnail width in pixels. Defaults to
@@ -164,7 +182,12 @@ async def generate_thumbnail(
         """
 
         def _create() -> bytes:
-            with Image.open(BytesIO(image_bytes)) as img:
+            if isinstance(image, np.ndarray):
+                img = Image.fromarray(image)
+            else:
+                img = Image.open(BytesIO(image))
+
+            with img:
                 # Preserve aspect ratio while fitting within the box
                 img.thumbnail((width_px, height_px))
                 with BytesIO() as buf:
diff --git a/application/backend/src/workers/dispatching.py b/application/backend/src/workers/dispatching.py
@@ -69,7 +69,7 @@ async def run_loop(self) -> None:
 
             passthrough_mode = not self._active_pipeline_service.is_running
             if passthrough_mode:
-                logger.debug("Passthrough mode; only dispatching to WebRTC stream")
+                logger.trace("Passthrough mode; only dispatching to WebRTC stream")
                 # Only dispatch to WebRTC stream
                 try:
                     self._rtc_stream_queue.put(stream_data.frame_data, block=False)
diff --git a/application/backend/src/workers/inference.py b/application/backend/src/workers/inference.py
@@ -71,6 +71,7 @@ async def _get_active_model(self) -> LoadedModel | None:
                 self._is_passthrough_mode = pipeline is None or (
                     pipeline.status.is_active and not pipeline.status.is_running
                 )
+                logger.info(f"Passthrough mode {'activated' if self._is_passthrough_mode else 'disabled'}.")
                 if pipeline is None or pipeline.model is None:
                     return None
 
@@ -160,7 +161,6 @@ async def _get_next_frame(self) -> StreamData | None:
 
     async def _handle_passthrough_mode(self, stream_data: StreamData) -> None:
         """Handle frame in passthrough mode (no model loaded)."""
-        logger.debug("No active model configured; frame passthrough mode")
         try:
             self._pred_queue.put(stream_data, timeout=1)
         except std_queue.Full:
diff --git a/application/backend/tests/unit/services/test_media_service.py b/application/backend/tests/unit/services/test_media_service.py
@@ -5,6 +5,7 @@
 from unittest.mock import AsyncMock, MagicMock, patch
 from uuid import uuid4
 
+import numpy as np
 import pytest
 from PIL import Image
 
@@ -43,6 +44,16 @@ def mock_db_context():
 
 
 class TestMediaService:
+    @pytest.fixture(autouse=True)
+    def mock_asyncio_to_thread(self):
+        """Mock asyncio.to_thread to run synchronously in tests."""
+
+        async def _mock_to_thread(func, *args, **kwargs):
+            return func(*args, **kwargs)
+
+        with patch("asyncio.to_thread", side_effect=_mock_to_thread):
+            yield
+
     def test_get_media_list(self, fxt_media_service, fxt_media_repository, fxt_media_list, fxt_project):
         """Test getting media list."""
         fxt_media_repository.get_all.return_value = fxt_media_list.media
@@ -116,13 +127,34 @@ def test_upload_image_success(self, fxt_media_service, fxt_upload_file, fxt_proj
             patch("services.media_service.MediaRepository", return_value=mock_media_repo),
         ):
             result = asyncio.run(
-                fxt_media_service.upload_image(fxt_project.id, fxt_upload_file, fxt_image_bytes, False)
+                fxt_media_service.upload_image(fxt_project.id, fxt_image_bytes, False, extension=".jpg")
             )
 
         assert result is not None
         assert mock_bin_repo.save_file.call_count == 1  # only original saved
         mock_media_repo.save.assert_called_once()
 
+    def test_upload_image_numpy_success(self, fxt_media_service, fxt_project):
+        """Test successful image upload with numpy array."""
+        # Create a random numpy image (100x100 RGB)
+        numpy_image = np.random.randint(0, 255, (100, 100, 3), dtype=np.uint8)
+
+        mock_bin_repo = MagicMock()
+        mock_bin_repo.save_file = AsyncMock(return_value="/path/to/file.png")
+
+        mock_media_repo = MagicMock()
+        mock_media_repo.save = AsyncMock(return_value=MagicMock())
+
+        with (
+            patch("services.media_service.ImageBinaryRepository", return_value=mock_bin_repo),
+            patch("services.media_service.MediaRepository", return_value=mock_media_repo),
+        ):
+            result = asyncio.run(fxt_media_service.upload_image(fxt_project.id, numpy_image, False, extension=".png"))
+
+        assert result is not None
+        assert mock_bin_repo.save_file.call_count == 1
+        mock_media_repo.save.assert_called_once()
+
     def test_upload_image_rollback_on_error(self, fxt_media_service, fxt_upload_file, fxt_project, fxt_image_bytes):
         """Test image upload rollback on error."""
         mock_bin_repo = MagicMock()
@@ -137,7 +169,7 @@ def test_upload_image_rollback_on_error(self, fxt_media_service, fxt_upload_file
             patch("services.media_service.MediaRepository", return_value=mock_media_repo),
             pytest.raises(Exception, match="Database error"),
         ):
-            asyncio.run(fxt_media_service.upload_image(fxt_project.id, fxt_upload_file, fxt_image_bytes, False))
+            asyncio.run(fxt_media_service.upload_image(fxt_project.id, fxt_image_bytes, False, extension=".jpg"))
 
         # Verify rollback deletes both files
         assert mock_bin_repo.delete_file.call_count == 1
@@ -158,7 +190,7 @@ def test_upload_image_rollback_file_not_found(
             patch("services.media_service.MediaRepository", return_value=mock_media_repo),
             pytest.raises(Exception, match="Database error"),
         ):
-            asyncio.run(fxt_media_service.upload_image(fxt_project.id, fxt_upload_file, fxt_image_bytes, False))
+            asyncio.run(fxt_media_service.upload_image(fxt_project.id, fxt_image_bytes, False, extension=".jpg"))
 
         # Verify rollback attempted to delete both files
         assert mock_bin_repo.delete_file.call_count == 1
@@ -263,7 +295,7 @@ async def _save_file(filename: str, content: bytes) -> str:  # type: ignore[over
                 fxt_media_service.generate_thumbnail(
                     project_id=fxt_project.id,
                     media_id=media_id,
-                    image_bytes=image_bytes,
+                    image=image_bytes,
                 )
             )
 
@@ -290,7 +322,7 @@ def test_thumbnail_upload_and_deletion(self, fxt_media_service, fxt_upload_file,
             patch("services.media_service.MediaRepository", return_value=mock_media_repo),
         ):
             # Upload saves only original
-            result = asyncio.run(fxt_media_service.upload_image(fxt_project.id, fxt_upload_file, image_bytes, False))
+            result = asyncio.run(fxt_media_service.upload_image(fxt_project.id, image_bytes, False, extension=".jpg"))
             assert result is not None
             assert mock_bin_repo.save_file.call_count == 1
 
@@ -299,7 +331,7 @@ def test_thumbnail_upload_and_deletion(self, fxt_media_service, fxt_upload_file,
                 fxt_media_service.generate_thumbnail(
                     project_id=fxt_project.id,
                     media_id=saved_media.id,
-                    image_bytes=image_bytes,
+                    image=image_bytes,
                 )
             )
 
diff --git a/application/backend/tests/unit/services/test_pipeline_metrics_service.py b/application/backend/tests/unit/services/test_pipeline_metrics_service.py
diff --git a/application/ui/src/features/inspect/dataset/dataset.component.tsx b/application/ui/src/features/inspect/dataset/dataset.component.tsx
