From c383fbdc996720ca37ca09158db1f99627440ae6 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Mon, 20 Mar 2023 12:29:43 +0400 Subject: [PATCH 1/2] Add lidar support --- src/superannotate/__init__.py | 2 +- src/superannotate/lib/core/__init__.py | 2 +- src/superannotate/lib/core/enums.py | 4 + .../lib/core/usecases/annotations.py | 24 ++- .../sample_project_point_cloud/3d_item_1.json | 136 +++++++++++++++ .../classes/classes.json | 157 ++++++++++++++++++ .../urls_template.csv | 2 + .../test_annotation_upload_point_cloud.py | 46 +++++ .../test_depricated_functions_video.py | 2 - 9 files changed, 366 insertions(+), 9 deletions(-) create mode 100644 tests/data_set/sample_project_point_cloud/3d_item_1.json create mode 100644 tests/data_set/sample_project_point_cloud/classes/classes.json create mode 100644 tests/data_set/sample_project_point_cloud/urls_template.csv create mode 100644 tests/integration/annotations/test_annotation_upload_point_cloud.py diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index 0838c956d..52e3f3bec 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -3,7 +3,7 @@ import sys import typing -__version__ = "4.4.10" +__version__ = "4.4.11-alpha" sys.path.append(os.path.split(os.path.realpath(__file__))[0]) diff --git a/src/superannotate/lib/core/__init__.py b/src/superannotate/lib/core/__init__.py index 7322412e4..4afa0e3e7 100644 --- a/src/superannotate/lib/core/__init__.py +++ b/src/superannotate/lib/core/__init__.py @@ -59,7 +59,7 @@ def setup_logging(level=DEFAULT_LOGGING_LEVEL, file_path=LOG_FILE_LOCATION): file_handler.setFormatter(file_formatter) logger.addHandler(file_handler) except OSError as e: - logging.error(e) + logging.debug(e) DEFAULT_IMAGE_EXTENSIONS = ["jpg", "jpeg", "png", "tif", "tiff", "webp", "bmp"] diff --git a/src/superannotate/lib/core/enums.py b/src/superannotate/lib/core/enums.py index 929ef13ef..36768aa06 100644 --- a/src/superannotate/lib/core/enums.py +++ b/src/superannotate/lib/core/enums.py @@ -98,6 +98,10 @@ class ProjectType(BaseTitledEnum): def images(self): return self.VECTOR.value, self.PIXEL.value, self.TILED.value + @classproperty + def unsupported_types(cls): + return cls.POINT_CLOUD.value, cls.OTHER.value + class UserRole(BaseTitledEnum): SUPER_ADMIN = "Superadmin", 1 # noqa diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index 9280a90f9..2bd16ac25 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -520,6 +520,8 @@ def get_annotation_from_s3(bucket, path: str): def prepare_annotation(self, annotation: dict, size) -> dict: errors = None + if self._project.type == constants.ProjectType.POINT_CLOUD.value: + return annotation if size < BIG_FILE_THRESHOLD: use_case = ValidateAnnotationUseCase( reporter=self.reporter, @@ -664,7 +666,13 @@ async def distribute_queues(self, items_to_upload: List[ItemToUpload]): json.dump(annotation, item_to_upload.file) item_to_upload.file.seek(0) while True: - if item_to_upload.file_size > BIG_FILE_THRESHOLD: + if ( + self._project.type + == constants.ProjectType.POINT_CLOUD.value + ): + self._big_files_queue.put_nowait(item_to_upload) + break + elif item_to_upload.file_size > BIG_FILE_THRESHOLD: if self._big_files_queue.qsize() > 32: await asyncio.sleep(3) continue @@ -1414,13 +1422,13 @@ def validate_item_names(self): def _prettify_annotations(self, annotations: List[dict]): re_struct = {} if self._item_names: - for annotation in annotations: - re_struct[annotation["metadata"]["name"]] = annotation try: + for annotation in annotations: + re_struct[annotation["metadata"]["name"]] = annotation return [re_struct[x] for x in self._item_names if x in re_struct] except KeyError: - raise AppException("Broken data.") - + if self._project.type not in constants.ProjectType.unsupported_types: + raise AppException("Broken data.") return annotations async def get_big_annotation(self): @@ -1587,6 +1595,12 @@ def __init__( self._big_file_queue = None self._small_file_queue = None + def validate_project_type(self): + if self._project.type == constants.ProjectType.POINT_CLOUD.value: + raise AppException( + f"The function is not supported for {constants.ProjectType(self._project.type).name} projects." + ) + def validate_item_names(self): if self._item_names: item_names = list(dict.fromkeys(self._item_names)) diff --git a/tests/data_set/sample_project_point_cloud/3d_item_1.json b/tests/data_set/sample_project_point_cloud/3d_item_1.json new file mode 100644 index 000000000..8c5c7c8b8 --- /dev/null +++ b/tests/data_set/sample_project_point_cloud/3d_item_1.json @@ -0,0 +1,136 @@ +{ + "000950": [ + { + "psr": { + "position": { + "x": -5.080617901849534, + "y": 4.7485066265361535, + "z": -0.5211173295974731 + }, + "scale": { + "x": 10.438753155962607, + "y": 7.852290496368532, + "z": 3.3729794025421143 + }, + "rotation": { + "x": 0, + "y": 0, + "z": 1.9247371622914418 + } + }, + "obj_type": "Personal vehicle", + "obj_id": "1" + }, + { + "psr": { + "position": { + "x": 15.25457933415853, + "y": -7.289155780814713, + "z": 0.3909074068069458 + }, + "scale": { + "x": 16.67372701148617, + "y": 12.997588199926543, + "z": 5.432509660720825 + }, + "rotation": { + "x": 0, + "y": 0, + "z": 2.993572126346171 + } + }, + "obj_type": "Personal vehicle", + "obj_id": "2" + } + ], + "000965": [ + { + "psr": { + "position": { + "x": 12.678969509007786, + "y": -12.72550632620182, + "z": -0.5258613526821136 + }, + "scale": { + "x": 14.873953074212206, + "y": 9.01690452224452, + "z": 2.72206574678421 + }, + "rotation": { + "x": 0, + "y": 0, + "z": 2.99357212634617 + } + }, + "obj_type": "Personal vehicle", + "obj_id": "3" + }, + { + "psr": { + "position": { + "x": 14.04525345440314, + "y": 3.6402911227157486, + "z": -0.04190933704376221 + }, + "scale": { + "x": 10.071495305599894, + "y": 7.171325113558155, + "z": 5.090493440628052 + }, + "rotation": { + "x": 0, + "y": 0, + "z": -2.6580474313269304 + } + }, + "obj_type": "Personal vehicle", + "obj_id": "4" + } + ], + "000970": [ + { + "psr": { + "position": { + "x": 2.4613493554371804, + "y": 8.947661626313172, + "z": 0.21572160720825195 + }, + "scale": { + "x": 62.35026185183082, + "y": 97.18034925311514, + "z": 8.726404190063477 + }, + "rotation": { + "x": 0, + "y": 0, + "z": -2.6580474313269304 + } + }, + "obj_type": "Personal vehicle", + "obj_id": "5" + } + ], + "000975": [ + { + "psr": { + "position": { + "x": 13.960599950371503, + "y": 0.7873128198567101, + "z": -1.5273447930812836 + }, + "scale": { + "x": 20.379303339511864, + "y": 14.074981597535814, + "z": 2.112749397754669 + }, + "rotation": { + "x": 0, + "y": 0, + "z": -0.39218261369803376 + } + }, + "obj_type": "Default", + "obj_id": "6" + } + ] +} \ No newline at end of file diff --git a/tests/data_set/sample_project_point_cloud/classes/classes.json b/tests/data_set/sample_project_point_cloud/classes/classes.json new file mode 100644 index 000000000..810a5a436 --- /dev/null +++ b/tests/data_set/sample_project_point_cloud/classes/classes.json @@ -0,0 +1,157 @@ +[ + { + "id": 55917, + "project_id": 11979, + "name": "Personal vehicle", + "color": "#ecb65f", + "count": 25, + "createdAt": "2020-10-12T11:35:20.000Z", + "updatedAt": "2020-10-12T11:48:19.000Z", + "attribute_groups": [ + { + "id": 17245, + "class_id": 55917, + "name": "Num doors", + "is_multiselect": 0, + "createdAt": "2020-10-12T11:35:20.000Z", + "updatedAt": "2020-10-12T11:35:20.000Z", + "attributes": [ + { + "id": 62792, + "group_id": 17245, + "project_id": 11979, + "name": "2", + "count": 1, + "createdAt": "2020-10-12T11:35:20.000Z", + "updatedAt": "2020-10-12T11:46:28.000Z" + }, + { + "id": 62793, + "group_id": 17245, + "project_id": 11979, + "name": "4", + "count": 1, + "createdAt": "2020-10-12T11:35:20.000Z", + "updatedAt": "2020-10-12T11:35:20.000Z" + } + ] + } + ] + }, + { + "id": 55918, + "project_id": 11979, + "name": "Large vehicle", + "color": "#737b28", + "count": 1, + "createdAt": "2020-10-12T11:35:20.000Z", + "updatedAt": "2020-10-12T11:48:19.000Z", + "attribute_groups": [ + { + "id": 17246, + "class_id": 55918, + "name": "swedish", + "is_multiselect": 0, + "createdAt": "2020-10-12T11:35:20.000Z", + "updatedAt": "2020-10-12T11:35:20.000Z", + "attributes": [ + { + "id": 62794, + "group_id": 17246, + "project_id": 11979, + "name": "yes", + "count": 0, + "createdAt": "2020-10-12T11:35:20.000Z", + "updatedAt": "2020-10-12T11:35:20.000Z" + }, + { + "id": 62795, + "group_id": 17246, + "project_id": 11979, + "name": "no", + "count": 1, + "createdAt": "2020-10-12T11:35:20.000Z", + "updatedAt": "2020-10-12T11:46:28.000Z" + } + ] + }, + { + "id": 17247, + "class_id": 55918, + "name": "Num doors", + "is_multiselect": 0, + "createdAt": "2020-10-12T11:35:20.000Z", + "updatedAt": "2020-10-12T11:35:20.000Z", + "attributes": [ + { + "id": 62796, + "group_id": 17247, + "project_id": 11979, + "name": "2", + "count": 0, + "createdAt": "2020-10-12T11:35:20.000Z", + "updatedAt": "2020-10-12T11:35:20.000Z" + }, + { + "id": 62797, + "group_id": 17247, + "project_id": 11979, + "name": "4", + "count": 1, + "createdAt": "2020-10-12T11:35:20.000Z", + "updatedAt": "2020-10-12T11:46:28.000Z" + } + ] + } + ] + }, + { + "id": 55919, + "project_id": 11979, + "name": "Human", + "color": "#e4542b", + "count": 9, + "createdAt": "2020-10-12T11:35:20.000Z", + "updatedAt": "2020-10-12T11:48:14.000Z", + "attribute_groups": [ + { + "id": 17248, + "class_id": 55919, + "name": "Height", + "is_multiselect": 0, + "createdAt": "2020-10-12T11:35:20.000Z", + "updatedAt": "2020-10-12T11:35:20.000Z", + "attributes": [ + { + "id": 62798, + "group_id": 17248, + "project_id": 11979, + "name": "Tall", + "count": 0, + "createdAt": "2020-10-12T11:35:20.000Z", + "updatedAt": "2020-10-12T11:35:20.000Z" + }, + { + "id": 62799, + "group_id": 17248, + "project_id": 11979, + "name": "Short", + "count": 0, + "createdAt": "2020-10-12T11:35:20.000Z", + "updatedAt": "2020-10-12T11:35:20.000Z" + } + ] + } + ] + }, + { + "id": 55920, + "project_id": 11979, + "name": "Plant", + "color": "#46ccb2", + "count": 0, + "createdAt": "2020-10-12T11:35:20.000Z", + "updatedAt": "2020-10-12T11:35:20.000Z", + "attribute_groups": [] + } +] diff --git a/tests/data_set/sample_project_point_cloud/urls_template.csv b/tests/data_set/sample_project_point_cloud/urls_template.csv new file mode 100644 index 000000000..b03d0211c --- /dev/null +++ b/tests/data_set/sample_project_point_cloud/urls_template.csv @@ -0,0 +1,2 @@ +url,name +https://sa-public-text-files.s3.us-west-2.amazonaws.com/davitb/lidar/data/3d_item_1.json,3d_item_1 diff --git a/tests/integration/annotations/test_annotation_upload_point_cloud.py b/tests/integration/annotations/test_annotation_upload_point_cloud.py new file mode 100644 index 000000000..ed32fa425 --- /dev/null +++ b/tests/integration/annotations/test_annotation_upload_point_cloud.py @@ -0,0 +1,46 @@ +import json +import os +import tempfile +from os.path import join + +from src.superannotate import AppException +from src.superannotate import SAClient +from tests import DATA_SET_PATH +from tests.integration.base import BaseTestCase + +sa = SAClient() + + +class TestAnnotationUploadVector(BaseTestCase): + PROJECT_NAME = "PointCloudAnnotationUploadDownload" + PROJECT_DESCRIPTION = "Desc" + PROJECT_TYPE = "PointCloud" + TEST_FOLDER_PATH = "sample_project_point_cloud" + CSV_PATH = f"{TEST_FOLDER_PATH}/urls_template.csv" + ITEM_NAME = "3d_item_1" + + @property + def folder_path(self): + return os.path.join(DATA_SET_PATH, self.TEST_FOLDER_PATH) + + def test_annotation_folder_upload_download(self): + sa.attach_items(self.PROJECT_NAME, os.path.join(DATA_SET_PATH, self.CSV_PATH)) + + sa.create_annotation_classes_from_classes_json( + self.PROJECT_NAME, f"{self.folder_path}/classes/classes.json" + ) + _, _, _ = sa.upload_annotations_from_folder_to_project( + self.PROJECT_NAME, self.folder_path + ) + with tempfile.TemporaryDirectory() as tmp_dir: + for item in sa.search_items(self.PROJECT_NAME): + item_name = item["name"] + annotation_path = join(self.folder_path, f"{item_name}.json") + annotations = sa.get_annotations(self.PROJECT_NAME, [item_name]) + origin_annotation = json.load(open(annotation_path)) + self.assertDictEqual(annotations[0], origin_annotation) + with self.assertRaisesRegexp( + AppException, + "The function is not supported for PointCloud projects.", + ): + sa.download_annotations(self.PROJECT_NAME, tmp_dir, [item_name]) diff --git a/tests/integration/test_depricated_functions_video.py b/tests/integration/test_depricated_functions_video.py index 8e16622d9..ded1461b5 100644 --- a/tests/integration/test_depricated_functions_video.py +++ b/tests/integration/test_depricated_functions_video.py @@ -2,7 +2,6 @@ from os.path import dirname from unittest import TestCase -import pytest from src.superannotate import AppException from src.superannotate import SAClient from src.superannotate.lib.core import DEPRICATED_DOCUMENT_VIDEO_MESSAGE @@ -61,7 +60,6 @@ def annotation_path(self): def image_path(self): return f"{self.folder_path}/example_image_1.jpg" - @pytest.mark.flaky(reruns=2) def test_deprecated_functions(self): try: sa.upload_images_from_folder_to_project(self.PROJECT_NAME, "some") From 117882e74a3e8302ce0d1d515b6a4c0b71f7bcb5 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Mon, 20 Mar 2023 18:18:04 +0400 Subject: [PATCH 2/2] Update __init__.py --- src/superannotate/__init__.py | 2 +- src/superannotate/lib/core/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index 52e3f3bec..c53240a1d 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -3,7 +3,7 @@ import sys import typing -__version__ = "4.4.11-alpha" +__version__ = "4.4.11dev1" sys.path.append(os.path.split(os.path.realpath(__file__))[0]) diff --git a/src/superannotate/lib/core/__init__.py b/src/superannotate/lib/core/__init__.py index 4afa0e3e7..8ed41dda5 100644 --- a/src/superannotate/lib/core/__init__.py +++ b/src/superannotate/lib/core/__init__.py @@ -59,7 +59,7 @@ def setup_logging(level=DEFAULT_LOGGING_LEVEL, file_path=LOG_FILE_LOCATION): file_handler.setFormatter(file_formatter) logger.addHandler(file_handler) except OSError as e: - logging.debug(e) + logger.debug(e) DEFAULT_IMAGE_EXTENSIONS = ["jpg", "jpeg", "png", "tif", "tiff", "webp", "bmp"]