From 329a25a8ad15a7484baf972b258d35010ca392f6 Mon Sep 17 00:00:00 2001 From: Narek Mkhitaryan Date: Mon, 24 Feb 2025 18:14:37 +0400 Subject: [PATCH 1/6] added github action for Test_PyPI --- .github/workflows/Test_PyPI.yml | 44 +++++++++++++++++++++++++++++++++ .github/workflows/release.yml | 4 +-- 2 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/Test_PyPI.yml diff --git a/.github/workflows/Test_PyPI.yml b/.github/workflows/Test_PyPI.yml new file mode 100644 index 000000000..cc69a2305 --- /dev/null +++ b/.github/workflows/Test_PyPI.yml @@ -0,0 +1,44 @@ +name: Publish Python 🐍 distributions 📦 to TestPyPI + +on: + push: + branches: + - develop + +jobs: + build-n-publish: + name: Build and publish Python 🐍 distributions 📦 to TestPyPI + runs-on: ubuntu-20.04 + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.8" + - name: Upgrade pip + run: >- + python -m + pip install + pip --upgrade + --user + - name: Install pypi/build + run: >- + python -m + pip install + build + --user + - name: Build a binary wheel and a source tarball + run: >- + python -m + build + --sdist + --wheel + --outdir dist/ + . + - name: Publish distribution 📦 to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + password: ${{ secrets.TEST_PYPI_API_TOKEN }} + verbose: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2efb16869..8a7e81912 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -name: Publish Python 🐍 distributions 📦 to PyPI and TestPyPI +name: Publish Python 🐍 distributions 📦 to PyPI on: release: @@ -6,7 +6,7 @@ on: jobs: build-n-publish: - name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI + name: Build and publish Python 🐍 distributions 📦 to PyPI runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v3 From 7be6631387d1e218181afed8a1f44aee46b38f85 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Thu, 27 Feb 2025 15:55:36 +0400 Subject: [PATCH 2/6] Add ability to query items count --- .../lib/core/serviceproviders.py | 8 ++++ src/superannotate/lib/core/usecases/items.py | 46 +++++++++++++++++++ .../lib/infrastructure/controller.py | 14 ++++++ .../lib/infrastructure/services/explore.py | 23 ++++++++++ tests/integration/items/test_saqul_query.py | 8 +++- 5 files changed, 98 insertions(+), 1 deletion(-) diff --git a/src/superannotate/lib/core/serviceproviders.py b/src/superannotate/lib/core/serviceproviders.py index 5fc302ec6..d8fbab28e 100644 --- a/src/superannotate/lib/core/serviceproviders.py +++ b/src/superannotate/lib/core/serviceproviders.py @@ -672,6 +672,14 @@ def saqul_query( ) -> ServiceResponse: raise NotImplementedError + @abstractmethod + def query_item_count( + self, + project: entities.ProjectEntity, + query: str = None, + ) -> ServiceResponse: + raise NotImplementedError + class BaseServiceProvider: projects: BaseProjectService diff --git a/src/superannotate/lib/core/usecases/items.py b/src/superannotate/lib/core/usecases/items.py index 650edfc0c..5c926a746 100644 --- a/src/superannotate/lib/core/usecases/items.py +++ b/src/superannotate/lib/core/usecases/items.py @@ -163,6 +163,52 @@ def execute(self) -> Response: return self._response +class QueryEntitiesCountUseCase(BaseReportableUseCase): + def __init__( + self, + reporter: Reporter, + project: ProjectEntity, + service_provider: BaseServiceProvider, + query: str, + ): + super().__init__(reporter) + self._project = project + self._service_provider = service_provider + self._query = query + + def validate_arguments(self): + if self._query: + response = self._service_provider.explore.validate_saqul_query( + project=self._project, query=self._query + ) + + if not response.ok: + raise AppException(response.error) + if response.data["isValidQuery"]: + self._query = response.data["parsedQuery"] + else: + raise AppException("Incorrect query.") + else: + response = self._service_provider.explore.validate_saqul_query( + self._project, "-" + ) + if not response.ok: + raise AppException(response.error) + + def execute(self) -> Response: + if self.is_valid(): + query_kwargs = {"query": self._query} + service_response = self._service_provider.explore.query_item_count( + self._project, + **query_kwargs, + ) + if service_response.ok: + self._response.data = service_response.data + else: + self._response.errors = service_response.data + return self._response + + class AssignItemsUseCase(BaseUseCase): CHUNK_SIZE = 500 diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index 57ae432e7..0e9f703b1 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -1678,3 +1678,17 @@ def query_entities( return ItemManager.process_response( self.service_provider, items, project, folder, map_fields=False ) + + def query_items_count(self, project_name: str, query: str = None) -> int: + project = self.get_project(project_name) + + use_case = usecases.QueryEntitiesCountUseCase( + reporter=self.get_default_reporter(), + project=project, + query=query, + service_provider=self.service_provider, + ) + response = use_case.execute() + if response.errors: + raise AppException(response.errors) + return response.data["count"] diff --git a/src/superannotate/lib/infrastructure/services/explore.py b/src/superannotate/lib/infrastructure/services/explore.py index dff6a2b3f..a646a3e07 100644 --- a/src/superannotate/lib/infrastructure/services/explore.py +++ b/src/superannotate/lib/infrastructure/services/explore.py @@ -10,6 +10,7 @@ from lib.core.service_types import SubsetListResponse from lib.core.service_types import UploadCustomFieldValuesResponse from lib.core.serviceproviders import BaseExploreService +from superannotate import AppException class ExploreService(BaseExploreService): @@ -25,6 +26,7 @@ class ExploreService(BaseExploreService): URL_UPLOAD_CUSTOM_VALUE = "custom/metadata/item" URL_SAQUL_QUERY = "items/search" URL_VALIDATE_SAQUL_QUERY = "items/parse/query" + URL_QUERY_COUNT = "items/count" @property def explore_service_url(self): @@ -201,3 +203,24 @@ def saqul_query( else: response = ServiceResponse(status=200, res_data=[]) return response + + def query_item_count( + self, + project: entities.ProjectEntity, + query: str = None, + ) -> ServiceResponse: + + params = { + "project_id": project.id, + "includeFolderNames": True, + } + data = {"query": query} + response = self.client.request( + urljoin(self.explore_service_url, self.URL_QUERY_COUNT), + "post", + params=params, + data=data, + ) + if not response.ok: + raise AppException(response.error) + return response diff --git a/tests/integration/items/test_saqul_query.py b/tests/integration/items/test_saqul_query.py index e63517ba7..d74281df6 100644 --- a/tests/integration/items/test_saqul_query.py +++ b/tests/integration/items/test_saqul_query.py @@ -59,7 +59,13 @@ def test_query(self): def test_query_on_100(self): sa.attach_items(self.PROJECT_NAME, os.path.join(DATA_SET_PATH, "100_urls.csv")) entities = sa.query(self.PROJECT_NAME, "metadata(status = NotStarted)") - print(len(entities)) + assert len(entities) == 100 + assert ( + sa.controller.query_items_count( + self.PROJECT_NAME, "metadata(status = NotStarted)" + ) + == 100 + ) def test_validate_saqul_query(self): try: From f8db11f269b9068673540ef956669918af9f088c Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Thu, 27 Feb 2025 15:57:17 +0400 Subject: [PATCH 3/6] Version update --- src/superannotate/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index c6a8a015d..12e2e5379 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -3,7 +3,7 @@ import sys -__version__ = "4.4.31dev2" +__version__ = "4.4.32dev1" os.environ.update({"sa_version": __version__}) sys.path.append(os.path.split(os.path.realpath(__file__))[0]) From f751a81a46617bdd7f7bfa1d6f0c78737a221e37 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Mon, 3 Mar 2025 10:09:05 +0400 Subject: [PATCH 4/6] Fix item_context set_value tod --- src/superannotate/__init__.py | 2 +- src/superannotate/lib/infrastructure/annotation_adapter.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index 12e2e5379..2e4c077fa 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -3,7 +3,7 @@ import sys -__version__ = "4.4.32dev1" +__version__ = "4.4.32dev2" os.environ.update({"sa_version": __version__}) sys.path.append(os.path.split(os.path.realpath(__file__))[0]) diff --git a/src/superannotate/lib/infrastructure/annotation_adapter.py b/src/superannotate/lib/infrastructure/annotation_adapter.py index e0e28aec0..c333231cf 100644 --- a/src/superannotate/lib/infrastructure/annotation_adapter.py +++ b/src/superannotate/lib/infrastructure/annotation_adapter.py @@ -44,7 +44,7 @@ def get_component_value(self, component_id: str): return None def set_component_value(self, component_id: str, value: Any): - self.annotation["data"][component_id] = {"value": value} + self.annotation.setdefault("data", {}).setdefault(component_id, {})["value"] = value return self From abfa5b39fbbfe0580838c8a5e89af29451be09a7 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Mon, 3 Mar 2025 18:10:14 +0400 Subject: [PATCH 5/6] Fix upload_annotations in folder handling --- docs/source/userguide/SDK_Functions_sheet.csv | 12 ++++++------ docs/source/userguide/utilities.rst | 10 +++++----- src/superannotate/__init__.py | 2 +- src/superannotate/lib/core/usecases/annotations.py | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/source/userguide/SDK_Functions_sheet.csv b/docs/source/userguide/SDK_Functions_sheet.csv index fd41d3342..7c1440fa8 100644 --- a/docs/source/userguide/SDK_Functions_sheet.csv +++ b/docs/source/userguide/SDK_Functions_sheet.csv @@ -8,7 +8,7 @@ Projects,create_project,No,Not Relevant,Not Relevant,Not Relevant,Not Relevant ,set_project_status,Yes,Not Relevant,Not Relevant,Not Relevant,Not Relevant ,get_project_metadata,Yes,Not Relevant,Not Relevant,Not Relevant,Not Relevant ,upload_images_to_project,Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not Relevant -,attach_items_from_integrated_storage,Yes,Not Relevant,Not Relevant,Not Relevant,"AWS, GCP, Azure" +,attach_items_from_integrated_storage,Yes,Not Relevant,Not Relevant,Not Relevant,"AWS, GCP, Azure, Databricks" ,upload_image_to_project,Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not Relevant ,upload_images_from_folder_to_project,Not Relevant,Not Relevant,Not Relevant,Not Relevant,AWS ,upload_video_to_project,Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not Relevant @@ -48,7 +48,7 @@ Annotations,upload_annotations(),Yes,Yes,Yes,Yes,Not Relevant ,set_annotation_statuses(),Yes,Not Relevant,Not Relevant,Not Relevant,Not Relevant ,delete_annotations(),Yes,Not Relevant,Not Relevant,Not Relevant,Not Relevant ,upload_annotations_from_folder_to_project(),No,No,Yes,No,AWS -"Annotation +"Annotation Classes",create_annotation_class(),Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not Relevant ,create_annotation_classes_from_classes_json(),Not Relevant,Not Relevant,Not Relevant,Not Relevant,AWS ,search_annotation_classes(),Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not Relevant @@ -57,8 +57,8 @@ Classes",create_annotation_class(),Not Relevant,Not Relevant,Not Relevant,Not Re Exports,prepare_export(),Yes,Yes,Yes,No,Not Relevant ,download_export(),Yes,Yes,Yes,Yes,AWS ,get_exports(),Yes,Not Relevant,Not Relevant,Not Relevant,Not Relevant -"Custom -Metadata +"Custom +Metadata ",create_custom_fields(),Yes,Not Relevant,Not Relevant,Not Relevant,Not Relevant ,get_custom_fields(),Yes,Not Relevant,Not Relevant,Not Relevant,Not Relevant ,delete_custom_fields(),Yes,Not Relevant,Not Relevant,Not Relevant,Not Relevant @@ -78,10 +78,10 @@ Team,get_team_metadata(),Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not ,get_user_metadata(),Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not Relevant ,set_user_custom_field(),Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not Relevant ,list_users(),Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not Relevant -"Converting +"Converting Annotations",import_annotation(),Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not Relevant ,export_annotation(),Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not Relevant ,convert_project_type(),Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not Relevant -"Working w/ +"Working w/ Annotations",validate_annotations(),Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not Relevant ,aggregate_annotations_as_df(),Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not Relevant \ No newline at end of file diff --git a/docs/source/userguide/utilities.rst b/docs/source/userguide/utilities.rst index e0e7df074..74c70985d 100644 --- a/docs/source/userguide/utilities.rst +++ b/docs/source/userguide/utilities.rst @@ -116,14 +116,14 @@ You can find more information annotation format conversion :ref:`here 1 or None not in distributed_items: + if len(distributed_items) > 1 or "" not in distributed_items: raise AppException( "You can't include a folder when uploading from within a folder." ) From a08e04f6dd1f72e7353010af2877d263b7de000c Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Tue, 4 Mar 2025 13:41:17 +0400 Subject: [PATCH 6/6] Update __init__.py --- src/superannotate/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index d96b06007..e6e8b7684 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -3,7 +3,7 @@ import sys -__version__ = "4.4.32dev3" +__version__ = "4.4.32" os.environ.update({"sa_version": __version__}) sys.path.append(os.path.split(os.path.realpath(__file__))[0])