Develop

.github/workflows/Test_PyPI.yml

@@ -0,0 +1,44 @@
+name: Publish Python 🐍 distributions 📦 to TestPyPI
+
+on:
+  push:
+    branches:
+      - develop
+
+jobs:
+  build-n-publish:
+    name: Build and publish Python 🐍 distributions 📦 to TestPyPI
+    runs-on: ubuntu-20.04
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.8"
+    - name: Upgrade pip
+      run: >-
+        python -m
+        pip install
+        pip --upgrade
+        --user
+    - name: Install pypi/build
+      run: >-
+        python -m
+        pip install
+        build
+        --user
+    - name: Build a binary wheel and a source tarball
+      run: >-
+        python -m
+        build
+        --sdist
+        --wheel
+        --outdir dist/
+        .
+    - name: Publish distribution 📦 to TestPyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        repository-url: https://test.pypi.org/legacy/
+        password: ${{ secrets.TEST_PYPI_API_TOKEN }}
+        verbose: true

.github/workflows/release.yml

@@ -1,12 +1,12 @@
-name: Publish Python 🐍 distributions 📦 to PyPI and TestPyPI
+name: Publish Python 🐍 distributions 📦 to PyPI
 
 on:
   release:
     types: [prereleased,released]
 
 jobs:
   build-n-publish:
-    name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI
+    name: Build and publish Python 🐍 distributions 📦 to PyPI
     runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v3

docs/source/userguide/SDK_Functions_sheet.csv

@@ -8,7 +8,7 @@ Projects,create_project,No,Not Relevant,Not Relevant,Not Relevant,Not Relevant
 ,set_project_status,Yes,Not Relevant,Not Relevant,Not Relevant,Not Relevant
 ,get_project_metadata,Yes,Not Relevant,Not Relevant,Not Relevant,Not Relevant
 ,upload_images_to_project,Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not Relevant
-,attach_items_from_integrated_storage,Yes,Not Relevant,Not Relevant,Not Relevant,"AWS, GCP, Azure"
+,attach_items_from_integrated_storage,Yes,Not Relevant,Not Relevant,Not Relevant,"AWS, GCP, Azure, Databricks"
 ,upload_image_to_project,Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not Relevant
 ,upload_images_from_folder_to_project,Not Relevant,Not Relevant,Not Relevant,Not Relevant,AWS
 ,upload_video_to_project,Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not Relevant
@@ -48,7 +48,7 @@ Annotations,upload_annotations(),Yes,Yes,Yes,Yes,Not Relevant
 ,set_annotation_statuses(),Yes,Not Relevant,Not Relevant,Not Relevant,Not Relevant
 ,delete_annotations(),Yes,Not Relevant,Not Relevant,Not Relevant,Not Relevant
 ,upload_annotations_from_folder_to_project(),No,No,Yes,No,AWS
-"Annotation
+"Annotation
 Classes",create_annotation_class(),Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not Relevant
 ,create_annotation_classes_from_classes_json(),Not Relevant,Not Relevant,Not Relevant,Not Relevant,AWS
 ,search_annotation_classes(),Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not Relevant
@@ -57,8 +57,8 @@ Classes",create_annotation_class(),Not Relevant,Not Relevant,Not Relevant,Not Re
 Exports,prepare_export(),Yes,Yes,Yes,No,Not Relevant
 ,download_export(),Yes,Yes,Yes,Yes,AWS
 ,get_exports(),Yes,Not Relevant,Not Relevant,Not Relevant,Not Relevant
-"Custom
-Metadata
+"Custom
+Metadata
 ",create_custom_fields(),Yes,Not Relevant,Not Relevant,Not Relevant,Not Relevant
 ,get_custom_fields(),Yes,Not Relevant,Not Relevant,Not Relevant,Not Relevant
 ,delete_custom_fields(),Yes,Not Relevant,Not Relevant,Not Relevant,Not Relevant
@@ -78,10 +78,10 @@ Team,get_team_metadata(),Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not
 ,get_user_metadata(),Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not Relevant
 ,set_user_custom_field(),Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not Relevant
 ,list_users(),Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not Relevant
-"Converting
+"Converting
 Annotations",import_annotation(),Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not Relevant
 ,export_annotation(),Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not Relevant
 ,convert_project_type(),Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not Relevant
-"Working w/
+"Working w/
 Annotations",validate_annotations(),Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not Relevant
 ,aggregate_annotations_as_df(),Not Relevant,Not Relevant,Not Relevant,Not Relevant,Not Relevant

docs/source/userguide/utilities.rst

@@ -116,14 +116,14 @@ You can find more information annotation format conversion :ref:`here <ref_conve
 
 Converting CSV and JSONL Formats for Annotation Management in SuperAnnotate
 ---------------------------------------------------------------------------
-SuperAnnotate primarily uses the **JSONL format** for annotation import/export. However,
-many external tools use **CSV**, requiring users to convert between these formats for seamless data management.
+SuperAnnotate primarily uses the JSONL format for annotation import/export. However,
+many external tools use CSV, requiring users to convert between these formats for seamless data management.
 
 This guide provides:
 
-- CSV to JSONL conversion** for annotation uploads.
-- Fetching annotations from SuperAnnotate** and converting them into JSONL/CSV.
-- Correct metadata mappings** to ensure consistency in the annotation format.
+- CSV to JSONL conversion for annotation uploads.
+- Fetching annotations from SuperAnnotate and converting them into JSONL/CSV.
+- Correct metadata mappings to ensure consistency in the annotation format.
 
 
 SuperAnnotate JSONL Schema Overview

src/superannotate/__init__.py

@@ -3,7 +3,7 @@
 import sys
 
 
-__version__ = "4.4.31"
+__version__ = "4.4.32"
 
 os.environ.update({"sa_version": __version__})
 sys.path.append(os.path.split(os.path.realpath(__file__))[0])

src/superannotate/lib/core/serviceproviders.py

@@ -672,6 +672,14 @@ def saqul_query(
     ) -> ServiceResponse:
         raise NotImplementedError
 
+    @abstractmethod
+    def query_item_count(
+        self,
+        project: entities.ProjectEntity,
+        query: str = None,
+    ) -> ServiceResponse:
+        raise NotImplementedError
+
 
 class BaseServiceProvider:
     projects: BaseProjectService

src/superannotate/lib/core/usecases/annotations.py

@@ -2046,7 +2046,7 @@ def execute(self):
                 f"annotations to the project {self._project.name}."
             )
             if not self._root_folder.is_root:
-                if len(distributed_items) > 1 or None not in distributed_items:
+                if len(distributed_items) > 1 or "" not in distributed_items:
                     raise AppException(
                         "You can't include a folder when uploading from within a folder."
                     )

src/superannotate/lib/core/usecases/items.py

@@ -163,6 +163,52 @@ def execute(self) -> Response:
         return self._response
 
 
+class QueryEntitiesCountUseCase(BaseReportableUseCase):
+    def __init__(
+        self,
+        reporter: Reporter,
+        project: ProjectEntity,
+        service_provider: BaseServiceProvider,
+        query: str,
+    ):
+        super().__init__(reporter)
+        self._project = project
+        self._service_provider = service_provider
+        self._query = query
+
+    def validate_arguments(self):
+        if self._query:
+            response = self._service_provider.explore.validate_saqul_query(
+                project=self._project, query=self._query
+            )
+
+            if not response.ok:
+                raise AppException(response.error)
+            if response.data["isValidQuery"]:
+                self._query = response.data["parsedQuery"]
+            else:
+                raise AppException("Incorrect query.")
+        else:
+            response = self._service_provider.explore.validate_saqul_query(
+                self._project, "-"
+            )
+            if not response.ok:
+                raise AppException(response.error)
+
+    def execute(self) -> Response:
+        if self.is_valid():
+            query_kwargs = {"query": self._query}
+            service_response = self._service_provider.explore.query_item_count(
+                self._project,
+                **query_kwargs,
+            )
+            if service_response.ok:
+                self._response.data = service_response.data
+            else:
+                self._response.errors = service_response.data
+        return self._response
+
+
 class AssignItemsUseCase(BaseUseCase):
     CHUNK_SIZE = 500
 

src/superannotate/lib/infrastructure/annotation_adapter.py

@@ -44,7 +44,7 @@ def get_component_value(self, component_id: str):
         return None
 
     def set_component_value(self, component_id: str, value: Any):
-        self.annotation["data"][component_id] = {"value": value}
+        self.annotation.setdefault("data", {}).setdefault(component_id, {})["value"] = value
         return self
 
 

src/superannotate/lib/infrastructure/controller.py

@@ -1678,3 +1678,17 @@ def query_entities(
         return ItemManager.process_response(
             self.service_provider, items, project, folder, map_fields=False
         )
+
+    def query_items_count(self, project_name: str, query: str = None) -> int:
+        project = self.get_project(project_name)
+
+        use_case = usecases.QueryEntitiesCountUseCase(
+            reporter=self.get_default_reporter(),
+            project=project,
+            query=query,
+            service_provider=self.service_provider,
+        )
+        response = use_case.execute()
+        if response.errors:
+            raise AppException(response.errors)
+        return response.data["count"]

src/superannotate/lib/infrastructure/services/explore.py

@@ -10,6 +10,7 @@
 from lib.core.service_types import SubsetListResponse
 from lib.core.service_types import UploadCustomFieldValuesResponse
 from lib.core.serviceproviders import BaseExploreService
+from superannotate import AppException
 
 
 class ExploreService(BaseExploreService):
@@ -25,6 +26,7 @@ class ExploreService(BaseExploreService):
     URL_UPLOAD_CUSTOM_VALUE = "custom/metadata/item"
     URL_SAQUL_QUERY = "items/search"
     URL_VALIDATE_SAQUL_QUERY = "items/parse/query"
+    URL_QUERY_COUNT = "items/count"
 
     @property
     def explore_service_url(self):
@@ -201,3 +203,24 @@ def saqul_query(
         else:
             response = ServiceResponse(status=200, res_data=[])
         return response
+
+    def query_item_count(
+        self,
+        project: entities.ProjectEntity,
+        query: str = None,
+    ) -> ServiceResponse:
+
+        params = {
+            "project_id": project.id,
+            "includeFolderNames": True,
+        }
+        data = {"query": query}
+        response = self.client.request(
+            urljoin(self.explore_service_url, self.URL_QUERY_COUNT),
+            "post",
+            params=params,
+            data=data,
+        )
+        if not response.ok:
+            raise AppException(response.error)
+        return response

tests/integration/items/test_saqul_query.py

@@ -59,7 +59,13 @@ def test_query(self):
     def test_query_on_100(self):
         sa.attach_items(self.PROJECT_NAME, os.path.join(DATA_SET_PATH, "100_urls.csv"))
         entities = sa.query(self.PROJECT_NAME, "metadata(status = NotStarted)")
-        print(len(entities))
+        assert len(entities) == 100
+        assert (
+            sa.controller.query_items_count(
+                self.PROJECT_NAME, "metadata(status = NotStarted)"
+            )
+            == 100
+        )
 
     def test_validate_saqul_query(self):
         try: