From 1526e086642c05fbe73f37cbc5d7207408a92c77 Mon Sep 17 00:00:00 2001 From: "Loic L." Date: Thu, 11 Jul 2024 18:26:55 +0200 Subject: [PATCH 1/5] fix: mypy errors and renamed primary errors --- naas_python/domains/storage/StorageDomain.py | 58 +++++++------- naas_python/domains/storage/StorageSchema.py | 74 ++++++++++------- .../adaptors/primary/SDKStorageAdaptor.py | 54 ++++++------- .../adaptors/primary/TyperStorageAdaptor.py | 79 +++++++++++-------- .../secondary/NaasStorageAPIAdaptor.py | 16 ++-- .../providers/S3StorageProviderAdaptor.py | 18 +++-- .../domains/storage/handlers/PythonHandler.py | 6 +- 7 files changed, 167 insertions(+), 138 deletions(-) diff --git a/naas_python/domains/storage/StorageDomain.py b/naas_python/domains/storage/StorageDomain.py index feaa924..8883e89 100644 --- a/naas_python/domains/storage/StorageDomain.py +++ b/naas_python/domains/storage/StorageDomain.py @@ -1,6 +1,6 @@ -from .models.Storage import Storage +from naas_python.domains.storage.models.Storage import Storage -from typing import Mapping +from typing import List, Mapping from naas_python.domains.storage.StorageSchema import ( IStorageDomain, @@ -11,18 +11,19 @@ StorageProviderNotFound ) class StorageDomain(IStorageDomain): - def __init__(self, adaptor: IStorageAdaptor, storage_provider_adaptors : Mapping[str, IStorageProviderAdaptor]): - # List[IStorageProviderAdaptor]) - #Map[str : IStorageProviderAdaptor]) - self.adaptor : IStorageAdaptor = adaptor + def __init__(self, adaptor: IStorageAdaptor, + storage_provider_adaptors : Mapping[str, IStorageProviderAdaptor] + ) : + + self.adaptor = adaptor self.storage_provider_adaptors : Mapping[str, IStorageProviderAdaptor] = storage_provider_adaptors ############### API ############### def create(self, - workspace_id: str, + workspace_id: str, storage_name: Storage.__fields__['name'], ) -> dict: - response = self.adaptor.create_workspace_storage( + response = self.adaptor.create( workspace_id=workspace_id, storage_name=storage_name, ) @@ -32,7 +33,7 @@ def delete(self, workspace_id: str, storage_name: Storage.__fields__['name'] ) -> dict: - response = self.adaptor.delete_workspace_storage( + response = self.adaptor.delete( workspace_id=workspace_id, storage_name=storage_name, ) @@ -41,7 +42,7 @@ def delete(self, def list(self, workspace_id: str, ) -> dict: - response = self.adaptor.list_workspace_storage( + response = self.adaptor.list( workspace_id=workspace_id, ) return response @@ -51,7 +52,7 @@ def list_objects(self, storage_name: Storage.__fields__['name'], storage_prefix: Object.__fields__['prefix'], ) -> dict: - response = self.adaptor.list_workspace_storage_object( + response = self.adaptor.list_objects( workspace_id=workspace_id, storage_name=storage_name, storage_prefix=storage_prefix, @@ -61,9 +62,9 @@ def list_objects(self, def delete_object(self, workspace_id: str, storage_name: Storage.__fields__['name'], - object_name: Object.__fields__['name'], + object_name: Storage.__fields__['name'], ) -> dict: - response = self.adaptor.delete_workspace_storage_object( + response = self.adaptor.delete_object( workspace_id=workspace_id, storage_name=storage_name, object_name=object_name, @@ -74,11 +75,12 @@ def create_credentials(self, workspace_id: str, storage_name: Storage.__fields__['name'], ) -> dict: - credentials = self.adaptor.generate_credentials(workspace_id, storage_name) - self.__get_storage_provider_adaptor(workspace_id=workspace_id, storage_name=storage_name).save_naas_credentials(workspace_id, storage_name, credentials) - return credentials + credentials = self.adaptor.create_credentials(workspace_id, storage_name) + storage_provider_adaptor = self.__get_storage_provider_adaptor(workspace_id=workspace_id, storage_name=storage_name) + storage_provider_adaptor.save_naas_credentials(workspace_id, storage_name, credentials) + return credentials -############### BOTO ############### +############### BOTO ############### def __get_storage_provider(self, workspace_id: str, storage_name: Storage.__fields__['name'] @@ -98,7 +100,7 @@ def __get_storage_provider_adaptor(self, def post_object(self, workspace_id: str, - storage_name: Storage.__fields__['name'], + storage_name: str, src_file: str, dst_file: str, ) -> dict: @@ -108,13 +110,13 @@ def post_object(self, if storage_provider_id not in self.storage_provider_adaptors: raise StorageProviderNotFound(f'Provider "{storage_provider_id}" is not implemented or not loaded.') - storage_provider : IStorageProviderAdaptor = self.storage_provider_adaptors[storage_provider_id] + storage_provider_adaptor : IStorageProviderAdaptor = self.storage_provider_adaptors[storage_provider_id] - if not storage_provider.valid_naas_credentials(workspace_id, storage_name): - credentials = self.adaptor.generate_credentials(workspace_id, storage_name) - storage_provider.save_naas_credentials(workspace_id, storage_name, credentials) + if not storage_provider_adaptor.valid_naas_credentials(workspace_id, storage_name): + credentials = self.adaptor.create_credentials(workspace_id, storage_name) + storage_provider_adaptor.save_naas_credentials(workspace_id, storage_name, credentials) - response = storage_provider.post_workspace_storage_object(workspace_id=workspace_id, storage_name=storage_name, src_file=src_file, dst_file=dst_file) + response = storage_provider_adaptor.post_object(workspace_id=workspace_id, storage_name=storage_name, src_file=src_file, dst_file=dst_file) return response def get_object(self, @@ -129,11 +131,11 @@ def get_object(self, if storage_provider_id not in self.storage_provider_adaptors: raise StorageProviderNotFound(f'Provider "{storage_provider_id}" is not implemented or not loaded.') - storage_provider : IStorageProviderAdaptor = self.storage_provider_adaptors[storage_provider_id] + storage_provider_adaptor : IStorageProviderAdaptor = self.storage_provider_adaptors[storage_provider_id] - if not storage_provider.valid_naas_credentials(workspace_id, storage_name): - credentials = self.adaptor.generate_credentials(workspace_id, storage_name) - storage_provider.save_naas_credentials(workspace_id, storage_name, credentials) + if not storage_provider_adaptor.valid_naas_credentials(workspace_id, storage_name): + credentials = self.adaptor.create_credentials(workspace_id, storage_name) + storage_provider_adaptor.save_naas_credentials(workspace_id, storage_name, credentials) - response = storage_provider.get_workspace_storage_object(workspace_id=workspace_id, storage_name=storage_name, src_file=src_file, dst_file=dst_file) + response = storage_provider_adaptor.get_object(workspace_id=workspace_id, storage_name=storage_name, src_file=src_file, dst_file=dst_file) return response diff --git a/naas_python/domains/storage/StorageSchema.py b/naas_python/domains/storage/StorageSchema.py index a8aa86f..a3c8c45 100644 --- a/naas_python/domains/storage/StorageSchema.py +++ b/naas_python/domains/storage/StorageSchema.py @@ -1,8 +1,9 @@ from abc import ABCMeta, abstractmethod from logging import getLogger -from typing import Mapping +from typing import List, Mapping, Any +from uuid import UUID -from naas_models.pydantic.storage_p2p import * +from naas_models.pydantic.storage_p2p import Storage from .models.Storage import Storage, Object @@ -13,27 +14,27 @@ class IStorageAdaptor(metaclass=ABCMeta): @abstractmethod - def create_workspace_storage(self, + def create(self, workspace_id: str, storage_name: Storage.__fields__['name'], - ) -> dict[str, str]: + ) -> dict: raise NotImplementedError @abstractmethod - def delete_workspace_storage(self, + def delete(self, workspace_id: str, storage_name: Storage.__fields__['name'], ) -> dict: raise NotImplementedError @abstractmethod - def list_workspace_storage(self, + def list(self, workspace_id: str, ) -> dict: raise NotImplementedError @abstractmethod - def list_workspace_storage_object(self, + def list_objects(self, workspace_id: str, storage_name: Storage.__fields__['name'], storage_prefix: Object.__fields__['prefix'], @@ -41,7 +42,7 @@ def list_workspace_storage_object(self, raise NotImplementedError @abstractmethod - def delete_workspace_storage_object(self, + def delete_object(self, workspace_id: str, storage_name: Storage.__fields__['name'], object_name: Object.__fields__['name'], @@ -49,7 +50,7 @@ def delete_workspace_storage_object(self, raise NotImplementedError @abstractmethod - def generate_credentials(self, workspace_id :str, storage_name: str) -> dict: + def create_credentials(self, workspace_id : str, storage_name: Storage.__fields__['name']) -> dict: raise NotImplementedError class IStorageProviderAdaptor(metaclass=ABCMeta): @@ -57,7 +58,7 @@ class IStorageProviderAdaptor(metaclass=ABCMeta): provider_id : str @abstractmethod - def post_workspace_storage_object(self, + def post_object(self, workspace_id: str, storage_name: Storage.__fields__['name'], src_file: str, @@ -66,7 +67,7 @@ def post_workspace_storage_object(self, raise NotImplementedError @abstractmethod - def get_workspace_storage_object(self, + def get_object(self, workspace_id: str, storage_name: Storage.__fields__['name'], src_file: str, @@ -82,8 +83,6 @@ def save_naas_credentials(self, workspace_id:str, storage_name:str, credentials: class IStorageDomain(metaclass=ABCMeta): adaptor: IStorageAdaptor storage_provider_adaptors : Mapping[str, IStorageProviderAdaptor] - # storage_provider_adaptors : Map[str, IStorageProviderAdaptor] - #TODO to be validated @abstractmethod def create(self, @@ -139,7 +138,6 @@ def get_object(self, ) -> bytes: raise NotImplementedError - @abstractmethod def create_credentials(self, workspace_id: str, @@ -150,35 +148,57 @@ def create_credentials(self, # Primary Adaptor class IStorageInvoker(metaclass=ABCMeta): @abstractmethod - def create_workspace_storage(self, **kwargs): + def create(self, workspace_id: str, storage_name: Storage.__fields__['name']) -> dict: raise NotImplementedError - + @abstractmethod - def delete_workspace_storage(self, **kwargs): - raise NotImplementedError + def delete(self, + workspace_id: str, + storage_name: str = Storage.__fields__['name'], + ) -> dict: + raise NotImplementedError @abstractmethod - def list_workspace_storage(self, **kwargs): + def list(self, workspace_id: str) -> dict: raise NotImplementedError @abstractmethod - def list_workspace_storage_object(self, **kwargs): + def create_credentials(self, + workspace_id : str, + storage_name = str, + ) -> dict: raise NotImplementedError @abstractmethod - def delete_workspace_storage_object(self, **kwargs): - raise NotImplementedError - + def list_objects(self, + workspace_id: str, + storage_name: str, + storage_prefix: str) -> dict: + raise NotImplementedError + @abstractmethod - def post_workspace_storage_object(self, **kwargs): + def delete_object(self, + workspace_id: str, + storage_name: str, + object_name: str) -> dict: raise NotImplementedError @abstractmethod - def get_workspace_storage_object(self, **kwargs): - raise NotImplementedError + def post_object(self, + workspace_id: str, + storage_name: str, + src_file: str, + dst_file: str, + ) -> dict: + raise NotImplementedError @abstractmethod - def create_workspace_storage_credentials(self, **kwargs): + def get_object(self, + workspace_id: str, + storage_name: str, + src_file: str, + dst_file: str, + ) -> bytes: raise NotImplementedError # Exceptions diff --git a/naas_python/domains/storage/adaptors/primary/SDKStorageAdaptor.py b/naas_python/domains/storage/adaptors/primary/SDKStorageAdaptor.py index 3e0aeae..fa52c92 100644 --- a/naas_python/domains/storage/adaptors/primary/SDKStorageAdaptor.py +++ b/naas_python/domains/storage/adaptors/primary/SDKStorageAdaptor.py @@ -13,88 +13,86 @@ def __init__(self, domain: IStorageDomain): ############### API ############### # Workspace Storage - def create_workspace_storage(self, workspace_id: str = "", storage_name: str = "") -> None: - response = self.domain.create_workspace_storage( + def create(self, workspace_id: str = "", storage_name: str = "") -> dict: + response = self.domain.create( workspace_id=workspace_id, storage_name=storage_name, ) return response - - def delete_workspace_storage(self, workspace_id: str = "", storage_name: str = "") -> None: - response = self.domain.delete_workspace_storage( + + def delete(self, workspace_id: str = "", storage_name: str = "") -> None: + response = self.domain.delete( workspace_id=workspace_id, storage_name=storage_name, - ) - return response + ) - def list_workspace_storage(self, workspace_id: str = "") -> str: - response = self.domain.list_workspace_storage( + def list(self, workspace_id: str = "") -> dict: + response = self.domain.list( workspace_id=workspace_id, ) return response - def create_workspace_storage_credentials(self, workspace_id: str = "", storage_name: str = ""): - response = self.domain.create_workspace_storage_credentials( + def create_credentials(self, workspace_id: str = "", storage_name: str = ""): + response = self.domain.create_credentials( workspace_id=workspace_id, storage_name=storage_name, - ) - return response + ) + return response # Workspace Storage Object - def list_workspace_storage_object(self, + def list_objects(self, workspace_id: str = "", storage_name: str = "", - storage_prefix: str = "") -> str: + storage_prefix: str = "") -> dict: - response = self.domain.list_workspace_storage_object( + response = self.domain.list_objects( workspace_id=workspace_id, storage_name=storage_name, storage_prefix=storage_prefix, ) return response - def delete_workspace_storage_object(self, + def delete_object(self, workspace_id: str = "", storage_name: str = "", object_name: str = "", ) -> None: - response = self.domain.delete_workspace_storage_object( + response = self.domain.delete_object( workspace_id=workspace_id, storage_name=storage_name, object_name=object_name, ) - return response - + ############### BOTO3 ############### - def post_workspace_storage_object(self, + def post_object(self, workspace_id: str = "", storage_name: str = "", src_file: str = "", dst_file: str = "", - ) -> bytes: + ) -> dict: if os.path.isfile(src_file): - response = self.domain.post_workspace_storage_object( + response = self.domain.post_object( workspace_id=workspace_id, storage_name=storage_name, src_file=src_file, dst_file=dst_file, ) - return response + return response else: raise FileNotFoundError(f"File not found: {src_file}") - def get_workspace_storage_object(self, + def get_object(self, workspace_id: str = "", storage_name: str = "", src_file: str = "", dst_file: str = "", - ) -> bytes: + ) -> bytes: - response = self.domain.get_workspace_storage_object( + response = self.domain.get_object( workspace_id=workspace_id, storage_name=storage_name, src_file=src_file, dst_file=dst_file, - ) + ) return response diff --git a/naas_python/domains/storage/adaptors/primary/TyperStorageAdaptor.py b/naas_python/domains/storage/adaptors/primary/TyperStorageAdaptor.py index 33b8fec..56c56be 100644 --- a/naas_python/domains/storage/adaptors/primary/TyperStorageAdaptor.py +++ b/naas_python/domains/storage/adaptors/primary/TyperStorageAdaptor.py @@ -39,17 +39,17 @@ def __init__(self, domain: IStorageDomain): context_settings={"help_option_names": ["-h", "--help"]}, ) - self.app.command("create")(self.create_workspace_storage) - self.app.command("delete")(self.delete_workspace_storage) - self.app.command("list")(self.list_workspace_storage) - self.app.command("list-object")(self.list_workspace_storage_object) - self.app.command("put-object")(self.post_workspace_storage_object) - self.app.command("get-object")(self.get_workspace_storage_object) - self.app.command("delete-object")(self.delete_workspace_storage_object) - self.app.command("connect")(self.create_workspace_storage_credentials) + self.app.command("create")(self.create) + self.app.command("delete")(self.delete) + self.app.command("list")(self.list) + self.app.command("list-object")(self.list_objects) + self.app.command("post-object")(self.post_object) + self.app.command("get-object")(self.get_object) + self.app.command("delete-object")(self.delete_object) + self.app.command("create-credentials")(self.create_credentials) ############### API ############### - def create_workspace_storage(self, + def create(self, workspace_id: str = typer.Option(..., "--workspace", "-w", help="ID of the workspace"), storage_name: str = typer.Option(..., "--storage", "-s", help="Name of the storage"), rich_preview: bool = typer.Option( @@ -58,16 +58,17 @@ def create_workspace_storage(self, "-rp", help="Rich preview of the information as a table", ) - ): + ) -> dict: """Create a Workspace Storage""" print("creating storage...") - storage = self.domain.create_workspace_storage( + storage = self.domain.create( workspace_id=workspace_id, storage_name=storage_name, ) print(f"Storage {storage_name} created.") + return storage - def delete_workspace_storage(self, + def delete(self, workspace_id: str = typer.Option(..., "--workspace", "-w", help="ID of the workspace"), storage_name: str = typer.Option(..., "--storage", "-s", help="Name of the storage"), rich_preview: bool = typer.Option( @@ -76,16 +77,17 @@ def delete_workspace_storage(self, "-rp", help="Rich preview of the information as a table", ) - ): + ) -> dict: """Delete a Workspace Storage""" print("deleting storage...") - storage = self.domain.delete_workspace_storage( + storage = self.domain.delete( workspace_id=workspace_id, storage_name=storage_name, ) print(f"Storage {storage_name} deleted.") + return storage - def list_workspace_storage(self, + def list(self, workspace_id: str = typer.Option(..., "--workspace", "-w", help="ID of the workspace"), rich_preview: bool = typer.Option( False, @@ -93,9 +95,9 @@ def list_workspace_storage(self, "-rp", help="Rich preview of the information as a table", ) - ): + ) -> dict: """List Workspace Storages""" - list_storage = self.domain.list_workspace_storage( + list_storage = self.domain.list( workspace_id=workspace_id, ) if rich_preview: @@ -107,10 +109,11 @@ def list_workspace_storage(self, table.add_row(storage['name']) console.print(table) + return {} else: - print(list_storage) + return list_storage - def list_workspace_storage_object(self, + def list_objects(self, workspace_id: str = typer.Option(..., "--workspace", "-w", help="ID of the workspace"), storage_name: str = typer.Option(..., "--storage", "-s", help="Name of the storage"), storage_prefix: str = typer.Option(..., "--prefix", "-p", help="Path prefix in the storage"), @@ -120,9 +123,9 @@ def list_workspace_storage_object(self, "-rp", help="Rich preview of the information as a table", ) - ): + ) -> dict: """List a Workspace Storage Objects""" - list_storage_object = self.domain.list_workspace_storage_object( + list_storage_object = self.domain.list_objects( workspace_id=workspace_id, storage_name=storage_name, storage_prefix=storage_prefix, @@ -140,10 +143,11 @@ def list_workspace_storage_object(self, table.add_row(object['name'], object['type'], object['prefix'], object['size'], object['lastmodified']) console.print(table) + return {} else: - print(list_storage_object) + return list_storage_object - def delete_workspace_storage_object(self, + def delete_object(self, workspace_id: str = typer.Option(..., "--workspace", "-w", help="ID of the workspace"), storage_name: str = typer.Option(..., "--storage", "-s", help="Name of the storage"), object_name: str = typer.Option(..., "--object", "-o", help="Name of Object or Folder to remove."), @@ -153,17 +157,18 @@ def delete_workspace_storage_object(self, "-rp", help="Rich preview of the information as a table", ) - ): + ) -> dict: """Delete a Workspace Storage Object""" print("Deleting object...") - response = self.domain.delete_workspace_storage_object( + response = self.domain.delete_object( workspace_id=workspace_id, storage_name=storage_name, object_name=object_name, ) print("Object deleted.") + return response - def create_workspace_storage_credentials(self, + def create_credentials(self, workspace_id: str = typer.Option(..., "--workspace", "-w", help="ID of the workspace"), storage_name: str = typer.Option(..., "--storage", "-s", help="Name of the storage"), rich_preview: bool = typer.Option( @@ -172,18 +177,18 @@ def create_workspace_storage_credentials(self, "-rp", help="Rich preview of the information as a table", ) - ): + ) -> dict: """Create Storage Credentials""" print("Creating credentials...") - response = self.domain.create_workspace_storage_credentials( + response = self.domain.create_credentials( workspace_id=workspace_id, storage_name=storage_name ) print("Credentials created.") - print(response) + return response ############### BOTO3 ############### - def post_workspace_storage_object(self, + def post_object(self, workspace_id: str = typer.Option(..., "--workspace", "-w", help="ID of the workspace"), storage_name: str = typer.Option(..., "--storage", "-s", help="Name of the storage"), src_file: str = typer.Option(..., "--source", "-src", help="File path to upload in the storage"), @@ -194,21 +199,23 @@ def post_workspace_storage_object(self, "-rp", help="Rich preview of the information as a table", ) - ) -> None: + ) -> dict: """Post a Workspace Storage Object""" if not os.path.isfile(src_file): print(f"File '{src_file}' does not exist.") + return {} else: print("Uploading object...") - response = self.domain.post_workspace_storage_object( + response = self.domain.post_object( workspace_id=workspace_id, storage_name=storage_name, src_file=src_file, dst_file=dst_file ) print("Object uploaded.") + return response - def get_workspace_storage_object(self, + def get_object(self, workspace_id: str = typer.Option(None, "--workspace", "-w", help="ID of the workspace"), storage_name: str = typer.Option(None, "--storage", "-s", help="Name of the storage"), src_file: str = typer.Option(None, "--source", "-src", help="File path to download in the storage"), @@ -219,16 +226,18 @@ def get_workspace_storage_object(self, "-rp", help="Rich preview of the information as a table", ) - ): + ) -> bytes: """Get a Workspace Storage Object""" if src_file.endswith("/"): print("this is not an object") + return b"" else : print("Downloading object...") - response = self.domain.get_workspace_storage_object( + response = self.domain.get_object( workspace_id=workspace_id, storage_name=storage_name, src_file=src_file, dst_file=dst_file ) print("Object downloaded.") + return response diff --git a/naas_python/domains/storage/adaptors/secondary/NaasStorageAPIAdaptor.py b/naas_python/domains/storage/adaptors/secondary/NaasStorageAPIAdaptor.py index 874b674..9cdca46 100644 --- a/naas_python/domains/storage/adaptors/secondary/NaasStorageAPIAdaptor.py +++ b/naas_python/domains/storage/adaptors/secondary/NaasStorageAPIAdaptor.py @@ -1,4 +1,3 @@ -import json import os from logging import getLogger import pydash as _ @@ -6,7 +5,6 @@ logger = getLogger(__name__) import requests -from requests.exceptions import ConnectionError from naas_python.utils.domains_base.secondary.BaseAPIAdaptor import BaseAPIAdaptor @@ -22,7 +20,7 @@ def __init__(self): def __handle_response(self, api_response: requests.Response) -> dict: if api_response.status_code == 201: - return None + return api_response.json() elif api_response.status_code == 200: return api_response.json() elif isinstance(api_response.json().get("error"), dict) and api_response.json().get("error")["error"] == 1: @@ -34,7 +32,7 @@ def __handle_response(self, api_response: requests.Response) -> dict: raise APIError(api_response.json()) @BaseAPIAdaptor.service_status_decorator - def create_workspace_storage(self, + def create(self, workspace_id: str, storage_name: Storage.__fields__['name'] ) -> dict: @@ -51,7 +49,7 @@ def create_workspace_storage(self, return self.__handle_response(api_response) @BaseAPIAdaptor.service_status_decorator - def delete_workspace_storage(self, + def delete(self, workspace_id: str, storage_name: str ) -> dict: @@ -64,7 +62,7 @@ def delete_workspace_storage(self, return self.__handle_response(api_response) @BaseAPIAdaptor.service_status_decorator - def list_workspace_storage(self, + def list(self, workspace_id: str, ) -> dict: _url = f"{self.host}/workspace/{workspace_id}/storage/" @@ -76,7 +74,7 @@ def list_workspace_storage(self, return self.__handle_response(api_response) @BaseAPIAdaptor.service_status_decorator - def list_workspace_storage_object(self, + def list_objects(self, workspace_id: str, storage_name: str, storage_prefix: str, @@ -90,7 +88,7 @@ def list_workspace_storage_object(self, return self.__handle_response(api_response) @BaseAPIAdaptor.service_status_decorator - def delete_workspace_storage_object(self, + def delete_object(self, workspace_id: str, storage_name: str, object_name: str, @@ -106,7 +104,7 @@ def delete_workspace_storage_object(self, return self.__handle_response(api_response) @BaseAPIAdaptor.service_status_decorator - def generate_credentials(self, workspace_id :str, storage_name: str) -> dict: + def create_credentials(self, workspace_id : str, storage_name: Storage.__fields__['name']) -> dict: _url = f"{self.host}/workspace/{workspace_id}/storage/credentials/" diff --git a/naas_python/domains/storage/adaptors/secondary/providers/S3StorageProviderAdaptor.py b/naas_python/domains/storage/adaptors/secondary/providers/S3StorageProviderAdaptor.py index 430eaa0..37963d2 100644 --- a/naas_python/domains/storage/adaptors/secondary/providers/S3StorageProviderAdaptor.py +++ b/naas_python/domains/storage/adaptors/secondary/providers/S3StorageProviderAdaptor.py @@ -1,4 +1,4 @@ -from naas_python.domains.storage.StorageSchema import IStorageProviderAdaptor, Storage, Object +from naas_python.domains.storage.StorageSchema import IStorageProviderAdaptor, Storage import boto3 import os, json, re @@ -42,7 +42,7 @@ def __init__(self): self.AWS_SESSION_EXPIRATION_TOKEN=None - def post_workspace_storage_object(self, + def post_object(self, workspace_id: str, storage_name: Storage.__fields__['name'], src_file: str, @@ -60,10 +60,12 @@ def post_workspace_storage_object(self, key = self.__clean_path(key) try: - if self.AWS_ACCESS_KEY_ID is None or self.__s3_token_is_expired(self.AWS_SESSION_EXPIRATION_TOKEN): - self.__read_naas_credentials(workspace_id, storage_name) + self.__read_naas_credentials(workspace_id, storage_name) content_type, _ = mimetypes.guess_type(src_file) + + print("AWS_ACCESS_KEY_ID: ", self.AWS_ACCESS_KEY_ID) + s3 = boto3.client('s3') response = s3.upload_file(Filename=src_file, Bucket=self.naas_bucket, Key=key, ExtraArgs={'ContentType': content_type}) return response @@ -72,7 +74,7 @@ def post_workspace_storage_object(self, return response - def get_workspace_storage_object(self, + def get_object(self, workspace_id:str, storage_name: str, src_file: str, @@ -91,8 +93,7 @@ def get_workspace_storage_object(self, object_key = self.__clean_path(object_key) try: - if self.AWS_ACCESS_KEY_ID is None or self.__s3_token_is_expired(self.AWS_SESSION_EXPIRATION_TOKEN): - self.__read_naas_credentials(workspace_id, storage_name) + self.__read_naas_credentials(workspace_id, storage_name) s3 = boto3.client('s3') response = s3.download_file(Bucket=self.naas_bucket , Key=object_key, Filename=filename) @@ -170,7 +171,8 @@ def __read_naas_credentials(self, workspace_id:str, storage_name:str)-> None: def valid_naas_credentials(self, workspace_id:str, storage_name:str)-> bool: # do not change setted env variables - if not self.AWS_ACCESS_KEY_ID: + print("VALID_AWS_ACCESS_KEY_ID: ", self.AWS_ACCESS_KEY_ID) + if not self.AWS_ACCESS_KEY_ID or self.__s3_token_is_expired(self.AWS_SESSION_EXPIRATION_TOKEN) : # try read env var from naas_credentials file if os.path.exists(self.naas_credentials): diff --git a/naas_python/domains/storage/handlers/PythonHandler.py b/naas_python/domains/storage/handlers/PythonHandler.py index 3f4aaf6..996896f 100644 --- a/naas_python/domains/storage/handlers/PythonHandler.py +++ b/naas_python/domains/storage/handlers/PythonHandler.py @@ -1,7 +1,7 @@ -from ..adaptors.secondary.NaasStorageAPIAdaptor import NaasStorageAPIAdaptor +from naas_python.domains.storage.adaptors.secondary.NaasStorageAPIAdaptor import NaasStorageAPIAdaptor from naas_python.domains.storage.adaptors.secondary.providers.S3StorageProviderAdaptor import S3StorageProviderAdaptor -from ..StorageDomain import StorageDomain -from ..adaptors.primary.SDKStorageAdaptor import SDKStorageAdaptor +from naas_python.domains.storage.StorageDomain import StorageDomain +from naas_python.domains.storage.adaptors.primary.SDKStorageAdaptor import SDKStorageAdaptor secondaryAdaptor = NaasStorageAPIAdaptor() From 00bd451f5f1e8b637e6664c936b14dfcb71d098a Mon Sep 17 00:00:00 2001 From: "Loic L." Date: Fri, 12 Jul 2024 10:13:14 +0200 Subject: [PATCH 2/5] fix: env issue and cli list output --- .../adaptors/primary/TyperStorageAdaptor.py | 60 ++++++++++--------- .../providers/S3StorageProviderAdaptor.py | 46 +++++++------- 2 files changed, 52 insertions(+), 54 deletions(-) diff --git a/naas_python/domains/storage/adaptors/primary/TyperStorageAdaptor.py b/naas_python/domains/storage/adaptors/primary/TyperStorageAdaptor.py index 56c56be..e3e46c5 100644 --- a/naas_python/domains/storage/adaptors/primary/TyperStorageAdaptor.py +++ b/naas_python/domains/storage/adaptors/primary/TyperStorageAdaptor.py @@ -100,18 +100,20 @@ def list(self, list_storage = self.domain.list( workspace_id=workspace_id, ) - if rich_preview: - console = Console() - table = Table(show_header=True, header_style="bold black") - table.add_column("Name") - - for storage in list_storage['storage']: - table.add_row(storage['name']) - - console.print(table) - return {} - else: - return list_storage + # Always Rich List + # if rich_preview: + console = Console() + table = Table(show_header=True, header_style="bold black") + table.add_column("Name") + + for storage in list_storage['storage']: + table.add_row(storage['name']) + + console.print(table) + return {} + # else: + # print(list_storage) + # return list_storage def list_objects(self, workspace_id: str = typer.Option(..., "--workspace", "-w", help="ID of the workspace"), @@ -130,22 +132,24 @@ def list_objects(self, storage_name=storage_name, storage_prefix=storage_prefix, ) - if rich_preview: - console = Console() - table = Table(show_header=True, header_style="bold black") - table.add_column("Name") - table.add_column("Type") - table.add_column("Prefix") - table.add_column("Size") - table.add_column("Last Modified") - - for object in list_storage_object["object"]: - table.add_row(object['name'], object['type'], object['prefix'], object['size'], object['lastmodified']) - - console.print(table) - return {} - else: - return list_storage_object + # Always Rich List + # if rich_preview: + console = Console() + table = Table(show_header=True, header_style="bold black") + table.add_column("Name") + table.add_column("Type") + table.add_column("Prefix") + table.add_column("Size") + table.add_column("Last Modified") + + for object in list_storage_object["object"]: + table.add_row(object['name'], object['type'], object['prefix'], object['size'], object['lastmodified']) + + console.print(table) + return {} + # else: + # print(list_storage_object) + # return list_storage_object def delete_object(self, workspace_id: str = typer.Option(..., "--workspace", "-w", help="ID of the workspace"), diff --git a/naas_python/domains/storage/adaptors/secondary/providers/S3StorageProviderAdaptor.py b/naas_python/domains/storage/adaptors/secondary/providers/S3StorageProviderAdaptor.py index 37963d2..a2587c2 100644 --- a/naas_python/domains/storage/adaptors/secondary/providers/S3StorageProviderAdaptor.py +++ b/naas_python/domains/storage/adaptors/secondary/providers/S3StorageProviderAdaptor.py @@ -64,8 +64,6 @@ def post_object(self, content_type, _ = mimetypes.guess_type(src_file) - print("AWS_ACCESS_KEY_ID: ", self.AWS_ACCESS_KEY_ID) - s3 = boto3.client('s3') response = s3.upload_file(Filename=src_file, Bucket=self.naas_bucket, Key=key, ExtraArgs={'ContentType': content_type}) return response @@ -170,35 +168,31 @@ def __read_naas_credentials(self, workspace_id:str, storage_name:str)-> None: raise BadCredentials("missing information in file, generate new credentials") def valid_naas_credentials(self, workspace_id:str, storage_name:str)-> bool: - # do not change setted env variables - print("VALID_AWS_ACCESS_KEY_ID: ", self.AWS_ACCESS_KEY_ID) - if not self.AWS_ACCESS_KEY_ID or self.__s3_token_is_expired(self.AWS_SESSION_EXPIRATION_TOKEN) : - # try read env var from naas_credentials file - if os.path.exists(self.naas_credentials): - with open(self.naas_credentials, 'r') as file: - json_credentials = json.load(file) + # try read env var from naas_credentials file + if os.path.exists(self.naas_credentials): + with open(self.naas_credentials, 'r') as file: + json_credentials = json.load(file) - # get the storages list - json_storages = json_credentials.get('storage', {}) + # get the storages list + json_storages = json_credentials.get('storage', {}) - # missing credentials in file - if workspace_id not in json_storages or storage_name not in json_storages[workspace_id] or 's3' not in json_storages[workspace_id][storage_name]: - return False - else: - json_credentials = json_storages[workspace_id][storage_name]['s3'] - access_key_id = json_credentials.get('AWS_ACCESS_KEY_ID') - expiration = json_credentials.get('AWS_SESSION_EXPIRATION_TOKEN') + # missing credentials in file + if workspace_id not in json_storages or storage_name not in json_storages[workspace_id] or 's3' not in json_storages[workspace_id][storage_name]: + return False + else: + json_credentials = json_storages[workspace_id][storage_name]['s3'] + access_key_id = json_credentials.get('AWS_ACCESS_KEY_ID') + expiration = json_credentials.get('AWS_SESSION_EXPIRATION_TOKEN') - if self.__s3_token_is_expired(expiration) : - return False - elif access_key_id is None: - return False - else: - return True + if self.__s3_token_is_expired(expiration) : + return False + elif access_key_id is None: + return False + else: + return True else: - return True - return True + return False def save_naas_credentials(self, workspace_id:str, storage_name:str, credentials:dict)-> str: From bfd17589330c48e5fd5d940335f519fbe3079a85 Mon Sep 17 00:00:00 2001 From: "Loic L." Date: Mon, 15 Jul 2024 14:49:08 +0200 Subject: [PATCH 3/5] bump naas-models to 1.11.6 --- poetry.lock | 10 +++++----- pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index b3536ec..0a19d12 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "annotated-types" @@ -441,13 +441,13 @@ types-protobuf = ">=4.24" [[package]] name = "naas-models" -version = "1.11.2" +version = "1.11.6" description = "Library containing all models used in the naas ecosystem. You can use this library to know how to talk to our micro services." optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "naas_models-1.11.2-py3-none-any.whl", hash = "sha256:e73791e38ad6915100bf59bef122ff1ec8e0a6c7d129680a8e5c5f9dc8977418"}, - {file = "naas_models-1.11.2.tar.gz", hash = "sha256:824b706da9f12c83ebc32b5bd19047db83a57618c999db65c080cbfce7d4967b"}, + {file = "naas_models-1.11.6-py3-none-any.whl", hash = "sha256:46678f8c3e2ca7e0467b1f153e3a50ae3946a1a976e1ddfc8f195be06d0a6d09"}, + {file = "naas_models-1.11.6.tar.gz", hash = "sha256:15ac5e12c21274ae24f4e491e9b401e51465773a0a8c58e1a52ab52ba436b845"}, ] [package.dependencies] @@ -871,4 +871,4 @@ zstd = ["zstandard (>=0.18.0)"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "1eb5ede1aacb3be0f0b20ecb8b4bd21069b7001936bf5e8d890add29551f07c3" +content-hash = "e12927804eb27a440be92e1bfe13ffc3e0a7445e8054793ca6fae7eed23a8add" diff --git a/pyproject.toml b/pyproject.toml index dc87446..36f1b2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ typer = { extras = ["all"], version = "^0.9.0" } requests = "^2.31.0" cachetools = "^5.3.1" jinja2 = "^3.0.1" -naas-models = "^1.11.2" +naas-models = "^1.11.6" grpcio = "^1.60.0" pydash = "^7.0.7" boto3 = "^1.34.128" From 3eb60af99815444b834ce275bf23c1b36b72e8b4 Mon Sep 17 00:00:00 2001 From: "Loic L." Date: Thu, 18 Jul 2024 16:55:56 +0200 Subject: [PATCH 4/5] fix: mypy error --- naas_python/domains/storage/StorageSchema.py | 2 +- .../storage/adaptors/primary/SDKStorageAdaptor.py | 8 +++++--- .../storage/adaptors/secondary/NaasStorageAPIAdaptor.py | 7 ++++--- naas_python/storage.py | 9 +++++++++ 4 files changed, 19 insertions(+), 7 deletions(-) create mode 100644 naas_python/storage.py diff --git a/naas_python/domains/storage/StorageSchema.py b/naas_python/domains/storage/StorageSchema.py index a3c8c45..3646b73 100644 --- a/naas_python/domains/storage/StorageSchema.py +++ b/naas_python/domains/storage/StorageSchema.py @@ -13,7 +13,7 @@ logger = getLogger(__name__) class IStorageAdaptor(metaclass=ABCMeta): - @abstractmethod + @abstractmethod def create(self, workspace_id: str, storage_name: Storage.__fields__['name'], diff --git a/naas_python/domains/storage/adaptors/primary/SDKStorageAdaptor.py b/naas_python/domains/storage/adaptors/primary/SDKStorageAdaptor.py index fa52c92..d5fa74c 100644 --- a/naas_python/domains/storage/adaptors/primary/SDKStorageAdaptor.py +++ b/naas_python/domains/storage/adaptors/primary/SDKStorageAdaptor.py @@ -20,11 +20,12 @@ def create(self, workspace_id: str = "", storage_name: str = "") -> dict: ) return response - def delete(self, workspace_id: str = "", storage_name: str = "") -> None: + def delete(self, workspace_id: str = "", storage_name: str = "") -> dict: response = self.domain.delete( workspace_id=workspace_id, storage_name=storage_name, ) + return response def list(self, workspace_id: str = "") -> dict: response = self.domain.list( @@ -56,13 +57,14 @@ def delete_object(self, workspace_id: str = "", storage_name: str = "", object_name: str = "", - ) -> None: + ) -> dict: response = self.domain.delete_object( workspace_id=workspace_id, storage_name=storage_name, object_name=object_name, ) + return response ############### BOTO3 ############### def post_object(self, @@ -95,4 +97,4 @@ def get_object(self, src_file=src_file, dst_file=dst_file, ) - return response + return response \ No newline at end of file diff --git a/naas_python/domains/storage/adaptors/secondary/NaasStorageAPIAdaptor.py b/naas_python/domains/storage/adaptors/secondary/NaasStorageAPIAdaptor.py index 9cdca46..e227cac 100644 --- a/naas_python/domains/storage/adaptors/secondary/NaasStorageAPIAdaptor.py +++ b/naas_python/domains/storage/adaptors/secondary/NaasStorageAPIAdaptor.py @@ -1,6 +1,7 @@ import os from logging import getLogger import pydash as _ +import json logger = getLogger(__name__) @@ -23,10 +24,10 @@ def __handle_response(self, api_response: requests.Response) -> dict: return api_response.json() elif api_response.status_code == 200: return api_response.json() - elif isinstance(api_response.json().get("error"), dict) and api_response.json().get("error")["error"] == 1: + elif api_response.status_code == 404: raise StorageNotFoundError(api_response.json().get("error")["message"]) - elif isinstance(api_response.json().get("error"), dict) and api_response.json().get("error")["error"] == 2: - raise StorageNotFoundError(api_response.json().get("error")["message"]) + elif api_response.json().get("error")["code"] == 0 and api_response.json().get("error")["message"] == "Success": + raise api_response.json() else: logger.error(api_response.json()) raise APIError(api_response.json()) diff --git a/naas_python/storage.py b/naas_python/storage.py new file mode 100644 index 0000000..bec9861 --- /dev/null +++ b/naas_python/storage.py @@ -0,0 +1,9 @@ +import os +import naas-python as naas_python +post_object = naas_python.storage.post_workspace_storage_object( + workspace_id=workspace_id, + storage_name="test", + src_file="Adele.png", + dst_file="Adele.png", +) +post_object \ No newline at end of file From 8b410ad0ef21da8a032fe085b2c1008e5a1b47ff Mon Sep 17 00:00:00 2001 From: "Loic L." Date: Fri, 19 Jul 2024 11:30:49 +0200 Subject: [PATCH 5/5] fix: NoneType' object is not subscriptable --- .../storage/adaptors/secondary/NaasStorageAPIAdaptor.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/naas_python/domains/storage/adaptors/secondary/NaasStorageAPIAdaptor.py b/naas_python/domains/storage/adaptors/secondary/NaasStorageAPIAdaptor.py index e227cac..a440ced 100644 --- a/naas_python/domains/storage/adaptors/secondary/NaasStorageAPIAdaptor.py +++ b/naas_python/domains/storage/adaptors/secondary/NaasStorageAPIAdaptor.py @@ -25,7 +25,7 @@ def __handle_response(self, api_response: requests.Response) -> dict: elif api_response.status_code == 200: return api_response.json() elif api_response.status_code == 404: - raise StorageNotFoundError(api_response.json().get("error")["message"]) + raise StorageNotFoundError('Not found.') elif api_response.json().get("error")["code"] == 0 and api_response.json().get("error")["message"] == "Success": raise api_response.json() else: @@ -95,13 +95,14 @@ def delete_object(self, object_name: str, ) -> dict: object=os.path.basename(object_name) - prefix=os.path.dirname(object_name) + prefix=os.path.dirname(object_name)+'/' _url = f"{self.host}/workspace/{workspace_id}/storage/{storage_name}?prefix={prefix}&object={object}" api_response = self.make_api_request( requests.delete, _url, ) + print(api_response) return self.__handle_response(api_response) @BaseAPIAdaptor.service_status_decorator