From 511923db8f7765386b000c884490392d3e134a9e Mon Sep 17 00:00:00 2001 From: toddn Date: Thu, 28 Mar 2024 13:00:34 -0500 Subject: [PATCH 01/91] adding new dependency and class --- backend/Pipfile | 1 + scripts/migration/migrate_users.py | 0 2 files changed, 1 insertion(+) create mode 100644 scripts/migration/migrate_users.py diff --git a/backend/Pipfile b/backend/Pipfile index e9360b975..6cafd2ed1 100644 --- a/backend/Pipfile +++ b/backend/Pipfile @@ -33,6 +33,7 @@ pytest = "7.3.1" pytest-asyncio = "0.21.0" black = "23.3.0" faker = "18.4.0" +python-dotenv = "1.0.1" [requires] python_version = "3.9" diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate_users.py new file mode 100644 index 000000000..e69de29bb From f763cb463f3f35fd023f69b4b550a2f47dcb20ed Mon Sep 17 00:00:00 2001 From: toddn Date: Thu, 28 Mar 2024 15:11:08 -0500 Subject: [PATCH 02/91] starting a user migration --- scripts/migration/migrate_users.py | 65 ++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate_users.py index e69de29bb..f9c27dc26 100644 --- a/scripts/migration/migrate_users.py +++ b/scripts/migration/migrate_users.py @@ -0,0 +1,65 @@ +import os +from faker import Faker +from dotenv import dotenv_values, load_dotenv +import requests + +fake = Faker() + +path_to_env = os.path.join(os.getcwd(), 'scripts', 'migration', '.env') +print(os.path.isfile(path_to_env)) +config = dotenv_values(dotenv_path=path_to_env) + +CLOWDER_V1 = config["CLOWDER_V1"] +ADMIN_KEY_V1 = config["ADMIN_KEY_V1"] + +CLOWDER_V2 = config["CLOWDER_V2"] +ADMIN_KEY_V2 = config["ADMIN_KEY_V2"] + +base_headers_v1 = {'X-API-key': ADMIN_KEY_V1} +clowder_headers_v1 = {**base_headers_v1, 'Content-type': 'application/json', + 'accept': 'application/json'} + +base_headers_v2 = {'X-API-key': ADMIN_KEY_V2} +clowder_headers_v2 = {**base_headers_v2, 'Content-type': 'application/json', + 'accept': 'application/json'} + +def email_user_new_login(user): + print("login to the new clowder instance") + +def generate_user_api_key(user): + print("Generate user api key") + +def get_clowder_v1_users(): + endpoint = CLOWDER_V1 + 'api/users' + r = requests.get(endpoint, headers=base_headers_v1, verify=False) + return r.json() + +def create_local_user(local_user_v1): + first_name = user_v1['firstName'] + last_name = user_v1['lastName'] + email = user_v1['email'] + password = fake.password(20) + user_json = { + "email": email, + "password": password, + "first_name": first_name, + "last_name": last_name + } + response = requests.post(f"{CLOWDER_V2}api/v2/users", json=user_json) + email_user_new_login(user_json) + api_key = generate_user_api_key(user_json) + print("Local user created and api key generated") + return api_key + +users_v1 = get_clowder_v1_users() + + +for user_v1 in users_v1: + print(user_v1) + id_provider = user_v1['identityProvider'] + if '[Local Account]' in user_v1['identityProvider']: + new_user_api_key = generate_user_api_key(user_v1) + else: + print("not a local account") + + From 84d43bc2101fbc7e416fa498e07ccb1a43e3e302 Mon Sep 17 00:00:00 2001 From: toddn Date: Thu, 28 Mar 2024 15:42:58 -0500 Subject: [PATCH 03/91] adding script for migrate metadata definitions --- .../migration/migrate_metadata_definitions.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 scripts/migration/migrate_metadata_definitions.py diff --git a/scripts/migration/migrate_metadata_definitions.py b/scripts/migration/migrate_metadata_definitions.py new file mode 100644 index 000000000..b85594b46 --- /dev/null +++ b/scripts/migration/migrate_metadata_definitions.py @@ -0,0 +1,34 @@ +import os +from faker import Faker +from dotenv import dotenv_values, load_dotenv +import requests + +fake = Faker() + +path_to_env = os.path.join(os.getcwd(), 'scripts', 'migration', '.env') +print(os.path.isfile(path_to_env)) +config = dotenv_values(dotenv_path=path_to_env) + +CLOWDER_V1 = config["CLOWDER_V1"] +ADMIN_KEY_V1 = config["ADMIN_KEY_V1"] + +CLOWDER_V2 = config["CLOWDER_V2"] +ADMIN_KEY_V2 = config["ADMIN_KEY_V2"] + +base_headers_v1 = {'X-API-key': ADMIN_KEY_V1} +clowder_headers_v1 = {**base_headers_v1, 'Content-type': 'application/json', + 'accept': 'application/json'} + +base_headers_v2 = {'X-API-key': ADMIN_KEY_V2} +clowder_headers_v2 = {**base_headers_v2, 'Content-type': 'application/json', + 'accept': 'application/json'} + + +def get_clowder_v1_metadata_definitions(): + endpoint = CLOWDER_V1 + 'api/metadata/definitions' + r = requests.get(endpoint, headers=base_headers_v1, verify=False) + return r.json() + + +md_definitions = get_clowder_v1_metadata_definitions() +print('got them') \ No newline at end of file From e82e02377b21edef4cc6f66cfe460db1de1fe01f Mon Sep 17 00:00:00 2001 From: toddn Date: Thu, 28 Mar 2024 15:58:14 -0500 Subject: [PATCH 04/91] adding option of temporary password (for use with users who are being migrated from old clowder instance --- backend/app/keycloak_auth.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/app/keycloak_auth.py b/backend/app/keycloak_auth.py index 4bfb093c7..f3172bc4a 100644 --- a/backend/app/keycloak_auth.py +++ b/backend/app/keycloak_auth.py @@ -312,7 +312,7 @@ async def get_current_user_id(identity: Json = Depends(get_token)) -> str: return keycloak_id -async def create_user(email: str, password: str, firstName: str, lastName: str): +async def create_user(email: str, password: str, firstName: str, lastName: str, temporary=False): """Create a user in Keycloak.""" keycloak_admin = KeycloakAdmin( server_url=settings.auth_server_url, @@ -336,6 +336,7 @@ async def create_user(email: str, password: str, firstName: str, lastName: str): { "value": password, "type": "password", + "temporary": temporary, } ], }, From dcdbea1dc05404c1fd3e41db37dc6569c5303d37 Mon Sep 17 00:00:00 2001 From: toddn Date: Thu, 28 Mar 2024 16:14:40 -0500 Subject: [PATCH 05/91] reverting changes --- backend/Pipfile | 1 - backend/app/keycloak_auth.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/backend/Pipfile b/backend/Pipfile index 6cafd2ed1..e9360b975 100644 --- a/backend/Pipfile +++ b/backend/Pipfile @@ -33,7 +33,6 @@ pytest = "7.3.1" pytest-asyncio = "0.21.0" black = "23.3.0" faker = "18.4.0" -python-dotenv = "1.0.1" [requires] python_version = "3.9" diff --git a/backend/app/keycloak_auth.py b/backend/app/keycloak_auth.py index f3172bc4a..4bfb093c7 100644 --- a/backend/app/keycloak_auth.py +++ b/backend/app/keycloak_auth.py @@ -312,7 +312,7 @@ async def get_current_user_id(identity: Json = Depends(get_token)) -> str: return keycloak_id -async def create_user(email: str, password: str, firstName: str, lastName: str, temporary=False): +async def create_user(email: str, password: str, firstName: str, lastName: str): """Create a user in Keycloak.""" keycloak_admin = KeycloakAdmin( server_url=settings.auth_server_url, @@ -336,7 +336,6 @@ async def create_user(email: str, password: str, firstName: str, lastName: str, { "value": password, "type": "password", - "temporary": temporary, } ], }, From 79e56b466bfce78ab100148b454e09476d682d23 Mon Sep 17 00:00:00 2001 From: toddn Date: Thu, 28 Mar 2024 16:51:34 -0500 Subject: [PATCH 06/91] temporary - for users that are migrated --- backend/app/keycloak_auth.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/app/keycloak_auth.py b/backend/app/keycloak_auth.py index 4bfb093c7..deb012c33 100644 --- a/backend/app/keycloak_auth.py +++ b/backend/app/keycloak_auth.py @@ -312,7 +312,7 @@ async def get_current_user_id(identity: Json = Depends(get_token)) -> str: return keycloak_id -async def create_user(email: str, password: str, firstName: str, lastName: str): +async def create_user(email: str, password: str, firstName: str, lastName: str, temporary: bool = False): """Create a user in Keycloak.""" keycloak_admin = KeycloakAdmin( server_url=settings.auth_server_url, @@ -336,6 +336,7 @@ async def create_user(email: str, password: str, firstName: str, lastName: str): { "value": password, "type": "password", + "temporary": temporary, } ], }, From 33f1cab0ac7f64a809f8959dd065846fb12a8f34 Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 23 Apr 2024 16:14:22 -0500 Subject: [PATCH 07/91] creates users NOTE - we need a feature that says 'reset password' if 'reset password' is set, then you have to reset password on logging in --- scripts/migration/migrate_users.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate_users.py index f9c27dc26..d7905483c 100644 --- a/scripts/migration/migrate_users.py +++ b/scripts/migration/migrate_users.py @@ -38,7 +38,8 @@ def create_local_user(local_user_v1): first_name = user_v1['firstName'] last_name = user_v1['lastName'] email = user_v1['email'] - password = fake.password(20) + # password = fake.password(20) + password = 'Password123&' user_json = { "email": email, "password": password, @@ -56,9 +57,13 @@ def create_local_user(local_user_v1): for user_v1 in users_v1: print(user_v1) + email = user_v1['email'] + firstName = user_v1['firstName'] + lastName = user_v1['lastName'] + id_provider = user_v1['identityProvider'] if '[Local Account]' in user_v1['identityProvider']: - new_user_api_key = generate_user_api_key(user_v1) + user_v2 = create_local_user(user_v1) else: print("not a local account") From 93509e7e55891e7b65ba1e05cceb55278adfa79a Mon Sep 17 00:00:00 2001 From: toddn Date: Thu, 25 Apr 2024 12:38:42 -0500 Subject: [PATCH 08/91] does not actually create the dataset will require some port changes for the running v1 instance (or v2) will work out a strategy --- scripts/migration/migrate_users.py | 55 +++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate_users.py index d7905483c..795ce63d3 100644 --- a/scripts/migration/migrate_users.py +++ b/scripts/migration/migrate_users.py @@ -2,6 +2,9 @@ from faker import Faker from dotenv import dotenv_values, load_dotenv import requests +from pymongo import MongoClient + +output_file = 'new_users.txt' fake = Faker() @@ -23,6 +26,15 @@ clowder_headers_v2 = {**base_headers_v2, 'Content-type': 'application/json', 'accept': 'application/json'} +TEST_DATASET_NAME = 'Migration Test Dataset' +# TODO this is just for testing +DEFAULT_PASSWORD = 'Password123&' +mongo_client = MongoClient("mongodb://localhost:27018", connect=False) +db = mongo_client["clowder"] +print('got the db') +v1_users = db["social.users"].find({}) +for u in v1_users: + print(u) def email_user_new_login(user): print("login to the new clowder instance") @@ -34,6 +46,32 @@ def get_clowder_v1_users(): r = requests.get(endpoint, headers=base_headers_v1, verify=False) return r.json() +def get_clowder_v2_users(): + endpoint = CLOWDER_V2 + 'api/v2/users' + r = requests.get(endpoint, headers=base_headers_v1, verify=False) + return r.json() + +def create_v2_dataset(headers, dataset): + print(dataset) + dataset_endpoint = CLOWDER_V1 + 'api/datasets/' + dataset['id'] + r = requests.get(dataset_endpoint, headers=base_headers_v1, verify=False) + dataset_result = r.json() + dataset_files_endpoint = CLOWDER_V1 + 'api/datasets/' + dataset['id'] + '/files' + r_files = requests.get(dataset_files_endpoint, headers=base_headers_v1, verify=False) + files_result = r_files.json() + print('here') + + +def get_clowder_v1_user_datasets(user_id): + user_datasets = [] + endpoint = CLOWDER_V1 + 'api/datasets?limit=0' + r = requests.get(endpoint, headers=base_headers_v1, verify=False) + request_json = r.json() + for dataset in request_json: + if dataset['authorId'] == user_id: + user_datasets.append(dataset) + return user_datasets + def create_local_user(local_user_v1): first_name = user_v1['firstName'] last_name = user_v1['lastName'] @@ -50,6 +88,9 @@ def create_local_user(local_user_v1): email_user_new_login(user_json) api_key = generate_user_api_key(user_json) print("Local user created and api key generated") + with open(output_file, 'a') as f: + entry = email + ',' + password + ',' + api_key + "\n" + f.write(entry) return api_key users_v1 = get_clowder_v1_users() @@ -57,13 +98,25 @@ def create_local_user(local_user_v1): for user_v1 in users_v1: print(user_v1) + id = user_v1['id'] email = user_v1['email'] firstName = user_v1['firstName'] lastName = user_v1['lastName'] id_provider = user_v1['identityProvider'] if '[Local Account]' in user_v1['identityProvider']: - user_v2 = create_local_user(user_v1) + # get the v2 users + if email != "a@a.com": + user_v1_datasets = get_clowder_v1_user_datasets(user_id=id) + # user_v2_api_key = create_local_user(user_v1) + # user_base_headers_v2 = {'X-API-key': user_v2_api_key} + # user_headers_v2 = {**user_base_headers_v2, 'Content-type': 'application/json', + # 'accept': 'application/json'} + for dataset in user_v1_datasets: + print('creating a dataset in v2') + dataset_id = create_v2_dataset("user_headers_v2", dataset) + + else: print("not a local account") From aba43a34fcab1d01dabcd791092fc37815ac1ea4 Mon Sep 17 00:00:00 2001 From: toddn Date: Thu, 25 Apr 2024 14:29:20 -0500 Subject: [PATCH 09/91] using beanie might not work here due to async issues --- scripts/migration/migrate_users.py | 136 +++++++++++++++++++++-------- 1 file changed, 102 insertions(+), 34 deletions(-) diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate_users.py index 795ce63d3..f3d3c29b4 100644 --- a/scripts/migration/migrate_users.py +++ b/scripts/migration/migrate_users.py @@ -2,10 +2,45 @@ from faker import Faker from dotenv import dotenv_values, load_dotenv import requests +import asyncio from pymongo import MongoClient - +from app.config import settings +from itsdangerous.url_safe import URLSafeSerializer +from secrets import token_urlsafe +from app.routers.files import add_file_entry +from app.models.files import ( + FileOut, + FileDB, + FileDBViewList, + LocalFileIn, + StorageType, +) +from app.models.users import UserDB, UserOut, UserAPIKeyDB +from beanie import init_beanie +from fastapi import FastAPI, APIRouter, Depends +from motor.motor_asyncio import AsyncIOMotorClient + +app = FastAPI() +@app.on_event("startup") +async def start_db(): + client = AsyncIOMotorClient(str(settings.MONGODB_URL)) + await init_beanie( + database=getattr(client, settings.MONGO_DATABASE), + # Make sure to include all models. If one depends on another that is not in the list it is not clear which one is missing. + document_models=[ + FileDB, + FileDBViewList, + UserDB, + UserAPIKeyDB, + ], + recreate_views=True, + ) + + +asyncio.run(start_db()) output_file = 'new_users.txt' + fake = Faker() path_to_env = os.path.join(os.getcwd(), 'scripts', 'migration', '.env') @@ -39,7 +74,11 @@ def email_user_new_login(user): print("login to the new clowder instance") def generate_user_api_key(user): - print("Generate user api key") + serializer = URLSafeSerializer(settings.local_auth_secret, salt="api_key") + unique_key = token_urlsafe(16) + hashed_key = serializer.dumps({"user": user, "key": unique_key}) + user_key = UserAPIKeyDB(user=user, key=unique_key, name='migrationKey') + return user_key.key def get_clowder_v1_users(): endpoint = CLOWDER_V1 + 'api/users' @@ -51,14 +90,32 @@ def get_clowder_v2_users(): r = requests.get(endpoint, headers=base_headers_v1, verify=False) return r.json() -def create_v2_dataset(headers, dataset): +async def create_v2_dataset(headers, dataset, user_email): print(dataset) + dataset_name = dataset['name'] + dataset_description = dataset['description'] dataset_endpoint = CLOWDER_V1 + 'api/datasets/' + dataset['id'] r = requests.get(dataset_endpoint, headers=base_headers_v1, verify=False) - dataset_result = r.json() dataset_files_endpoint = CLOWDER_V1 + 'api/datasets/' + dataset['id'] + '/files' r_files = requests.get(dataset_files_endpoint, headers=base_headers_v1, verify=False) + dataset_in_v2_endpoint = CLOWDER_V2 + 'api/v2/datasets' + # create dataset + dataset_example = { + "name": dataset_name, + "description": dataset_description, + } + response = requests.post( + dataset_in_v2_endpoint, headers=headers, json=dataset_example + ) files_result = r_files.json() + if (user := await UserDB.find_one(UserDB.email == user_email)) is not None: + print('we got a user!') + for file in files_result: + new_file = FileDB( + name=file['filename'], + creator=user, + dataset_id=dataset.id, + ) print('here') @@ -72,7 +129,7 @@ def get_clowder_v1_user_datasets(user_id): user_datasets.append(dataset) return user_datasets -def create_local_user(local_user_v1): +def create_local_user(user_v1): first_name = user_v1['firstName'] last_name = user_v1['lastName'] email = user_v1['email'] @@ -84,40 +141,51 @@ def create_local_user(local_user_v1): "first_name": first_name, "last_name": last_name } - response = requests.post(f"{CLOWDER_V2}api/v2/users", json=user_json) - email_user_new_login(user_json) - api_key = generate_user_api_key(user_json) + # response = requests.post(f"{CLOWDER_V2}api/v2/users", json=user_json) + email_user_new_login(email) + api_key = generate_user_api_key(email) print("Local user created and api key generated") + if os.path.exists(output_file): + print('it exists.') + else: + f = open(output_file, "x") with open(output_file, 'a') as f: entry = email + ',' + password + ',' + api_key + "\n" f.write(entry) return api_key -users_v1 = get_clowder_v1_users() - - -for user_v1 in users_v1: - print(user_v1) - id = user_v1['id'] - email = user_v1['email'] - firstName = user_v1['firstName'] - lastName = user_v1['lastName'] - - id_provider = user_v1['identityProvider'] - if '[Local Account]' in user_v1['identityProvider']: - # get the v2 users - if email != "a@a.com": - user_v1_datasets = get_clowder_v1_user_datasets(user_id=id) - # user_v2_api_key = create_local_user(user_v1) - # user_base_headers_v2 = {'X-API-key': user_v2_api_key} - # user_headers_v2 = {**user_base_headers_v2, 'Content-type': 'application/json', - # 'accept': 'application/json'} - for dataset in user_v1_datasets: - print('creating a dataset in v2') - dataset_id = create_v2_dataset("user_headers_v2", dataset) - - - else: - print("not a local account") +async def process_users(): + users_v1 = get_clowder_v1_users() + + for user_v1 in users_v1: + print(user_v1) + id = user_v1['id'] + email = user_v1['email'] + firstName = user_v1['firstName'] + lastName = user_v1['lastName'] + + id_provider = user_v1['identityProvider'] + if '[Local Account]' in user_v1['identityProvider']: + # get the v2 users + if email != "a@a.com": + user_v1_datasets = get_clowder_v1_user_datasets(user_id=id) + # TODO check if there is already a local user + # user_v2_api_key = create_local_user(user_v1) + user_v2_api_key = 'aZM2QXJ_lvw_5FKNUB89Vg' + user_base_headers_v2 = {'X-API-key': user_v2_api_key} + user_headers_v2 = {**user_base_headers_v2, 'Content-type': 'application/json', + 'accept': 'application/json'} + for dataset in user_v1_datasets: + print('creating a dataset in v2') + dataset_id = await create_v2_dataset(user_headers_v2, dataset, email) + dataset_files_endpoint = CLOWDER_V1 + 'api/datasets/' + dataset['id'] + '/files' + r_files = requests.get(dataset_files_endpoint, headers=base_headers_v1, verify=False) + r_files_json = r_files.json() + print('here') + + else: + print("not a local account") + +asyncio.run(process_users()) From 727685f96c34e9d42f427fcd36a0ca63ae52a937 Mon Sep 17 00:00:00 2001 From: toddn Date: Thu, 25 Apr 2024 16:40:42 -0500 Subject: [PATCH 10/91] using nest asyncio to fix problem with even loop closing --- scripts/migration/migrate_users.py | 58 +++++++++++++++++++++--------- 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate_users.py index f3d3c29b4..6040e2889 100644 --- a/scripts/migration/migrate_users.py +++ b/scripts/migration/migrate_users.py @@ -15,11 +15,22 @@ LocalFileIn, StorageType, ) +import nest_asyncio +nest_asyncio.apply() from app.models.users import UserDB, UserOut, UserAPIKeyDB from beanie import init_beanie from fastapi import FastAPI, APIRouter, Depends from motor.motor_asyncio import AsyncIOMotorClient +mongo_client = MongoClient("mongodb://localhost:27018", connect=False) +mongo_client_v2 = MongoClient("mongodb://localhost:27017", connect=False) +db = mongo_client["clowder"] +db_v2 = mongo_client_v2["clowder2"] +print('got the db') +v1_users = db["social.users"].find({}) +for u in v1_users: + print(u) + app = FastAPI() @app.on_event("startup") async def start_db(): @@ -64,12 +75,7 @@ async def start_db(): TEST_DATASET_NAME = 'Migration Test Dataset' # TODO this is just for testing DEFAULT_PASSWORD = 'Password123&' -mongo_client = MongoClient("mongodb://localhost:27018", connect=False) -db = mongo_client["clowder"] -print('got the db') -v1_users = db["social.users"].find({}) -for u in v1_users: - print(u) + def email_user_new_login(user): print("login to the new clowder instance") @@ -78,6 +84,9 @@ def generate_user_api_key(user): unique_key = token_urlsafe(16) hashed_key = serializer.dumps({"user": user, "key": unique_key}) user_key = UserAPIKeyDB(user=user, key=unique_key, name='migrationKey') + # user_key = {'user': user, 'key': unique_key, 'name': 'migrationKey'} + user_key_collection = mongo_client["user_keys"] + result = mongo_client["user_keys"].insert_one({user_key}) return user_key.key def get_clowder_v1_users(): @@ -107,16 +116,7 @@ async def create_v2_dataset(headers, dataset, user_email): response = requests.post( dataset_in_v2_endpoint, headers=headers, json=dataset_example ) - files_result = r_files.json() - if (user := await UserDB.find_one(UserDB.email == user_email)) is not None: - print('we got a user!') - for file in files_result: - new_file = FileDB( - name=file['filename'], - creator=user, - dataset_id=dataset.id, - ) - print('here') + return response.json() def get_clowder_v1_user_datasets(user_id): @@ -143,7 +143,8 @@ def create_local_user(user_v1): } # response = requests.post(f"{CLOWDER_V2}api/v2/users", json=user_json) email_user_new_login(email) - api_key = generate_user_api_key(email) + # api_key = generate_user_api_key(email) + api_key = 'aZM2QXJ_lvw_5FKNUB89Vg' print("Local user created and api key generated") if os.path.exists(output_file): print('it exists.') @@ -179,8 +180,31 @@ async def process_users(): print('creating a dataset in v2') dataset_id = await create_v2_dataset(user_headers_v2, dataset, email) dataset_files_endpoint = CLOWDER_V1 + 'api/datasets/' + dataset['id'] + '/files' + # move file stuff here + print('we got a dataset id') + r_files = requests.get(dataset_files_endpoint, headers=base_headers_v1, verify=False) r_files_json = r_files.json() + files_result = r_files.json() + user_collection = db_v2["users"] + user = user_collection.find_one({"email": email}) + print('got a user') + userDB = await UserDB.find_one({"email": email}) + for file in files_result: + new_file = FileDB( + name=file['filename'], + creator=user, + dataset_id=dataset.id, + ) + print('here') + for file in r_files_json: + file_json = requests.get(CLOWDER_V1 + 'api/files/' + file['id'], headers=base_headers_v1).json() + filename = file['filename'] + file_id = file['id'] + loader_id = file['loader'] + add_file_entry( + + ) print('here') else: From d4e9526f340a15e8f849c98d95d0525828596a23 Mon Sep 17 00:00:00 2001 From: toddn Date: Fri, 26 Apr 2024 13:56:45 -0500 Subject: [PATCH 11/91] entry from right table for file bytes need to move to new db and minio --- scripts/migration/migrate_users.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate_users.py index 6040e2889..cd131e035 100644 --- a/scripts/migration/migrate_users.py +++ b/scripts/migration/migrate_users.py @@ -1,4 +1,6 @@ import os + +from bson import ObjectId from faker import Faker from dotenv import dotenv_values, load_dotenv import requests @@ -179,11 +181,11 @@ async def process_users(): for dataset in user_v1_datasets: print('creating a dataset in v2') dataset_id = await create_v2_dataset(user_headers_v2, dataset, email) - dataset_files_endpoint = CLOWDER_V1 + 'api/datasets/' + dataset['id'] + '/files' + dataset_files_endpoint = CLOWDER_V1 + 'api/datasets/' + dataset['id'] + '/files?=superAdmin=true' # move file stuff here print('we got a dataset id') - r_files = requests.get(dataset_files_endpoint, headers=base_headers_v1, verify=False) + r_files = requests.get(dataset_files_endpoint, headers=clowder_headers_v1, verify=False) r_files_json = r_files.json() files_result = r_files.json() user_collection = db_v2["users"] @@ -193,15 +195,20 @@ async def process_users(): for file in files_result: new_file = FileDB( name=file['filename'], - creator=user, - dataset_id=dataset.id, + creator=userDB, + dataset_id=dataset["id"], ) - print('here') - for file in r_files_json: - file_json = requests.get(CLOWDER_V1 + 'api/files/' + file['id'], headers=base_headers_v1).json() - filename = file['filename'] + print('here') file_id = file['id'] - loader_id = file['loader'] + file = db["uploads"].find_one({"_id": ObjectId(file_id)}) + filename = file['filename'] + loader_id = file["loader_id"] + content_type = file["contentType"] + upload_chunks_entry = db["uploads.chunks"].find_one({"files_id": ObjectId(loader_id)}) + data_bytes = upload_chunks_entry['data'] + print('got the entry') + + add_file_entry( ) From 8c7d6bdf3ac02294a00cf2ceb58eda6b8c1136d7 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 20 May 2024 12:56:22 -0500 Subject: [PATCH 12/91] dependencies for migration, adding file entry and indexing files --- scripts/migration/migrate_users.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate_users.py index cd131e035..7ea3d32ed 100644 --- a/scripts/migration/migrate_users.py +++ b/scripts/migration/migrate_users.py @@ -17,12 +17,16 @@ LocalFileIn, StorageType, ) +from minio import Minio import nest_asyncio nest_asyncio.apply() from app.models.users import UserDB, UserOut, UserAPIKeyDB from beanie import init_beanie from fastapi import FastAPI, APIRouter, Depends +from elasticsearch import Elasticsearch +from app import dependencies from motor.motor_asyncio import AsyncIOMotorClient +from pika.adapters.blocking_connection import BlockingChannel mongo_client = MongoClient("mongodb://localhost:27018", connect=False) mongo_client_v2 = MongoClient("mongodb://localhost:27017", connect=False) @@ -157,7 +161,11 @@ def create_local_user(user_v1): f.write(entry) return api_key -async def process_users(): +async def process_users( + fs: Minio = Depends(dependencies.get_fs), + es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), + rabbitmq_client: BlockingChannel = Depends(dependencies.get_rabbitmq), + ): users_v1 = get_clowder_v1_users() for user_v1 in users_v1: @@ -206,13 +214,15 @@ async def process_users(): content_type = file["contentType"] upload_chunks_entry = db["uploads.chunks"].find_one({"files_id": ObjectId(loader_id)}) data_bytes = upload_chunks_entry['data'] - print('got the entry') - - - add_file_entry( - + await add_file_entry( + new_file, + user, + fs, + es, + rabbitmq_client, + file.file, + content_type=file.content_type, ) - print('here') else: print("not a local account") From 2ba7df5d9017fd569edb9cbea3661f478cb0ba45 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 20 May 2024 14:51:54 -0500 Subject: [PATCH 13/91] wrong api key --- scripts/migration/migrate_users.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate_users.py index 7ea3d32ed..64076dffe 100644 --- a/scripts/migration/migrate_users.py +++ b/scripts/migration/migrate_users.py @@ -97,12 +97,18 @@ def generate_user_api_key(user): def get_clowder_v1_users(): endpoint = CLOWDER_V1 + 'api/users' - r = requests.get(endpoint, headers=base_headers_v1, verify=False) + print(base_headers_v1) + r = requests.get(endpoint, headers=clowder_headers_v1, verify=False) return r.json() def get_clowder_v2_users(): endpoint = CLOWDER_V2 + 'api/v2/users' - r = requests.get(endpoint, headers=base_headers_v1, verify=False) + r = requests.get(endpoint, headers=base_headers_v2, verify=False) + return r.json() + +def get_clowder_v2_user_by_name(username): + endpoint = CLOWDER_V2 + 'api/v2/users/username/' + username + r = requests.get(endpoint, headers=base_headers_v2, verify=False) return r.json() async def create_v2_dataset(headers, dataset, user_email): @@ -147,10 +153,10 @@ def create_local_user(user_v1): "first_name": first_name, "last_name": last_name } - # response = requests.post(f"{CLOWDER_V2}api/v2/users", json=user_json) + response = requests.post(f"{CLOWDER_V2}api/v2/users", json=user_json) email_user_new_login(email) - # api_key = generate_user_api_key(email) - api_key = 'aZM2QXJ_lvw_5FKNUB89Vg' + api_key = generate_user_api_key(email) + # api_key = 'aZM2QXJ_lvw_5FKNUB89Vg' print("Local user created and api key generated") if os.path.exists(output_file): print('it exists.') @@ -178,10 +184,12 @@ async def process_users( id_provider = user_v1['identityProvider'] if '[Local Account]' in user_v1['identityProvider']: # get the v2 users + # i create a user account in v2 with this username if email != "a@a.com": user_v1_datasets = get_clowder_v1_user_datasets(user_id=id) # TODO check if there is already a local user - # user_v2_api_key = create_local_user(user_v1) + user_v2 = get_clowder_v2_user_by_name(email) + user_v2_api_key = create_local_user(user_v1) user_v2_api_key = 'aZM2QXJ_lvw_5FKNUB89Vg' user_base_headers_v2 = {'X-API-key': user_v2_api_key} user_headers_v2 = {**user_base_headers_v2, 'Content-type': 'application/json', From f624784c8c602e65b4b0ffe91377d0c9317144d1 Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 21 May 2024 15:31:49 -0500 Subject: [PATCH 14/91] api key works for user, file upload broken, need to fix --- scripts/migration/migrate_users.py | 40 ++++++++++++++++++------------ 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate_users.py index 64076dffe..c396490c6 100644 --- a/scripts/migration/migrate_users.py +++ b/scripts/migration/migrate_users.py @@ -85,15 +85,22 @@ async def start_db(): def email_user_new_login(user): print("login to the new clowder instance") -def generate_user_api_key(user): - serializer = URLSafeSerializer(settings.local_auth_secret, salt="api_key") - unique_key = token_urlsafe(16) - hashed_key = serializer.dumps({"user": user, "key": unique_key}) - user_key = UserAPIKeyDB(user=user, key=unique_key, name='migrationKey') - # user_key = {'user': user, 'key': unique_key, 'name': 'migrationKey'} - user_key_collection = mongo_client["user_keys"] - result = mongo_client["user_keys"].insert_one({user_key}) - return user_key.key +def generate_user_api_key(user, password): + user_example = { + "email": user["email"], + "password": DEFAULT_PASSWORD, + "first_name": user["first_name"], + "last_name": user["last_name"], + } + login_endpoint = CLOWDER_V2 + 'api/v2/login' + response = requests.post(login_endpoint, json=user_example) + token = response.json().get("token") + current_headers = {"Authorization": "Bearer " + token} + auth = {'username': user["email"], 'password': password} + api_key_endpoint = CLOWDER_V2 + 'api/v2/users/keys?name=migration&mins=0' + result = requests.post(api_key_endpoint, headers=current_headers) + api_key = result.json() + return api_key def get_clowder_v1_users(): endpoint = CLOWDER_V1 + 'api/users' @@ -153,9 +160,9 @@ def create_local_user(user_v1): "first_name": first_name, "last_name": last_name } - response = requests.post(f"{CLOWDER_V2}api/v2/users", json=user_json) + # response = requests.post(f"{CLOWDER_V2}api/v2/users", json=user_json) email_user_new_login(email) - api_key = generate_user_api_key(email) + api_key = generate_user_api_key(email, DEFAULT_PASSWORD) # api_key = 'aZM2QXJ_lvw_5FKNUB89Vg' print("Local user created and api key generated") if os.path.exists(output_file): @@ -173,7 +180,6 @@ async def process_users( rabbitmq_client: BlockingChannel = Depends(dependencies.get_rabbitmq), ): users_v1 = get_clowder_v1_users() - for user_v1 in users_v1: print(user_v1) id = user_v1['id'] @@ -189,8 +195,9 @@ async def process_users( user_v1_datasets = get_clowder_v1_user_datasets(user_id=id) # TODO check if there is already a local user user_v2 = get_clowder_v2_user_by_name(email) - user_v2_api_key = create_local_user(user_v1) - user_v2_api_key = 'aZM2QXJ_lvw_5FKNUB89Vg' + # user_v2_api_key = create_local_user(user_v1) + user_v2_api_key = generate_user_api_key(user_v2, DEFAULT_PASSWORD) + # user_v2_api_key = 'aZM2QXJ_lvw_5FKNUB89Vg' user_base_headers_v2 = {'X-API-key': user_v2_api_key} user_headers_v2 = {**user_base_headers_v2, 'Content-type': 'application/json', 'accept': 'application/json'} @@ -228,9 +235,10 @@ async def process_users( fs, es, rabbitmq_client, - file.file, - content_type=file.content_type, + data_bytes, + content_type=content_type, ) + print('done') else: print("not a local account") From adc6c63eb758d91b2c553def6844a32a78af8f6c Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 21 May 2024 15:48:14 -0500 Subject: [PATCH 15/91] file upload works, sloppy needs fixed --- scripts/migration/migrate_users.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate_users.py index c396490c6..cf36d76fa 100644 --- a/scripts/migration/migrate_users.py +++ b/scripts/migration/migrate_users.py @@ -229,15 +229,14 @@ async def process_users( content_type = file["contentType"] upload_chunks_entry = db["uploads.chunks"].find_one({"files_id": ObjectId(loader_id)}) data_bytes = upload_chunks_entry['data'] - await add_file_entry( - new_file, - user, - fs, - es, - rabbitmq_client, - data_bytes, - content_type=content_type, - ) + current_path = os.path.join(os.getcwd(),'scripts','migration') + path_to_temp_file = os.path.join(current_path, filename) + with open(path_to_temp_file, "wb") as f: + f.write(data_bytes) + file_data = {"file": open(path_to_temp_file, "rb")} + dataset_file_upload_endoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_id['id'] + '/files' + response = requests.post(dataset_file_upload_endoint, files=file_data, headers=user_base_headers_v2) + result = response.json() print('done') else: From 171b2b303d3b9ac922171b44000b73b568da1de9 Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 21 May 2024 16:21:23 -0500 Subject: [PATCH 16/91] we are only getting partial files, not sure how to fix yet --- scripts/migration/migrate_users.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate_users.py index cf36d76fa..b705cf017 100644 --- a/scripts/migration/migrate_users.py +++ b/scripts/migration/migrate_users.py @@ -88,7 +88,7 @@ def email_user_new_login(user): def generate_user_api_key(user, password): user_example = { "email": user["email"], - "password": DEFAULT_PASSWORD, + "password": password, "first_name": user["first_name"], "last_name": user["last_name"], } @@ -160,9 +160,9 @@ def create_local_user(user_v1): "first_name": first_name, "last_name": last_name } - # response = requests.post(f"{CLOWDER_V2}api/v2/users", json=user_json) + response = requests.post(f"{CLOWDER_V2}api/v2/users", json=user_json) email_user_new_login(email) - api_key = generate_user_api_key(email, DEFAULT_PASSWORD) + api_key = generate_user_api_key(user_json, DEFAULT_PASSWORD) # api_key = 'aZM2QXJ_lvw_5FKNUB89Vg' print("Local user created and api key generated") if os.path.exists(output_file): @@ -174,11 +174,25 @@ def create_local_user(user_v1): f.write(entry) return api_key +def create_admin_user(): + user_json = { + "email": "a@a.com", + "password": "admin", + "first_name": "aa", + "last_name": "aa" + } + response = requests.post(f"{CLOWDER_V2}api/v2/users", json=user_json) + api_key = generate_user_api_key(user_json, "admin") + return api_key + async def process_users( fs: Minio = Depends(dependencies.get_fs), es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), rabbitmq_client: BlockingChannel = Depends(dependencies.get_rabbitmq), ): + print("We create a v2 admin user") + NEW_ADMIN_KEY_V2 = create_admin_user() + print('here') users_v1 = get_clowder_v1_users() for user_v1 in users_v1: print(user_v1) @@ -194,8 +208,8 @@ async def process_users( if email != "a@a.com": user_v1_datasets = get_clowder_v1_user_datasets(user_id=id) # TODO check if there is already a local user - user_v2 = get_clowder_v2_user_by_name(email) - # user_v2_api_key = create_local_user(user_v1) + # user_v2 = get_clowder_v2_user_by_name(email) + user_v2 = create_local_user(user_v1) user_v2_api_key = generate_user_api_key(user_v2, DEFAULT_PASSWORD) # user_v2_api_key = 'aZM2QXJ_lvw_5FKNUB89Vg' user_base_headers_v2 = {'X-API-key': user_v2_api_key} @@ -227,6 +241,9 @@ async def process_users( filename = file['filename'] loader_id = file["loader_id"] content_type = file["contentType"] + # TODO download the file from v1 using api routes + v1_download_url = CLOWDER_V1 + 'api/files' + file_id + download = requests.get(v1_download_url, headers=clowder_headers_v1) upload_chunks_entry = db["uploads.chunks"].find_one({"files_id": ObjectId(loader_id)}) data_bytes = upload_chunks_entry['data'] current_path = os.path.join(os.getcwd(),'scripts','migration') From 1f6a2ed9807dd01c8a4d2f2c9acb432e4a83503e Mon Sep 17 00:00:00 2001 From: toddn Date: Sun, 26 May 2024 16:37:24 -0500 Subject: [PATCH 17/91] files upload right now --- scripts/migration/migrate_users.py | 40 ++++++++++++++++-------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate_users.py index b705cf017..73c5ccf7e 100644 --- a/scripts/migration/migrate_users.py +++ b/scripts/migration/migrate_users.py @@ -60,7 +60,7 @@ async def start_db(): fake = Faker() -path_to_env = os.path.join(os.getcwd(), 'scripts', 'migration', '.env') +path_to_env = os.path.join(os.getcwd(), '.env') print(os.path.isfile(path_to_env)) config = dotenv_values(dotenv_path=path_to_env) @@ -208,10 +208,10 @@ async def process_users( if email != "a@a.com": user_v1_datasets = get_clowder_v1_user_datasets(user_id=id) # TODO check if there is already a local user - # user_v2 = get_clowder_v2_user_by_name(email) + user_v2 = get_clowder_v2_user_by_name(email) user_v2 = create_local_user(user_v1) - user_v2_api_key = generate_user_api_key(user_v2, DEFAULT_PASSWORD) - # user_v2_api_key = 'aZM2QXJ_lvw_5FKNUB89Vg' + # user_v2_api_key = 'eyJ1c2VyIjoiYkBiLmNvbSIsImtleSI6Ik5yNUd1clFmNGhTZFd5ZEVlQ2FmSEEifQ.FTvhQrDgvmSgnwBGwafRNAXkxH8' + user_v2_api_key = user_v2 user_base_headers_v2 = {'X-API-key': user_v2_api_key} user_headers_v2 = {**user_base_headers_v2, 'Content-type': 'application/json', 'accept': 'application/json'} @@ -230,27 +230,31 @@ async def process_users( print('got a user') userDB = await UserDB.find_one({"email": email}) for file in files_result: - new_file = FileDB( - name=file['filename'], - creator=userDB, - dataset_id=dataset["id"], - ) - print('here') + # new_file = FileDB( + # name=file['filename'], + # creator=userDB, + # dataset_id=dataset["id"], + # ) + # print('here') file_id = file['id'] file = db["uploads"].find_one({"_id": ObjectId(file_id)}) filename = file['filename'] loader_id = file["loader_id"] content_type = file["contentType"] # TODO download the file from v1 using api routes - v1_download_url = CLOWDER_V1 + 'api/files' + file_id + v1_download_url = CLOWDER_V1 + 'api/files/' + file_id + '?superAdmin=true' + print('downloading file', filename) download = requests.get(v1_download_url, headers=clowder_headers_v1) - upload_chunks_entry = db["uploads.chunks"].find_one({"files_id": ObjectId(loader_id)}) - data_bytes = upload_chunks_entry['data'] - current_path = os.path.join(os.getcwd(),'scripts','migration') - path_to_temp_file = os.path.join(current_path, filename) - with open(path_to_temp_file, "wb") as f: - f.write(data_bytes) - file_data = {"file": open(path_to_temp_file, "rb")} + with open(filename, 'wb') as f: + f.write(download.content) + # print('after the file download') + # upload_chunks_entry = db["uploads.chunks"].find_one({"files_id": ObjectId(loader_id)}) + # data_bytes = upload_chunks_entry['data'] + # current_path = os.path.join(os.getcwd(),'scripts','migration') + # path_to_temp_file = os.path.join(current_path, filename) + # with open(path_to_temp_file, "wb") as f: + # f.write(data_bytes) + file_data = {"file": open(filename, "rb")} dataset_file_upload_endoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_id['id'] + '/files' response = requests.post(dataset_file_upload_endoint, files=file_data, headers=user_base_headers_v2) result = response.json() From 2bf100cb32e6e4e6b8ec67e0b75eaf3e9c0575d6 Mon Sep 17 00:00:00 2001 From: toddn Date: Sun, 26 May 2024 16:38:07 -0500 Subject: [PATCH 18/91] delete file --- scripts/migration/migrate_users.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate_users.py index 73c5ccf7e..5ad54f2b4 100644 --- a/scripts/migration/migrate_users.py +++ b/scripts/migration/migrate_users.py @@ -258,6 +258,11 @@ async def process_users( dataset_file_upload_endoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_id['id'] + '/files' response = requests.post(dataset_file_upload_endoint, files=file_data, headers=user_base_headers_v2) result = response.json() + try: + os.remove(filename) + except Exception as e: + print("could not delete locally downloaded file") + print(e) print('done') else: From 4eccae139352a8560de1919be45620fd586595ef Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 27 May 2024 15:50:43 -0500 Subject: [PATCH 19/91] adding folders --- scripts/migration/migrate_users.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate_users.py index 5ad54f2b4..9207c94db 100644 --- a/scripts/migration/migrate_users.py +++ b/scripts/migration/migrate_users.py @@ -124,8 +124,11 @@ async def create_v2_dataset(headers, dataset, user_email): dataset_description = dataset['description'] dataset_endpoint = CLOWDER_V1 + 'api/datasets/' + dataset['id'] r = requests.get(dataset_endpoint, headers=base_headers_v1, verify=False) - dataset_files_endpoint = CLOWDER_V1 + 'api/datasets/' + dataset['id'] + '/files' + dataset_files_endpoint = CLOWDER_V1 + 'api/datasets/' + dataset['id'] + '/files?superAdmin=true' r_files = requests.get(dataset_files_endpoint, headers=base_headers_v1, verify=False) + dataset_folders_endpoint = CLOWDER_V1 + 'api/datasets/' + dataset['id'] + '/folders?superAdmin=true' + dataset_folders = requests.get(dataset_folders_endpoint, base_headers_v1) + dataset_folders_json = dataset_folders.json() dataset_in_v2_endpoint = CLOWDER_V2 + 'api/v2/datasets' # create dataset dataset_example = { @@ -218,6 +221,9 @@ async def process_users( for dataset in user_v1_datasets: print('creating a dataset in v2') dataset_id = await create_v2_dataset(user_headers_v2, dataset, email) + dataset_folders_endpoint = CLOWDER_V1 + 'api/datasets/' + dataset['id'] + '/folders?superAdmin=true' + dataset_folders = requests.get(dataset_folders_endpoint, base_headers_v1) + dataset_folders_json = dataset_folders.json() dataset_files_endpoint = CLOWDER_V1 + 'api/datasets/' + dataset['id'] + '/files?=superAdmin=true' # move file stuff here print('we got a dataset id') From c01ce96309ddae0f2e43c3e9b8ba59ef24f10776 Mon Sep 17 00:00:00 2001 From: toddn Date: Thu, 30 May 2024 18:15:57 -0500 Subject: [PATCH 20/91] folder hierarchy - this should work --- scripts/migration/migrate_users.py | 132 +++++++++++++++++++++++++---- 1 file changed, 114 insertions(+), 18 deletions(-) diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate_users.py index 9207c94db..c32905c72 100644 --- a/scripts/migration/migrate_users.py +++ b/scripts/migration/migrate_users.py @@ -122,13 +122,6 @@ async def create_v2_dataset(headers, dataset, user_email): print(dataset) dataset_name = dataset['name'] dataset_description = dataset['description'] - dataset_endpoint = CLOWDER_V1 + 'api/datasets/' + dataset['id'] - r = requests.get(dataset_endpoint, headers=base_headers_v1, verify=False) - dataset_files_endpoint = CLOWDER_V1 + 'api/datasets/' + dataset['id'] + '/files?superAdmin=true' - r_files = requests.get(dataset_files_endpoint, headers=base_headers_v1, verify=False) - dataset_folders_endpoint = CLOWDER_V1 + 'api/datasets/' + dataset['id'] + '/folders?superAdmin=true' - dataset_folders = requests.get(dataset_folders_endpoint, base_headers_v1) - dataset_folders_json = dataset_folders.json() dataset_in_v2_endpoint = CLOWDER_V2 + 'api/v2/datasets' # create dataset dataset_example = { @@ -138,7 +131,7 @@ async def create_v2_dataset(headers, dataset, user_email): response = requests.post( dataset_in_v2_endpoint, headers=headers, json=dataset_example ) - return response.json() + return response.json()['id'] def get_clowder_v1_user_datasets(user_id): @@ -188,13 +181,115 @@ def create_admin_user(): api_key = generate_user_api_key(user_json, "admin") return api_key +async def create_or_get_folder(dataset, folder_name, current_headers): + dataset_folder_url = CLOWDER_V2 + 'api/v2/datasets/' + dataset['id'] + '/folders' + + +async def add_folder_entry_to_dataset(dataset_id, folder_name, current_headers): + current_dataset_folders = [] + dataset_folder_url = CLOWDER_V2 + 'api/v2/datasets/' + dataset_id + '/folders' + response = requests.get(dataset_folder_url, headers=current_headers) + response_json = response.json() + existing_folder_names = dict() + if 'data' in response_json: + existing_folders = response_json['data'] + for existing_folder in existing_folders: + existing_folder_names[existing_folder['name']] = existing_folder['id'] + if folder_name.startswith('/'): + folder_name = folder_name.lstrip('/') + folder_parts = folder_name.split('/') + parent = None + for folder_part in folder_parts: + folder_data = {"name": folder_part} + # TODO create or get folder + if folder_part not in existing_folder_names: + create_folder_endpoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_id + '/folders' + folder_api_call = requests.post(create_folder_endpoint, json=folder_data, headers=current_headers) + print("created folder") + else: + parent = folder_part + print("this one already exists") + print('got folder parts') + + +async def create_folder_if_not_exists_or_get(folder, parent, dataset_v2, current_headers): + clowder_v2_folder_endpoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_v2 + '/folders' + current_dataset_folders = requests.get(clowder_v2_folder_endpoint, headers=current_headers) + folder_json = current_dataset_folders.json() + folder_json_data = folder_json['data'] + current_folder_data = {"name": folder} + if parent: + print('we have a parent to check for') + else: + for each in folder_json_data: + if each['name'] == folder: + print('we found this folder') + return each + response = requests.post( + f"{CLOWDER_V2}api/v2/datasets/{dataset_v2}/folders", + json=current_folder_data, + headers=current_headers, + ) + return response + +async def add_folder_hierarchy(folder_hierarchy, dataset_v2, current_headers): + clowder_v2_folder_endpoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_v2 + '/folders' + current_dataset_folders = requests.get(clowder_v2_folder_endpoint, headers=current_headers) + folder_json = current_dataset_folders.json() + folder_json_data = folder_json['data'] + hierarchy_parts = folder_hierarchy.split('/') + hierarchy_parts.remove('') + current_parent = None + for part in hierarchy_parts: + result = await create_folder_if_not_exists_or_get(part, current_parent, dataset_v2, current_headers=current_headers) + current_parent = result + print('got result') + +async def add_dataset_folders(dataset_v1, dataset_v2, current_headers): + dataset_folders_endpoint = CLOWDER_V1 + 'api/datasets/' + dataset_v1['id'] + '/folders?superAdmin=true' + dataset_folders = requests.get(dataset_folders_endpoint, headers=base_headers_v1) + dataset_folders_json = dataset_folders.json() + folder_names = [] + for folder in dataset_folders_json: + folder_names.append(folder["name"]) + for folder in folder_names: + new = await add_folder_entry_to_dataset(dataset_v2, folder, current_headers) + + + async def process_users( fs: Minio = Depends(dependencies.get_fs), es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), rabbitmq_client: BlockingChannel = Depends(dependencies.get_rabbitmq), ): + + test_admin_key = 'eyJ1c2VyIjoiYUBhLmNvbSIsImtleSI6IkI3RDVJdl85WURQRHVnVXJXS3RlLWcifQ.bKYm8OuOovYKl-YvvBgzi54A_wA' + user_base_headers_v2 = {'X-API-key': test_admin_key} + user_headers_v2 = {**user_base_headers_v2, 'Content-type': 'application/json', + 'accept': 'application/json'} + + # create a dataset + dataset_name = "test" + dataset_description = "just a test" + dataset_in_v2_endpoint = CLOWDER_V2 + 'api/v2/datasets' + # create dataset + dataset_example = { + "name": dataset_name, + "description": dataset_description, + } + # response = requests.post( + # dataset_in_v2_endpoint, headers=user_headers_v2, json=dataset_example + # ) + # test_dataset = response.json()['id'] + + test_datast_id = '6659047bd8ec0c3da1f19b73' + # add folder hierarchy + print('created a dataset') + result = await add_folder_hierarchy('/root/child/subchild', test_datast_id, current_headers=user_headers_v2) + print("We create a v2 admin user") - NEW_ADMIN_KEY_V2 = create_admin_user() + # NEW_ADMIN_KEY_V2 = create_admin_user() + NEW_ADMIN_KEY_V2 = 'eyJ1c2VyIjoiYUBhLmNvbSIsImtleSI6IjlZdWxlcmxhbDlyODF5WDYwTVE5dVEifQ.0ygTBVGeStf7zUl7CBq7jDyc4ZI' print('here') users_v1 = get_clowder_v1_users() for user_v1 in users_v1: @@ -211,19 +306,20 @@ async def process_users( if email != "a@a.com": user_v1_datasets = get_clowder_v1_user_datasets(user_id=id) # TODO check if there is already a local user - user_v2 = get_clowder_v2_user_by_name(email) - user_v2 = create_local_user(user_v1) - # user_v2_api_key = 'eyJ1c2VyIjoiYkBiLmNvbSIsImtleSI6Ik5yNUd1clFmNGhTZFd5ZEVlQ2FmSEEifQ.FTvhQrDgvmSgnwBGwafRNAXkxH8' - user_v2_api_key = user_v2 + # user_v2 = get_clowder_v2_user_by_name(email) + # user_v2 = create_local_user(user_v1) + # # user_v2_api_key = 'eyJ1c2VyIjoiYkBiLmNvbSIsImtleSI6Ik5yNUd1clFmNGhTZFd5ZEVlQ2FmSEEifQ.FTvhQrDgvmSgnwBGwafRNAXkxH8' + # user_v2_api_key = user_v2 + user_v2_api_key = 'eyJ1c2VyIjoiYkBiLmNvbSIsImtleSI6ImRvLUQtcG5kVWg1a3ZQVWVtWWNFTFEifQ.i_0jvyHKX0UmHrcps_pH4N2nru0' user_base_headers_v2 = {'X-API-key': user_v2_api_key} user_headers_v2 = {**user_base_headers_v2, 'Content-type': 'application/json', 'accept': 'application/json'} for dataset in user_v1_datasets: print('creating a dataset in v2') - dataset_id = await create_v2_dataset(user_headers_v2, dataset, email) - dataset_folders_endpoint = CLOWDER_V1 + 'api/datasets/' + dataset['id'] + '/folders?superAdmin=true' - dataset_folders = requests.get(dataset_folders_endpoint, base_headers_v1) - dataset_folders_json = dataset_folders.json() + dataset_v2_id = await create_v2_dataset(user_headers_v2, dataset, email) + dataset_v2_id = '66563fd645c9e9039f41faf7' + folders = await add_dataset_folders(dataset, dataset_v2_id, user_headers_v2) + dataset_files_endpoint = CLOWDER_V1 + 'api/datasets/' + dataset['id'] + '/files?=superAdmin=true' # move file stuff here print('we got a dataset id') @@ -261,7 +357,7 @@ async def process_users( # with open(path_to_temp_file, "wb") as f: # f.write(data_bytes) file_data = {"file": open(filename, "rb")} - dataset_file_upload_endoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_id['id'] + '/files' + dataset_file_upload_endoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_v2['id'] + '/files' response = requests.post(dataset_file_upload_endoint, files=file_data, headers=user_base_headers_v2) result = response.json() try: From c2ab289636927ccf8bf0f4f461affa35f5842abc Mon Sep 17 00:00:00 2001 From: toddn Date: Fri, 31 May 2024 10:44:35 -0500 Subject: [PATCH 21/91] works for now, folders need to be fixed --- scripts/migration/migrate_users.py | 54 +++++++++++++++--------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate_users.py index c32905c72..5ec00e9e9 100644 --- a/scripts/migration/migrate_users.py +++ b/scripts/migration/migrate_users.py @@ -263,34 +263,34 @@ async def process_users( rabbitmq_client: BlockingChannel = Depends(dependencies.get_rabbitmq), ): - test_admin_key = 'eyJ1c2VyIjoiYUBhLmNvbSIsImtleSI6IkI3RDVJdl85WURQRHVnVXJXS3RlLWcifQ.bKYm8OuOovYKl-YvvBgzi54A_wA' - user_base_headers_v2 = {'X-API-key': test_admin_key} - user_headers_v2 = {**user_base_headers_v2, 'Content-type': 'application/json', - 'accept': 'application/json'} + # test_admin_key = 'eyJ1c2VyIjoiYUBhLmNvbSIsImtleSI6IkI3RDVJdl85WURQRHVnVXJXS3RlLWcifQ.bKYm8OuOovYKl-YvvBgzi54A_wA' + # user_base_headers_v2 = {'X-API-key': test_admin_key} + # user_headers_v2 = {**user_base_headers_v2, 'Content-type': 'application/json', + # 'accept': 'application/json'} # create a dataset - dataset_name = "test" - dataset_description = "just a test" - dataset_in_v2_endpoint = CLOWDER_V2 + 'api/v2/datasets' - # create dataset - dataset_example = { - "name": dataset_name, - "description": dataset_description, - } + # dataset_name = "test" + # dataset_description = "just a test" + # dataset_in_v2_endpoint = CLOWDER_V2 + 'api/v2/datasets' + # # create dataset + # dataset_example = { + # "name": dataset_name, + # "description": dataset_description, + # } # response = requests.post( # dataset_in_v2_endpoint, headers=user_headers_v2, json=dataset_example # ) # test_dataset = response.json()['id'] - test_datast_id = '6659047bd8ec0c3da1f19b73' - # add folder hierarchy - print('created a dataset') - result = await add_folder_hierarchy('/root/child/subchild', test_datast_id, current_headers=user_headers_v2) - + # test_datast_id = '6659047bd8ec0c3da1f19b73' + # # add folder hierarchy + # print('created a dataset') + # result = await add_folder_hierarchy('/root/child/subchild', test_datast_id, current_headers=user_headers_v2) + # print("We create a v2 admin user") - # NEW_ADMIN_KEY_V2 = create_admin_user() - NEW_ADMIN_KEY_V2 = 'eyJ1c2VyIjoiYUBhLmNvbSIsImtleSI6IjlZdWxlcmxhbDlyODF5WDYwTVE5dVEifQ.0ygTBVGeStf7zUl7CBq7jDyc4ZI' - print('here') + NEW_ADMIN_KEY_V2 = create_admin_user() + # NEW_ADMIN_KEY_V2 = 'eyJ1c2VyIjoiYUBhLmNvbSIsImtleSI6IjlZdWxlcmxhbDlyODF5WDYwTVE5dVEifQ.0ygTBVGeStf7zUl7CBq7jDyc4ZI' + # print('here') users_v1 = get_clowder_v1_users() for user_v1 in users_v1: print(user_v1) @@ -307,18 +307,18 @@ async def process_users( user_v1_datasets = get_clowder_v1_user_datasets(user_id=id) # TODO check if there is already a local user # user_v2 = get_clowder_v2_user_by_name(email) - # user_v2 = create_local_user(user_v1) + user_v2 = create_local_user(user_v1) # # user_v2_api_key = 'eyJ1c2VyIjoiYkBiLmNvbSIsImtleSI6Ik5yNUd1clFmNGhTZFd5ZEVlQ2FmSEEifQ.FTvhQrDgvmSgnwBGwafRNAXkxH8' - # user_v2_api_key = user_v2 - user_v2_api_key = 'eyJ1c2VyIjoiYkBiLmNvbSIsImtleSI6ImRvLUQtcG5kVWg1a3ZQVWVtWWNFTFEifQ.i_0jvyHKX0UmHrcps_pH4N2nru0' + user_v2_api_key = user_v2 + # user_v2_api_key = 'eyJ1c2VyIjoiYkBiLmNvbSIsImtleSI6ImRvLUQtcG5kVWg1a3ZQVWVtWWNFTFEifQ.i_0jvyHKX0UmHrcps_pH4N2nru0' user_base_headers_v2 = {'X-API-key': user_v2_api_key} user_headers_v2 = {**user_base_headers_v2, 'Content-type': 'application/json', 'accept': 'application/json'} for dataset in user_v1_datasets: print('creating a dataset in v2') dataset_v2_id = await create_v2_dataset(user_headers_v2, dataset, email) - dataset_v2_id = '66563fd645c9e9039f41faf7' - folders = await add_dataset_folders(dataset, dataset_v2_id, user_headers_v2) + # dataset_v2_id = '66563fd645c9e9039f41faf7' + # folders = await add_dataset_folders(dataset, dataset_v2_id, user_headers_v2) dataset_files_endpoint = CLOWDER_V1 + 'api/datasets/' + dataset['id'] + '/files?=superAdmin=true' # move file stuff here @@ -357,7 +357,7 @@ async def process_users( # with open(path_to_temp_file, "wb") as f: # f.write(data_bytes) file_data = {"file": open(filename, "rb")} - dataset_file_upload_endoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_v2['id'] + '/files' + dataset_file_upload_endoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_v2_id + '/files' response = requests.post(dataset_file_upload_endoint, files=file_data, headers=user_base_headers_v2) result = response.json() try: @@ -365,7 +365,7 @@ async def process_users( except Exception as e: print("could not delete locally downloaded file") print(e) - print('done') + print('done with file upload') else: print("not a local account") From 2d7cdce782bc2b6e6df497e6625e010651053117 Mon Sep 17 00:00:00 2001 From: toddn Date: Sat, 1 Jun 2024 15:51:16 -0500 Subject: [PATCH 22/91] previous is no longer selectable when moving to my datasets. we start on the first page --- scripts/migration/migrate_users.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate_users.py index 5ec00e9e9..98e52d639 100644 --- a/scripts/migration/migrate_users.py +++ b/scripts/migration/migrate_users.py @@ -68,8 +68,8 @@ async def start_db(): ADMIN_KEY_V1 = config["ADMIN_KEY_V1"] CLOWDER_V2 = config["CLOWDER_V2"] -ADMIN_KEY_V2 = config["ADMIN_KEY_V2"] - +# ADMIN_KEY_V2 = config["ADMIN_KEY_V2"] +ADMIN_KEY_V2 = 'eyJ1c2VyIjoiYUBhLmNvbSIsImtleSI6IlU1dllaWnB4elNDREl1Q0xObDZ3TWcifQ.LRiLqSH0fJlFSObKrNz-qexkoHw' base_headers_v1 = {'X-API-key': ADMIN_KEY_V1} clowder_headers_v1 = {**base_headers_v1, 'Content-type': 'application/json', 'accept': 'application/json'} @@ -219,7 +219,7 @@ async def create_folder_if_not_exists_or_get(folder, parent, dataset_v2, current folder_json_data = folder_json['data'] current_folder_data = {"name": folder} if parent: - print('we have a parent to check for') + current_folder_data["parent"] = parent else: for each in folder_json_data: if each['name'] == folder: @@ -282,10 +282,10 @@ async def process_users( # ) # test_dataset = response.json()['id'] - # test_datast_id = '6659047bd8ec0c3da1f19b73' + test_datast_id = '665b888d2038e8d9bd4b3a9b' # # add folder hierarchy # print('created a dataset') - # result = await add_folder_hierarchy('/root/child/subchild', test_datast_id, current_headers=user_headers_v2) + result = await add_folder_hierarchy('/root/child/subchild', test_datast_id, current_headers=clowder_headers_v2) # print("We create a v2 admin user") NEW_ADMIN_KEY_V2 = create_admin_user() From 5be97bfda33f6d42087a729d9b4fa2ca8c56b614 Mon Sep 17 00:00:00 2001 From: toddn Date: Sat, 1 Jun 2024 16:01:10 -0500 Subject: [PATCH 23/91] new folder hierarchy method works --- scripts/migration/migrate_users.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate_users.py index 98e52d639..a5e1a44fd 100644 --- a/scripts/migration/migrate_users.py +++ b/scripts/migration/migrate_users.py @@ -219,12 +219,12 @@ async def create_folder_if_not_exists_or_get(folder, parent, dataset_v2, current folder_json_data = folder_json['data'] current_folder_data = {"name": folder} if parent: - current_folder_data["parent"] = parent + current_folder_data["parent_folder"] = parent else: for each in folder_json_data: if each['name'] == folder: print('we found this folder') - return each + return each['id'] response = requests.post( f"{CLOWDER_V2}api/v2/datasets/{dataset_v2}/folders", json=current_folder_data, @@ -242,7 +242,8 @@ async def add_folder_hierarchy(folder_hierarchy, dataset_v2, current_headers): current_parent = None for part in hierarchy_parts: result = await create_folder_if_not_exists_or_get(part, current_parent, dataset_v2, current_headers=current_headers) - current_parent = result + if result.status_code == 200: + current_parent = result.json()['id'] print('got result') async def add_dataset_folders(dataset_v1, dataset_v2, current_headers): From f479284273ec9c1e8ef0028cf101dae64e283cfc Mon Sep 17 00:00:00 2001 From: toddn Date: Sun, 2 Jun 2024 13:36:36 -0500 Subject: [PATCH 24/91] need a license for datasets removing unused code --- scripts/migration/migrate_users.py | 62 ++++++++---------------------- 1 file changed, 17 insertions(+), 45 deletions(-) diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate_users.py index a5e1a44fd..9c325832b 100644 --- a/scripts/migration/migrate_users.py +++ b/scripts/migration/migrate_users.py @@ -68,8 +68,8 @@ async def start_db(): ADMIN_KEY_V1 = config["ADMIN_KEY_V1"] CLOWDER_V2 = config["CLOWDER_V2"] -# ADMIN_KEY_V2 = config["ADMIN_KEY_V2"] -ADMIN_KEY_V2 = 'eyJ1c2VyIjoiYUBhLmNvbSIsImtleSI6IlU1dllaWnB4elNDREl1Q0xObDZ3TWcifQ.LRiLqSH0fJlFSObKrNz-qexkoHw' +ADMIN_KEY_V2 = config["ADMIN_KEY_V2"] +# ADMIN_KEY_V2 = 'eyJ1c2VyIjoiYUBhLmNvbSIsImtleSI6IlU1dllaWnB4elNDREl1Q0xObDZ3TWcifQ.LRiLqSH0fJlFSObKrNz-qexkoHw' base_headers_v1 = {'X-API-key': ADMIN_KEY_V1} clowder_headers_v1 = {**base_headers_v1, 'Content-type': 'application/json', 'accept': 'application/json'} @@ -120,9 +120,10 @@ def get_clowder_v2_user_by_name(username): async def create_v2_dataset(headers, dataset, user_email): print(dataset) + default_license_id = "CC BY" dataset_name = dataset['name'] dataset_description = dataset['description'] - dataset_in_v2_endpoint = CLOWDER_V2 + 'api/v2/datasets' + dataset_in_v2_endpoint = CLOWDER_V2 + f'api/v2/datasets?license_id={default_license_id}' # create dataset dataset_example = { "name": dataset_name, @@ -181,9 +182,6 @@ def create_admin_user(): api_key = generate_user_api_key(user_json, "admin") return api_key -async def create_or_get_folder(dataset, folder_name, current_headers): - dataset_folder_url = CLOWDER_V2 + 'api/v2/datasets/' + dataset['id'] + '/folders' - async def add_folder_entry_to_dataset(dataset_id, folder_name, current_headers): current_dataset_folders = [] @@ -254,7 +252,8 @@ async def add_dataset_folders(dataset_v1, dataset_v2, current_headers): for folder in dataset_folders_json: folder_names.append(folder["name"]) for folder in folder_names: - new = await add_folder_entry_to_dataset(dataset_v2, folder, current_headers) + new = await add_folder_hierarchy(folder_hierarchy=folder, dataset_v2=dataset_v2, current_headers=current_headers) + print("added a folder") @@ -264,32 +263,13 @@ async def process_users( rabbitmq_client: BlockingChannel = Depends(dependencies.get_rabbitmq), ): - # test_admin_key = 'eyJ1c2VyIjoiYUBhLmNvbSIsImtleSI6IkI3RDVJdl85WURQRHVnVXJXS3RlLWcifQ.bKYm8OuOovYKl-YvvBgzi54A_wA' - # user_base_headers_v2 = {'X-API-key': test_admin_key} - # user_headers_v2 = {**user_base_headers_v2, 'Content-type': 'application/json', - # 'accept': 'application/json'} - - # create a dataset - # dataset_name = "test" - # dataset_description = "just a test" - # dataset_in_v2_endpoint = CLOWDER_V2 + 'api/v2/datasets' - # # create dataset - # dataset_example = { - # "name": dataset_name, - # "description": dataset_description, - # } - # response = requests.post( - # dataset_in_v2_endpoint, headers=user_headers_v2, json=dataset_example - # ) - # test_dataset = response.json()['id'] - test_datast_id = '665b888d2038e8d9bd4b3a9b' # # add folder hierarchy # print('created a dataset') - result = await add_folder_hierarchy('/root/child/subchild', test_datast_id, current_headers=clowder_headers_v2) + # result = await add_folder_hierarchy('/root/child/subchild', test_datast_id, current_headers=clowder_headers_v2) # print("We create a v2 admin user") - NEW_ADMIN_KEY_V2 = create_admin_user() + # NEW_ADMIN_KEY_V2 = create_admin_user() # NEW_ADMIN_KEY_V2 = 'eyJ1c2VyIjoiYUBhLmNvbSIsImtleSI6IjlZdWxlcmxhbDlyODF5WDYwTVE5dVEifQ.0ygTBVGeStf7zUl7CBq7jDyc4ZI' # print('here') users_v1 = get_clowder_v1_users() @@ -308,7 +288,8 @@ async def process_users( user_v1_datasets = get_clowder_v1_user_datasets(user_id=id) # TODO check if there is already a local user # user_v2 = get_clowder_v2_user_by_name(email) - user_v2 = create_local_user(user_v1) + # user_v2 = create_local_user(user_v1) + user_v2 = 'eyJ1c2VyIjoiYkBiLmNvbSIsImtleSI6ImkyNmJ6MXdPNVpiejJYQTA5ZlRMbXcifQ.JuLZJ6zOBj6fBXoEmX97cs_bx5c' # # user_v2_api_key = 'eyJ1c2VyIjoiYkBiLmNvbSIsImtleSI6Ik5yNUd1clFmNGhTZFd5ZEVlQ2FmSEEifQ.FTvhQrDgvmSgnwBGwafRNAXkxH8' user_v2_api_key = user_v2 # user_v2_api_key = 'eyJ1c2VyIjoiYkBiLmNvbSIsImtleSI6ImRvLUQtcG5kVWg1a3ZQVWVtWWNFTFEifQ.i_0jvyHKX0UmHrcps_pH4N2nru0' @@ -317,10 +298,13 @@ async def process_users( 'accept': 'application/json'} for dataset in user_v1_datasets: print('creating a dataset in v2') - dataset_v2_id = await create_v2_dataset(user_headers_v2, dataset, email) + dataset_v2_id = await create_v2_dataset(user_base_headers_v2, dataset, email) # dataset_v2_id = '66563fd645c9e9039f41faf7' - # folders = await add_dataset_folders(dataset, dataset_v2_id, user_headers_v2) - + folders = await add_dataset_folders(dataset, dataset_v2_id, user_headers_v2) + print('we got folders') + folder_v2_endpoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_v2_id + '/folders' + folders_v2_dataset = requests.get(folder_v2_endpoint, user_headers_v2) + folders_v2_dataset_json = folders_v2_dataset.json() dataset_files_endpoint = CLOWDER_V1 + 'api/datasets/' + dataset['id'] + '/files?=superAdmin=true' # move file stuff here print('we got a dataset id') @@ -333,12 +317,7 @@ async def process_users( print('got a user') userDB = await UserDB.find_one({"email": email}) for file in files_result: - # new_file = FileDB( - # name=file['filename'], - # creator=userDB, - # dataset_id=dataset["id"], - # ) - # print('here') + file_id = file['id'] file = db["uploads"].find_one({"_id": ObjectId(file_id)}) filename = file['filename'] @@ -350,13 +329,6 @@ async def process_users( download = requests.get(v1_download_url, headers=clowder_headers_v1) with open(filename, 'wb') as f: f.write(download.content) - # print('after the file download') - # upload_chunks_entry = db["uploads.chunks"].find_one({"files_id": ObjectId(loader_id)}) - # data_bytes = upload_chunks_entry['data'] - # current_path = os.path.join(os.getcwd(),'scripts','migration') - # path_to_temp_file = os.path.join(current_path, filename) - # with open(path_to_temp_file, "wb") as f: - # f.write(data_bytes) file_data = {"file": open(filename, "rb")} dataset_file_upload_endoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_v2_id + '/files' response = requests.post(dataset_file_upload_endoint, files=file_data, headers=user_base_headers_v2) From cb4b8354d4a0a6c8b4e2bfe643d2c242251751e4 Mon Sep 17 00:00:00 2001 From: toddn Date: Sun, 2 Jun 2024 16:34:08 -0500 Subject: [PATCH 25/91] method for getting all the folders --- scripts/migration/migrate_users.py | 54 ++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate_users.py index 9c325832b..e789b971d 100644 --- a/scripts/migration/migrate_users.py +++ b/scripts/migration/migrate_users.py @@ -209,6 +209,28 @@ async def add_folder_entry_to_dataset(dataset_id, folder_name, current_headers): print("this one already exists") print('got folder parts') +async def get_folder_and_subfolders(dataset_id, folder, current_headers): + total_folders = [] + if folder: + path_url = CLOWDER_V2 + 'api/v2/datasets/' + dataset_id + '/folders_and_files?folder_id=' + folder['id'] + folder_result = requests.get(path_url, headers=current_headers) + folder_json_data = folder_result.json()['data'] + for data in folder_json_data: + if data['object_type'] == 'folder': + total_folders.append(data) + else: + path_url = CLOWDER_V2 + 'api/v2/datasets/' + dataset_id + '/folders' + folder_result = requests.get(path_url, headers=current_headers) + folder_json_data = folder_result.json()['data'] + for data in folder_json_data: + total_folders.append(data) + print('we got the base level now') + current_subfolders = [] + for current_folder in total_folders: + subfolders = await get_folder_and_subfolders(dataset_id, current_folder, current_headers) + current_subfolders += subfolders + total_folders += current_subfolders + return total_folders async def create_folder_if_not_exists_or_get(folder, parent, dataset_v2, current_headers): clowder_v2_folder_endpoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_v2 + '/folders' @@ -216,19 +238,21 @@ async def create_folder_if_not_exists_or_get(folder, parent, dataset_v2, current folder_json = current_dataset_folders.json() folder_json_data = folder_json['data'] current_folder_data = {"name": folder} + found_folder = None if parent: current_folder_data["parent_folder"] = parent - else: - for each in folder_json_data: - if each['name'] == folder: - print('we found this folder') - return each['id'] - response = requests.post( - f"{CLOWDER_V2}api/v2/datasets/{dataset_v2}/folders", - json=current_folder_data, - headers=current_headers, - ) - return response + for each in folder_json_data: + if each['name'] == folder: + found_folder = each + if not found_folder: + response = requests.post( + f"{CLOWDER_V2}api/v2/datasets/{dataset_v2}/folders", + json=current_folder_data, + headers=current_headers, + ) + found_folder = response.json() + print("We just created", found_folder) + return found_folder async def add_folder_hierarchy(folder_hierarchy, dataset_v2, current_headers): clowder_v2_folder_endpoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_v2 + '/folders' @@ -240,8 +264,8 @@ async def add_folder_hierarchy(folder_hierarchy, dataset_v2, current_headers): current_parent = None for part in hierarchy_parts: result = await create_folder_if_not_exists_or_get(part, current_parent, dataset_v2, current_headers=current_headers) - if result.status_code == 200: - current_parent = result.json()['id'] + if result: + current_parent = result['id'] print('got result') async def add_dataset_folders(dataset_v1, dataset_v2, current_headers): @@ -253,7 +277,7 @@ async def add_dataset_folders(dataset_v1, dataset_v2, current_headers): folder_names.append(folder["name"]) for folder in folder_names: new = await add_folder_hierarchy(folder_hierarchy=folder, dataset_v2=dataset_v2, current_headers=current_headers) - print("added a folder") + print('added', folder) @@ -296,6 +320,7 @@ async def process_users( user_base_headers_v2 = {'X-API-key': user_v2_api_key} user_headers_v2 = {**user_base_headers_v2, 'Content-type': 'application/json', 'accept': 'application/json'} + folders_from_test = await get_folder_and_subfolders(dataset_id='665cdb1b7af29eb74e40bc86',folder=None, current_headers=user_headers_v2) for dataset in user_v1_datasets: print('creating a dataset in v2') dataset_v2_id = await create_v2_dataset(user_base_headers_v2, dataset, email) @@ -316,6 +341,7 @@ async def process_users( user = user_collection.find_one({"email": email}) print('got a user') userDB = await UserDB.find_one({"email": email}) + print('BEFORE FILES') for file in files_result: file_id = file['id'] From e885a2abb0f9f7cc1843ab5a3b7c25865651a0d8 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 3 Jun 2024 13:07:02 -0500 Subject: [PATCH 26/91] need to remove logging, but folders are created and files uploaded to correct folders --- scripts/migration/migrate_users.py | 58 ++++++++++++++++-------------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate_users.py index e789b971d..5542e65ce 100644 --- a/scripts/migration/migrate_users.py +++ b/scripts/migration/migrate_users.py @@ -209,6 +209,7 @@ async def add_folder_entry_to_dataset(dataset_id, folder_name, current_headers): print("this one already exists") print('got folder parts') +# gets all folders and subfolders in a dataset async def get_folder_and_subfolders(dataset_id, folder, current_headers): total_folders = [] if folder: @@ -234,14 +235,12 @@ async def get_folder_and_subfolders(dataset_id, folder, current_headers): async def create_folder_if_not_exists_or_get(folder, parent, dataset_v2, current_headers): clowder_v2_folder_endpoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_v2 + '/folders' - current_dataset_folders = requests.get(clowder_v2_folder_endpoint, headers=current_headers) - folder_json = current_dataset_folders.json() - folder_json_data = folder_json['data'] + current_dataset_folders = await get_folder_and_subfolders(dataset_id=dataset_v2,folder=None, current_headers=current_headers) current_folder_data = {"name": folder} found_folder = None if parent: current_folder_data["parent_folder"] = parent - for each in folder_json_data: + for each in current_dataset_folders: if each['name'] == folder: found_folder = each if not found_folder: @@ -255,10 +254,8 @@ async def create_folder_if_not_exists_or_get(folder, parent, dataset_v2, current return found_folder async def add_folder_hierarchy(folder_hierarchy, dataset_v2, current_headers): - clowder_v2_folder_endpoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_v2 + '/folders' - current_dataset_folders = requests.get(clowder_v2_folder_endpoint, headers=current_headers) - folder_json = current_dataset_folders.json() - folder_json_data = folder_json['data'] + current_dataset_folders = await get_folder_and_subfolders(dataset_id=dataset_v2, folder=None, current_headers=current_headers) + folder_json_data = current_dataset_folders hierarchy_parts = folder_hierarchy.split('/') hierarchy_parts.remove('') current_parent = None @@ -266,7 +263,6 @@ async def add_folder_hierarchy(folder_hierarchy, dataset_v2, current_headers): result = await create_folder_if_not_exists_or_get(part, current_parent, dataset_v2, current_headers=current_headers) if result: current_parent = result['id'] - print('got result') async def add_dataset_folders(dataset_v1, dataset_v2, current_headers): dataset_folders_endpoint = CLOWDER_V1 + 'api/datasets/' + dataset_v1['id'] + '/folders?superAdmin=true' @@ -286,8 +282,6 @@ async def process_users( es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), rabbitmq_client: BlockingChannel = Depends(dependencies.get_rabbitmq), ): - - test_datast_id = '665b888d2038e8d9bd4b3a9b' # # add folder hierarchy # print('created a dataset') # result = await add_folder_hierarchy('/root/child/subchild', test_datast_id, current_headers=clowder_headers_v2) @@ -320,35 +314,33 @@ async def process_users( user_base_headers_v2 = {'X-API-key': user_v2_api_key} user_headers_v2 = {**user_base_headers_v2, 'Content-type': 'application/json', 'accept': 'application/json'} - folders_from_test = await get_folder_and_subfolders(dataset_id='665cdb1b7af29eb74e40bc86',folder=None, current_headers=user_headers_v2) for dataset in user_v1_datasets: print('creating a dataset in v2') dataset_v2_id = await create_v2_dataset(user_base_headers_v2, dataset, email) - # dataset_v2_id = '66563fd645c9e9039f41faf7' + # dataset_v2_id = '665dfe7649c5719830e2c0e0' folders = await add_dataset_folders(dataset, dataset_v2_id, user_headers_v2) print('we got folders') - folder_v2_endpoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_v2_id + '/folders' - folders_v2_dataset = requests.get(folder_v2_endpoint, user_headers_v2) - folders_v2_dataset_json = folders_v2_dataset.json() + all_dataset_folders = await get_folder_and_subfolders(dataset_id=dataset_v2_id, folder=None, current_headers=user_headers_v2) dataset_files_endpoint = CLOWDER_V1 + 'api/datasets/' + dataset['id'] + '/files?=superAdmin=true' # move file stuff here print('we got a dataset id') r_files = requests.get(dataset_files_endpoint, headers=clowder_headers_v1, verify=False) - r_files_json = r_files.json() files_result = r_files.json() - user_collection = db_v2["users"] - user = user_collection.find_one({"email": email}) print('got a user') - userDB = await UserDB.find_one({"email": email}) print('BEFORE FILES') + files_with_folders = [] for file in files_result: - + if 'folders' in file: + print('a folder') + files_with_folders.append(file) + for file in files_result: + file_folder = None file_id = file['id'] - file = db["uploads"].find_one({"_id": ObjectId(file_id)}) filename = file['filename'] - loader_id = file["loader_id"] - content_type = file["contentType"] + if 'folders' in file: + file_folder = file['folders'] + file_folder_name = file['folders']['name'] # TODO download the file from v1 using api routes v1_download_url = CLOWDER_V1 + 'api/files/' + file_id + '?superAdmin=true' print('downloading file', filename) @@ -356,8 +348,22 @@ async def process_users( with open(filename, 'wb') as f: f.write(download.content) file_data = {"file": open(filename, "rb")} - dataset_file_upload_endoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_v2_id + '/files' - response = requests.post(dataset_file_upload_endoint, files=file_data, headers=user_base_headers_v2) + matching_folder = None + if file_folder: + for folder in all_dataset_folders: + if folder['name'] == file_folder['name']: + matching_folder = folder + if matching_folder: + upload_files = {"files":open(filename,'rb')} + dataset_file_upload_endoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_v2_id + '/filesMultiple?folder_id=' + matching_folder['id'] + response = requests.post(dataset_file_upload_endoint, files=upload_files, + headers=user_base_headers_v2) + + else: + dataset_file_upload_endoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_v2_id + '/files' + response = requests.post(dataset_file_upload_endoint, files=file_data, + headers=user_base_headers_v2) + result = response.json() try: os.remove(filename) From b4db0f4981bb92af013587c48321bf3c9ff80de0 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 3 Jun 2024 13:31:48 -0500 Subject: [PATCH 27/91] should work now --- scripts/migration/migrate_users.py | 42 ++++++++---------------------- 1 file changed, 11 insertions(+), 31 deletions(-) diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate_users.py index 5542e65ce..44c329693 100644 --- a/scripts/migration/migrate_users.py +++ b/scripts/migration/migrate_users.py @@ -282,17 +282,11 @@ async def process_users( es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), rabbitmq_client: BlockingChannel = Depends(dependencies.get_rabbitmq), ): - # # add folder hierarchy - # print('created a dataset') - # result = await add_folder_hierarchy('/root/child/subchild', test_datast_id, current_headers=clowder_headers_v2) - # print("We create a v2 admin user") - # NEW_ADMIN_KEY_V2 = create_admin_user() - # NEW_ADMIN_KEY_V2 = 'eyJ1c2VyIjoiYUBhLmNvbSIsImtleSI6IjlZdWxlcmxhbDlyODF5WDYwTVE5dVEifQ.0ygTBVGeStf7zUl7CBq7jDyc4ZI' - # print('here') + NEW_ADMIN_KEY_V2 = create_admin_user() users_v1 = get_clowder_v1_users() for user_v1 in users_v1: - print(user_v1) + print("migrating v1 user", user_v1) id = user_v1['id'] email = user_v1['email'] firstName = user_v1['firstName'] @@ -301,46 +295,31 @@ async def process_users( id_provider = user_v1['identityProvider'] if '[Local Account]' in user_v1['identityProvider']: # get the v2 users - # i create a user account in v2 with this username + # create a user account in v2 with this username if email != "a@a.com": user_v1_datasets = get_clowder_v1_user_datasets(user_id=id) # TODO check if there is already a local user - # user_v2 = get_clowder_v2_user_by_name(email) - # user_v2 = create_local_user(user_v1) - user_v2 = 'eyJ1c2VyIjoiYkBiLmNvbSIsImtleSI6ImkyNmJ6MXdPNVpiejJYQTA5ZlRMbXcifQ.JuLZJ6zOBj6fBXoEmX97cs_bx5c' - # # user_v2_api_key = 'eyJ1c2VyIjoiYkBiLmNvbSIsImtleSI6Ik5yNUd1clFmNGhTZFd5ZEVlQ2FmSEEifQ.FTvhQrDgvmSgnwBGwafRNAXkxH8' + user_v2 = create_local_user(user_v1) user_v2_api_key = user_v2 - # user_v2_api_key = 'eyJ1c2VyIjoiYkBiLmNvbSIsImtleSI6ImRvLUQtcG5kVWg1a3ZQVWVtWWNFTFEifQ.i_0jvyHKX0UmHrcps_pH4N2nru0' user_base_headers_v2 = {'X-API-key': user_v2_api_key} user_headers_v2 = {**user_base_headers_v2, 'Content-type': 'application/json', 'accept': 'application/json'} for dataset in user_v1_datasets: - print('creating a dataset in v2') + print('creating a dataset in v2', dataset['id'], dataset['name']) dataset_v2_id = await create_v2_dataset(user_base_headers_v2, dataset, email) - # dataset_v2_id = '665dfe7649c5719830e2c0e0' folders = await add_dataset_folders(dataset, dataset_v2_id, user_headers_v2) - print('we got folders') + print("Created folders in new dataset") + all_dataset_folders = await get_folder_and_subfolders(dataset_id=dataset_v2_id, folder=None, current_headers=user_headers_v2) dataset_files_endpoint = CLOWDER_V1 + 'api/datasets/' + dataset['id'] + '/files?=superAdmin=true' - # move file stuff here - print('we got a dataset id') - r_files = requests.get(dataset_files_endpoint, headers=clowder_headers_v1, verify=False) files_result = r_files.json() - print('got a user') - print('BEFORE FILES') - files_with_folders = [] - for file in files_result: - if 'folders' in file: - print('a folder') - files_with_folders.append(file) for file in files_result: file_folder = None file_id = file['id'] filename = file['filename'] if 'folders' in file: file_folder = file['folders'] - file_folder_name = file['folders']['name'] # TODO download the file from v1 using api routes v1_download_url = CLOWDER_V1 + 'api/files/' + file_id + '?superAdmin=true' print('downloading file', filename) @@ -363,8 +342,9 @@ async def process_users( dataset_file_upload_endoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_v2_id + '/files' response = requests.post(dataset_file_upload_endoint, files=file_data, headers=user_base_headers_v2) - - result = response.json() + if response.status_code == 200: + result = response.json() + print("added file", result) try: os.remove(filename) except Exception as e: @@ -373,7 +353,7 @@ async def process_users( print('done with file upload') else: - print("not a local account") + print("not a local account, not migrated at this time") asyncio.run(process_users()) From 72e084580c55837dada321ccfb531720e56d7499 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 3 Jun 2024 13:34:08 -0500 Subject: [PATCH 28/91] fixing typos --- scripts/migration/migrate_users.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate_users.py index 44c329693..abfe7a71e 100644 --- a/scripts/migration/migrate_users.py +++ b/scripts/migration/migrate_users.py @@ -334,13 +334,13 @@ async def process_users( matching_folder = folder if matching_folder: upload_files = {"files":open(filename,'rb')} - dataset_file_upload_endoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_v2_id + '/filesMultiple?folder_id=' + matching_folder['id'] - response = requests.post(dataset_file_upload_endoint, files=upload_files, + dataset_file_upload_endpoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_v2_id + '/filesMultiple?folder_id=' + matching_folder['id'] + response = requests.post(dataset_file_upload_endpoint, files=upload_files, headers=user_base_headers_v2) else: - dataset_file_upload_endoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_v2_id + '/files' - response = requests.post(dataset_file_upload_endoint, files=file_data, + dataset_file_upload_endpoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_v2_id + '/files' + response = requests.post(dataset_file_upload_endpoint, files=file_data, headers=user_base_headers_v2) if response.status_code == 200: result = response.json() From a6cd25a28861034c18eebe5e2688abd49b805994 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 3 Jun 2024 13:35:35 -0500 Subject: [PATCH 29/91] formatting --- .../migration/migrate_metadata_definitions.py | 24 +- scripts/migration/migrate_users.py | 294 ++++++++++++------ 2 files changed, 206 insertions(+), 112 deletions(-) diff --git a/scripts/migration/migrate_metadata_definitions.py b/scripts/migration/migrate_metadata_definitions.py index b85594b46..53db9c7ab 100644 --- a/scripts/migration/migrate_metadata_definitions.py +++ b/scripts/migration/migrate_metadata_definitions.py @@ -5,7 +5,7 @@ fake = Faker() -path_to_env = os.path.join(os.getcwd(), 'scripts', 'migration', '.env') +path_to_env = os.path.join(os.getcwd(), "scripts", "migration", ".env") print(os.path.isfile(path_to_env)) config = dotenv_values(dotenv_path=path_to_env) @@ -15,20 +15,26 @@ CLOWDER_V2 = config["CLOWDER_V2"] ADMIN_KEY_V2 = config["ADMIN_KEY_V2"] -base_headers_v1 = {'X-API-key': ADMIN_KEY_V1} -clowder_headers_v1 = {**base_headers_v1, 'Content-type': 'application/json', - 'accept': 'application/json'} +base_headers_v1 = {"X-API-key": ADMIN_KEY_V1} +clowder_headers_v1 = { + **base_headers_v1, + "Content-type": "application/json", + "accept": "application/json", +} -base_headers_v2 = {'X-API-key': ADMIN_KEY_V2} -clowder_headers_v2 = {**base_headers_v2, 'Content-type': 'application/json', - 'accept': 'application/json'} +base_headers_v2 = {"X-API-key": ADMIN_KEY_V2} +clowder_headers_v2 = { + **base_headers_v2, + "Content-type": "application/json", + "accept": "application/json", +} def get_clowder_v1_metadata_definitions(): - endpoint = CLOWDER_V1 + 'api/metadata/definitions' + endpoint = CLOWDER_V1 + "api/metadata/definitions" r = requests.get(endpoint, headers=base_headers_v1, verify=False) return r.json() md_definitions = get_clowder_v1_metadata_definitions() -print('got them') \ No newline at end of file +print("got them") diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate_users.py index abfe7a71e..781105377 100644 --- a/scripts/migration/migrate_users.py +++ b/scripts/migration/migrate_users.py @@ -19,6 +19,7 @@ ) from minio import Minio import nest_asyncio + nest_asyncio.apply() from app.models.users import UserDB, UserOut, UserAPIKeyDB from beanie import init_beanie @@ -32,12 +33,14 @@ mongo_client_v2 = MongoClient("mongodb://localhost:27017", connect=False) db = mongo_client["clowder"] db_v2 = mongo_client_v2["clowder2"] -print('got the db') +print("got the db") v1_users = db["social.users"].find({}) for u in v1_users: print(u) app = FastAPI() + + @app.on_event("startup") async def start_db(): client = AsyncIOMotorClient(str(settings.MONGODB_URL)) @@ -55,12 +58,12 @@ async def start_db(): asyncio.run(start_db()) -output_file = 'new_users.txt' +output_file = "new_users.txt" fake = Faker() -path_to_env = os.path.join(os.getcwd(), '.env') +path_to_env = os.path.join(os.getcwd(), ".env") print(os.path.isfile(path_to_env)) config = dotenv_values(dotenv_path=path_to_env) @@ -70,21 +73,29 @@ async def start_db(): CLOWDER_V2 = config["CLOWDER_V2"] ADMIN_KEY_V2 = config["ADMIN_KEY_V2"] # ADMIN_KEY_V2 = 'eyJ1c2VyIjoiYUBhLmNvbSIsImtleSI6IlU1dllaWnB4elNDREl1Q0xObDZ3TWcifQ.LRiLqSH0fJlFSObKrNz-qexkoHw' -base_headers_v1 = {'X-API-key': ADMIN_KEY_V1} -clowder_headers_v1 = {**base_headers_v1, 'Content-type': 'application/json', - 'accept': 'application/json'} - -base_headers_v2 = {'X-API-key': ADMIN_KEY_V2} -clowder_headers_v2 = {**base_headers_v2, 'Content-type': 'application/json', - 'accept': 'application/json'} - -TEST_DATASET_NAME = 'Migration Test Dataset' +base_headers_v1 = {"X-API-key": ADMIN_KEY_V1} +clowder_headers_v1 = { + **base_headers_v1, + "Content-type": "application/json", + "accept": "application/json", +} + +base_headers_v2 = {"X-API-key": ADMIN_KEY_V2} +clowder_headers_v2 = { + **base_headers_v2, + "Content-type": "application/json", + "accept": "application/json", +} + +TEST_DATASET_NAME = "Migration Test Dataset" # TODO this is just for testing -DEFAULT_PASSWORD = 'Password123&' +DEFAULT_PASSWORD = "Password123&" + def email_user_new_login(user): print("login to the new clowder instance") + def generate_user_api_key(user, password): user_example = { "email": user["email"], @@ -92,38 +103,44 @@ def generate_user_api_key(user, password): "first_name": user["first_name"], "last_name": user["last_name"], } - login_endpoint = CLOWDER_V2 + 'api/v2/login' + login_endpoint = CLOWDER_V2 + "api/v2/login" response = requests.post(login_endpoint, json=user_example) token = response.json().get("token") current_headers = {"Authorization": "Bearer " + token} - auth = {'username': user["email"], 'password': password} - api_key_endpoint = CLOWDER_V2 + 'api/v2/users/keys?name=migration&mins=0' + auth = {"username": user["email"], "password": password} + api_key_endpoint = CLOWDER_V2 + "api/v2/users/keys?name=migration&mins=0" result = requests.post(api_key_endpoint, headers=current_headers) api_key = result.json() return api_key + def get_clowder_v1_users(): - endpoint = CLOWDER_V1 + 'api/users' + endpoint = CLOWDER_V1 + "api/users" print(base_headers_v1) - r = requests.get(endpoint, headers=clowder_headers_v1, verify=False) + r = requests.get(endpoint, headers=clowder_headers_v1, verify=False) return r.json() + def get_clowder_v2_users(): - endpoint = CLOWDER_V2 + 'api/v2/users' + endpoint = CLOWDER_V2 + "api/v2/users" r = requests.get(endpoint, headers=base_headers_v2, verify=False) return r.json() + def get_clowder_v2_user_by_name(username): - endpoint = CLOWDER_V2 + 'api/v2/users/username/' + username + endpoint = CLOWDER_V2 + "api/v2/users/username/" + username r = requests.get(endpoint, headers=base_headers_v2, verify=False) return r.json() + async def create_v2_dataset(headers, dataset, user_email): print(dataset) default_license_id = "CC BY" - dataset_name = dataset['name'] - dataset_description = dataset['description'] - dataset_in_v2_endpoint = CLOWDER_V2 + f'api/v2/datasets?license_id={default_license_id}' + dataset_name = dataset["name"] + dataset_description = dataset["description"] + dataset_in_v2_endpoint = ( + CLOWDER_V2 + f"api/v2/datasets?license_id={default_license_id}" + ) # create dataset dataset_example = { "name": dataset_name, @@ -132,30 +149,31 @@ async def create_v2_dataset(headers, dataset, user_email): response = requests.post( dataset_in_v2_endpoint, headers=headers, json=dataset_example ) - return response.json()['id'] + return response.json()["id"] def get_clowder_v1_user_datasets(user_id): user_datasets = [] - endpoint = CLOWDER_V1 + 'api/datasets?limit=0' + endpoint = CLOWDER_V1 + "api/datasets?limit=0" r = requests.get(endpoint, headers=base_headers_v1, verify=False) request_json = r.json() for dataset in request_json: - if dataset['authorId'] == user_id: + if dataset["authorId"] == user_id: user_datasets.append(dataset) return user_datasets + def create_local_user(user_v1): - first_name = user_v1['firstName'] - last_name = user_v1['lastName'] - email = user_v1['email'] + first_name = user_v1["firstName"] + last_name = user_v1["lastName"] + email = user_v1["email"] # password = fake.password(20) - password = 'Password123&' + password = "Password123&" user_json = { "email": email, "password": password, "first_name": first_name, - "last_name": last_name + "last_name": last_name, } response = requests.post(f"{CLOWDER_V2}api/v2/users", json=user_json) email_user_new_login(email) @@ -163,20 +181,21 @@ def create_local_user(user_v1): # api_key = 'aZM2QXJ_lvw_5FKNUB89Vg' print("Local user created and api key generated") if os.path.exists(output_file): - print('it exists.') + print("it exists.") else: f = open(output_file, "x") - with open(output_file, 'a') as f: - entry = email + ',' + password + ',' + api_key + "\n" + with open(output_file, "a") as f: + entry = email + "," + password + "," + api_key + "\n" f.write(entry) return api_key + def create_admin_user(): user_json = { "email": "a@a.com", "password": "admin", "first_name": "aa", - "last_name": "aa" + "last_name": "aa", } response = requests.post(f"{CLOWDER_V2}api/v2/users", json=user_json) api_key = generate_user_api_key(user_json, "admin") @@ -185,63 +204,83 @@ def create_admin_user(): async def add_folder_entry_to_dataset(dataset_id, folder_name, current_headers): current_dataset_folders = [] - dataset_folder_url = CLOWDER_V2 + 'api/v2/datasets/' + dataset_id + '/folders' + dataset_folder_url = CLOWDER_V2 + "api/v2/datasets/" + dataset_id + "/folders" response = requests.get(dataset_folder_url, headers=current_headers) response_json = response.json() existing_folder_names = dict() - if 'data' in response_json: - existing_folders = response_json['data'] + if "data" in response_json: + existing_folders = response_json["data"] for existing_folder in existing_folders: - existing_folder_names[existing_folder['name']] = existing_folder['id'] - if folder_name.startswith('/'): - folder_name = folder_name.lstrip('/') - folder_parts = folder_name.split('/') + existing_folder_names[existing_folder["name"]] = existing_folder["id"] + if folder_name.startswith("/"): + folder_name = folder_name.lstrip("/") + folder_parts = folder_name.split("/") parent = None for folder_part in folder_parts: folder_data = {"name": folder_part} # TODO create or get folder if folder_part not in existing_folder_names: - create_folder_endpoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_id + '/folders' - folder_api_call = requests.post(create_folder_endpoint, json=folder_data, headers=current_headers) + create_folder_endpoint = ( + CLOWDER_V2 + "api/v2/datasets/" + dataset_id + "/folders" + ) + folder_api_call = requests.post( + create_folder_endpoint, json=folder_data, headers=current_headers + ) print("created folder") else: parent = folder_part print("this one already exists") - print('got folder parts') + print("got folder parts") + # gets all folders and subfolders in a dataset async def get_folder_and_subfolders(dataset_id, folder, current_headers): total_folders = [] if folder: - path_url = CLOWDER_V2 + 'api/v2/datasets/' + dataset_id + '/folders_and_files?folder_id=' + folder['id'] + path_url = ( + CLOWDER_V2 + + "api/v2/datasets/" + + dataset_id + + "/folders_and_files?folder_id=" + + folder["id"] + ) folder_result = requests.get(path_url, headers=current_headers) - folder_json_data = folder_result.json()['data'] + folder_json_data = folder_result.json()["data"] for data in folder_json_data: - if data['object_type'] == 'folder': + if data["object_type"] == "folder": total_folders.append(data) else: - path_url = CLOWDER_V2 + 'api/v2/datasets/' + dataset_id + '/folders' + path_url = CLOWDER_V2 + "api/v2/datasets/" + dataset_id + "/folders" folder_result = requests.get(path_url, headers=current_headers) - folder_json_data = folder_result.json()['data'] + folder_json_data = folder_result.json()["data"] for data in folder_json_data: total_folders.append(data) - print('we got the base level now') + print("we got the base level now") current_subfolders = [] for current_folder in total_folders: - subfolders = await get_folder_and_subfolders(dataset_id, current_folder, current_headers) + subfolders = await get_folder_and_subfolders( + dataset_id, current_folder, current_headers + ) current_subfolders += subfolders total_folders += current_subfolders return total_folders -async def create_folder_if_not_exists_or_get(folder, parent, dataset_v2, current_headers): - clowder_v2_folder_endpoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_v2 + '/folders' - current_dataset_folders = await get_folder_and_subfolders(dataset_id=dataset_v2,folder=None, current_headers=current_headers) + +async def create_folder_if_not_exists_or_get( + folder, parent, dataset_v2, current_headers +): + clowder_v2_folder_endpoint = ( + CLOWDER_V2 + "api/v2/datasets/" + dataset_v2 + "/folders" + ) + current_dataset_folders = await get_folder_and_subfolders( + dataset_id=dataset_v2, folder=None, current_headers=current_headers + ) current_folder_data = {"name": folder} found_folder = None if parent: current_folder_data["parent_folder"] = parent for each in current_dataset_folders: - if each['name'] == folder: + if each["name"] == folder: found_folder = each if not found_folder: response = requests.post( @@ -253,47 +292,58 @@ async def create_folder_if_not_exists_or_get(folder, parent, dataset_v2, current print("We just created", found_folder) return found_folder + async def add_folder_hierarchy(folder_hierarchy, dataset_v2, current_headers): - current_dataset_folders = await get_folder_and_subfolders(dataset_id=dataset_v2, folder=None, current_headers=current_headers) + current_dataset_folders = await get_folder_and_subfolders( + dataset_id=dataset_v2, folder=None, current_headers=current_headers + ) folder_json_data = current_dataset_folders - hierarchy_parts = folder_hierarchy.split('/') - hierarchy_parts.remove('') + hierarchy_parts = folder_hierarchy.split("/") + hierarchy_parts.remove("") current_parent = None for part in hierarchy_parts: - result = await create_folder_if_not_exists_or_get(part, current_parent, dataset_v2, current_headers=current_headers) + result = await create_folder_if_not_exists_or_get( + part, current_parent, dataset_v2, current_headers=current_headers + ) if result: - current_parent = result['id'] + current_parent = result["id"] + async def add_dataset_folders(dataset_v1, dataset_v2, current_headers): - dataset_folders_endpoint = CLOWDER_V1 + 'api/datasets/' + dataset_v1['id'] + '/folders?superAdmin=true' + dataset_folders_endpoint = ( + CLOWDER_V1 + "api/datasets/" + dataset_v1["id"] + "/folders?superAdmin=true" + ) dataset_folders = requests.get(dataset_folders_endpoint, headers=base_headers_v1) dataset_folders_json = dataset_folders.json() folder_names = [] for folder in dataset_folders_json: folder_names.append(folder["name"]) for folder in folder_names: - new = await add_folder_hierarchy(folder_hierarchy=folder, dataset_v2=dataset_v2, current_headers=current_headers) - print('added', folder) - + new = await add_folder_hierarchy( + folder_hierarchy=folder, + dataset_v2=dataset_v2, + current_headers=current_headers, + ) + print("added", folder) async def process_users( - fs: Minio = Depends(dependencies.get_fs), - es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), - rabbitmq_client: BlockingChannel = Depends(dependencies.get_rabbitmq), - ): + fs: Minio = Depends(dependencies.get_fs), + es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), + rabbitmq_client: BlockingChannel = Depends(dependencies.get_rabbitmq), +): print("We create a v2 admin user") NEW_ADMIN_KEY_V2 = create_admin_user() users_v1 = get_clowder_v1_users() for user_v1 in users_v1: print("migrating v1 user", user_v1) - id = user_v1['id'] - email = user_v1['email'] - firstName = user_v1['firstName'] - lastName = user_v1['lastName'] + id = user_v1["id"] + email = user_v1["email"] + firstName = user_v1["firstName"] + lastName = user_v1["lastName"] - id_provider = user_v1['identityProvider'] - if '[Local Account]' in user_v1['identityProvider']: + id_provider = user_v1["identityProvider"] + if "[Local Account]" in user_v1["identityProvider"]: # get the v2 users # create a user account in v2 with this username if email != "a@a.com": @@ -301,47 +351,86 @@ async def process_users( # TODO check if there is already a local user user_v2 = create_local_user(user_v1) user_v2_api_key = user_v2 - user_base_headers_v2 = {'X-API-key': user_v2_api_key} - user_headers_v2 = {**user_base_headers_v2, 'Content-type': 'application/json', - 'accept': 'application/json'} + user_base_headers_v2 = {"X-API-key": user_v2_api_key} + user_headers_v2 = { + **user_base_headers_v2, + "Content-type": "application/json", + "accept": "application/json", + } for dataset in user_v1_datasets: - print('creating a dataset in v2', dataset['id'], dataset['name']) - dataset_v2_id = await create_v2_dataset(user_base_headers_v2, dataset, email) - folders = await add_dataset_folders(dataset, dataset_v2_id, user_headers_v2) + print("creating a dataset in v2", dataset["id"], dataset["name"]) + dataset_v2_id = await create_v2_dataset( + user_base_headers_v2, dataset, email + ) + folders = await add_dataset_folders( + dataset, dataset_v2_id, user_headers_v2 + ) print("Created folders in new dataset") - all_dataset_folders = await get_folder_and_subfolders(dataset_id=dataset_v2_id, folder=None, current_headers=user_headers_v2) - dataset_files_endpoint = CLOWDER_V1 + 'api/datasets/' + dataset['id'] + '/files?=superAdmin=true' - r_files = requests.get(dataset_files_endpoint, headers=clowder_headers_v1, verify=False) + all_dataset_folders = await get_folder_and_subfolders( + dataset_id=dataset_v2_id, + folder=None, + current_headers=user_headers_v2, + ) + dataset_files_endpoint = ( + CLOWDER_V1 + + "api/datasets/" + + dataset["id"] + + "/files?=superAdmin=true" + ) + r_files = requests.get( + dataset_files_endpoint, headers=clowder_headers_v1, verify=False + ) files_result = r_files.json() for file in files_result: file_folder = None - file_id = file['id'] - filename = file['filename'] - if 'folders' in file: - file_folder = file['folders'] + file_id = file["id"] + filename = file["filename"] + if "folders" in file: + file_folder = file["folders"] # TODO download the file from v1 using api routes - v1_download_url = CLOWDER_V1 + 'api/files/' + file_id + '?superAdmin=true' - print('downloading file', filename) - download = requests.get(v1_download_url, headers=clowder_headers_v1) - with open(filename, 'wb') as f: + v1_download_url = ( + CLOWDER_V1 + "api/files/" + file_id + "?superAdmin=true" + ) + print("downloading file", filename) + download = requests.get( + v1_download_url, headers=clowder_headers_v1 + ) + with open(filename, "wb") as f: f.write(download.content) file_data = {"file": open(filename, "rb")} matching_folder = None if file_folder: for folder in all_dataset_folders: - if folder['name'] == file_folder['name']: + if folder["name"] == file_folder["name"]: matching_folder = folder if matching_folder: - upload_files = {"files":open(filename,'rb')} - dataset_file_upload_endpoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_v2_id + '/filesMultiple?folder_id=' + matching_folder['id'] - response = requests.post(dataset_file_upload_endpoint, files=upload_files, - headers=user_base_headers_v2) + upload_files = {"files": open(filename, "rb")} + dataset_file_upload_endpoint = ( + CLOWDER_V2 + + "api/v2/datasets/" + + dataset_v2_id + + "/filesMultiple?folder_id=" + + matching_folder["id"] + ) + response = requests.post( + dataset_file_upload_endpoint, + files=upload_files, + headers=user_base_headers_v2, + ) else: - dataset_file_upload_endpoint = CLOWDER_V2 + 'api/v2/datasets/' + dataset_v2_id + '/files' - response = requests.post(dataset_file_upload_endpoint, files=file_data, - headers=user_base_headers_v2) + dataset_file_upload_endpoint = ( + CLOWDER_V2 + + "api/v2/datasets/" + + dataset_v2_id + + "/files" + ) + response = requests.post( + dataset_file_upload_endpoint, + files=file_data, + headers=user_base_headers_v2, + ) if response.status_code == 200: result = response.json() print("added file", result) @@ -350,11 +439,10 @@ async def process_users( except Exception as e: print("could not delete locally downloaded file") print(e) - print('done with file upload') + print("done with file upload") else: print("not a local account, not migrated at this time") -asyncio.run(process_users()) - +asyncio.run(process_users()) From 54747cace1d49ac09630480c0c599e94df3ba49e Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 3 Jun 2024 13:38:22 -0500 Subject: [PATCH 30/91] sample .env --- scripts/migration/.env | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 scripts/migration/.env diff --git a/scripts/migration/.env b/scripts/migration/.env new file mode 100644 index 000000000..389573fea --- /dev/null +++ b/scripts/migration/.env @@ -0,0 +1,4 @@ +CLOWDER_V1=http://0.0.0.0:9002/ +ADMIN_KEY_V1=YOUR_KEY_HERE +CLOWDER_V2=http://0.0.0.0:8000/ +ADMIN_KEY_V2=ANOTHER_KEY_HERE \ No newline at end of file From 55ff21aafe02b411b8217a23db23a0c88a320a6a Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 3 Jun 2024 13:40:41 -0500 Subject: [PATCH 31/91] formatting --- backend/app/keycloak_auth.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/app/keycloak_auth.py b/backend/app/keycloak_auth.py index 4c89d59a8..01cc1c9f7 100644 --- a/backend/app/keycloak_auth.py +++ b/backend/app/keycloak_auth.py @@ -318,7 +318,9 @@ async def get_current_user_id(identity: Json = Depends(get_token)) -> str: return keycloak_id -async def create_user(email: str, password: str, firstName: str, lastName: str, temporary: bool = False): +async def create_user( + email: str, password: str, firstName: str, lastName: str, temporary: bool = False +): """Create a user in Keycloak.""" keycloak_admin = KeycloakAdmin( server_url=settings.auth_server_url, From 851503e36d2208b95014380dd48460a336bff728 Mon Sep 17 00:00:00 2001 From: toddn Date: Sat, 13 Jul 2024 11:35:35 -0500 Subject: [PATCH 32/91] refactoring migrate_user this might need to be modified for CILogon, but this will work for local users --- scripts/migration/migrate_users.py | 208 ++++++++++++++++------------- 1 file changed, 114 insertions(+), 94 deletions(-) diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate_users.py index 781105377..0eb22fbd5 100644 --- a/scripts/migration/migrate_users.py +++ b/scripts/migration/migrate_users.py @@ -327,119 +327,139 @@ async def add_dataset_folders(dataset_v1, dataset_v2, current_headers): print("added", folder) -async def process_users( +async def process_user( + user_v1, fs: Minio = Depends(dependencies.get_fs), es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), rabbitmq_client: BlockingChannel = Depends(dependencies.get_rabbitmq), ): - print("We create a v2 admin user") - NEW_ADMIN_KEY_V2 = create_admin_user() - users_v1 = get_clowder_v1_users() - for user_v1 in users_v1: - print("migrating v1 user", user_v1) - id = user_v1["id"] - email = user_v1["email"] - firstName = user_v1["firstName"] - lastName = user_v1["lastName"] - - id_provider = user_v1["identityProvider"] - if "[Local Account]" in user_v1["identityProvider"]: - # get the v2 users - # create a user account in v2 with this username - if email != "a@a.com": - user_v1_datasets = get_clowder_v1_user_datasets(user_id=id) - # TODO check if there is already a local user - user_v2 = create_local_user(user_v1) - user_v2_api_key = user_v2 - user_base_headers_v2 = {"X-API-key": user_v2_api_key} - user_headers_v2 = { - **user_base_headers_v2, - "Content-type": "application/json", - "accept": "application/json", - } - for dataset in user_v1_datasets: - print("creating a dataset in v2", dataset["id"], dataset["name"]) - dataset_v2_id = await create_v2_dataset( - user_base_headers_v2, dataset, email - ) - folders = await add_dataset_folders( - dataset, dataset_v2_id, user_headers_v2 - ) - print("Created folders in new dataset") - - all_dataset_folders = await get_folder_and_subfolders( - dataset_id=dataset_v2_id, - folder=None, - current_headers=user_headers_v2, - ) - dataset_files_endpoint = ( + print("migrating v1 user", user_v1) + id = user_v1["id"] + email = user_v1["email"] + firstName = user_v1["firstName"] + lastName = user_v1["lastName"] + + id_provider = user_v1["identityProvider"] + if "[Local Account]" in user_v1["identityProvider"]: + # get the v2 users + # create a user account in v2 with this username + if email != "a@a.com": + user_v1_datasets = get_clowder_v1_user_datasets(user_id=id) + # TODO check if there is already a local user + user_v2 = create_local_user(user_v1) + user_v2_api_key = user_v2 + user_base_headers_v2 = {"X-API-key": user_v2_api_key} + user_headers_v2 = { + **user_base_headers_v2, + "Content-type": "application/json", + "accept": "application/json", + } + for dataset in user_v1_datasets: + print("creating a dataset in v2", dataset["id"], dataset["name"]) + dataset_v2_id = await create_v2_dataset( + user_base_headers_v2, dataset, email + ) + folders = await add_dataset_folders( + dataset, dataset_v2_id, user_headers_v2 + ) + print("Created folders in new dataset") + + all_dataset_folders = await get_folder_and_subfolders( + dataset_id=dataset_v2_id, + folder=None, + current_headers=user_headers_v2, + ) + dataset_files_endpoint = ( CLOWDER_V1 + "api/datasets/" + dataset["id"] + "/files?=superAdmin=true" + ) + r_files = requests.get( + dataset_files_endpoint, headers=clowder_headers_v1, verify=False + ) + files_result = r_files.json() + for file in files_result: + file_folder = None + file_id = file["id"] + filename = file["filename"] + if "folders" in file: + file_folder = file["folders"] + # TODO download the file from v1 using api routes + v1_download_url = ( + CLOWDER_V1 + "api/files/" + file_id + "?superAdmin=true" ) - r_files = requests.get( - dataset_files_endpoint, headers=clowder_headers_v1, verify=False + print("downloading file", filename) + download = requests.get( + v1_download_url, headers=clowder_headers_v1 ) - files_result = r_files.json() - for file in files_result: - file_folder = None - file_id = file["id"] - filename = file["filename"] - if "folders" in file: - file_folder = file["folders"] - # TODO download the file from v1 using api routes - v1_download_url = ( - CLOWDER_V1 + "api/files/" + file_id + "?superAdmin=true" - ) - print("downloading file", filename) - download = requests.get( - v1_download_url, headers=clowder_headers_v1 - ) - with open(filename, "wb") as f: - f.write(download.content) - file_data = {"file": open(filename, "rb")} - matching_folder = None - if file_folder: - for folder in all_dataset_folders: - if folder["name"] == file_folder["name"]: - matching_folder = folder - if matching_folder: - upload_files = {"files": open(filename, "rb")} - dataset_file_upload_endpoint = ( + with open(filename, "wb") as f: + f.write(download.content) + file_data = {"file": open(filename, "rb")} + matching_folder = None + if file_folder: + for folder in all_dataset_folders: + if folder["name"] == file_folder["name"]: + matching_folder = folder + if matching_folder: + upload_files = {"files": open(filename, "rb")} + dataset_file_upload_endpoint = ( CLOWDER_V2 + "api/v2/datasets/" + dataset_v2_id + "/filesMultiple?folder_id=" + matching_folder["id"] - ) - response = requests.post( - dataset_file_upload_endpoint, - files=upload_files, - headers=user_base_headers_v2, - ) - - else: - dataset_file_upload_endpoint = ( + ) + response = requests.post( + dataset_file_upload_endpoint, + files=upload_files, + headers=user_base_headers_v2, + ) + + else: + dataset_file_upload_endpoint = ( CLOWDER_V2 + "api/v2/datasets/" + dataset_v2_id + "/files" - ) - response = requests.post( - dataset_file_upload_endpoint, - files=file_data, - headers=user_base_headers_v2, - ) - if response.status_code == 200: - result = response.json() - print("added file", result) - try: - os.remove(filename) - except Exception as e: - print("could not delete locally downloaded file") - print(e) - print("done with file upload") + ) + response = requests.post( + dataset_file_upload_endpoint, + files=file_data, + headers=user_base_headers_v2, + ) + if response.status_code == 200: + result = response.json() + print("added file", result) + try: + os.remove(filename) + except Exception as e: + print("could not delete locally downloaded file") + print(e) + print("done with file upload") + + +async def process_users( + fs: Minio = Depends(dependencies.get_fs), + es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), + rabbitmq_client: BlockingChannel = Depends(dependencies.get_rabbitmq), +): + print("We create a v2 admin user") + NEW_ADMIN_KEY_V2 = create_admin_user() + users_v1 = get_clowder_v1_users() + for user_v1 in users_v1: + print("migrating v1 user", user_v1) + id = user_v1["id"] + email = user_v1["email"] + firstName = user_v1["firstName"] + lastName = user_v1["lastName"] + + id_provider = user_v1["identityProvider"] + if "[Local Account]" in user_v1["identityProvider"]: + # get the v2 users + # create a user account in v2 with this username + await process_user(user_v1, fs=fs, es=es, rabbitmq_client=rabbitmq_client) + print("Migrated user", user_v1) else: print("not a local account, not migrated at this time") From 074154ecd502c6bdc2989134dba8f2a9c4235e66 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 15 Jul 2024 16:44:33 -0500 Subject: [PATCH 33/91] formatting --- scripts/migration/migrate_users.py | 59 ++++++++++++------------------ 1 file changed, 24 insertions(+), 35 deletions(-) diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate_users.py index 0eb22fbd5..f682f3c49 100644 --- a/scripts/migration/migrate_users.py +++ b/scripts/migration/migrate_users.py @@ -1,31 +1,25 @@ +import asyncio import os +from secrets import token_urlsafe -from bson import ObjectId -from faker import Faker -from dotenv import dotenv_values, load_dotenv +import nest_asyncio import requests -import asyncio -from pymongo import MongoClient from app.config import settings -from itsdangerous.url_safe import URLSafeSerializer -from secrets import token_urlsafe +from app.models.files import FileDB, FileDBViewList, FileOut, LocalFileIn, StorageType from app.routers.files import add_file_entry -from app.models.files import ( - FileOut, - FileDB, - FileDBViewList, - LocalFileIn, - StorageType, -) +from bson import ObjectId +from dotenv import dotenv_values, load_dotenv +from faker import Faker +from itsdangerous.url_safe import URLSafeSerializer from minio import Minio -import nest_asyncio +from pymongo import MongoClient nest_asyncio.apply() -from app.models.users import UserDB, UserOut, UserAPIKeyDB +from app import dependencies +from app.models.users import UserAPIKeyDB, UserDB, UserOut from beanie import init_beanie -from fastapi import FastAPI, APIRouter, Depends from elasticsearch import Elasticsearch -from app import dependencies +from fastapi import APIRouter, Depends, FastAPI from motor.motor_asyncio import AsyncIOMotorClient from pika.adapters.blocking_connection import BlockingChannel @@ -370,10 +364,10 @@ async def process_user( current_headers=user_headers_v2, ) dataset_files_endpoint = ( - CLOWDER_V1 - + "api/datasets/" - + dataset["id"] - + "/files?=superAdmin=true" + CLOWDER_V1 + + "api/datasets/" + + dataset["id"] + + "/files?=superAdmin=true" ) r_files = requests.get( dataset_files_endpoint, headers=clowder_headers_v1, verify=False @@ -387,12 +381,10 @@ async def process_user( file_folder = file["folders"] # TODO download the file from v1 using api routes v1_download_url = ( - CLOWDER_V1 + "api/files/" + file_id + "?superAdmin=true" + CLOWDER_V1 + "api/files/" + file_id + "?superAdmin=true" ) print("downloading file", filename) - download = requests.get( - v1_download_url, headers=clowder_headers_v1 - ) + download = requests.get(v1_download_url, headers=clowder_headers_v1) with open(filename, "wb") as f: f.write(download.content) file_data = {"file": open(filename, "rb")} @@ -404,11 +396,11 @@ async def process_user( if matching_folder: upload_files = {"files": open(filename, "rb")} dataset_file_upload_endpoint = ( - CLOWDER_V2 - + "api/v2/datasets/" - + dataset_v2_id - + "/filesMultiple?folder_id=" - + matching_folder["id"] + CLOWDER_V2 + + "api/v2/datasets/" + + dataset_v2_id + + "/filesMultiple?folder_id=" + + matching_folder["id"] ) response = requests.post( dataset_file_upload_endpoint, @@ -418,10 +410,7 @@ async def process_user( else: dataset_file_upload_endpoint = ( - CLOWDER_V2 - + "api/v2/datasets/" - + dataset_v2_id - + "/files" + CLOWDER_V2 + "api/v2/datasets/" + dataset_v2_id + "/files" ) response = requests.post( dataset_file_upload_endpoint, From 62d5b027d534eb544635bed2499d75a868b6bf78 Mon Sep 17 00:00:00 2001 From: toddn Date: Thu, 22 Aug 2024 09:35:26 -0500 Subject: [PATCH 34/91] rename --- scripts/migration/{migrate_users.py => migrate.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/migration/{migrate_users.py => migrate.py} (100%) diff --git a/scripts/migration/migrate_users.py b/scripts/migration/migrate.py similarity index 100% rename from scripts/migration/migrate_users.py rename to scripts/migration/migrate.py From ec56aa40f2a880e6392a91366bfeff601fdf4fcd Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Mon, 26 Aug 2024 09:31:21 -0500 Subject: [PATCH 35/91] migrate metadata definition; modify migration script (#1184) * migrate metadata definition; modify migration script * local migrate changes (#1186) --------- Co-authored-by: Todd Nicholson <40038535+tcnichol@users.noreply.github.com> --- .gitignore | 4 + scripts/migration/.env | 4 - scripts/migration/migrate.py | 685 ++++++++---------- .../migration/migrate_metadata_definitions.py | 168 ++++- 4 files changed, 458 insertions(+), 403 deletions(-) delete mode 100644 scripts/migration/.env diff --git a/.gitignore b/.gitignore index ce36a8619..6370e29a7 100644 --- a/.gitignore +++ b/.gitignore @@ -62,6 +62,7 @@ deployments/kubernetes/charts/clowder2/*clowder2-software-dev.yaml *secret*.yaml # Environments +**/*.env .env .venv env/ @@ -69,6 +70,9 @@ venv/ ENV/ env.bak/ venv.bak/ +*/.env +scripts/migration/.env + # Test folder contect except for test source code and metadata document. backend/app/tests/* diff --git a/scripts/migration/.env b/scripts/migration/.env deleted file mode 100644 index 389573fea..000000000 --- a/scripts/migration/.env +++ /dev/null @@ -1,4 +0,0 @@ -CLOWDER_V1=http://0.0.0.0:9002/ -ADMIN_KEY_V1=YOUR_KEY_HERE -CLOWDER_V2=http://0.0.0.0:8000/ -ADMIN_KEY_V2=ANOTHER_KEY_HERE \ No newline at end of file diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index f682f3c49..814e02901 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -1,457 +1,372 @@ -import asyncio import os -from secrets import token_urlsafe +from datetime import datetime -import nest_asyncio import requests -from app.config import settings -from app.models.files import FileDB, FileDBViewList, FileOut, LocalFileIn, StorageType -from app.routers.files import add_file_entry -from bson import ObjectId -from dotenv import dotenv_values, load_dotenv -from faker import Faker -from itsdangerous.url_safe import URLSafeSerializer -from minio import Minio -from pymongo import MongoClient - -nest_asyncio.apply() -from app import dependencies -from app.models.users import UserAPIKeyDB, UserDB, UserOut -from beanie import init_beanie -from elasticsearch import Elasticsearch -from fastapi import APIRouter, Depends, FastAPI -from motor.motor_asyncio import AsyncIOMotorClient -from pika.adapters.blocking_connection import BlockingChannel - -mongo_client = MongoClient("mongodb://localhost:27018", connect=False) -mongo_client_v2 = MongoClient("mongodb://localhost:27017", connect=False) -db = mongo_client["clowder"] -db_v2 = mongo_client_v2["clowder2"] -print("got the db") -v1_users = db["social.users"].find({}) -for u in v1_users: - print(u) - -app = FastAPI() - - -@app.on_event("startup") -async def start_db(): - client = AsyncIOMotorClient(str(settings.MONGODB_URL)) - await init_beanie( - database=getattr(client, settings.MONGO_DATABASE), - # Make sure to include all models. If one depends on another that is not in the list it is not clear which one is missing. - document_models=[ - FileDB, - FileDBViewList, - UserDB, - UserAPIKeyDB, - ], - recreate_views=True, - ) - - -asyncio.run(start_db()) -output_file = "new_users.txt" +from dotenv import dotenv_values +# Configuration and Constants +DEFAULT_PASSWORD = "Password123&" -fake = Faker() +# Get the current timestamp +timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") +OUTPUT_FILE = f"migrated_new_users_{timestamp}.log" +# Load environment variables path_to_env = os.path.join(os.getcwd(), ".env") -print(os.path.isfile(path_to_env)) config = dotenv_values(dotenv_path=path_to_env) CLOWDER_V1 = config["CLOWDER_V1"] ADMIN_KEY_V1 = config["ADMIN_KEY_V1"] - CLOWDER_V2 = config["CLOWDER_V2"] ADMIN_KEY_V2 = config["ADMIN_KEY_V2"] -# ADMIN_KEY_V2 = 'eyJ1c2VyIjoiYUBhLmNvbSIsImtleSI6IlU1dllaWnB4elNDREl1Q0xObDZ3TWcifQ.LRiLqSH0fJlFSObKrNz-qexkoHw' + base_headers_v1 = {"X-API-key": ADMIN_KEY_V1} +base_headers_v2 = {"X-API-key": ADMIN_KEY_V2} + clowder_headers_v1 = { **base_headers_v1, "Content-type": "application/json", "accept": "application/json", } -base_headers_v2 = {"X-API-key": ADMIN_KEY_V2} clowder_headers_v2 = { **base_headers_v2, "Content-type": "application/json", "accept": "application/json", } -TEST_DATASET_NAME = "Migration Test Dataset" -# TODO this is just for testing -DEFAULT_PASSWORD = "Password123&" +admin_user = { + "email": "admin@example.com", + "password": "admin", + "first_name": "admin", + "last_name": "admin", +} -def email_user_new_login(user): - print("login to the new clowder instance") +def email_user_new_login(user_email): + """Send an email to the user with the new login details.""" + print(f"Login to the new Clowder instance: {user_email}") def generate_user_api_key(user, password): - user_example = { - "email": user["email"], - "password": password, - "first_name": user["first_name"], - "last_name": user["last_name"], - } - login_endpoint = CLOWDER_V2 + "api/v2/login" - response = requests.post(login_endpoint, json=user_example) + """Generate an API key for a user.""" + login_endpoint = f"{CLOWDER_V2}/api/v2/login" + response = requests.post(login_endpoint, json=user) token = response.json().get("token") - current_headers = {"Authorization": "Bearer " + token} - auth = {"username": user["email"], "password": password} - api_key_endpoint = CLOWDER_V2 + "api/v2/users/keys?name=migration&mins=0" + current_headers = {"Authorization": f"Bearer {token}"} + + api_key_endpoint = f"{CLOWDER_V2}/api/v2/users/keys?name=migration&mins=0" result = requests.post(api_key_endpoint, headers=current_headers) - api_key = result.json() - return api_key + return result.json() def get_clowder_v1_users(): - endpoint = CLOWDER_V1 + "api/users" - print(base_headers_v1) - r = requests.get(endpoint, headers=clowder_headers_v1, verify=False) - return r.json() + """Retrieve all users from Clowder v1.""" + endpoint = f"{CLOWDER_V1}/api/users" + response = requests.get(endpoint, headers=base_headers_v1, verify=False) + return response.json() -def get_clowder_v2_users(): - endpoint = CLOWDER_V2 + "api/v2/users" - r = requests.get(endpoint, headers=base_headers_v2, verify=False) - return r.json() +def get_clowder_v1_user_datasets(user_id): + """Retrieve datasets created by a specific user in Clowder v1.""" + # TODO what about pagination + endpoint = f"{CLOWDER_V1}/api/datasets?limit=0" + response = requests.get(endpoint, headers=clowder_headers_v1, verify=False) + return [dataset for dataset in response.json() if dataset["authorId"] == user_id] -def get_clowder_v2_user_by_name(username): - endpoint = CLOWDER_V2 + "api/v2/users/username/" + username - r = requests.get(endpoint, headers=base_headers_v2, verify=False) - return r.json() +def get_clowder_v1_user_spaces(user_v1): + endpoint = f"{CLOWDER_V1}/api/spaces" + response = requests.get(endpoint, headers=clowder_headers_v1, verify=False) + return [space for space in response.json() if space["creator"] == user_v1["id"]] -async def create_v2_dataset(headers, dataset, user_email): - print(dataset) - default_license_id = "CC BY" - dataset_name = dataset["name"] - dataset_description = dataset["description"] - dataset_in_v2_endpoint = ( - CLOWDER_V2 + f"api/v2/datasets?license_id={default_license_id}" - ) - # create dataset - dataset_example = { - "name": dataset_name, - "description": dataset_description, - } - response = requests.post( - dataset_in_v2_endpoint, headers=headers, json=dataset_example - ) - return response.json()["id"] +def get_clowder_v1_user_spaces_members(space_id): + endpoint = f"{CLOWDER_V1}/api/spaces/{space_id}/users" + response = requests.get(endpoint, headers=clowder_headers_v1, verify=False) + return response.json() -def get_clowder_v1_user_datasets(user_id): - user_datasets = [] - endpoint = CLOWDER_V1 + "api/datasets?limit=0" - r = requests.get(endpoint, headers=base_headers_v1, verify=False) - request_json = r.json() - for dataset in request_json: - if dataset["authorId"] == user_id: - user_datasets.append(dataset) - return user_datasets +def get_clowder_v2_space_datasets(space_id): + endpoint = f"{CLOWDER_V1}/api/spaces/{space_id}/datasets" + response = requests.get(endpoint, headers=clowder_headers_v1, verify=False) + return response.json() + + +def share_dataset_with_group(group_id, dataset, headers): + endpoint = f"{CLOWDER_V2}/authorizations/datasets/{dataset['id']}/group_role/{group_id}/viewer" + response = requests.get(endpoint, headers=headers, verify=False) + return response.json() + + +def add_v1_space_members_to_v2_group(space, group_id, headers): + space_members = get_clowder_v1_user_spaces_members(space["id"]) + for member in space_members: + member_email = member["email"] + endpoint = f"{CLOWDER_V2}/api/v2/groups/{group_id}/add/{member_email}" + response = requests.post( + endpoint, + headers=headers, + ) def create_local_user(user_v1): - first_name = user_v1["firstName"] - last_name = user_v1["lastName"] - email = user_v1["email"] - # password = fake.password(20) - password = "Password123&" + """Create a local user in Clowder v2 if they don't already exist, and generate an API key.""" + # Search for the user by email + search_endpoint = f"{CLOWDER_V2}/api/v2/users/search" + search_params = {"text": user_v1["email"]} + search_response = requests.get( + search_endpoint, headers=clowder_headers_v2, params=search_params + ) + + # Check if user already exists + if search_response.status_code == 200: + search_data = search_response.json() + if search_data.get("metadata", {}).get("total_count", 0) > 0: + for existing_user in search_response.json().get("data", []): + if existing_user.get("email") == user_v1["email"]: + print(f"User {user_v1['email']} already exists in Clowder v2.") + return generate_user_api_key( + user_v1, DEFAULT_PASSWORD + ) # Return the existing user's API key + + # User does not exist, proceed to create a new user user_json = { - "email": email, - "password": password, - "first_name": first_name, - "last_name": last_name, + "email": user_v1["email"], + "password": DEFAULT_PASSWORD, + "first_name": user_v1["firstName"], + "last_name": user_v1["lastName"], } - response = requests.post(f"{CLOWDER_V2}api/v2/users", json=user_json) - email_user_new_login(email) - api_key = generate_user_api_key(user_json, DEFAULT_PASSWORD) - # api_key = 'aZM2QXJ_lvw_5FKNUB89Vg' - print("Local user created and api key generated") - if os.path.exists(output_file): - print("it exists.") + + # Create the user + create_user_response = requests.post(f"{CLOWDER_V2}/api/v2/users", json=user_json) + if create_user_response.status_code == 200: + print(f"Created user {user_v1['email']} in Clowder v2.") + email_user_new_login(user_v1["email"]) + + # Generate and return API key for the new user + api_key = generate_user_api_key(user_json, DEFAULT_PASSWORD) + with open(OUTPUT_FILE, "a") as f: + f.write(f"{user_v1['email']},{DEFAULT_PASSWORD},{api_key}\n") + return api_key else: - f = open(output_file, "x") - with open(output_file, "a") as f: - entry = email + "," + password + "," + api_key + "\n" - f.write(entry) - return api_key + print( + f"Failed to create user {user_v1['email']}. Status code: {create_user_response.status_code}" + ) + return None def create_admin_user(): - user_json = { - "email": "a@a.com", - "password": "admin", - "first_name": "aa", - "last_name": "aa", - } - response = requests.post(f"{CLOWDER_V2}api/v2/users", json=user_json) - api_key = generate_user_api_key(user_json, "admin") - return api_key - - -async def add_folder_entry_to_dataset(dataset_id, folder_name, current_headers): - current_dataset_folders = [] - dataset_folder_url = CLOWDER_V2 + "api/v2/datasets/" + dataset_id + "/folders" - response = requests.get(dataset_folder_url, headers=current_headers) - response_json = response.json() - existing_folder_names = dict() - if "data" in response_json: - existing_folders = response_json["data"] - for existing_folder in existing_folders: - existing_folder_names[existing_folder["name"]] = existing_folder["id"] - if folder_name.startswith("/"): - folder_name = folder_name.lstrip("/") - folder_parts = folder_name.split("/") - parent = None - for folder_part in folder_parts: - folder_data = {"name": folder_part} - # TODO create or get folder - if folder_part not in existing_folder_names: - create_folder_endpoint = ( - CLOWDER_V2 + "api/v2/datasets/" + dataset_id + "/folders" - ) - folder_api_call = requests.post( - create_folder_endpoint, json=folder_data, headers=current_headers - ) - print("created folder") - else: - parent = folder_part - print("this one already exists") - print("got folder parts") - - -# gets all folders and subfolders in a dataset -async def get_folder_and_subfolders(dataset_id, folder, current_headers): - total_folders = [] - if folder: - path_url = ( - CLOWDER_V2 - + "api/v2/datasets/" - + dataset_id - + "/folders_and_files?folder_id=" - + folder["id"] - ) - folder_result = requests.get(path_url, headers=current_headers) - folder_json_data = folder_result.json()["data"] - for data in folder_json_data: - if data["object_type"] == "folder": - total_folders.append(data) - else: - path_url = CLOWDER_V2 + "api/v2/datasets/" + dataset_id + "/folders" - folder_result = requests.get(path_url, headers=current_headers) - folder_json_data = folder_result.json()["data"] - for data in folder_json_data: - total_folders.append(data) - print("we got the base level now") - current_subfolders = [] - for current_folder in total_folders: - subfolders = await get_folder_and_subfolders( - dataset_id, current_folder, current_headers - ) - current_subfolders += subfolders - total_folders += current_subfolders - return total_folders + """Create an admin user and return the API key.""" + requests.post(f"{CLOWDER_V2}/api/v2/users", json=admin_user) + return generate_user_api_key(admin_user, admin_user["password"]) -async def create_folder_if_not_exists_or_get( - folder, parent, dataset_v2, current_headers -): - clowder_v2_folder_endpoint = ( - CLOWDER_V2 + "api/v2/datasets/" + dataset_v2 + "/folders" - ) - current_dataset_folders = await get_folder_and_subfolders( - dataset_id=dataset_v2, folder=None, current_headers=current_headers +def create_v2_dataset(dataset, headers): + """Create a dataset in Clowder v2.""" + # TODO: GET correct license + dataset_in_v2_endpoint = f"{CLOWDER_V2}/api/v2/datasets?license_id=CC BY" + dataset_example = { + "name": dataset["name"], + "description": dataset["description"], + } + response = requests.post( + dataset_in_v2_endpoint, headers=headers, json=dataset_example ) - current_folder_data = {"name": folder} - found_folder = None - if parent: - current_folder_data["parent_folder"] = parent - for each in current_dataset_folders: - if each["name"] == folder: - found_folder = each - if not found_folder: - response = requests.post( - f"{CLOWDER_V2}api/v2/datasets/{dataset_v2}/folders", - json=current_folder_data, - headers=current_headers, - ) - found_folder = response.json() - print("We just created", found_folder) - return found_folder + return response.json()["id"] -async def add_folder_hierarchy(folder_hierarchy, dataset_v2, current_headers): - current_dataset_folders = await get_folder_and_subfolders( - dataset_id=dataset_v2, folder=None, current_headers=current_headers - ) - folder_json_data = current_dataset_folders +def create_v2_group(space, headers): + group = {"name": space["name"], "description": space["description"]} + group_in_v2_endpoint = f"{CLOWDER_V2}/api/v2/groups" + response = requests.post(group_in_v2_endpoint, json=group, headers=headers) + return response.json()["id"] + + +def add_folder_hierarchy(folder_hierarchy, dataset_v2, headers): + """Add folder hierarchy to a dataset in Clowder v2.""" hierarchy_parts = folder_hierarchy.split("/") - hierarchy_parts.remove("") current_parent = None for part in hierarchy_parts: - result = await create_folder_if_not_exists_or_get( - part, current_parent, dataset_v2, current_headers=current_headers + result = create_folder_if_not_exists_or_get( + part, current_parent, dataset_v2, headers ) if result: current_parent = result["id"] -async def add_dataset_folders(dataset_v1, dataset_v2, current_headers): - dataset_folders_endpoint = ( - CLOWDER_V1 + "api/datasets/" + dataset_v1["id"] + "/folders?superAdmin=true" +def create_folder_if_not_exists_or_get(folder, parent, dataset_v2, headers): + """Create a folder if it does not exist or return the existing folder.""" + current_folders = get_folder_and_subfolders(dataset_v2, headers) + folder_data = ( + {"name": folder, "parent_folder": parent} if parent else {"name": folder} + ) + + for existing_folder in current_folders: + if existing_folder["name"] == folder: + return existing_folder + + response = requests.post( + f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2}/folders", + json=folder_data, + headers=headers, ) - dataset_folders = requests.get(dataset_folders_endpoint, headers=base_headers_v1) - dataset_folders_json = dataset_folders.json() - folder_names = [] - for folder in dataset_folders_json: - folder_names.append(folder["name"]) - for folder in folder_names: - new = await add_folder_hierarchy( - folder_hierarchy=folder, - dataset_v2=dataset_v2, - current_headers=current_headers, + return response.json() + + +def get_folder_and_subfolders(dataset_id, headers): + """Retrieve all folders and subfolders in a dataset.""" + endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_id}/folders_and_files" + response = requests.get(endpoint, headers=headers) + return [ + folder + for folder in response.json().get("data", []) + if folder["object_type"] == "folder" + ] + + +def add_dataset_folders(dataset_v1, dataset_v2, headers): + """Add folders from a Clowder v1 dataset to a Clowder v2 dataset.""" + endpoint = f"{CLOWDER_V1}/api/datasets/{dataset_v1['id']}/folders?superAdmin=true" + folders = requests.get(endpoint, headers=clowder_headers_v1).json() + + for folder in folders: + add_folder_hierarchy(folder["name"], dataset_v2, headers) + + +def download_and_upload_file(file, all_dataset_folders, dataset_v2_id, headers_v2): + """Download a file from Clowder v1 and upload it to Clowder v2.""" + filename = file["filename"] + file_id = file["id"] + file_folder = file.get("folders", None) + + # Download the file from Clowder v1 + v1_download_url = f"{CLOWDER_V1}/api/files/{file_id}?superAdmin=true" + print(f"Downloading file: {filename}") + download_response = requests.get(v1_download_url, headers=clowder_headers_v1) + + with open(filename, "wb") as f: + f.write(download_response.content) + + # Determine the correct folder in Clowder v2 for the upload + matching_folder = None + if file_folder: + matching_folder = next( + ( + folder + for folder in all_dataset_folders + if folder["name"] == file_folder["name"] + ), + None, ) - print("added", folder) - - -async def process_user( - user_v1, - fs: Minio = Depends(dependencies.get_fs), - es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), - rabbitmq_client: BlockingChannel = Depends(dependencies.get_rabbitmq), -): - print("migrating v1 user", user_v1) - id = user_v1["id"] - email = user_v1["email"] - firstName = user_v1["firstName"] - lastName = user_v1["lastName"] - - id_provider = user_v1["identityProvider"] - if "[Local Account]" in user_v1["identityProvider"]: - # get the v2 users - # create a user account in v2 with this username - if email != "a@a.com": - user_v1_datasets = get_clowder_v1_user_datasets(user_id=id) - # TODO check if there is already a local user - user_v2 = create_local_user(user_v1) - user_v2_api_key = user_v2 - user_base_headers_v2 = {"X-API-key": user_v2_api_key} - user_headers_v2 = { - **user_base_headers_v2, - "Content-type": "application/json", - "accept": "application/json", - } - for dataset in user_v1_datasets: - print("creating a dataset in v2", dataset["id"], dataset["name"]) - dataset_v2_id = await create_v2_dataset( - user_base_headers_v2, dataset, email - ) - folders = await add_dataset_folders( - dataset, dataset_v2_id, user_headers_v2 - ) - print("Created folders in new dataset") - all_dataset_folders = await get_folder_and_subfolders( - dataset_id=dataset_v2_id, - folder=None, - current_headers=user_headers_v2, - ) - dataset_files_endpoint = ( - CLOWDER_V1 - + "api/datasets/" - + dataset["id"] - + "/files?=superAdmin=true" - ) - r_files = requests.get( - dataset_files_endpoint, headers=clowder_headers_v1, verify=False - ) - files_result = r_files.json() - for file in files_result: - file_folder = None - file_id = file["id"] - filename = file["filename"] - if "folders" in file: - file_folder = file["folders"] - # TODO download the file from v1 using api routes - v1_download_url = ( - CLOWDER_V1 + "api/files/" + file_id + "?superAdmin=true" - ) - print("downloading file", filename) - download = requests.get(v1_download_url, headers=clowder_headers_v1) - with open(filename, "wb") as f: - f.write(download.content) - file_data = {"file": open(filename, "rb")} - matching_folder = None - if file_folder: - for folder in all_dataset_folders: - if folder["name"] == file_folder["name"]: - matching_folder = folder - if matching_folder: - upload_files = {"files": open(filename, "rb")} - dataset_file_upload_endpoint = ( - CLOWDER_V2 - + "api/v2/datasets/" - + dataset_v2_id - + "/filesMultiple?folder_id=" - + matching_folder["id"] - ) - response = requests.post( - dataset_file_upload_endpoint, - files=upload_files, - headers=user_base_headers_v2, - ) - - else: - dataset_file_upload_endpoint = ( - CLOWDER_V2 + "api/v2/datasets/" + dataset_v2_id + "/files" - ) - response = requests.post( - dataset_file_upload_endpoint, - files=file_data, - headers=user_base_headers_v2, - ) - if response.status_code == 200: - result = response.json() - print("added file", result) - try: - os.remove(filename) - except Exception as e: - print("could not delete locally downloaded file") - print(e) - print("done with file upload") - - -async def process_users( - fs: Minio = Depends(dependencies.get_fs), - es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), - rabbitmq_client: BlockingChannel = Depends(dependencies.get_rabbitmq), -): - print("We create a v2 admin user") - NEW_ADMIN_KEY_V2 = create_admin_user() - users_v1 = get_clowder_v1_users() - for user_v1 in users_v1: - print("migrating v1 user", user_v1) - id = user_v1["id"] - email = user_v1["email"] - firstName = user_v1["firstName"] - lastName = user_v1["lastName"] - - id_provider = user_v1["identityProvider"] - if "[Local Account]" in user_v1["identityProvider"]: - # get the v2 users - # create a user account in v2 with this username - await process_user(user_v1, fs=fs, es=es, rabbitmq_client=rabbitmq_client) - print("Migrated user", user_v1) + # Upload the file to Clowder v2 + dataset_file_upload_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/files" + if matching_folder: + dataset_file_upload_endpoint += f"Multiple?folder_id={matching_folder['id']}" - else: - print("not a local account, not migrated at this time") + with open(filename, "rb") as file_data: + response = requests.post( + dataset_file_upload_endpoint, files={"file": file_data}, headers=headers_v2 + ) + + if response.status_code == 200: + print(f"Uploaded file: {filename} to dataset {dataset_v2_id}") + + # Clean up the local file after upload + try: + os.remove(filename) + except Exception as e: + print(f"Could not delete locally downloaded file: {filename}") + print(e) + print(f"Completed upload for file: {filename}") + + +def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): + """Process user resources from Clowder v1 to Clowder v2.""" + user_v1_datasets = get_clowder_v1_user_datasets(user_id=user_v1["id"]) + user_v2_api_key = create_local_user(user_v1) + USER_MAP[user_v1["id"]] = user_v2_api_key + user_headers_v2 = { + "x-api-key": user_v2_api_key, + "content-type": "application/json", + "accept": "application/json", + } + + for dataset in user_v1_datasets: + print(f"Creating dataset in v2: {dataset['id']} - {dataset['name']}") + dataset_v2_id = create_v2_dataset(dataset, user_headers_v2) + DATASET_MAP[dataset["id"]] = dataset_v2_id + add_dataset_folders(dataset, dataset_v2_id, user_headers_v2) + print("Created folders in the new dataset") + + all_dataset_folders = get_folder_and_subfolders(dataset_v2_id, user_headers_v2) + # Retrieve files for the dataset in Clowder v1 + dataset_files_endpoint = ( + f"{CLOWDER_V1}/api/datasets/{dataset['id']}/files?superAdmin=true" + ) + files_response = requests.get( + dataset_files_endpoint, headers=clowder_headers_v1, verify=False + ) + files_result = files_response.json() -asyncio.run(process_users()) + for file in files_result: + download_and_upload_file( + file, all_dataset_folders, dataset_v2_id, user_headers_v2 + ) + return [USER_MAP, DATASET_MAP] + + +if __name__ == "__main__": + # users_v1 = get_clowder_v1_users() + USER_MAP = {} + DATASET_MAP = {} + users_v1 = [ + { + "@context": { + "firstName": "http://schema.org/Person/givenName", + "lastName": "http://schema.org/Person/familyName", + "email": "http://schema.org/Person/email", + "affiliation": "http://schema.org/Person/affiliation", + }, + "id": "576313ce1407b25fe19fc381", + "firstName": "Chen", + "lastName": "Wang", + "fullName": "Chen Wang", + "email": "cwang138-clowder2@illinois.edu", + "avatar": "http://www.gravatar.com/avatar/2f97a52f2214949c4172d7fb796f173e?d=404", + "profile": {}, + "identityProvider": "Chen Wang (cwang138@illinois.edu) [Local Account]", + } + ] + users_v1 = get_clowder_v1_users() + for user_v1 in users_v1: + if ( + "[Local Account]" in user_v1["identityProvider"] + and user_v1["email"] != admin_user["email"] + ): + [USER_MAP, DATASET_MAP] = process_user_and_resources( + user_v1, USER_MAP, DATASET_MAP + ) + print(f"Migrated user {user_v1['email']} and associated resources.") + else: + print(f"Skipping user {user_v1['email']} as it is not a local account.") + print(f"Now migrating spaces") + for user_v1 in users_v1: + print(f"Migrating spaces of user {user_v1['email']}") + user_v1_spaces = get_clowder_v1_user_spaces(user_v1) + user_v2_api_key = USER_MAP[user_v1["id"]] + for space in user_v1_spaces: + group_id = create_v2_group(space, headers={"X-API-key": user_v2_api_key}) + add_v1_space_members_to_v2_group( + space, group_id, headers={"X-API-key": user_v2_api_key} + ) + space_datasets = get_clowder_v2_space_datasets(space["id"]) + for space_dataset in space_datasets: + dataset_v2_id = DATASET_MAP[space_dataset["id"]] + share_dataset_with_group( + group_id, space, headers={"X-API-key": user_v2_api_key} + ) + print(f"Migrated spaces of user {user_v1['email']}") + print("Migration complete.") diff --git a/scripts/migration/migrate_metadata_definitions.py b/scripts/migration/migrate_metadata_definitions.py index 53db9c7ab..4daf8a26c 100644 --- a/scripts/migration/migrate_metadata_definitions.py +++ b/scripts/migration/migrate_metadata_definitions.py @@ -1,18 +1,16 @@ import os -from faker import Faker -from dotenv import dotenv_values, load_dotenv -import requests +from datetime import datetime -fake = Faker() +import requests +from dotenv import dotenv_values -path_to_env = os.path.join(os.getcwd(), "scripts", "migration", ".env") -print(os.path.isfile(path_to_env)) +path_to_env = os.path.join(os.getcwd(), ".env") config = dotenv_values(dotenv_path=path_to_env) -CLOWDER_V1 = config["CLOWDER_V1"] +CLOWDER_V1_URL = config["CLOWDER_V1"] ADMIN_KEY_V1 = config["ADMIN_KEY_V1"] -CLOWDER_V2 = config["CLOWDER_V2"] +CLOWDER_V2_URL = config["CLOWDER_V2"] ADMIN_KEY_V2 = config["ADMIN_KEY_V2"] base_headers_v1 = {"X-API-key": ADMIN_KEY_V1} @@ -22,7 +20,7 @@ "accept": "application/json", } -base_headers_v2 = {"X-API-key": ADMIN_KEY_V2} +base_headers_v2 = {"x-api-key": ADMIN_KEY_V2} clowder_headers_v2 = { **base_headers_v2, "Content-type": "application/json", @@ -30,11 +28,153 @@ } -def get_clowder_v1_metadata_definitions(): - endpoint = CLOWDER_V1 + "api/metadata/definitions" - r = requests.get(endpoint, headers=base_headers_v1, verify=False) +def get_clowder_v1_metadata_definitions( + clowder_v1_url=CLOWDER_V1_URL, headers=clowder_headers_v1 +): + r = requests.get( + f"{clowder_v1_url}/api/metadata/definitions", headers=headers, verify=False + ) return r.json() -md_definitions = get_clowder_v1_metadata_definitions() -print("got them") +def map_widget_type(field_type): + if field_type == "string": + return "TextField" + elif field_type == "datetime": + return "DateTimePicker" + elif field_type in ["list", "listjquery", "annotation", "scientific_variable"]: + return "Select" + else: + return "TextField" # default widget type if not specified + + +def fetch_definitions(definitions_url, headers=base_headers_v1): + response = requests.get(definitions_url, headers=headers, verify=False) + + if response.status_code == 200: + data = response.json() + + # If the data is a simple list, return it + if isinstance(data, list): + return data + + # If the data is a dictionary, check if its values are lists + elif isinstance(data, dict): + flattened = [] + for key, value in data.items(): + if isinstance(value, list): + flattened.extend(value) + else: + print( + f"Value for key '{key}' is not a list. Skipping flattening for this key." + ) + return [] + return flattened + + else: + print("Unexpected structure in response data. Returning empty list.") + return [] + + else: + print( + f"Failed to fetch definitions from {definitions_url}. Status code: {response.status_code}" + ) + return [] + + +def transform_metadata_v1_to_v2(v1_metadata): + # Extracting data from v1 format + label = v1_metadata.get("json", {}).get("label", "") + uri = v1_metadata.get("json", {}).get("uri", "") + type_ = v1_metadata.get("json", {}).get("type", "string") + definitions_url = v1_metadata.get("json", {}).get("definitions_url", "") + + widget_type = map_widget_type(type_) + field_config = {"type": type_} + is_list = False + + if widget_type == "Select": + options = fetch_definitions(definitions_url) + field_config = {"type": "enum", "options": options} + + # Constructing v2 format + v2_metadata = { + "name": label, + "description": v1_metadata.get("json", {}).get( + "description", f"Metadata for {label}" + ), + "required_for_items": {"datasets": False, "files": False}, + "context": [{label.lower(): uri}], + "fields": [ + { + "name": label.lower(), + "list": is_list, + "widgetType": widget_type, + "config": field_config, + "required": False, + } + ], + } + + return v2_metadata + + +def post_metadata_definition( + v1_metadata, clowder_v2_url=CLOWDER_V2_URL, headers=clowder_headers_v2 +): + # Transform v1 to v2 + v2_metadata = transform_metadata_v1_to_v2(v1_metadata) + + # Post to Clowder v2 + response = requests.post( + f"{clowder_v2_url}/api/v2/metadata/definition", + json=v2_metadata, + headers=headers, + ) + + if response.status_code == 200: + return response.json().get("id") + else: + print( + f"Failed to post metadata definition. Status code: {response.status_code}" + ) + return None + + +def cleanup_metadata_definition( + definition_id, clowder_v2_url=CLOWDER_V2_URL, headers=clowder_headers_v2 +): + delete_url = f"{clowder_v2_url}/metadata/definition/{definition_id}" + response = requests.delete(delete_url, headers=headers) + + if response.status_code == 204: + print(f"Successfully deleted metadata definition with ID: {definition_id}") + else: + print( + f"Failed to delete metadata definition with ID: {definition_id}. Status code: {response.status_code}" + ) + + +if __name__ == "__main__": + v1_md_definitions = get_clowder_v1_metadata_definitions( + CLOWDER_V1_URL, base_headers_v1 + ) + posted_ids = [] + + for v1_md in v1_md_definitions: + definition_id = post_metadata_definition( + v1_md, CLOWDER_V2_URL, clowder_headers_v2 + ) + if definition_id: + posted_ids.append(definition_id) + + # Get the current timestamp + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = f"migrated_metadata_definition_{timestamp}.log" + with open(filename, "w") as file: + for id in posted_ids: + file.write(f"{id}\n") + + # Uncomment the lines below if you need to clean up (delete) the posted metadata definitions + # for id in posted_ids: + # cleanup_metadata_definition(id, CLOWDER_V2, clowder_headers_v2) From 8050460d475f5343b57969227a7507e24331f98e Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Tue, 27 Aug 2024 11:26:41 -0500 Subject: [PATCH 36/91] fix clowder user api key creation issue --- scripts/migration/migrate.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 814e02901..123bd8e35 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -48,9 +48,10 @@ def email_user_new_login(user_email): print(f"Login to the new Clowder instance: {user_email}") -def generate_user_api_key(user, password): +def generate_user_api_key(user, password=DEFAULT_PASSWORD): """Generate an API key for a user.""" login_endpoint = f"{CLOWDER_V2}/api/v2/login" + user.update({"password": password}) response = requests.post(login_endpoint, json=user) token = response.json().get("token") current_headers = {"Authorization": f"Bearer {token}"} @@ -104,7 +105,7 @@ def add_v1_space_members_to_v2_group(space, group_id, headers): for member in space_members: member_email = member["email"] endpoint = f"{CLOWDER_V2}/api/v2/groups/{group_id}/add/{member_email}" - response = requests.post( + requests.post( endpoint, headers=headers, ) @@ -127,7 +128,7 @@ def create_local_user(user_v1): if existing_user.get("email") == user_v1["email"]: print(f"User {user_v1['email']} already exists in Clowder v2.") return generate_user_api_key( - user_v1, DEFAULT_PASSWORD + existing_user, DEFAULT_PASSWORD ) # Return the existing user's API key # User does not exist, proceed to create a new user @@ -340,7 +341,7 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): "identityProvider": "Chen Wang (cwang138@illinois.edu) [Local Account]", } ] - users_v1 = get_clowder_v1_users() + # users_v1 = get_clowder_v1_users() for user_v1 in users_v1: if ( "[Local Account]" in user_v1["identityProvider"] @@ -352,7 +353,8 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): print(f"Migrated user {user_v1['email']} and associated resources.") else: print(f"Skipping user {user_v1['email']} as it is not a local account.") - print(f"Now migrating spaces") + + print("Now migrating spaces.") for user_v1 in users_v1: print(f"Migrating spaces of user {user_v1['email']}") user_v1_spaces = get_clowder_v1_user_spaces(user_v1) From c936760d4d479b5ff4cf6c1921584ae5a8580f59 Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 27 Aug 2024 13:07:12 -0500 Subject: [PATCH 37/91] fix user api key if user already exists --- scripts/migration/migrate.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 814e02901..78c48c5f3 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -51,6 +51,7 @@ def email_user_new_login(user_email): def generate_user_api_key(user, password): """Generate an API key for a user.""" login_endpoint = f"{CLOWDER_V2}/api/v2/login" + user['password'] = DEFAULT_PASSWORD response = requests.post(login_endpoint, json=user) token = response.json().get("token") current_headers = {"Authorization": f"Bearer {token}"} @@ -285,7 +286,10 @@ def download_and_upload_file(file, all_dataset_folders, dataset_v2_id, headers_v def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): """Process user resources from Clowder v1 to Clowder v2.""" user_v1_datasets = get_clowder_v1_user_datasets(user_id=user_v1["id"]) - user_v2_api_key = create_local_user(user_v1) + try: + user_v2_api_key = create_local_user(user_v1) + except Exception as e: + print(e) USER_MAP[user_v1["id"]] = user_v2_api_key user_headers_v2 = { "x-api-key": user_v2_api_key, From 86c84937f00f9a483dfb21739e45e07156dd8b1b Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 27 Aug 2024 13:22:38 -0500 Subject: [PATCH 38/91] switch headers and files --- scripts/migration/migrate.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 123bd8e35..e49580fc2 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -265,11 +265,11 @@ def download_and_upload_file(file, all_dataset_folders, dataset_v2_id, headers_v dataset_file_upload_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/files" if matching_folder: dataset_file_upload_endpoint += f"Multiple?folder_id={matching_folder['id']}" - - with open(filename, "rb") as file_data: - response = requests.post( - dataset_file_upload_endpoint, files={"file": file_data}, headers=headers_v2 - ) + file_exists = os.path.exists(filename) + # with open(filename, "rb") as file_data: + response = requests.post( + dataset_file_upload_endpoint, headers=headers_v2, files={"file": open(filename, "rb")} + ) if response.status_code == 200: print(f"Uploaded file: {filename} to dataset {dataset_v2_id}") @@ -288,6 +288,9 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): user_v1_datasets = get_clowder_v1_user_datasets(user_id=user_v1["id"]) user_v2_api_key = create_local_user(user_v1) USER_MAP[user_v1["id"]] = user_v2_api_key + base_user_headers_v2 = { + "x-api-key": user_v2_api_key + } user_headers_v2 = { "x-api-key": user_v2_api_key, "content-type": "application/json", @@ -314,7 +317,7 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): for file in files_result: download_and_upload_file( - file, all_dataset_folders, dataset_v2_id, user_headers_v2 + file, all_dataset_folders, dataset_v2_id, base_user_headers_v2 ) return [USER_MAP, DATASET_MAP] @@ -341,7 +344,7 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): "identityProvider": "Chen Wang (cwang138@illinois.edu) [Local Account]", } ] - # users_v1 = get_clowder_v1_users() + users_v1 = get_clowder_v1_users() for user_v1 in users_v1: if ( "[Local Account]" in user_v1["identityProvider"] From 2a52c7c581c8704805baac812256ed7125de4e80 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Thu, 29 Aug 2024 09:12:03 -0500 Subject: [PATCH 39/91] Migrate metadata (#1192) * dataset metadata is working * register migration extractor and successfully migrate machine metadata --- backend/app/models/metadata.py | 1 + scripts/migration/migrate.py | 260 ++++++++++++++---- .../migration/migrate_metadata_definitions.py | 25 +- 3 files changed, 230 insertions(+), 56 deletions(-) diff --git a/backend/app/models/metadata.py b/backend/app/models/metadata.py index 8d81c6311..7f25cd499 100644 --- a/backend/app/models/metadata.py +++ b/backend/app/models/metadata.py @@ -20,6 +20,7 @@ "int": int, "float": float, "str": str, + "string": str, "TextField": str, "bool": bool, # TODO figure out how to parse "yyyymmdd hh:mm:ssssssz" into datetime object diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index e49580fc2..144e84e84 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -4,6 +4,12 @@ import requests from dotenv import dotenv_values +from scripts.migration.migrate_metadata_definitions import ( + check_metadata_definition_exists, + get_clowder_v1_metadata_definitions, + post_metadata_definition, +) + # Configuration and Constants DEFAULT_PASSWORD = "Password123&" @@ -63,7 +69,7 @@ def generate_user_api_key(user, password=DEFAULT_PASSWORD): def get_clowder_v1_users(): """Retrieve all users from Clowder v1.""" - endpoint = f"{CLOWDER_V1}/api/users" + endpoint = f"{CLOWDER_V1}/api/users?superAdmin=true" response = requests.get(endpoint, headers=base_headers_v1, verify=False) return response.json() @@ -71,25 +77,25 @@ def get_clowder_v1_users(): def get_clowder_v1_user_datasets(user_id): """Retrieve datasets created by a specific user in Clowder v1.""" # TODO what about pagination - endpoint = f"{CLOWDER_V1}/api/datasets?limit=0" + endpoint = f"{CLOWDER_V1}/api/datasets?limit=0&superAdmin=true" response = requests.get(endpoint, headers=clowder_headers_v1, verify=False) return [dataset for dataset in response.json() if dataset["authorId"] == user_id] def get_clowder_v1_user_spaces(user_v1): - endpoint = f"{CLOWDER_V1}/api/spaces" + endpoint = f"{CLOWDER_V1}/api/spaces?superAdmin=true" response = requests.get(endpoint, headers=clowder_headers_v1, verify=False) return [space for space in response.json() if space["creator"] == user_v1["id"]] def get_clowder_v1_user_spaces_members(space_id): - endpoint = f"{CLOWDER_V1}/api/spaces/{space_id}/users" + endpoint = f"{CLOWDER_V1}/api/spaces/{space_id}/users?superAdmin=true" response = requests.get(endpoint, headers=clowder_headers_v1, verify=False) return response.json() def get_clowder_v2_space_datasets(space_id): - endpoint = f"{CLOWDER_V1}/api/spaces/{space_id}/datasets" + endpoint = f"{CLOWDER_V1}/api/spaces/{space_id}/datasets?superAdmin=true" response = requests.get(endpoint, headers=clowder_headers_v1, verify=False) return response.json() @@ -265,22 +271,172 @@ def download_and_upload_file(file, all_dataset_folders, dataset_v2_id, headers_v dataset_file_upload_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/files" if matching_folder: dataset_file_upload_endpoint += f"Multiple?folder_id={matching_folder['id']}" - file_exists = os.path.exists(filename) - # with open(filename, "rb") as file_data: response = requests.post( - dataset_file_upload_endpoint, headers=headers_v2, files={"file": open(filename, "rb")} + dataset_file_upload_endpoint, + headers=headers_v2, + files={"file": open(filename, "rb")}, ) - if response.status_code == 200: - print(f"Uploaded file: {filename} to dataset {dataset_v2_id}") - # Clean up the local file after upload try: os.remove(filename) except Exception as e: print(f"Could not delete locally downloaded file: {filename}") print(e) - print(f"Completed upload for file: {filename}") + + if response.status_code == 200: + print(f"Uploaded file: {filename} to dataset {dataset_v2_id}") + return response.json().get("id") + else: + print(f"Failed to upload file: {filename} to dataset {dataset_v2_id}") + + return None + + +def add_file_metadata(file_v1, file_v2_id, headers_v1, headers_v2): + # Get metadata from Clowder V1 + endpoint = f"{CLOWDER_V1}/api/files/{file_v1['id']}/metadata.jsonld?superAdmin=true" + metadata_v1 = requests.get(endpoint, headers=headers_v1).json() + + # Iterate through the metadata and post it to Clowder V2 + for metadata in metadata_v1: + # Extract and map each key-value pair from the metadata's content + if "content" in metadata: + for key, value in metadata["content"].items(): + # Define the payload to send to V2 + metadata_payload_v2 = { + "definition": key, + "content": metadata["content"], + } + + # Check if the metadata definition exists; + # if exists, post to user metadat; otherwise, post to machine metadata + v2_metadata_endpoint = ( + f"{CLOWDER_V2}/api/v2/files/{file_v2_id}/metadata" + ) + if check_metadata_definition_exists( + CLOWDER_V2, key, headers=headers_v2 + ): + response = requests.post( + v2_metadata_endpoint, + json=metadata_payload_v2, + headers=headers_v2, + ) + + if response.status_code != 200: + print(f"Failed to post file metadata to V2: {response.text}") + else: + print(f"Successfully posted file metadata to V2: {key}") + else: + if "agent" in metadata and "listener" not in metadata: + metadata["listener"] = { + "name": "migration", + "version": "1", + "description": "Migration of metadata from Clowder v1 to Clowder v2", + } + response = requests.post( + v2_metadata_endpoint, json=metadata, headers=headers_v2 + ) + + if response.status_code != 200: + print(f"Failed to post file metadata to V2: {response.text}") + else: + print("Successfully posted file machine metadata to V2") + break # machine metadata no need to iterate through all the keys + + +def add_dataset_metadata(dataset_v1, dataset_v2_id, headers_v1, headers_v2): + # Get metadata from Clowder V1 + endpoint = ( + f"{CLOWDER_V1}/api/datasets/{dataset_v1['id']}/metadata.jsonld?superAdmin=true" + ) + metadata_v1 = requests.get(endpoint, headers=headers_v1).json() + + # Iterate through the metadata and post it to Clowder V2 + for metadata in metadata_v1: + # Extract and map each key-value pair from the metadata's content + if "content" in metadata: + for key, value in metadata["content"].items(): + # Define the payload to send to V2 + metadata_payload_v2 = { + "definition": key, + "content": metadata["content"], + } + + # Check if the metadata definition exists; + # if exists, post to user metadat; otherwise, post to machine metadata + v2_metadata_endpoint = ( + f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/metadata" + ) + if check_metadata_definition_exists( + CLOWDER_V2, key, headers=headers_v2 + ): + response = requests.post( + v2_metadata_endpoint, + json=metadata_payload_v2, + headers=headers_v2, + ) + + if response.status_code != 200: + print(f"Failed to post dataset metadata to V2: {response.text}") + else: + print(f"Successfully posted dataset metadata to V2: {key}") + else: + if "agent" in metadata and "listener" not in metadata: + metadata["listener"] = { + "name": "migration", + "version": "1", + "description": "Migration of metadata from Clowder v1 to Clowder v2", + } + response = requests.post( + v2_metadata_endpoint, json=metadata, headers=headers_v2 + ) + + if response.status_code != 200: + print(f"Failed to post dataset metadata to V2: {response.text}") + else: + print("Successfully posted dataset machine metadata to V2") + break # machine metadata no need to iterate through all the keys + + +def register_migration_extractor(): + """Register the migration extractor in Clowder v2.""" + migration_extractor = { + "name": "migration", + "description": "Migration of metadata from Clowder v1 to Clowder v2", + "version": "1", + "author": "Clowder Devs", + } + + # check if migration extractor already exists + search_endpoint = f"{CLOWDER_V2}/api/v2/listeners/search" + search_params = {"text": migration_extractor["name"]} + search_response = requests.get( + search_endpoint, headers=clowder_headers_v2, params=search_params + ) + + # Check if extractor already exists + if search_response.status_code == 200: + search_data = search_response.json() + if search_data.get("metadata", {}).get("total_count", 0) > 0: + for existing_extractor in search_response.json().get("data", []): + if existing_extractor.get("name") == migration_extractor["name"]: + print( + f"Extractor {migration_extractor['name']} already exists in Clowder v2." + ) + return + + endpoint = f"{CLOWDER_V2}/api/v2/extractors" + response = requests.post( + endpoint, json=migration_extractor, headers=clowder_headers_v2 + ) + + if response.status_code == 200: + print("Successfully registered migration extractor in Clowder v2.") + else: + print( + f"Failed to register migration extractor in Clowder v2. Status code: {response.status_code}" + ) def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): @@ -288,9 +444,7 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): user_v1_datasets = get_clowder_v1_user_datasets(user_id=user_v1["id"]) user_v2_api_key = create_local_user(user_v1) USER_MAP[user_v1["id"]] = user_v2_api_key - base_user_headers_v2 = { - "x-api-key": user_v2_api_key - } + base_user_headers_v2 = {"x-api-key": user_v2_api_key} user_headers_v2 = { "x-api-key": user_v2_api_key, "content-type": "application/json", @@ -301,6 +455,7 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): print(f"Creating dataset in v2: {dataset['id']} - {dataset['name']}") dataset_v2_id = create_v2_dataset(dataset, user_headers_v2) DATASET_MAP[dataset["id"]] = dataset_v2_id + add_dataset_metadata(dataset, dataset_v2_id, base_headers_v1, user_headers_v2) add_dataset_folders(dataset, dataset_v2_id, user_headers_v2) print("Created folders in the new dataset") @@ -316,34 +471,33 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): files_result = files_response.json() for file in files_result: - download_and_upload_file( + file_v2_id = download_and_upload_file( file, all_dataset_folders, dataset_v2_id, base_user_headers_v2 ) + if file_v2_id is not None: + add_file_metadata(file, file_v2_id, clowder_headers_v1, user_headers_v2) + return [USER_MAP, DATASET_MAP] if __name__ == "__main__": - # users_v1 = get_clowder_v1_users() + ############################################################################################################## + # migrate metadata definition + v1_md_definitions = get_clowder_v1_metadata_definitions(CLOWDER_V1, base_headers_v1) + posted_ids = [] + for v1_md in v1_md_definitions: + definition_id = post_metadata_definition(v1_md, CLOWDER_V2, clowder_headers_v2) + if definition_id: + posted_ids.append(definition_id) + + ############################################################################################################## + # Register the migration extractor in Clowder v2 + register_migration_extractor() + + ############################################################################################################## + # migrate users and resources USER_MAP = {} DATASET_MAP = {} - users_v1 = [ - { - "@context": { - "firstName": "http://schema.org/Person/givenName", - "lastName": "http://schema.org/Person/familyName", - "email": "http://schema.org/Person/email", - "affiliation": "http://schema.org/Person/affiliation", - }, - "id": "576313ce1407b25fe19fc381", - "firstName": "Chen", - "lastName": "Wang", - "fullName": "Chen Wang", - "email": "cwang138-clowder2@illinois.edu", - "avatar": "http://www.gravatar.com/avatar/2f97a52f2214949c4172d7fb796f173e?d=404", - "profile": {}, - "identityProvider": "Chen Wang (cwang138@illinois.edu) [Local Account]", - } - ] users_v1 = get_clowder_v1_users() for user_v1 in users_v1: if ( @@ -357,21 +511,23 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): else: print(f"Skipping user {user_v1['email']} as it is not a local account.") - print("Now migrating spaces.") - for user_v1 in users_v1: - print(f"Migrating spaces of user {user_v1['email']}") - user_v1_spaces = get_clowder_v1_user_spaces(user_v1) - user_v2_api_key = USER_MAP[user_v1["id"]] - for space in user_v1_spaces: - group_id = create_v2_group(space, headers={"X-API-key": user_v2_api_key}) - add_v1_space_members_to_v2_group( - space, group_id, headers={"X-API-key": user_v2_api_key} - ) - space_datasets = get_clowder_v2_space_datasets(space["id"]) - for space_dataset in space_datasets: - dataset_v2_id = DATASET_MAP[space_dataset["id"]] - share_dataset_with_group( - group_id, space, headers={"X-API-key": user_v2_api_key} - ) - print(f"Migrated spaces of user {user_v1['email']}") + ############################################################################################################## + # migrate spaces + # print("Now migrating spaces.") + # for user_v1 in users_v1: + # print(f"Migrating spaces of user {user_v1['email']}") + # user_v1_spaces = get_clowder_v1_user_spaces(user_v1) + # user_v2_api_key = USER_MAP[user_v1["id"]] + # for space in user_v1_spaces: + # group_id = create_v2_group(space, headers={"X-API-key": user_v2_api_key}) + # add_v1_space_members_to_v2_group( + # space, group_id, headers={"X-API-key": user_v2_api_key} + # ) + # space_datasets = get_clowder_v2_space_datasets(space["id"]) + # for space_dataset in space_datasets: + # dataset_v2_id = DATASET_MAP[space_dataset["id"]] + # share_dataset_with_group( + # group_id, space, headers={"X-API-key": user_v2_api_key} + # ) + # print(f"Migrated spaces of user {user_v1['email']}") print("Migration complete.") diff --git a/scripts/migration/migrate_metadata_definitions.py b/scripts/migration/migrate_metadata_definitions.py index 4daf8a26c..0b096f940 100644 --- a/scripts/migration/migrate_metadata_definitions.py +++ b/scripts/migration/migrate_metadata_definitions.py @@ -84,8 +84,8 @@ def fetch_definitions(definitions_url, headers=base_headers_v1): def transform_metadata_v1_to_v2(v1_metadata): # Extracting data from v1 format - label = v1_metadata.get("json", {}).get("label", "") uri = v1_metadata.get("json", {}).get("uri", "") + label = uri.split("/")[-1] type_ = v1_metadata.get("json", {}).get("type", "string") definitions_url = v1_metadata.get("json", {}).get("definitions_url", "") @@ -104,10 +104,10 @@ def transform_metadata_v1_to_v2(v1_metadata): "description", f"Metadata for {label}" ), "required_for_items": {"datasets": False, "files": False}, - "context": [{label.lower(): uri}], + "context": [{label: uri}], "fields": [ { - "name": label.lower(), + "name": label, "list": is_list, "widgetType": widget_type, "config": field_config, @@ -144,7 +144,7 @@ def post_metadata_definition( def cleanup_metadata_definition( definition_id, clowder_v2_url=CLOWDER_V2_URL, headers=clowder_headers_v2 ): - delete_url = f"{clowder_v2_url}/metadata/definition/{definition_id}" + delete_url = f"{clowder_v2_url}/api/v2/metadata/definition/{definition_id}" response = requests.delete(delete_url, headers=headers) if response.status_code == 204: @@ -155,6 +155,23 @@ def cleanup_metadata_definition( ) +def check_metadata_definition_exists(clowder_v2_url, definition_name, headers): + # Construct the API endpoint URL + endpoint = f"{clowder_v2_url}/api/v2/metadata/definition/search/{definition_name}?skip=0&limit=100000" + + # Make the GET request to the API + response = requests.get(endpoint, headers=headers) + + if response.status_code == 200: + data = response.json() + # Check if the definition is present in the response + for item in data.get("data", []): + if item.get("name") == definition_name: + return True + + return False + + if __name__ == "__main__": v1_md_definitions = get_clowder_v1_metadata_definitions( CLOWDER_V1_URL, base_headers_v1 From 4e68f21e52e90a969ceb19a925e80d60adf9f197 Mon Sep 17 00:00:00 2001 From: Todd Nicholson <40038535+tcnichol@users.noreply.github.com> Date: Tue, 3 Sep 2024 13:14:54 -0500 Subject: [PATCH 40/91] 1187 migrate collections and spaces as v2 metadata. (#1190) * match other branch * adding new collection metadata * str not float * getting collections for a dataset in v1, fixing metadata for collections * posts collection name and id * adding routes for getting collections * making a method like the one in v1 for self and ancestors. it will be easier to build a collection hierarchy from this * sample json for mdata * posts collection name and id * building the data for collections * something works now * matching with other branch * methods for migrating collections as metadata * need to post it as metadata * change name * adding the metadata for collections * adding context url and right endpoint * getting spaces as well as collections * change name * remaning method * created v2 license based on v1 license details (#1193) Co-authored-by: Chen Wang * removing print statements * better error logging --------- Co-authored-by: Dipannita Co-authored-by: Chen Wang --- .../metadata/definitions/collectionv1.json | 54 ++++ scripts/migration/migrate.py | 286 +++++++++++++++++- 2 files changed, 339 insertions(+), 1 deletion(-) create mode 100644 scripts/metadata/definitions/collectionv1.json diff --git a/scripts/metadata/definitions/collectionv1.json b/scripts/metadata/definitions/collectionv1.json new file mode 100644 index 000000000..dddc397e3 --- /dev/null +++ b/scripts/metadata/definitions/collectionv1.json @@ -0,0 +1,54 @@ +{ + "name" : "Collection", + "description" : "Collection information from v1", + "required_for_items": { + "datasets": false, + "files": false + }, + "context" : [ + { + "collection_name" : "https://schema.org/colname", + "collection_id" : "https://schema.org/colid", + "parent_collection_name": "https://schema.org/parentcolname", + "parent_collection_id": "https://schema.org/parentcolid" + } + ], + "fields" : [ + { + "name" : "collection_name", + "list" : false, + "widgetType": "TextField", + "config": { + "type" : "str" + }, + "required" : false + }, + { + "name" : "collection_id", + "list" : false, + "widgetType": "TextField", + "config": { + "type" : "str" + }, + "required" : false + }, + { + "name" : "parent_collection_name", + "list" : false, + "widgetType": "TextField", + "config": { + "type" : "str" + }, + "required" : false + }, + { + "name" : "parent_collection_id", + "list" : false, + "widgetType": "TextField", + "config": { + "type" : "str" + }, + "required" : false + } + ] +} diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 144e84e84..ba9764346 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -117,6 +117,102 @@ def add_v1_space_members_to_v2_group(space, group_id, headers): ) +def get_clowder_v1_user_collections(headers, user_v1): + endpoint = f"{CLOWDER_V1}/api/collections" + response = requests.get(endpoint, headers=headers) + return [col for col in response.json() if col["authorId"] == user_v1["id"]] + + +def get_clowder_v1_dataset_collections(headers, user_v1, dataset_id): + matching_collections = [] + endpoint = f"{CLOWDER_V1}/api/collections/allCollections" + response = requests.get(endpoint, headers=headers) + user_collections = response.json() + for collection in user_collections: + collection_id = collection["id"] + collection_dataset_endpoint = ( + f"{CLOWDER_V1}/api/collections/{collection_id}/datasets" + ) + try: + dataset_response = requests.get( + collection_dataset_endpoint, headers=headers + ) + datasets = dataset_response.json() + for ds in datasets: + if ds["id"] == dataset_id: + matching_collections.append(collection) + except Exception as e: + print("Exception", e) + return matching_collections + + +def get_clowder_v1_collection(collection_id, headers): + endpoint = f"{CLOWDER_V1}/api/collections/{collection_id}" + response = requests.get(endpoint, headers=headers) + return response.json() + + +def get_clowder_v1_collections(collection_ids, headers): + collections = [] + for collection_id in collection_ids: + endpoint = f"{CLOWDER_V1}/api/collections/{collection_id}" + response = requests.get(endpoint, headers=headers) + collections.append(response.json()) + return collections + + +def get_clowder_v1_collection_self_and_ancestors( + collection_id, self_and_ancestors, headers +): + endpoint = f"{CLOWDER_V1}/api/collections/{collection_id}" + response = requests.get(endpoint, headers=headers) + self = response.json() + if self["id"] not in self_and_ancestors: + self_and_ancestors.append(self["id"]) + parents_entry = self["parent_collection_ids"] + parents_entry = parents_entry.lstrip("List(") + parents_entry = parents_entry.rstrip(")") + if parents_entry != "": + parents = parents_entry.split(",") + for parent in parents: + # replace empty space + parent = parent.lstrip(" ") + parent = parent.rstrip(" ") + if parent not in self_and_ancestors: + self_and_ancestors.append(parent) + for parent in parents: + parent = parent.lstrip(" ") + parent = parent.rstrip(" ") + if parent != "" and parent is not None: + current_self_and_ancestors = ( + get_clowder_v1_collection_self_and_ancestors( + parent, self_and_ancestors, headers=headers + ) + ) + for col_id in current_self_and_ancestors: + if col_id not in self_and_ancestors: + self_and_ancestors.append(col_id) + return self_and_ancestors + + +def get_clowder_v1_parent_collection_ids(current_collection_id, headers): + parents = [] + all_collections_v1_endpoint = ( + f"{CLOWDER_V1}/api/collections/allCollections?limit=0&showAll=true" + ) + response = requests.get(all_collections_v1_endpoint, headers=headers) + all_collections = response.json() + for collection in all_collections: + children_entry = collection["child_collection_ids"] + children_entry = children_entry.lstrip("List(") + children_entry = children_entry.rstrip(")") + child_ids = children_entry.split(",") + for child in child_ids: + if child == current_collection_id: + parents.append(collection["id"]) + return parents + + def create_local_user(user_v1): """Create a local user in Clowder v2 if they don't already exist, and generate an API key.""" # Search for the user by email @@ -169,10 +265,74 @@ def create_admin_user(): return generate_user_api_key(admin_user, admin_user["password"]) +def add_dataset_license(v1_license, headers): + """Create appropriate license (standard/custom) based on v1 license details""" + license_id = "CC-BY" + # standard licenses + if v1_license["license_type"] == "license2": + if ( + not v1_license["ccAllowCommercial"] + and not v1_license["ccAllowDerivative"] + and not v1_license["ccRequireShareAlike"] + ): + license_id = "CC BY-NC-ND" + elif ( + v1_license["ccAllowCommercial"] + and not v1_license["ccAllowDerivative"] + and not v1_license["ccRequireShareAlike"] + ): + license_id = "CC BY-ND" + elif ( + not v1_license["ccAllowCommercial"] + and v1_license["ccAllowDerivative"] + and not v1_license["ccRequireShareAlike"] + ): + license_id = "CC BY-NC" + elif ( + not v1_license["ccAllowCommercial"] + and v1_license["ccAllowDerivative"] + and v1_license["ccRequireShareAlike"] + ): + license_id = "CC BY-NC-SA" + elif ( + v1_license["ccAllowCommercial"] + and v1_license["ccAllowDerivative"] + and v1_license["ccRequireShareAlike"] + ): + license_id = "CC BY-SA" + elif ( + v1_license["ccAllowCommercial"] + and v1_license["ccAllowDerivative"] + and not v1_license["ccRequireShareAlike"] + ): + license_id = "CC BY" + elif v1_license["license_type"] == "license3": + license_id = "CCO Public Domain Dedication" + else: + # custom license + license_body = { + "name": v1_license["license_text"], + "url": v1_license["license_url"], + "holders": v1_license["holders"], + } + if license_body["url"] == "": + license_body["url"] = "https://dbpedia.org/page/All_rights_reserved" + license_v2_endpoint = f"{CLOWDER_V2}/api/v2/licenses?" + response = requests.post( + license_v2_endpoint, headers=headers, json=license_body + ) + print(response.json()) + license_id = response.json()["id"] + return license_id + + def create_v2_dataset(dataset, headers): """Create a dataset in Clowder v2.""" # TODO: GET correct license - dataset_in_v2_endpoint = f"{CLOWDER_V2}/api/v2/datasets?license_id=CC BY" + print("Creating dataset license in Clowder v2.") + v2_license_id = add_dataset_license(dataset["license"], headers) + + dataset_in_v2_endpoint = f"{CLOWDER_V2}/api/v2/datasets?license_id={v2_license_id}" dataset_example = { "name": dataset["name"], "description": dataset["description"], @@ -439,6 +599,101 @@ def register_migration_extractor(): ) +def add_children(collection_hierarchy_json, remaining_collections): + new_json = [] + new_remaining_collections = [] + for collection in remaining_collections: + collection_parents = collection["parent_collection_ids"] + current_collection_parents = [] + for entry in collection_hierarchy_json: + if entry["id"] in collection_parents: + current_collection_parents.append(entry) + print("We got the parents now") + if len(current_collection_parents) > 0: + current_collection_entry = { + "id": collection["id"], + "name": collection["name"], + "parents": current_collection_parents, + } + new_json.append(current_collection_entry) + else: + new_remaining_collections.append(collection) + return new_json, new_remaining_collections + + +def build_collection_hierarchy(collection_id, headers): + self_and_ancestors = get_clowder_v1_collection_self_and_ancestors( + collection_id=collection_id, self_and_ancestors=[], headers=headers + ) + self_and_ancestors_collections = get_clowder_v1_collections( + self_and_ancestors, headers=clowder_headers_v1 + ) + children = [] + remaining_collections = [] + for col in self_and_ancestors_collections: + parent_collection_ids = col["parent_collection_ids"] + parent_collection_ids = parent_collection_ids.lstrip("List(") + parent_collection_ids = parent_collection_ids.rstrip(")") + parent_collection_ids = parent_collection_ids.lstrip(" ") + parent_collection_ids = parent_collection_ids.rstrip(" ") + if parent_collection_ids == "": + root_col_entry = {"name": col["name"], "id": col["id"], "parents": []} + children.append(root_col_entry) + else: + remaining_collections.append(col) + + while len(remaining_collections) > 0: + children, remaining_collections = add_children(children, remaining_collections) + print("Now we are done") + return children + + +def build_collection_metadata_for_v1_dataset(dataset_id, user_v1, headers): + dataset_collections = get_clowder_v1_dataset_collections( + headers=headers, user_v1=user_v1, dataset_id=dataset_id + ) + return dataset_collections + + +def build_collection_space_metadata_for_v1_dataset(dataset, user_v1, headers): + dataset_id = dataset["id"] + dataset_collections = get_clowder_v1_dataset_collections( + headers=headers, user_v1=user_v1, dataset_id=dataset_id + ) + dataset_spaces = dataset["spaces"] + space_entries = [] + for space_id in dataset_spaces: + space_endpoint = f"{CLOWDER_V1}/api/spaces/{space_id}" + response = requests.get(space_endpoint, headers=headers) + space = response.json() + try: + space_entry = { + "id": space["id"], + "name": space["name"], + "creator": space["creator"], + } + space_entries.append(space_entry) + except Exception as e: + print(f"Error in getting space entry.") + print(e) + try: + space_entry = {"id": space["id"], "name": space["name"]} + space_entries.append(space_entry) + except Exception as e: + print(f"Error in getting space entry") + print(e) + collection_data = [] + for collection in dataset_collections: + collection_children = build_collection_hierarchy( + collection_id=collection["id"], headers=headers + ) + for child in collection_children: + collection_data.append(child) + metadata = {"spaces": space_entries, "collections": collection_data} + print(f"Got space and collection metadata from dataset {dataset_id}") + return metadata + + def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): """Process user resources from Clowder v1 to Clowder v2.""" user_v1_datasets = get_clowder_v1_user_datasets(user_id=user_v1["id"]) @@ -476,6 +731,35 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): ) if file_v2_id is not None: add_file_metadata(file, file_v2_id, clowder_headers_v1, user_headers_v2) + # posting the collection hierarchy as metadata + collection_space_metadata_dict = build_collection_space_metadata_for_v1_dataset( + dataset=dataset, user_v1=user_v1, headers=clowder_headers_v1 + ) + migration_extractor_collection_metadata = { + "listener": { + "name": "migration", + "version": "1", + "description": "Migration of metadata from Clowder v1 to Clowder v2", + }, + "context_url": "https://clowder.ncsa.illinois.edu/contexts/metadata.jsonld", + "content": collection_space_metadata_dict, + "contents": collection_space_metadata_dict, + } + v2_metadata_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/metadata" + response = requests.post( + v2_metadata_endpoint, + json=migration_extractor_collection_metadata, + headers=clowder_headers_v2, + ) + if response.status_code == 200: + print("Successfully added collection info as metadata in v2.") + else: + print( + f"Failed to add collection info as metadata in Clowder v2. Status code: {response.status_code}" + ) + + if file_v2_id is not None: + add_file_metadata(file, file_v2_id, clowder_headers_v1, user_headers_v2) return [USER_MAP, DATASET_MAP] From 70ea8afe0382aaf942a7a8402ee2cf35f4985822 Mon Sep 17 00:00:00 2001 From: toddn Date: Sat, 16 Aug 2025 15:43:51 -0500 Subject: [PATCH 41/91] removed - already processed --- scripts/migration/migrate.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index ba9764346..9b477a638 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -758,9 +758,6 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): f"Failed to add collection info as metadata in Clowder v2. Status code: {response.status_code}" ) - if file_v2_id is not None: - add_file_metadata(file, file_v2_id, clowder_headers_v1, user_headers_v2) - return [USER_MAP, DATASET_MAP] From 92f4f2cc57b81bbd39c0618ad3b42868fb43cba4 Mon Sep 17 00:00:00 2001 From: toddn Date: Sun, 17 Aug 2025 12:21:32 -0500 Subject: [PATCH 42/91] new readme, using tomli and a config file to limit or set options for migration --- scripts/README.md | 2 ++ scripts/migration/README.md | 8 ++++++++ scripts/migration/migrate.py | 6 ++++++ 3 files changed, 16 insertions(+) create mode 100644 scripts/migration/README.md diff --git a/scripts/README.md b/scripts/README.md index 3db467683..11bffb1ac 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -4,3 +4,5 @@ or delete existing test data. * ``/keylcloak`` includes Keycloak configuration files and Clowder theme. Used when creating a new instance. * ``/metadata`` includes examples of metadata definitions. Used to populate database for new or testing instances. + +* ``/migration`` Migrate from clowder v2 to v2. It has its own readme file. diff --git a/scripts/migration/README.md b/scripts/migration/README.md new file mode 100644 index 000000000..ea7e38c41 --- /dev/null +++ b/scripts/migration/README.md @@ -0,0 +1,8 @@ +# Migration + +these scripts are used to migrate data from a clowder v1 to v2 instance + +## config.toml + +This file can be used to limit what gets migrated by space or user. + diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 9b477a638..b543994b4 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -4,6 +4,12 @@ import requests from dotenv import dotenv_values +try: + import tomllib # Python 3.11+ +except ImportError: + import tomli as tomllib + + from scripts.migration.migrate_metadata_definitions import ( check_metadata_definition_exists, get_clowder_v1_metadata_definitions, From 80d41b07b331379650cf3a67cba5a12c798f2a5c Mon Sep 17 00:00:00 2001 From: toddn Date: Sun, 17 Aug 2025 12:37:26 -0500 Subject: [PATCH 43/91] adding example for config.toml --- scripts/migration/config.example.toml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 scripts/migration/config.example.toml diff --git a/scripts/migration/config.example.toml b/scripts/migration/config.example.toml new file mode 100644 index 000000000..e6cbe1cab --- /dev/null +++ b/scripts/migration/config.example.toml @@ -0,0 +1,8 @@ +[spaces] +space_ids = [ + +] + +[users] +user_email = [ +] From bc67fc731136a0c38dab7fa2204af75c631409a2 Mon Sep 17 00:00:00 2001 From: toddn Date: Sun, 17 Aug 2025 12:39:00 -0500 Subject: [PATCH 44/91] exclude users - users we do not migrate (or their files) --- scripts/migration/config.example.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/migration/config.example.toml b/scripts/migration/config.example.toml index e6cbe1cab..a4c1d8c4b 100644 --- a/scripts/migration/config.example.toml +++ b/scripts/migration/config.example.toml @@ -6,3 +6,8 @@ space_ids = [ [users] user_email = [ ] + +[exclude_users] +exclude_user_emails = [ + +] \ No newline at end of file From d86a756a9b0348112c0bc82b1dfc932dd8c2cf9d Mon Sep 17 00:00:00 2001 From: toddn Date: Sun, 17 Aug 2025 13:21:34 -0500 Subject: [PATCH 45/91] adding example env --- scripts/migration/example.env | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 scripts/migration/example.env diff --git a/scripts/migration/example.env b/scripts/migration/example.env new file mode 100644 index 000000000..434136961 --- /dev/null +++ b/scripts/migration/example.env @@ -0,0 +1,4 @@ +CLOWDER_V1={clowder_v1_url} +ADMIN_KEY_V1=admin_key_v1 +CLOWDER_V2={clowder_v2_url} +ADMIN_KEY_V2=admin_key_v2 \ No newline at end of file From a4530681f37d277053c6accfd7c8c2d70f2515ce Mon Sep 17 00:00:00 2001 From: toddn Date: Sun, 17 Aug 2025 13:33:35 -0500 Subject: [PATCH 46/91] proper path to .env file --- scripts/migration/migrate.py | 7 ++++++- scripts/migration/migrate_metadata_definitions.py | 11 ++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index b543994b4..27f68addc 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -24,7 +24,7 @@ OUTPUT_FILE = f"migrated_new_users_{timestamp}.log" # Load environment variables -path_to_env = os.path.join(os.getcwd(), ".env") +path_to_env = os.path.join(os.getcwd(),"scripts","migration", ".env") config = dotenv_values(dotenv_path=path_to_env) CLOWDER_V1 = config["CLOWDER_V1"] @@ -32,6 +32,11 @@ CLOWDER_V2 = config["CLOWDER_V2"] ADMIN_KEY_V2 = config["ADMIN_KEY_V2"] +if not CLOWDER_V1 or not ADMIN_KEY_V1 or not CLOWDER_V2 or not ADMIN_KEY_V2: + print("MISSING SOME ENVIRONMENT VARIABLES") +else: + print("WE HAVE THEM ALL") + base_headers_v1 = {"X-API-key": ADMIN_KEY_V1} base_headers_v2 = {"X-API-key": ADMIN_KEY_V2} diff --git a/scripts/migration/migrate_metadata_definitions.py b/scripts/migration/migrate_metadata_definitions.py index 0b096f940..8dd747b56 100644 --- a/scripts/migration/migrate_metadata_definitions.py +++ b/scripts/migration/migrate_metadata_definitions.py @@ -4,7 +4,11 @@ import requests from dotenv import dotenv_values -path_to_env = os.path.join(os.getcwd(), ".env") +path_to_env = os.path.join(os.getcwd(),"scripts","migration", ".env") +if not os.path.exists(path_to_env): + raise FileNotFoundError( + f"Environment file not found at {path_to_env}. Please ensure it exists." + ) config = dotenv_values(dotenv_path=path_to_env) CLOWDER_V1_URL = config["CLOWDER_V1"] @@ -13,6 +17,11 @@ CLOWDER_V2_URL = config["CLOWDER_V2"] ADMIN_KEY_V2 = config["ADMIN_KEY_V2"] +if not CLOWDER_V1_URL or not ADMIN_KEY_V1 or not CLOWDER_V2_URL or not ADMIN_KEY_V2: + print("MISSING SOME ENVIRONMENT VARIABLES") +else: + print("WE HAVE THEM ALL") + base_headers_v1 = {"X-API-key": ADMIN_KEY_V1} clowder_headers_v1 = { **base_headers_v1, From c64be064aad8cbcb190d7e8fde71fa7ec87b83d2 Mon Sep 17 00:00:00 2001 From: toddn Date: Sun, 17 Aug 2025 13:54:34 -0500 Subject: [PATCH 47/91] excluding items --- scripts/migration/config.example.toml | 22 ++++++++++++++++++---- scripts/migration/migrate.py | 24 ++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/scripts/migration/config.example.toml b/scripts/migration/config.example.toml index a4c1d8c4b..c873b447b 100644 --- a/scripts/migration/config.example.toml +++ b/scripts/migration/config.example.toml @@ -1,13 +1,27 @@ [spaces] space_ids = [ + "66ec6210e4b08e5e86b30c20", + "62d992dce4b0e32fd16081f4" +] +exclude_space_ids = [ ] [users] -user_email = [ +user_emails = [ + "tcnichol@illinois.edu", + "cwang138@illinois.edu", + "lmarini@illinois.edu" + +] +exclude_space_ids = [ + ] -[exclude_users] -exclude_user_emails = [ +[datasets] +dataset_ids = [ -] \ No newline at end of file +] +exclude_dataset_ids = [ + "66eb1ddfe4b08e5e86b2e124" +] diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 27f68addc..7e03dd247 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -25,8 +25,24 @@ # Load environment variables path_to_env = os.path.join(os.getcwd(),"scripts","migration", ".env") +path_to_toml = os.path.join(os.getcwd(),"scripts","migration", "config.toml") config = dotenv_values(dotenv_path=path_to_env) +toml_space_ids=None +toml_exclude_space_ids=None +toml_users=None +toml_exclude_users=None +toml_exclude_dataset_ids=None + +# Load configuration from toml file +if os.path.exists(path_to_toml): + toml_config = tomllib.loads(open(path_to_toml).read()) + print(f"Loaded toml config") + toml_space_ids = toml_config["spaces"]["space_ids"] + toml_users = toml_config["users"]["user_emails"] + toml_exclude_dataset_ids = toml_config["datasets"]["exclude_dataset_ids"] + + CLOWDER_V1 = config["CLOWDER_V1"] ADMIN_KEY_V1 = config["ADMIN_KEY_V1"] CLOWDER_V2 = config["CLOWDER_V2"] @@ -791,6 +807,14 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): USER_MAP = {} DATASET_MAP = {} users_v1 = get_clowder_v1_users() + # TODO filter if toml users + if toml_users is not None and len(toml_users) > 0: + print(f"Using spaces from config.toml: {toml_users}") + users_v1 = [ + user for user in users_v1 if user["email"] in toml_users + ] + else: + print("No spaces specified in config.toml, migrating all users.") for user_v1 in users_v1: if ( "[Local Account]" in user_v1["identityProvider"] From c7e218c3d48c58ffadb3b1392730f2edbe4b932b Mon Sep 17 00:00:00 2001 From: toddn Date: Sun, 17 Aug 2025 14:06:07 -0500 Subject: [PATCH 48/91] check to see if we should migrate a dataset --- scripts/migration/migrate.py | 122 ++++++++++++++++++++++------------- 1 file changed, 76 insertions(+), 46 deletions(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 7e03dd247..d441d0473 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -735,55 +735,85 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): for dataset in user_v1_datasets: print(f"Creating dataset in v2: {dataset['id']} - {dataset['name']}") - dataset_v2_id = create_v2_dataset(dataset, user_headers_v2) - DATASET_MAP[dataset["id"]] = dataset_v2_id - add_dataset_metadata(dataset, dataset_v2_id, base_headers_v1, user_headers_v2) - add_dataset_folders(dataset, dataset_v2_id, user_headers_v2) - print("Created folders in the new dataset") - - all_dataset_folders = get_folder_and_subfolders(dataset_v2_id, user_headers_v2) - - # Retrieve files for the dataset in Clowder v1 - dataset_files_endpoint = ( - f"{CLOWDER_V1}/api/datasets/{dataset['id']}/files?superAdmin=true" - ) - files_response = requests.get( - dataset_files_endpoint, headers=clowder_headers_v1, verify=False - ) - files_result = files_response.json() + # TODO: check if dataset is in toml_exclude_dataset_id + dataset_v1_id = dataset["id"] + dataset_v1_spaces = dataset["spaces"] + # TODO check if dataset is in toml_space_ids or exclude_space_ids + MIGRATE_DATASET = True + print(toml_exclude_dataset_ids) + print(toml_space_ids) + print(toml_exclude_space_ids) + # Check if dataset is in the excluded dataset list + if dataset_v1_id in toml_exclude_dataset_ids: + print(f"Skipping dataset {dataset_v1_id} as it is in the exclude list.") + MIGRATE_DATASET = False + # Check if dataset is in the specified space list + if toml_space_ids is not None and len(toml_space_ids) > 0: + if not any( + space_id in dataset_v1_spaces for space_id in toml_space_ids + ): + print( + f"Skipping dataset {dataset_v1_id} as it is not in the specified spaces." + ) + MIGRATE_DATASET = False + if toml_exclude_space_ids is not None and len(toml_exclude_space_ids) > 0: + if any( + space_id in dataset_v1_spaces for space_id in toml_exclude_space_ids + ): + print( + f"Skipping dataset {dataset_v1_id} as it is in the excluded spaces." + ) + MIGRATE_DATASET = False + if MIGRATE_DATASET: + dataset_v2_id = create_v2_dataset(dataset, user_headers_v2) + DATASET_MAP[dataset["id"]] = dataset_v2_id + add_dataset_metadata(dataset, dataset_v2_id, base_headers_v1, user_headers_v2) + add_dataset_folders(dataset, dataset_v2_id, user_headers_v2) + print("Created folders in the new dataset") + + all_dataset_folders = get_folder_and_subfolders(dataset_v2_id, user_headers_v2) + + # Retrieve files for the dataset in Clowder v1 + dataset_files_endpoint = ( + f"{CLOWDER_V1}/api/datasets/{dataset['id']}/files?superAdmin=true" + ) + files_response = requests.get( + dataset_files_endpoint, headers=clowder_headers_v1, verify=False + ) + files_result = files_response.json() - for file in files_result: - file_v2_id = download_and_upload_file( - file, all_dataset_folders, dataset_v2_id, base_user_headers_v2 + for file in files_result: + file_v2_id = download_and_upload_file( + file, all_dataset_folders, dataset_v2_id, base_user_headers_v2 + ) + if file_v2_id is not None: + add_file_metadata(file, file_v2_id, clowder_headers_v1, user_headers_v2) + # posting the collection hierarchy as metadata + collection_space_metadata_dict = build_collection_space_metadata_for_v1_dataset( + dataset=dataset, user_v1=user_v1, headers=clowder_headers_v1 ) - if file_v2_id is not None: - add_file_metadata(file, file_v2_id, clowder_headers_v1, user_headers_v2) - # posting the collection hierarchy as metadata - collection_space_metadata_dict = build_collection_space_metadata_for_v1_dataset( - dataset=dataset, user_v1=user_v1, headers=clowder_headers_v1 - ) - migration_extractor_collection_metadata = { - "listener": { - "name": "migration", - "version": "1", - "description": "Migration of metadata from Clowder v1 to Clowder v2", - }, - "context_url": "https://clowder.ncsa.illinois.edu/contexts/metadata.jsonld", - "content": collection_space_metadata_dict, - "contents": collection_space_metadata_dict, - } - v2_metadata_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/metadata" - response = requests.post( - v2_metadata_endpoint, - json=migration_extractor_collection_metadata, - headers=clowder_headers_v2, - ) - if response.status_code == 200: - print("Successfully added collection info as metadata in v2.") - else: - print( - f"Failed to add collection info as metadata in Clowder v2. Status code: {response.status_code}" + migration_extractor_collection_metadata = { + "listener": { + "name": "migration", + "version": "1", + "description": "Migration of metadata from Clowder v1 to Clowder v2", + }, + "context_url": "https://clowder.ncsa.illinois.edu/contexts/metadata.jsonld", + "content": collection_space_metadata_dict, + "contents": collection_space_metadata_dict, + } + v2_metadata_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/metadata" + response = requests.post( + v2_metadata_endpoint, + json=migration_extractor_collection_metadata, + headers=clowder_headers_v2, ) + if response.status_code == 200: + print("Successfully added collection info as metadata in v2.") + else: + print( + f"Failed to add collection info as metadata in Clowder v2. Status code: {response.status_code}" + ) return [USER_MAP, DATASET_MAP] From c4d3cdb8c9d93983783d812f2d193bb5ae8ba287 Mon Sep 17 00:00:00 2001 From: toddn Date: Sun, 17 Aug 2025 15:11:15 -0500 Subject: [PATCH 49/91] using the excluded space ids --- scripts/migration/migrate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index d441d0473..44bbd4064 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -39,6 +39,7 @@ toml_config = tomllib.loads(open(path_to_toml).read()) print(f"Loaded toml config") toml_space_ids = toml_config["spaces"]["space_ids"] + toml_exclude_space_ids = toml_config["spaces"]["exclude_space_ids"] toml_users = toml_config["users"]["user_emails"] toml_exclude_dataset_ids = toml_config["datasets"]["exclude_dataset_ids"] From ca80dbde229e1a0ab09231bbc354148e713ad3f4 Mon Sep 17 00:00:00 2001 From: toddn Date: Sun, 17 Aug 2025 15:52:15 -0500 Subject: [PATCH 50/91] fix - if license absent in v1 assign a default license for v2 --- scripts/migration/migrate.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 44bbd4064..417365a07 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -358,7 +358,11 @@ def create_v2_dataset(dataset, headers): """Create a dataset in Clowder v2.""" # TODO: GET correct license print("Creating dataset license in Clowder v2.") - v2_license_id = add_dataset_license(dataset["license"], headers) + try: + v2_license_id = add_dataset_license(dataset["license"], headers) + except Exception as e: + print(f"Error creating dataset license: {e}") + v2_license_id = "CC-BY" dataset_in_v2_endpoint = f"{CLOWDER_V2}/api/v2/datasets?license_id={v2_license_id}" dataset_example = { From f98eddae1aa8215e5372d1ac12389807384d4a7c Mon Sep 17 00:00:00 2001 From: toddn Date: Sun, 17 Aug 2025 16:12:20 -0500 Subject: [PATCH 51/91] fix type in license --- scripts/migration/migrate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 417365a07..2816925e5 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -295,7 +295,7 @@ def create_admin_user(): def add_dataset_license(v1_license, headers): """Create appropriate license (standard/custom) based on v1 license details""" - license_id = "CC-BY" + license_id = "CC BY" # standard licenses if v1_license["license_type"] == "license2": if ( @@ -362,7 +362,7 @@ def create_v2_dataset(dataset, headers): v2_license_id = add_dataset_license(dataset["license"], headers) except Exception as e: print(f"Error creating dataset license: {e}") - v2_license_id = "CC-BY" + v2_license_id = "CC BY" dataset_in_v2_endpoint = f"{CLOWDER_V2}/api/v2/datasets?license_id={v2_license_id}" dataset_example = { From 5a84349e3f4b931e0f8929d0faeab9f4ffc18de0 Mon Sep 17 00:00:00 2001 From: toddn Date: Sun, 17 Aug 2025 17:20:36 -0500 Subject: [PATCH 52/91] adding TODO items the sheer number of collections in some instances makes this way of getting collections incredibly slow. instead we should start at the top and then check each level, this should mean fewer collections --- scripts/migration/migrate.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 2816925e5..3e335ef56 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -150,7 +150,7 @@ def get_clowder_v1_user_collections(headers, user_v1): response = requests.get(endpoint, headers=headers) return [col for col in response.json() if col["authorId"] == user_v1["id"]] - +# TODO this is too slow, we need to optimize it def get_clowder_v1_dataset_collections(headers, user_v1, dataset_id): matching_collections = [] endpoint = f"{CLOWDER_V1}/api/collections/allCollections" @@ -687,8 +687,10 @@ def build_collection_metadata_for_v1_dataset(dataset_id, user_v1, headers): return dataset_collections +# TODO test this method def build_collection_space_metadata_for_v1_dataset(dataset, user_v1, headers): dataset_id = dataset["id"] + # TODO this is too slow we need a way to sort through collection hierarchy better dataset_collections = get_clowder_v1_dataset_collections( headers=headers, user_v1=user_v1, dataset_id=dataset_id ) From 1932362ee9d65fcf1b59258bbbc09a71a24d9964 Mon Sep 17 00:00:00 2001 From: toddn Date: Sun, 17 Aug 2025 17:32:07 -0500 Subject: [PATCH 53/91] working on this, need a better way to check what collections a dataset is contained in --- scripts/migration/migrate.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 3e335ef56..fdbbd0e5e 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -153,9 +153,19 @@ def get_clowder_v1_user_collections(headers, user_v1): # TODO this is too slow, we need to optimize it def get_clowder_v1_dataset_collections(headers, user_v1, dataset_id): matching_collections = [] - endpoint = f"{CLOWDER_V1}/api/collections/allCollections" - response = requests.get(endpoint, headers=headers) + endpoint1 = f"{CLOWDER_V1}/api/collections/rootCollections?superAdmin=true" + endpint2 = f"{CLOWDER_V1}/api/collections/topLevelCollections?superAdmin=true" + response = requests.get(endpoint1, headers=headers) + response2 = requests.get(endpint2, headers=headers) user_collections = response.json() + user_collections_ids = [] + user_collections_ids_2 = [] + user_collections_2 = response2.json() + for collection in user_collections_2: + id = collection['id'] + user_collections_ids_2.append(id) + for collection in user_collections: + user_collections_ids.append(collection['id']) for collection in user_collections: collection_id = collection["id"] collection_dataset_endpoint = ( From 21629c7b0517820f459758d079a0787ec756c81c Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 18 Aug 2025 10:57:10 -0500 Subject: [PATCH 54/91] sloppy hack - keeping for commit since it's got the descendant collections right - can check from there --- scripts/migration/migrate.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index fdbbd0e5e..8b9cc67fb 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -145,6 +145,22 @@ def add_v1_space_members_to_v2_group(space, group_id, headers): ) +def get_collection_v1_descendants(headers, collection_id): + descendant_ids = [] + + collection_endpoint = f"{CLOWDER_V1}/api/collections/{collection_id}" + response = requests.get(collection_endpoint, headers=headers, verify=False) + collection_json = response.json() + print(collection_json["child_collection_ids"]) + if int(collection_json["childCollectionsCount"]) > 0: + child_collections_ids = collection_json["child_collection_ids"] + descendant_ids = child_collections_ids[5:-1].split(', ') + for id in descendant_ids: + sub_descendants = get_collection_v1_descendants(headers, id) + descendant_ids.extend(sub_descendants) + return descendant_ids + print('we got collection') + def get_clowder_v1_user_collections(headers, user_v1): endpoint = f"{CLOWDER_V1}/api/collections" response = requests.get(endpoint, headers=headers) @@ -154,6 +170,7 @@ def get_clowder_v1_user_collections(headers, user_v1): def get_clowder_v1_dataset_collections(headers, user_v1, dataset_id): matching_collections = [] endpoint1 = f"{CLOWDER_V1}/api/collections/rootCollections?superAdmin=true" + # use this one below endpint2 = f"{CLOWDER_V1}/api/collections/topLevelCollections?superAdmin=true" response = requests.get(endpoint1, headers=headers) response2 = requests.get(endpint2, headers=headers) @@ -163,7 +180,10 @@ def get_clowder_v1_dataset_collections(headers, user_v1, dataset_id): user_collections_2 = response2.json() for collection in user_collections_2: id = collection['id'] - user_collections_ids_2.append(id) + descendants = get_collection_v1_descendants(headers, id) + # test_descendants = get_collection_v1_descendants(headers, "68a34b28e4b0cc7386c091a4") + # TODO check here if the dataset is in a descendant + print('got descendants') for collection in user_collections: user_collections_ids.append(collection['id']) for collection in user_collections: From b600cc95478a1595241279f5da3466162ae0fc50 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 18 Aug 2025 14:17:34 -0500 Subject: [PATCH 55/91] new classes (to be consolidated later) to make it easier to map datasets to collections and their parent hierarchcy, we now will generate a json that will help with lookup. this will be run BEFORE the migration script --- scripts/migration/dataset_collection_json.py | 32 +++++ scripts/migration/get_collections.py | 115 ++++++++++++++++++ scripts/migration/get_collections_datasets.py | 97 +++++++++++++++ scripts/migration/migrate.py | 54 ++++---- 4 files changed, 275 insertions(+), 23 deletions(-) create mode 100644 scripts/migration/dataset_collection_json.py create mode 100644 scripts/migration/get_collections.py create mode 100644 scripts/migration/get_collections_datasets.py diff --git a/scripts/migration/dataset_collection_json.py b/scripts/migration/dataset_collection_json.py new file mode 100644 index 000000000..4c25eb17c --- /dev/null +++ b/scripts/migration/dataset_collection_json.py @@ -0,0 +1,32 @@ +import os +from datetime import datetime +import json +import requests +from dotenv import dotenv_values + +try: + import tomllib # Python 3.11+ +except ImportError: + import tomli as tomllib + + + +DEFAULT_PASSWORD = "Password123&" + +# Get the current timestamp +timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") +COLLECTIONS_FILE = "collections_datasets.json" + +def get_dataset_collections_map(): + print("Getting collections and datasets from Clowder v1...") + + with open(COLLECTIONS_FILE, "r") as jf: + data = json.load(jf) + print(f"Loaded {len(data)} collections from {COLLECTIONS_FILE}") + dataset_to_collection = {} + + for collection, datasets in data.items(): + for dataset in datasets: + dataset_to_collection[dataset] = collection + return dataset_to_collection + diff --git a/scripts/migration/get_collections.py b/scripts/migration/get_collections.py new file mode 100644 index 000000000..5656164dd --- /dev/null +++ b/scripts/migration/get_collections.py @@ -0,0 +1,115 @@ +import os +from datetime import datetime + +import requests +from dotenv import dotenv_values + +try: + import tomllib # Python 3.11+ +except ImportError: + import tomli as tomllib + + +from scripts.migration.migrate_metadata_definitions import ( + check_metadata_definition_exists, + get_clowder_v1_metadata_definitions, + post_metadata_definition, +) + +# Configuration and Constants +DEFAULT_PASSWORD = "Password123&" + +# Get the current timestamp +timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") +OUTPUT_FILE = "collections_ids.txt" + +# Load environment variables +path_to_env = os.path.join(os.getcwd(),"scripts","migration", ".env") +config = dotenv_values(dotenv_path=path_to_env) + + +CLOWDER_V1 = config["CLOWDER_V1"] +ADMIN_KEY_V1 = config["ADMIN_KEY_V1"] +CLOWDER_V2 = config["CLOWDER_V2"] +ADMIN_KEY_V2 = config["ADMIN_KEY_V2"] + +if not CLOWDER_V1 or not ADMIN_KEY_V1 or not CLOWDER_V2 or not ADMIN_KEY_V2: + print("MISSING SOME ENVIRONMENT VARIABLES") +else: + print("WE HAVE THEM ALL") + +base_headers_v1 = {"X-API-key": ADMIN_KEY_V1} +base_headers_v2 = {"X-API-key": ADMIN_KEY_V2} + +clowder_headers_v1 = { + **base_headers_v1, + "Content-type": "application/json", + "accept": "application/json", +} + +clowder_headers_v2 = { + **base_headers_v2, + "Content-type": "application/json", + "accept": "application/json", +} + +admin_user = { + "email": "admin@example.com", + "password": "admin", + "first_name": "admin", + "last_name": "admin", +} + +def get_clowder_v1_top_level_collections(headers): + endpoint = f"{CLOWDER_V1}/api/collections/topLevelCollections?superAdmin=true" + response = requests.get(endpoint, headers=headers) + user_collections = response.json() + return user_collections + +def get_collection_v1_descendants(headers, collection_id): + descendant_ids = [] + + collection_endpoint = f"{CLOWDER_V1}/api/collections/{collection_id}" + response = requests.get(collection_endpoint, headers=headers, verify=False) + collection_json = response.json() + print(collection_json["child_collection_ids"]) + if int(collection_json["childCollectionsCount"]) > 0: + child_collections_ids = collection_json["child_collection_ids"] + descendant_ids = child_collections_ids[5:-1].split(', ') + for i in range(0, len(descendant_ids)): + id = descendant_ids[i] + descendent_endpoint = f"{CLOWDER_V1}/api/collections/{id}" + descendent_response = requests.get(descendent_endpoint, headers=headers, verify=False) + descendent_json = descendent_response.json() + if int(descendent_json["childCollectionsCount"]) > 0: + sub_descendants = get_collection_v1_descendants(headers, id) + descendant_ids.extend(sub_descendants) + return descendant_ids + +def get_dataset_ids_in_v1_collection(headers, collection_id): + dataset_ids = [] + collection_endpoint = f"{CLOWDER_V1}/api/collections/{collection_id}/datasets" + response = requests.get(collection_endpoint, headers=headers, verify=False) + datasets_json = response.json() + for dataset in datasets_json: + dataset_ids.append(dataset["id"]) + return dataset_ids + +if __name__ == "__main__": + top_level_collections = get_clowder_v1_top_level_collections(clowder_headers_v1) + all_v1_collections = [] + for collection in top_level_collections: + print(f"Getting descendents for collection {collection['name']} ({collection['id']})") + all_v1_collections.append(collection["id"]) + if int(collection["childCollectionsCount"]) > 0: + descendant_ids = get_collection_v1_descendants(clowder_headers_v1, collection["id"]) + all_v1_collections.extend(descendant_ids) + print(f"Added descendents for collection {collection['name']} ({collection['id']})") + + + print(f"TOTAL V1 COLLECTIONS TO MIGRATE: {len(all_v1_collections)}") + + with open(OUTPUT_FILE, "w") as outfile: + for v1_collection in all_v1_collections: + outfile.write(v1_collection + "\n") + print(f"Migration complete. New users logged to {OUTPUT_FILE}") diff --git a/scripts/migration/get_collections_datasets.py b/scripts/migration/get_collections_datasets.py new file mode 100644 index 000000000..436dc5bad --- /dev/null +++ b/scripts/migration/get_collections_datasets.py @@ -0,0 +1,97 @@ +import os +from datetime import datetime +import json +import requests +from dotenv import dotenv_values + +try: + import tomllib # Python 3.11+ +except ImportError: + import tomli as tomllib + + + +DEFAULT_PASSWORD = "Password123&" + +# Get the current timestamp +timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") +COLLECTIONS_FILE = "collections_ids.txt" + + + +# Load environment variables +path_to_env = os.path.join(os.getcwd(),"scripts","migration", ".env") +config = dotenv_values(dotenv_path=path_to_env) + + +CLOWDER_V1 = config["CLOWDER_V1"] +ADMIN_KEY_V1 = config["ADMIN_KEY_V1"] +CLOWDER_V2 = config["CLOWDER_V2"] +ADMIN_KEY_V2 = config["ADMIN_KEY_V2"] + +if not CLOWDER_V1 or not ADMIN_KEY_V1 or not CLOWDER_V2 or not ADMIN_KEY_V2: + print("MISSING SOME ENVIRONMENT VARIABLES") +else: + print("WE HAVE THEM ALL") + +base_headers_v1 = {"X-API-key": ADMIN_KEY_V1} +base_headers_v2 = {"X-API-key": ADMIN_KEY_V2} + +clowder_headers_v1 = { + **base_headers_v1, + "Content-type": "application/json", + "accept": "application/json", +} + +clowder_headers_v2 = { + **base_headers_v2, + "Content-type": "application/json", + "accept": "application/json", +} + +admin_user = { + "email": "admin@example.com", + "password": "admin", + "first_name": "admin", + "last_name": "admin", +} + +def get_collections_datasets(headers, collection_id): + collection_dataset_endpoint = ( + f"{CLOWDER_V1}/api/collections/{collection_id}/datasets?superAdmin=true" + ) + collection_dataset_response = requests.get( + collection_dataset_endpoint, headers=headers + ) + collection_dataset_json = collection_dataset_response.json() + return collection_dataset_json + + +if __name__ == "__main__": + print("Getting collections and datasets from Clowder v1...") + + collection_ids =[] + if os.path.exists(COLLECTIONS_FILE): + print('exists') + else: + print('does not exist') + + with open(COLLECTIONS_FILE, "r") as outfile: + lines = outfile.readlines() + for line in lines: + collection_ids.append(line.rstrip('\n')) + print(f"Found {len(collection_ids)} collections in {COLLECTIONS_FILE}") + collection_dataset_dict = dict() + for id in collection_ids: + print(f"Getting datasets for collection id {id}...") + datasets = get_collections_datasets(clowder_headers_v1, id) + if len(datasets) > 0: + dataset_ids = [] + for ds in datasets: + dataset_ids.append(ds["id"]) + collection_dataset_dict[id] = dataset_ids + + json_file = "collections_datasets.json" + with open(json_file, "w") as jf: + json.dump(collection_dataset_dict, jf) + print("dumped to a file") \ No newline at end of file diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 8b9cc67fb..709c4737d 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -156,10 +156,13 @@ def get_collection_v1_descendants(headers, collection_id): child_collections_ids = collection_json["child_collection_ids"] descendant_ids = child_collections_ids[5:-1].split(', ') for id in descendant_ids: - sub_descendants = get_collection_v1_descendants(headers, id) - descendant_ids.extend(sub_descendants) + descendent_endpoint = f"{CLOWDER_V1}/api/collections/{id}" + descendent_response = requests.get(descendent_endpoint, headers=headers, verify=False) + descendent_json = descendent_response.json() + if int(descendent_json["childCollectionsCount"]) > 0: + sub_descendants = get_collection_v1_descendants(headers, id) + descendant_ids.extend(sub_descendants) return descendant_ids - print('we got collection') def get_clowder_v1_user_collections(headers, user_v1): endpoint = f"{CLOWDER_V1}/api/collections" @@ -169,23 +172,10 @@ def get_clowder_v1_user_collections(headers, user_v1): # TODO this is too slow, we need to optimize it def get_clowder_v1_dataset_collections(headers, user_v1, dataset_id): matching_collections = [] - endpoint1 = f"{CLOWDER_V1}/api/collections/rootCollections?superAdmin=true" - # use this one below - endpint2 = f"{CLOWDER_V1}/api/collections/topLevelCollections?superAdmin=true" - response = requests.get(endpoint1, headers=headers) - response2 = requests.get(endpint2, headers=headers) + endpoint = f"{CLOWDER_V1}/api/collections/topLevelCollections?superAdmin=true" + response = requests.get(endpoint, headers=headers) user_collections = response.json() - user_collections_ids = [] - user_collections_ids_2 = [] - user_collections_2 = response2.json() - for collection in user_collections_2: - id = collection['id'] - descendants = get_collection_v1_descendants(headers, id) - # test_descendants = get_collection_v1_descendants(headers, "68a34b28e4b0cc7386c091a4") - # TODO check here if the dataset is in a descendant - print('got descendants') - for collection in user_collections: - user_collections_ids.append(collection['id']) + for collection in user_collections: collection_id = collection["id"] collection_dataset_endpoint = ( @@ -198,9 +188,24 @@ def get_clowder_v1_dataset_collections(headers, user_v1, dataset_id): datasets = dataset_response.json() for ds in datasets: if ds["id"] == dataset_id: - matching_collections.append(collection) + if collection not in matching_collections: + matching_collections.append(collection) except Exception as e: print("Exception", e) + if int(collection["childCollectionsCount"]) > 0: + collection_descendants = get_collection_v1_descendants(headers, collection_id) + for descendant in collection_descendants: + collection_dataset_endpoint = ( + f"{CLOWDER_V1}/api/collections/{descendant}/datasets?superAdmin=true" + ) + collection_dataset_response = requests.get( + collection_dataset_endpoint, headers=headers + ) + collection_dataset_json = collection_dataset_response.json() + for ds in collection_dataset_json: + if ds['id'] == dataset_id: + if descendant not in matching_collections: + matching_collections.append(descendant) return matching_collections @@ -826,9 +831,12 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): if file_v2_id is not None: add_file_metadata(file, file_v2_id, clowder_headers_v1, user_headers_v2) # posting the collection hierarchy as metadata - collection_space_metadata_dict = build_collection_space_metadata_for_v1_dataset( - dataset=dataset, user_v1=user_v1, headers=clowder_headers_v1 - ) + try: + collection_space_metadata_dict = build_collection_space_metadata_for_v1_dataset( + dataset=dataset, user_v1=user_v1, headers=clowder_headers_v1 + ) + except Exception as e: + print(e) migration_extractor_collection_metadata = { "listener": { "name": "migration", From 734c9f1e0574e56944c2eb9e30396e11a4e033a6 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 18 Aug 2025 14:46:59 -0500 Subject: [PATCH 56/91] this seems to be working the map for the dataset collections seems to work will add documentation next commit --- scripts/migration/dataset_collection_json.py | 7 ++++++- scripts/migration/migrate.py | 17 ++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/scripts/migration/dataset_collection_json.py b/scripts/migration/dataset_collection_json.py index 4c25eb17c..74230dc0c 100644 --- a/scripts/migration/dataset_collection_json.py +++ b/scripts/migration/dataset_collection_json.py @@ -27,6 +27,11 @@ def get_dataset_collections_map(): for collection, datasets in data.items(): for dataset in datasets: - dataset_to_collection[dataset] = collection + if dataset not in dataset_to_collection: + dataset_to_collection[dataset] = [collection] + else: + current_value = dataset_to_collection[dataset] + current_value.append(collection) + dataset_to_collection[dataset] = current_value return dataset_to_collection diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 709c4737d..d3d070238 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -16,6 +16,10 @@ post_metadata_definition, ) +from scripts.migration.dataset_collection_json import get_dataset_collections_map + +DATASET_COLLECTIONS_MAP = get_dataset_collections_map() + # Configuration and Constants DEFAULT_PASSWORD = "Password123&" @@ -725,10 +729,12 @@ def build_collection_metadata_for_v1_dataset(dataset_id, user_v1, headers): # TODO test this method def build_collection_space_metadata_for_v1_dataset(dataset, user_v1, headers): dataset_id = dataset["id"] - # TODO this is too slow we need a way to sort through collection hierarchy better - dataset_collections = get_clowder_v1_dataset_collections( - headers=headers, user_v1=user_v1, dataset_id=dataset_id - ) + dataset_collections = [] + if dataset_id in DATASET_COLLECTIONS_MAP: + dataset_collections_ids = DATASET_COLLECTIONS_MAP[dataset_id] + for col_id in dataset_collections_ids: + collection = get_clowder_v1_collection(col_id, headers=headers) + dataset_collections.append(collection) dataset_spaces = dataset["spaces"] space_entries = [] for space_id in dataset_spaces: @@ -739,7 +745,8 @@ def build_collection_space_metadata_for_v1_dataset(dataset, user_v1, headers): space_entry = { "id": space["id"], "name": space["name"], - "creator": space["creator"], + # TODO this is not part of the json + # "creator": space["creator"], } space_entries.append(space_entry) except Exception as e: From 87324ecaf088801b4d957a405938e8cec10463a3 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 18 Aug 2025 14:49:30 -0500 Subject: [PATCH 57/91] instructions for now (rough draft) --- scripts/migration/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/migration/README.md b/scripts/migration/README.md index ea7e38c41..3aa2b5801 100644 --- a/scripts/migration/README.md +++ b/scripts/migration/README.md @@ -6,3 +6,12 @@ these scripts are used to migrate data from a clowder v1 to v2 instance This file can be used to limit what gets migrated by space or user. +## steps to run migration + + +1. setup a clowder v2 instance and make sure it is running +2. add your values to the .env and the config.toml file +3. run the script `get_collections.py` this gets all collections from the v1 instance +4. run the script `get_collection_datasets.py` this gets all the datasets in the collections +5. Now you are ready to run `migrate.py` - it uses the data from the previous scripts to place datasets into the right collections + From 55a54bb01446a75c612c11e55b2ed0114ec0d32d Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 18 Aug 2025 15:56:08 -0500 Subject: [PATCH 58/91] metadata posts limiting datasets to 2 per user for testing, will remove later --- scripts/migration/migrate.py | 51 +++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index d3d070238..d1bce773f 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -746,18 +746,18 @@ def build_collection_space_metadata_for_v1_dataset(dataset, user_v1, headers): "id": space["id"], "name": space["name"], # TODO this is not part of the json - # "creator": space["creator"], + "creator": space["creator"], } space_entries.append(space_entry) except Exception as e: print(f"Error in getting space entry.") print(e) - try: - space_entry = {"id": space["id"], "name": space["name"]} - space_entries.append(space_entry) - except Exception as e: - print(f"Error in getting space entry") - print(e) + try: + space_entry = {"id": space["id"], "name": space["name"]} + space_entries.append(space_entry) + except Exception as e: + print(f"Error in getting space entry") + print(e) collection_data = [] for collection in dataset_collections: collection_children = build_collection_hierarchy( @@ -773,6 +773,7 @@ def build_collection_space_metadata_for_v1_dataset(dataset, user_v1, headers): def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): """Process user resources from Clowder v1 to Clowder v2.""" user_v1_datasets = get_clowder_v1_user_datasets(user_id=user_v1["id"]) + user_v1_datasets = user_v1_datasets[:2] user_v2_api_key = create_local_user(user_v1) USER_MAP[user_v1["id"]] = user_v2_api_key base_user_headers_v2 = {"x-api-key": user_v2_api_key} @@ -866,6 +867,8 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): print( f"Failed to add collection info as metadata in Clowder v2. Status code: {response.status_code}" ) + else: + print(f"Skipping dataset {dataset_v1_id} as it does not meet the criteria.") return [USER_MAP, DATASET_MAP] @@ -911,21 +914,21 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): ############################################################################################################## # migrate spaces - # print("Now migrating spaces.") - # for user_v1 in users_v1: - # print(f"Migrating spaces of user {user_v1['email']}") - # user_v1_spaces = get_clowder_v1_user_spaces(user_v1) - # user_v2_api_key = USER_MAP[user_v1["id"]] - # for space in user_v1_spaces: - # group_id = create_v2_group(space, headers={"X-API-key": user_v2_api_key}) - # add_v1_space_members_to_v2_group( - # space, group_id, headers={"X-API-key": user_v2_api_key} - # ) - # space_datasets = get_clowder_v2_space_datasets(space["id"]) - # for space_dataset in space_datasets: - # dataset_v2_id = DATASET_MAP[space_dataset["id"]] - # share_dataset_with_group( - # group_id, space, headers={"X-API-key": user_v2_api_key} - # ) - # print(f"Migrated spaces of user {user_v1['email']}") + print("Now migrating spaces.") + for user_v1 in users_v1: + print(f"Migrating spaces of user {user_v1['email']}") + user_v1_spaces = get_clowder_v1_user_spaces(user_v1) + user_v2_api_key = USER_MAP[user_v1["id"]] + for space in user_v1_spaces: + group_id = create_v2_group(space, headers={"X-API-key": user_v2_api_key}) + add_v1_space_members_to_v2_group( + space, group_id, headers={"X-API-key": user_v2_api_key} + ) + space_datasets = get_clowder_v2_space_datasets(space["id"]) + for space_dataset in space_datasets: + dataset_v2_id = DATASET_MAP[space_dataset["id"]] + share_dataset_with_group( + group_id, space, headers={"X-API-key": user_v2_api_key} + ) + print(f"Migrated spaces of user {user_v1['email']}") print("Migration complete.") From dbe3016fb10b9d869aa853d640441d9d99181302 Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 19 Aug 2025 10:36:42 -0500 Subject: [PATCH 59/91] log metadata def - errors are when there are duplicates --- scripts/migration/migrate_metadata_definitions.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/migration/migrate_metadata_definitions.py b/scripts/migration/migrate_metadata_definitions.py index 8dd747b56..a31cbfc32 100644 --- a/scripts/migration/migrate_metadata_definitions.py +++ b/scripts/migration/migrate_metadata_definitions.py @@ -142,8 +142,12 @@ def post_metadata_definition( ) if response.status_code == 200: + print( + f"Success posting metadata definition. Metadata name: {v2_metadata['name']}" + ) return response.json().get("id") else: + print(f"Error posting metadata, definitio, name: {v2_metadata['name']}") print( f"Failed to post metadata definition. Status code: {response.status_code}" ) From 73609d2e812141f49b5d8f3c564db1833573e291 Mon Sep 17 00:00:00 2001 From: toddn Date: Thu, 21 Aug 2025 16:10:23 -0500 Subject: [PATCH 60/91] get top level collections these will become datasets --- scripts/migration/migrate.py | 132 ++++++++++++++++++++++++++++++++++- 1 file changed, 130 insertions(+), 2 deletions(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index d1bce773f..947a5dc7d 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -173,6 +173,22 @@ def get_clowder_v1_user_collections(headers, user_v1): response = requests.get(endpoint, headers=headers) return [col for col in response.json() if col["authorId"] == user_v1["id"]] + +def get_clowder_v1_user_collections_top_level(headers, user_v1): + top_level_collections = [] + + endpoint = f"{CLOWDER_V1}/api/collections/topLevelCollections" + response = requests.get(endpoint, headers=headers) + response_json = response.json() + for col in response_json: + author = col["author"] + author_id = author.lstrip('MiniUser(') + author_id = author_id[:author_id.index(',')] + if author_id == user_v1["id"]: + top_level_collections.append(col) + return top_level_collections + + # TODO this is too slow, we need to optimize it def get_clowder_v1_dataset_collections(headers, user_v1, dataset_id): matching_collections = [] @@ -769,6 +785,117 @@ def build_collection_space_metadata_for_v1_dataset(dataset, user_v1, headers): print(f"Got space and collection metadata from dataset {dataset_id}") return metadata +def process_user_and_resources_collections(user_v1, USER_MAP, DATASET_MAP, COLLETIONS_MAP): + """Process user resources from Clowder v1 to Clowder v2.""" + + # get collections of the user + + user_v1_datasets = get_clowder_v1_user_datasets(user_id=user_v1["id"]) + user_v1_datasets = user_v1_datasets[:2] + user_v2_api_key = create_local_user(user_v1) + USER_MAP[user_v1["id"]] = user_v2_api_key + base_user_headers_v2 = {"x-api-key": user_v2_api_key} + user_headers_v2 = { + "x-api-key": user_v2_api_key, + "content-type": "application/json", + "accept": "application/json", + } + + user_v1_collections = get_clowder_v1_user_collections_top_level( + headers=clowder_headers_v1, user_v1=user_v1 + ) + + print(f"Got {len(user_v1_collections)} user collections in the top level") + + for dataset in user_v1_datasets: + print(f"Creating dataset in v2: {dataset['id']} - {dataset['name']}") + # TODO: check if dataset is in toml_exclude_dataset_id + dataset_v1_id = dataset["id"] + dataset_v1_spaces = dataset["spaces"] + # TODO check if dataset is in toml_space_ids or exclude_space_ids + MIGRATE_DATASET = True + print(toml_exclude_dataset_ids) + print(toml_space_ids) + print(toml_exclude_space_ids) + # Check if dataset is in the excluded dataset list + if dataset_v1_id in toml_exclude_dataset_ids: + print(f"Skipping dataset {dataset_v1_id} as it is in the exclude list.") + MIGRATE_DATASET = False + # Check if dataset is in the specified space list + if toml_space_ids is not None and len(toml_space_ids) > 0: + if not any( + space_id in dataset_v1_spaces for space_id in toml_space_ids + ): + print( + f"Skipping dataset {dataset_v1_id} as it is not in the specified spaces." + ) + MIGRATE_DATASET = False + if toml_exclude_space_ids is not None and len(toml_exclude_space_ids) > 0: + if any( + space_id in dataset_v1_spaces for space_id in toml_exclude_space_ids + ): + print( + f"Skipping dataset {dataset_v1_id} as it is in the excluded spaces." + ) + MIGRATE_DATASET = False + if MIGRATE_DATASET: + dataset_v2_id = create_v2_dataset(dataset, user_headers_v2) + DATASET_MAP[dataset["id"]] = dataset_v2_id + add_dataset_metadata(dataset, dataset_v2_id, base_headers_v1, user_headers_v2) + add_dataset_folders(dataset, dataset_v2_id, user_headers_v2) + print("Created folders in the new dataset") + + all_dataset_folders = get_folder_and_subfolders(dataset_v2_id, user_headers_v2) + + # Retrieve files for the dataset in Clowder v1 + dataset_files_endpoint = ( + f"{CLOWDER_V1}/api/datasets/{dataset['id']}/files?superAdmin=true" + ) + files_response = requests.get( + dataset_files_endpoint, headers=clowder_headers_v1, verify=False + ) + files_result = files_response.json() + + for file in files_result: + file_v2_id = download_and_upload_file( + file, all_dataset_folders, dataset_v2_id, base_user_headers_v2 + ) + if file_v2_id is not None: + add_file_metadata(file, file_v2_id, clowder_headers_v1, user_headers_v2) + # posting the collection hierarchy as metadata + try: + collection_space_metadata_dict = build_collection_space_metadata_for_v1_dataset( + dataset=dataset, user_v1=user_v1, headers=clowder_headers_v1 + ) + except Exception as e: + print(e) + migration_extractor_collection_metadata = { + "listener": { + "name": "migration", + "version": "1", + "description": "Migration of metadata from Clowder v1 to Clowder v2", + }, + "context_url": "https://clowder.ncsa.illinois.edu/contexts/metadata.jsonld", + "content": collection_space_metadata_dict, + "contents": collection_space_metadata_dict, + } + v2_metadata_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/metadata" + response = requests.post( + v2_metadata_endpoint, + json=migration_extractor_collection_metadata, + headers=clowder_headers_v2, + ) + if response.status_code == 200: + print("Successfully added collection info as metadata in v2.") + else: + print( + f"Failed to add collection info as metadata in Clowder v2. Status code: {response.status_code}" + ) + else: + print(f"Skipping dataset {dataset_v1_id} as it does not meet the criteria.") + + return [USER_MAP, DATASET_MAP] + def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): """Process user resources from Clowder v1 to Clowder v2.""" @@ -891,6 +1018,7 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): # migrate users and resources USER_MAP = {} DATASET_MAP = {} + COLLECTIONS_MAP = {} users_v1 = get_clowder_v1_users() # TODO filter if toml users if toml_users is not None and len(toml_users) > 0: @@ -905,8 +1033,8 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): "[Local Account]" in user_v1["identityProvider"] and user_v1["email"] != admin_user["email"] ): - [USER_MAP, DATASET_MAP] = process_user_and_resources( - user_v1, USER_MAP, DATASET_MAP + [USER_MAP, DATASET_MAP] = process_user_and_resources_collections( + user_v1, USER_MAP, DATASET_MAP, COLLECTIONS_MAP ) print(f"Migrated user {user_v1['email']} and associated resources.") else: From 093bfb0e2668016186a972e1265a846e18a93d61 Mon Sep 17 00:00:00 2001 From: toddn Date: Fri, 22 Aug 2025 10:55:39 -0500 Subject: [PATCH 61/91] method stubs for new way of processing --- scripts/migration/migrate.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 947a5dc7d..99b94f596 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -188,6 +188,18 @@ def get_clowder_v1_user_collections_top_level(headers, user_v1): top_level_collections.append(col) return top_level_collections +def create_folder_from_dataset(dataset, dataset_v2, headers): + pass + +def create_folder_from_collection(collection, parent_folder, dataset_v2, headers): + pass + +def create_v2_dataset_from_collection(collection, user_v1, headers): + # create the dataset + + # go through sub collections creating folders + print("Creating v2-dataset from collection") + # TODO this is too slow, we need to optimize it def get_clowder_v1_dataset_collections(headers, user_v1, dataset_id): From 8477cbacc60ce6776f8fdf50950f672a157769a7 Mon Sep 17 00:00:00 2001 From: toddn Date: Fri, 22 Aug 2025 11:07:23 -0500 Subject: [PATCH 62/91] more method stubs --- scripts/migration/migrate.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 99b94f596..da685444b 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -188,6 +188,13 @@ def get_clowder_v1_user_collections_top_level(headers, user_v1): top_level_collections.append(col) return top_level_collections +def process_collection_descendants(collection, headers): + sub_collections_endpoint = "" + datasets_endpoint = f"{CLOWDER_V1}/api/datasets" + +def process_dataset_folders(dataset, headers): + pass + def create_folder_from_dataset(dataset, dataset_v2, headers): pass @@ -196,6 +203,23 @@ def create_folder_from_collection(collection, parent_folder, dataset_v2, headers def create_v2_dataset_from_collection(collection, user_v1, headers): # create the dataset + collection_name = collection["name"] + collection_description = collection["description"] + v2_license_id = "CC BY" + + dataset_in_v2_endpoint = f"{CLOWDER_V2}/api/v2/datasets?license_id={v2_license_id}" + dataset_example = { + "name":collection_name, + "description": collection_description + } + response = requests.post( + dataset_in_v2_endpoint, headers=headers, json=dataset_example + ) + + process_collection_descendants(collection, headers) + + return response.json()["id"] + # go through sub collections creating folders print("Creating v2-dataset from collection") From f2a48a7e554aa1c566fd34230fbc26495ce50adf Mon Sep 17 00:00:00 2001 From: toddn Date: Sat, 23 Aug 2025 14:59:08 -0500 Subject: [PATCH 63/91] new methods --- scripts/migration/migrate.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index da685444b..22a34ff7c 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -189,9 +189,24 @@ def get_clowder_v1_user_collections_top_level(headers, user_v1): return top_level_collections def process_collection_descendants(collection, headers): - sub_collections_endpoint = "" + child_collections_endpoint = f"{CLOWDER_V1}/api/getChildCollections" datasets_endpoint = f"{CLOWDER_V1}/api/datasets" + child_col_response = requests.get(child_collections_endpoint, headers=headers) + dataset_response = requests.get(datasets_endpoint, headers=headers) + child_col_json = child_col_response.json() + dataset_json = dataset_response.json() + + print(f"Got child collections and datasets") + for child in child_col_json: + print(f"Make a folder for this child") + print(f"Call process descendants") + process_collection_descendants(child, headers) + + for dataset in dataset_json: + print(f"Make a folder for this dataset") + print(f"Add folders and subfolders") + def process_dataset_folders(dataset, headers): pass From 9b9cbede9e8e2dd9d1f666d189228904a0128bf0 Mon Sep 17 00:00:00 2001 From: toddn Date: Sat, 23 Aug 2025 15:06:36 -0500 Subject: [PATCH 64/91] organize based on the type of parent, need to work out this logic --- scripts/migration/migrate.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 22a34ff7c..bc7985497 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -188,9 +188,9 @@ def get_clowder_v1_user_collections_top_level(headers, user_v1): top_level_collections.append(col) return top_level_collections -def process_collection_descendants(collection, headers): - child_collections_endpoint = f"{CLOWDER_V1}/api/getChildCollections" - datasets_endpoint = f"{CLOWDER_V1}/api/datasets" +def process_collection_descendants(collection, headers, v2_parent_id, v2_parent_type, v2_dataset_id): + child_collections_endpoint = f"{CLOWDER_V1}/api/{collection['id']}/getChildCollections" + datasets_endpoint = f"{CLOWDER_V1}/api/collections/{collection['id']}/datasets" child_col_response = requests.get(child_collections_endpoint, headers=headers) dataset_response = requests.get(datasets_endpoint, headers=headers) @@ -199,6 +199,10 @@ def process_collection_descendants(collection, headers): print(f"Got child collections and datasets") for child in child_col_json: + if v2_parent_type == "dataset": + print(f"Add folder to the dataset") + else: + print(f"parent was a folder") print(f"Make a folder for this child") print(f"Call process descendants") process_collection_descendants(child, headers) @@ -231,7 +235,10 @@ def create_v2_dataset_from_collection(collection, user_v1, headers): dataset_in_v2_endpoint, headers=headers, json=dataset_example ) - process_collection_descendants(collection, headers) + new_dataset_json = response.json() + v2_dataset_id = new_dataset_json["id"] + + process_collection_descendants(collection, headers, new_dataset_json["id"], "dataset", v2_dataset_id) return response.json()["id"] From b34eb84c49ec37eaa5fac311675e828aafd7fcba Mon Sep 17 00:00:00 2001 From: toddn Date: Sat, 23 Aug 2025 15:23:36 -0500 Subject: [PATCH 65/91] this should work up to the datasets in a root collection, need testing and strategy for the folders --- scripts/migration/migrate.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index bc7985497..07b7f44c6 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -201,15 +201,26 @@ def process_collection_descendants(collection, headers, v2_parent_id, v2_parent_ for child in child_col_json: if v2_parent_type == "dataset": print(f"Add folder to the dataset") + folder_name = child["name"] + new_folder = create_folder_if_not_exists_or_get(folder_name, None, v2_dataset_id, headers) + process_collection_descendants(child, headers, new_folder['id'], 'folder', v2_dataset_id ) else: print(f"parent was a folder") - print(f"Make a folder for this child") - print(f"Call process descendants") - process_collection_descendants(child, headers) + print(f"Add folder to the dataset") + folder_name = child["name"] + new_folder = create_folder_if_not_exists_or_get(folder_name, v2_parent_id, v2_dataset_id, headers) + process_collection_descendants(child, headers, new_folder['id'], 'folder', v2_dataset_id) for dataset in dataset_json: - print(f"Make a folder for this dataset") - print(f"Add folders and subfolders") + if v2_parent_type == "dataset": + print(f"Parent is a dataset") + new_folder = create_folder_if_not_exists_or_get(dataset["name"], v2_parent_id, v2_dataset_id, headers) + print(f"Now we need to add the sub folders of this dataset") + else: + print(f"Parent is a folder") + new_folder = create_folder_if_not_exists_or_get(dataset["name"], v2_parent_id, v2_dataset_id, headers) + + def process_dataset_folders(dataset, headers): pass From a5dd8ea3e5d151cf2499d95dc514f27a0502e5b3 Mon Sep 17 00:00:00 2001 From: toddn Date: Sun, 24 Aug 2025 13:16:35 -0500 Subject: [PATCH 66/91] remove unused methods --- scripts/migration/migrate.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 07b7f44c6..df195858d 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -221,15 +221,9 @@ def process_collection_descendants(collection, headers, v2_parent_id, v2_parent_ new_folder = create_folder_if_not_exists_or_get(dataset["name"], v2_parent_id, v2_dataset_id, headers) - -def process_dataset_folders(dataset, headers): - pass - -def create_folder_from_dataset(dataset, dataset_v2, headers): +def process_dataset_folders(dataset, headers, parent_type, parent_id): pass -def create_folder_from_collection(collection, parent_folder, dataset_v2, headers): - pass def create_v2_dataset_from_collection(collection, user_v1, headers): # create the dataset From f534160342c74c113ff087b89d2e1e0e606c94b4 Mon Sep 17 00:00:00 2001 From: toddn Date: Sun, 24 Aug 2025 14:23:02 -0500 Subject: [PATCH 67/91] too many folders created test with new 'test space' with fewer options first --- scripts/migration/migrate.py | 38 ++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index df195858d..81bbd954a 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -188,12 +188,12 @@ def get_clowder_v1_user_collections_top_level(headers, user_v1): top_level_collections.append(col) return top_level_collections -def process_collection_descendants(collection, headers, v2_parent_id, v2_parent_type, v2_dataset_id): - child_collections_endpoint = f"{CLOWDER_V1}/api/{collection['id']}/getChildCollections" +def process_collection_descendants(collection, headers_v1,headers_v2, v2_parent_id, v2_parent_type, v2_dataset_id): + child_collections_endpoint = f"{CLOWDER_V1}/api/collections/{collection['id']}/getChildCollections" datasets_endpoint = f"{CLOWDER_V1}/api/collections/{collection['id']}/datasets" - child_col_response = requests.get(child_collections_endpoint, headers=headers) - dataset_response = requests.get(datasets_endpoint, headers=headers) + child_col_response = requests.get(child_collections_endpoint, headers=headers_v1) + dataset_response = requests.get(datasets_endpoint, headers=headers_v1) child_col_json = child_col_response.json() dataset_json = dataset_response.json() @@ -202,30 +202,34 @@ def process_collection_descendants(collection, headers, v2_parent_id, v2_parent_ if v2_parent_type == "dataset": print(f"Add folder to the dataset") folder_name = child["name"] - new_folder = create_folder_if_not_exists_or_get(folder_name, None, v2_dataset_id, headers) - process_collection_descendants(child, headers, new_folder['id'], 'folder', v2_dataset_id ) + new_folder = create_folder_if_not_exists_or_get(folder_name, None, v2_dataset_id, headers_v2) + process_collection_descendants(child, headers_v1,headers_v2, new_folder['id'], 'folder', v2_dataset_id ) else: print(f"parent was a folder") print(f"Add folder to the dataset") folder_name = child["name"] - new_folder = create_folder_if_not_exists_or_get(folder_name, v2_parent_id, v2_dataset_id, headers) - process_collection_descendants(child, headers, new_folder['id'], 'folder', v2_dataset_id) + new_folder = create_folder_if_not_exists_or_get(folder_name, v2_parent_id, v2_dataset_id, headers_v2) + process_collection_descendants(child, headers_v1, headers_v2, new_folder['id'], 'folder', v2_dataset_id) for dataset in dataset_json: if v2_parent_type == "dataset": print(f"Parent is a dataset") - new_folder = create_folder_if_not_exists_or_get(dataset["name"], v2_parent_id, v2_dataset_id, headers) + new_folder = create_folder_if_not_exists_or_get(dataset["name"], v2_parent_id, v2_dataset_id, headers_v2) print(f"Now we need to add the sub folders of this dataset") else: print(f"Parent is a folder") - new_folder = create_folder_if_not_exists_or_get(dataset["name"], v2_parent_id, v2_dataset_id, headers) + new_folder = create_folder_if_not_exists_or_get(dataset["name"], v2_parent_id, v2_dataset_id, headers_v2) -def process_dataset_folders(dataset, headers, parent_type, parent_id): - pass +def process_dataset_folders(dataset, headers_v1, headers_v2, parent_type, parent_id): + folder_endpoint = f"{CLOWDER_V1}/api/datasets/{dataset['id']}/folders" + folder_response = requests.get(folder_endpoint, headers=headers_v1) + folder_json = folder_response.json() + print(f"Got dataset folders") -def create_v2_dataset_from_collection(collection, user_v1, headers): + +def create_v2_dataset_from_collection(collection, user_v1, headers_v1, headers_v2): # create the dataset collection_name = collection["name"] collection_description = collection["description"] @@ -237,13 +241,13 @@ def create_v2_dataset_from_collection(collection, user_v1, headers): "description": collection_description } response = requests.post( - dataset_in_v2_endpoint, headers=headers, json=dataset_example + dataset_in_v2_endpoint, headers=headers_v2, json=dataset_example ) new_dataset_json = response.json() v2_dataset_id = new_dataset_json["id"] - process_collection_descendants(collection, headers, new_dataset_json["id"], "dataset", v2_dataset_id) + process_collection_descendants(collection, headers_v1, headers_v2, new_dataset_json["id"], "dataset", v2_dataset_id) return response.json()["id"] @@ -870,6 +874,10 @@ def process_user_and_resources_collections(user_v1, USER_MAP, DATASET_MAP, COLLE print(f"Got {len(user_v1_collections)} user collections in the top level") + for top_level_col in user_v1_collections: + dataset_v2 = create_v2_dataset_from_collection(top_level_col, user_v1, clowder_headers_v1, user_headers_v2) + print('did this') + for dataset in user_v1_datasets: print(f"Creating dataset in v2: {dataset['id']} - {dataset['name']}") # TODO: check if dataset is in toml_exclude_dataset_id From a00e600d752112be4582ed85da736963e2a668d9 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 25 Aug 2025 16:04:12 -0500 Subject: [PATCH 68/91] progress for dataset folders and files need to create folders with a map, then upload files to correct folder --- scripts/migration/migrate.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 81bbd954a..8a2fc8918 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -212,13 +212,21 @@ def process_collection_descendants(collection, headers_v1,headers_v2, v2_parent_ process_collection_descendants(child, headers_v1, headers_v2, new_folder['id'], 'folder', v2_dataset_id) for dataset in dataset_json: + # TODO TODO DATASET PROCESSING if v2_parent_type == "dataset": print(f"Parent is a dataset") new_folder = create_folder_if_not_exists_or_get(dataset["name"], v2_parent_id, v2_dataset_id, headers_v2) print(f"Now we need to add the sub folders of this dataset") + # TODO get DATASET FOLDERS HERE FROM v1 + process_dataset_folders(dataset, headers_v1, headers_v2, new_folder['id'], v2_dataset_id) + process_dataset_files(dataset, headers_v1, headers_v2, new_folder['id'], v2_dataset_id) else: print(f"Parent is a folder") new_folder = create_folder_if_not_exists_or_get(dataset["name"], v2_parent_id, v2_dataset_id, headers_v2) + # TODO GET DATASET FOLDERS HERE FROM v1 + process_dataset_folders(dataset, headers_v1, headers_v2, new_folder['id'], v2_dataset_id) + process_dataset_files(dataset, headers_v1, headers_v2, new_folder['id'], v2_dataset_id) + def process_dataset_folders(dataset, headers_v1, headers_v2, parent_type, parent_id): @@ -227,6 +235,12 @@ def process_dataset_folders(dataset, headers_v1, headers_v2, parent_type, parent folder_json = folder_response.json() print(f"Got dataset folders") +def process_dataset_files(dataset, headers_v1, headers_v2, parent_type, parent_id): + files_endpoint = f"{CLOWDER_V1}/api/datasets/{dataset['id']}/files" + files_response = requests.get(files_endpoint, headers=headers_v1) + files_json = files_response.json() + print(f"Got dataset files") + def create_v2_dataset_from_collection(collection, user_v1, headers_v1, headers_v2): @@ -518,6 +532,8 @@ def add_folder_hierarchy(folder_hierarchy, dataset_v2, headers): def create_folder_if_not_exists_or_get(folder, parent, dataset_v2, headers): """Create a folder if it does not exist or return the existing folder.""" + # TODO if this is a dataset, we should create the subfolders + # TODO or write another method for processing datasets current_folders = get_folder_and_subfolders(dataset_v2, headers) folder_data = ( {"name": folder, "parent_folder": parent} if parent else {"name": folder} From 2fb33c4c780d6c639fc523603d1eb08e7a48b233 Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 26 Aug 2025 17:12:25 -0500 Subject: [PATCH 69/91] the folder hierarchy part is not working for v2 --- scripts/migration/migrate.py | 49 +++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 8a2fc8918..7f3c00dad 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -219,13 +219,13 @@ def process_collection_descendants(collection, headers_v1,headers_v2, v2_parent_ print(f"Now we need to add the sub folders of this dataset") # TODO get DATASET FOLDERS HERE FROM v1 process_dataset_folders(dataset, headers_v1, headers_v2, new_folder['id'], v2_dataset_id) - process_dataset_files(dataset, headers_v1, headers_v2, new_folder['id'], v2_dataset_id) + process_dataset_files(dataset, headers_v1, headers_v2, 'folder', new_folder['id'], v2_dataset_id) else: print(f"Parent is a folder") new_folder = create_folder_if_not_exists_or_get(dataset["name"], v2_parent_id, v2_dataset_id, headers_v2) # TODO GET DATASET FOLDERS HERE FROM v1 process_dataset_folders(dataset, headers_v1, headers_v2, new_folder['id'], v2_dataset_id) - process_dataset_files(dataset, headers_v1, headers_v2, new_folder['id'], v2_dataset_id) + process_dataset_files(dataset, headers_v1, headers_v2, 'folder', new_folder['id'], v2_dataset_id) @@ -235,10 +235,40 @@ def process_dataset_folders(dataset, headers_v1, headers_v2, parent_type, parent folder_json = folder_response.json() print(f"Got dataset folders") -def process_dataset_files(dataset, headers_v1, headers_v2, parent_type, parent_id): +def get_v1_dataset_folders(dataset, headers_v1, headers_v2, parent_type, parent_id): + folder_endpoint = f"{CLOWDER_V1}/api/datasets/{dataset['id']}/folders" + folder_response = requests.get(folder_endpoint, headers=headers_v1) + folder_json = folder_response.json() + return folder_json + +def process_dataset_files(dataset, headers_v1, headers_v2, parent_type, parent_id, dataset_v2_id): + dataset_v1_folders = get_v1_dataset_folders(dataset, headers_v1, headers_v2, parent_type, parent_id) + + for folder_v1 in dataset_v1_folders: + current_folder_hierarchy = folder_v1['name'] + new_parent = add_folder_hierarchy_to_migration_folder(folder_hierarchy=current_folder_hierarchy, + dataset_v2=dataset_v2_id, + folder_id_v2=parent_id, + headers=headers_v2 + ) + parent_id = new_parent + print(f"This is the folder") + files_endpoint = f"{CLOWDER_V1}/api/datasets/{dataset['id']}/files" files_response = requests.get(files_endpoint, headers=headers_v1) files_json = files_response.json() + for file in files_json: + if 'folders' in file: + print(f"This file is in a folder") + # TODO get folder + folder_endpoint = f"{CLOWDER_V1}/api/datasets/{dataset['id']}/files" + else: + print(f"This file is not in a folder") + # TODO upload it to the folder + if parent_type == "dataset": + print(f"Upload to a dataset") + if parent_type == "folder": + print(f"Upload to a folder") print(f"Got dataset files") @@ -517,6 +547,19 @@ def create_v2_group(space, headers): response = requests.post(group_in_v2_endpoint, json=group, headers=headers) return response.json()["id"] +# TODO try this +def add_folder_hierarchy_to_migration_folder(folder_hierarchy, dataset_v2, folder_id_v2, headers): + """Add folder hierarchy to a dataset in Clowder v2.""" + hierarchy_parts = folder_hierarchy.split("/") + if hierarchy_parts[0] == '': + hierarchy_parts = hierarchy_parts[1:] + current_parent = folder_id_v2 + for part in hierarchy_parts: + result = create_folder_if_not_exists_or_get( + part, current_parent, dataset_v2, headers + ) + if result: + current_parent = result["id"] def add_folder_hierarchy(folder_hierarchy, dataset_v2, headers): """Add folder hierarchy to a dataset in Clowder v2.""" From 10f2026c228e228b5c6b65dcbfaf92a905c9d58d Mon Sep 17 00:00:00 2001 From: toddn Date: Wed, 27 Aug 2025 11:39:32 -0500 Subject: [PATCH 70/91] new route should return all folders in a dataset --- backend/app/routers/datasets.py | 34 +++++++++++++++++++++++++++++++++ scripts/migration/migrate.py | 22 ++++++++++----------- 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/backend/app/routers/datasets.py b/backend/app/routers/datasets.py index ae608174b..ede07d801 100644 --- a/backend/app/routers/datasets.py +++ b/backend/app/routers/datasets.py @@ -837,6 +837,40 @@ async def get_dataset_folders_and_files( return page.dict() raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found") +@router.get("/{dataset_id}/folders", response_model=Paged) +async def get_dataset_folders( + dataset_id: str, + authenticated: bool = Depends(CheckStatus("AUTHENTICATED")), + public: bool = Depends(CheckStatus("PUBLIC")), + user_id=Depends(get_user), + skip: int = 0, + limit: int = 10, + admin=Depends(get_admin), + enable_admin: bool = False, + admin_mode: bool = Depends(get_admin_mode), + allow: bool = Depends(Authorization("viewer")), +): + if ( + await DatasetDBViewList.find_one( + Or( + DatasetDBViewList.id == PydanticObjectId(dataset_id), + ) + ) + ) is not None: + if authenticated or public or (admin and admin_mode): + query = [ + FolderFileViewList.dataset_id == PydanticObjectId(dataset_id), + ] + else: + query = [ + FolderFileViewList.dataset_id == PydanticObjectId(dataset_id), + ] + + folders = (await FolderFileViewList.find(*query).to_list()) + + return folders.dict() + raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found") + @router.delete("/{dataset_id}/folders/{folder_id}") async def delete_folder( diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 7f3c00dad..93876f872 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -212,7 +212,6 @@ def process_collection_descendants(collection, headers_v1,headers_v2, v2_parent_ process_collection_descendants(child, headers_v1, headers_v2, new_folder['id'], 'folder', v2_dataset_id) for dataset in dataset_json: - # TODO TODO DATASET PROCESSING if v2_parent_type == "dataset": print(f"Parent is a dataset") new_folder = create_folder_if_not_exists_or_get(dataset["name"], v2_parent_id, v2_dataset_id, headers_v2) @@ -246,12 +245,12 @@ def process_dataset_files(dataset, headers_v1, headers_v2, parent_type, parent_i for folder_v1 in dataset_v1_folders: current_folder_hierarchy = folder_v1['name'] - new_parent = add_folder_hierarchy_to_migration_folder(folder_hierarchy=current_folder_hierarchy, + # TODO CHECK HERE + add_folder_hierarchy_to_migration_folder(folder_hierarchy=current_folder_hierarchy, dataset_v2=dataset_v2_id, folder_id_v2=parent_id, headers=headers_v2 ) - parent_id = new_parent print(f"This is the folder") files_endpoint = f"{CLOWDER_V1}/api/datasets/{dataset['id']}/files" @@ -575,9 +574,8 @@ def add_folder_hierarchy(folder_hierarchy, dataset_v2, headers): def create_folder_if_not_exists_or_get(folder, parent, dataset_v2, headers): """Create a folder if it does not exist or return the existing folder.""" - # TODO if this is a dataset, we should create the subfolders - # TODO or write another method for processing datasets current_folders = get_folder_and_subfolders(dataset_v2, headers) + current_all_folders = get_all_folder_and_subfolders(dataset_v2, headers) folder_data = ( {"name": folder, "parent_folder": parent} if parent else {"name": folder} ) @@ -593,6 +591,13 @@ def create_folder_if_not_exists_or_get(folder, parent, dataset_v2, headers): ) return response.json() +def get_all_folder_and_subfolders(dataset_id, headers): + """Retrieve all folders and subfolders in a dataset.""" + endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_id}/folders" + response = requests.get(endpoint, headers=headers) + folder_response = response.json() + return folder_response + def get_folder_and_subfolders(dataset_id, headers): """Retrieve all folders and subfolders in a dataset.""" @@ -868,7 +873,6 @@ def build_collection_metadata_for_v1_dataset(dataset_id, user_v1, headers): return dataset_collections -# TODO test this method def build_collection_space_metadata_for_v1_dataset(dataset, user_v1, headers): dataset_id = dataset["id"] dataset_collections = [] @@ -939,10 +943,8 @@ def process_user_and_resources_collections(user_v1, USER_MAP, DATASET_MAP, COLLE for dataset in user_v1_datasets: print(f"Creating dataset in v2: {dataset['id']} - {dataset['name']}") - # TODO: check if dataset is in toml_exclude_dataset_id dataset_v1_id = dataset["id"] dataset_v1_spaces = dataset["spaces"] - # TODO check if dataset is in toml_space_ids or exclude_space_ids MIGRATE_DATASET = True print(toml_exclude_dataset_ids) print(toml_space_ids) @@ -1042,10 +1044,8 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): for dataset in user_v1_datasets: print(f"Creating dataset in v2: {dataset['id']} - {dataset['name']}") - # TODO: check if dataset is in toml_exclude_dataset_id dataset_v1_id = dataset["id"] dataset_v1_spaces = dataset["spaces"] - # TODO check if dataset is in toml_space_ids or exclude_space_ids MIGRATE_DATASET = True print(toml_exclude_dataset_ids) print(toml_space_ids) @@ -1150,7 +1150,7 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): DATASET_MAP = {} COLLECTIONS_MAP = {} users_v1 = get_clowder_v1_users() - # TODO filter if toml users + if toml_users is not None and len(toml_users) > 0: print(f"Using spaces from config.toml: {toml_users}") users_v1 = [ From 19fb46b1823b60d608769f5cf7bbf55805be861d Mon Sep 17 00:00:00 2001 From: toddn Date: Thu, 28 Aug 2025 11:54:07 -0500 Subject: [PATCH 71/91] new route for returning all folders in a dataset --- backend/app/routers/datasets.py | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/backend/app/routers/datasets.py b/backend/app/routers/datasets.py index ede07d801..558973322 100644 --- a/backend/app/routers/datasets.py +++ b/backend/app/routers/datasets.py @@ -663,7 +663,7 @@ async def delete_freeze_dataset_version( if ( frozen_dataset := await DatasetFreezeDB.find_one( DatasetFreezeDB.origin_id == PydanticObjectId(dataset_id), - DatasetFreezeDB.frozen_version_num == frozen_version_num, + DatasetFreezeDxB.frozen_version_num == frozen_version_num, ) ) is not None: return await _delete_frozen_dataset(frozen_dataset, fs, hard_delete=False) @@ -837,8 +837,8 @@ async def get_dataset_folders_and_files( return page.dict() raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found") -@router.get("/{dataset_id}/folders", response_model=Paged) -async def get_dataset_folders( +@router.get("/{dataset_id}/all_folders") +async def get_dataset_folders_all( dataset_id: str, authenticated: bool = Depends(CheckStatus("AUTHENTICATED")), public: bool = Depends(CheckStatus("PUBLIC")), @@ -866,7 +866,31 @@ async def get_dataset_folders( FolderFileViewList.dataset_id == PydanticObjectId(dataset_id), ] - folders = (await FolderFileViewList.find(*query).to_list()) + folders = (await FolderFileViewList.find(*query).aggregate( + [ + _get_page_query( + skip, + limit, + sort_clause={ + "$sort": { + "object_type": -1, # folder first + "created": -1, # then sort by created descendingly + } + }, + ) + ], + ).to_list()) + page_metadata = _construct_page_metadata(folders, skip, limit) + + page = Paged( + metadata=page_metadata, + data=[ + FolderOut(id=item.pop("_id"), **item) + for item in folders[0]["data"] + ], + ) + + return page.dict() return folders.dict() raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found") From 4640c19ff12a6db7d2ef73d7ee175b4451137fd4 Mon Sep 17 00:00:00 2001 From: toddn Date: Thu, 28 Aug 2025 13:12:17 -0500 Subject: [PATCH 72/91] folder hierarchy is now properly created --- scripts/migration/migrate.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 93876f872..501e2e0e7 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -245,22 +245,23 @@ def process_dataset_files(dataset, headers_v1, headers_v2, parent_type, parent_i for folder_v1 in dataset_v1_folders: current_folder_hierarchy = folder_v1['name'] - # TODO CHECK HERE add_folder_hierarchy_to_migration_folder(folder_hierarchy=current_folder_hierarchy, dataset_v2=dataset_v2_id, folder_id_v2=parent_id, headers=headers_v2 ) - print(f"This is the folder") + all_v2_dataset_folders = get_all_folder_and_subfolders(dataset_v2_id, headers_v2) files_endpoint = f"{CLOWDER_V1}/api/datasets/{dataset['id']}/files" files_response = requests.get(files_endpoint, headers=headers_v1) files_json = files_response.json() for file in files_json: if 'folders' in file: print(f"This file is in a folder") - # TODO get folder - folder_endpoint = f"{CLOWDER_V1}/api/datasets/{dataset['id']}/files" + matching_folder = None + for folder_v2 in all_v2_dataset_folders: + if folder_v2['name'] == file['folders']['name']: + print(f"Upload this file to a folder") else: print(f"This file is not in a folder") # TODO upload it to the folder @@ -574,13 +575,13 @@ def add_folder_hierarchy(folder_hierarchy, dataset_v2, headers): def create_folder_if_not_exists_or_get(folder, parent, dataset_v2, headers): """Create a folder if it does not exist or return the existing folder.""" - current_folders = get_folder_and_subfolders(dataset_v2, headers) + # current_folders = get_folder_and_subfolders(dataset_v2, headers) current_all_folders = get_all_folder_and_subfolders(dataset_v2, headers) folder_data = ( {"name": folder, "parent_folder": parent} if parent else {"name": folder} ) - for existing_folder in current_folders: + for existing_folder in current_all_folders: if existing_folder["name"] == folder: return existing_folder @@ -593,9 +594,9 @@ def create_folder_if_not_exists_or_get(folder, parent, dataset_v2, headers): def get_all_folder_and_subfolders(dataset_id, headers): """Retrieve all folders and subfolders in a dataset.""" - endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_id}/folders" + endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_id}/all_folders" response = requests.get(endpoint, headers=headers) - folder_response = response.json() + folder_response = response.json().get("data", []) return folder_response From ff8c62ca67a7e5a49b63df6fe6a8bd5a024cdd9d Mon Sep 17 00:00:00 2001 From: toddn Date: Thu, 28 Aug 2025 13:20:54 -0500 Subject: [PATCH 73/91] new method upload to folder - we find the folder, may need to deprecate old method since it looks like it won't find the right folders --- scripts/migration/migrate.py | 48 +++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 501e2e0e7..8fc488e14 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -255,13 +255,17 @@ def process_dataset_files(dataset, headers_v1, headers_v2, parent_type, parent_i files_endpoint = f"{CLOWDER_V1}/api/datasets/{dataset['id']}/files" files_response = requests.get(files_endpoint, headers=headers_v1) files_json = files_response.json() + # TODO WORK HERE for file in files_json: if 'folders' in file: print(f"This file is in a folder") + current_file_folder_name = file['folders']['name'] matching_folder = None for folder_v2 in all_v2_dataset_folders: if folder_v2['name'] == file['folders']['name']: - print(f"Upload this file to a folder") + print(f"Upload this file to a folder") + matching_folder = folder_v2 + download_and_upload_file() else: print(f"This file is not in a folder") # TODO upload it to the folder @@ -620,6 +624,48 @@ def add_dataset_folders(dataset_v1, dataset_v2, headers): add_folder_hierarchy(folder["name"], dataset_v2, headers) +def download_and_upload_file_to_folder(file, folder, dataset_v2_id, headers_v2): + """Download a file from Clowder v1 and upload it to Clowder v2.""" + filename = file["filename"] + file_id = file["id"] + file_folder = folder + + # Download the file from Clowder v1 + v1_download_url = f"{CLOWDER_V1}/api/files/{file_id}?superAdmin=true" + print(f"Downloading file: {filename}") + download_response = requests.get(v1_download_url, headers=clowder_headers_v1) + + with open(filename, "wb") as f: + f.write(download_response.content) + + # Upload the file to Clowder v2 + dataset_file_upload_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/files" + if folder: + dataset_file_upload_endpoint += f"Multiple?folder_id={folder['id']}" + response = requests.post( + dataset_file_upload_endpoint, + headers=headers_v2, + files={"file": open(filename, "rb")}, + ) + else: + print(f"This file is not in a folder") + + # Clean up the local file after upload + try: + os.remove(filename) + except Exception as e: + print(f"Could not delete locally downloaded file: {filename}") + print(e) + + if response.status_code == 200: + print(f"Uploaded file: {filename} to dataset {dataset_v2_id}") + return response.json().get("id") + else: + print(f"Failed to upload file: {filename} to dataset {dataset_v2_id}") + + return None + + def download_and_upload_file(file, all_dataset_folders, dataset_v2_id, headers_v2): """Download a file from Clowder v1 and upload it to Clowder v2.""" filename = file["filename"] From 1a5e2b7ffde0d0dbd46cae1d0208b15324930910 Mon Sep 17 00:00:00 2001 From: toddn Date: Fri, 29 Aug 2025 11:10:29 -0500 Subject: [PATCH 74/91] new method should work for uploading files to either a dataset or a folder, most of these will be uploaded to folders following the 'collection becomes dataset' model --- scripts/migration/migrate.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 8fc488e14..9b4f8a5c4 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -640,15 +640,19 @@ def download_and_upload_file_to_folder(file, folder, dataset_v2_id, headers_v2): # Upload the file to Clowder v2 dataset_file_upload_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/files" - if folder: + if folder is not None: + # add folder if it is not None dataset_file_upload_endpoint += f"Multiple?folder_id={folder['id']}" - response = requests.post( - dataset_file_upload_endpoint, - headers=headers_v2, - files={"file": open(filename, "rb")}, - ) + response = requests.post( + dataset_file_upload_endpoint, + headers=headers_v2, + files={"file": open(filename, "rb")}, + ) + if response.status_code == 200: + print(f"Uploaded file: {filename} to dataset {dataset_v2_id}") + return response.json().get("id") else: - print(f"This file is not in a folder") + print(f"Failed to upload file: {filename} to dataset {dataset_v2_id}") # Clean up the local file after upload try: @@ -656,13 +660,6 @@ def download_and_upload_file_to_folder(file, folder, dataset_v2_id, headers_v2): except Exception as e: print(f"Could not delete locally downloaded file: {filename}") print(e) - - if response.status_code == 200: - print(f"Uploaded file: {filename} to dataset {dataset_v2_id}") - return response.json().get("id") - else: - print(f"Failed to upload file: {filename} to dataset {dataset_v2_id}") - return None From 04fc4b2956c3e1e56d210b18ca53d579a550d028 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 1 Sep 2025 15:39:57 -0500 Subject: [PATCH 75/91] partial fix? --- scripts/migration/migrate.py | 235 +++++++++++++++++++++++++++++++++-- scripts/migration/test.py | 72 +++++++++++ 2 files changed, 297 insertions(+), 10 deletions(-) create mode 100644 scripts/migration/test.py diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 9b4f8a5c4..7df66d528 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -265,7 +265,7 @@ def process_dataset_files(dataset, headers_v1, headers_v2, parent_type, parent_i if folder_v2['name'] == file['folders']['name']: print(f"Upload this file to a folder") matching_folder = folder_v2 - download_and_upload_file() + download_and_upload_file_to_folder_id(file, matching_folder, dataset_v2_id, headers_v2) else: print(f"This file is not in a folder") # TODO upload it to the folder @@ -568,6 +568,8 @@ def add_folder_hierarchy_to_migration_folder(folder_hierarchy, dataset_v2, folde def add_folder_hierarchy(folder_hierarchy, dataset_v2, headers): """Add folder hierarchy to a dataset in Clowder v2.""" hierarchy_parts = folder_hierarchy.split("/") + if hierarchy_parts[0] == '': + hierarchy_parts = hierarchy_parts[1:] current_parent = None for part in hierarchy_parts: result = create_folder_if_not_exists_or_get( @@ -642,11 +644,15 @@ def download_and_upload_file_to_folder(file, folder, dataset_v2_id, headers_v2): dataset_file_upload_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/files" if folder is not None: # add folder if it is not None - dataset_file_upload_endpoint += f"Multiple?folder_id={folder['id']}" + folder_id = folder["id"] + dataset_file_upload_endpoint += f"Multiple?folder_id={folder_id}" + files = [ + ("files", open(filename, "rb")), + ] response = requests.post( dataset_file_upload_endpoint, headers=headers_v2, - files={"file": open(filename, "rb")}, + files=files, ) if response.status_code == 200: print(f"Uploaded file: {filename} to dataset {dataset_v2_id}") @@ -662,6 +668,134 @@ def download_and_upload_file_to_folder(file, folder, dataset_v2_id, headers_v2): print(e) return None +def download_and_upload_file_to_folder_id(file, folder_v2, dataset_v2_id, headers_v2): + """Download a file from Clowder v1 and upload it to Clowder v2.""" + filename = file["filename"] + file_id = file["id"] + file_folder = file.get("folders", None) + + # Download the file from Clowder v1 + v1_download_url = f"{CLOWDER_V1}/api/files/{file_id}?superAdmin=true" + print(f"Downloading file: {filename}") + download_response = requests.get(v1_download_url, headers=clowder_headers_v1) + + with open(filename, "wb") as f: + f.write(download_response.content) + + file_exists = os.path.exists(filename) + # Upload the file to Clowder v2 + dataset_file_upload_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/files" + if folder_v2 is not None: + dataset_file_upload_endpoint += f"?folder_id={folder_v2['id']}" + response = requests.post( + dataset_file_upload_endpoint, + headers=headers_v2, + files={"file": open(filename, "rb")}, + ) + + # Clean up the local file after upload + # try: + # os.remove(filename) + # except Exception as e: + # print(f"Could not delete locally downloaded file: {filename}") + # print(e) + + if response.status_code == 200: + print(f"Uploaded file: {filename} to dataset {dataset_v2_id}") + return response.json().get("id") + else: + print(f"Failed to upload file: {filename} to dataset {dataset_v2_id}") + + return None + +# def download_and_upload_file_to_folder_id(file, folder_v2, dataset_v2_id, headers_v2): +# """Download a file from Clowder v1 and upload it to Clowder v2.""" +# filename = file["filename"] +# file_id = file["id"] +# +# # DEBUG: Print all inputs +# print(f"=== DEBUG START ===") +# print(f"File: {file}") +# print(f"Folder_v2: {folder_v2}") +# print(f"Dataset_v2_id: {dataset_v2_id}") +# print(f"Headers_v2 keys: {list(headers_v2.keys()) if headers_v2 else 'None'}") +# +# # Download the file from Clowder v1 +# v1_download_url = f"{CLOWDER_V1}/api/files/{file_id}?superAdmin=true" +# print(f"Downloading file: {filename} from {v1_download_url}") +# download_response = requests.get(v1_download_url, headers=clowder_headers_v1) +# print(f"Download status: {download_response.status_code}") +# +# with open(filename, "wb") as f: +# f.write(download_response.content) +# +# # Check file exists and has content +# file_size = os.path.getsize(filename) +# print(f"Local file size: {file_size} bytes") +# +# # Upload the file to Clowder v2 +# dataset_file_upload_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/files" +# +# if folder_v2 is not None: +# folder_id = folder_v2['id'] if isinstance(folder_v2, dict) else folder_v2.id +# dataset_file_upload_endpoint += f"Multiple?folder_id={folder_id}" +# +# print(f"Upload endpoint: {dataset_file_upload_endpoint}") +# +# # Read file content to verify it's not corrupted +# with open(filename, "rb") as f: +# file_content = f.read() +# print(f"File content length: {len(file_content)}") +# print(f"File content starts with: {file_content[:100]}...") +# +# # Make the upload request with detailed debugging +# with open(filename, "rb") as file_obj: +# files = {"file": (filename, file_obj)} +# +# print(f"Final files dict: {files}") +# # Create headers without content-type for file uploads +# upload_headers = headers_v2.copy() +# upload_headers.pop('content-type', None) +# print(f"Final headers: {upload_headers}") +# +# # Use a session to see raw request +# session = requests.Session() +# prepared_request = requests.Request( +# 'POST', +# dataset_file_upload_endpoint, +# headers=upload_headers, +# files=files +# ).prepare() +# +# print(f"Prepared request URL: {prepared_request.url}") +# print(f"Prepared request headers: {dict(prepared_request.headers)}") +# # Don't print body as it's binary, but we can check content-type +# print(f"Content-Type header: {prepared_request.headers.get('Content-Type')}") +# +# response = session.send(prepared_request) +# +# # DEBUG: Full response analysis +# print(f"Response status: {response.status_code}") +# print(f"Response headers: {dict(response.headers)}") +# print(f"Response text: {response.text}") +# print(f"=== DEBUG END ===") +# +# # Clean up the local file after upload +# try: +# os.remove(filename) +# except Exception as e: +# print(f"Could not delete locally downloaded file: {filename}") +# print(e) +# +# if response.status_code == 200: +# print(f"Uploaded file: {filename} to dataset {dataset_v2_id}") +# return response.json().get("id") +# else: +# print(f"Failed to upload file: {filename} to dataset {dataset_v2_id}") +# +# return None + + def download_and_upload_file(file, all_dataset_folders, dataset_v2_id, headers_v2): """Download a file from Clowder v1 and upload it to Clowder v2.""" @@ -693,11 +827,17 @@ def download_and_upload_file(file, all_dataset_folders, dataset_v2_id, headers_v dataset_file_upload_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/files" if matching_folder: dataset_file_upload_endpoint += f"Multiple?folder_id={matching_folder['id']}" - response = requests.post( - dataset_file_upload_endpoint, - headers=headers_v2, - files={"file": open(filename, "rb")}, - ) + response = requests.post( + dataset_file_upload_endpoint, + headers=headers_v2, + files=[("files", (filename, open(filename, "rb")))], + ) + else: + response = requests.post( + dataset_file_upload_endpoint, + headers=headers_v2, + files={"file": open(filename, "rb")}, + ) # Clean up the local file after upload try: @@ -714,6 +854,78 @@ def download_and_upload_file(file, all_dataset_folders, dataset_v2_id, headers_v return None +def download_and_upload_file_1(file, all_dataset_folders, dataset_v2_id, headers_v2): + """Download a file from Clowder v1 and upload it to Clowder v2.""" + filename = file["filename"] + file_id = file["id"] + file_folder = file.get("folders", None) + + # Download the file from Clowder v1 + v1_download_url = f"{CLOWDER_V1}/api/files/{file_id}?superAdmin=true" + print(f"Downloading file: {filename}") + download_response = requests.get(v1_download_url, headers=clowder_headers_v1) + + with open(filename, "wb") as f: + f.write(download_response.content) + + # Determine the correct folder in Clowder v2 for the upload + matching_folder = None + if file_folder: + matching_folder = next( + ( + folder + for folder in all_dataset_folders + if folder["name"] == file_folder["name"] + ), + None, + ) + + # Upload the file to Clowder v2 + dataset_file_upload_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/files" + if matching_folder: + dataset_file_upload_endpoint += f"Multiple?folder_id={matching_folder['id']}" + + # DEBUG: Add the same debugging as the new method + print(f"=== WORKING METHOD DEBUG ===") + print(f"Upload endpoint: {dataset_file_upload_endpoint}") + print(f"Headers: {headers_v2}") + + with open(filename, "rb") as file_obj: + files = {"file": (filename, file_obj)} + + # Use a session to see raw request + session = requests.Session() + prepared_request = requests.Request( + 'POST', + dataset_file_upload_endpoint, + headers=headers_v2, + files=files + ).prepare() + + print(f"Prepared request URL: {prepared_request.url}") + print(f"Prepared request headers: {dict(prepared_request.headers)}") + print(f"Content-Type header: {prepared_request.headers.get('Content-Type')}") + + response = session.send(prepared_request) + + print(f"Response status: {response.status_code}") + print(f"Response text: {response.text}") + print(f"=== WORKING METHOD DEBUG END ===") + + # Clean up the local file after upload + try: + os.remove(filename) + except Exception as e: + print(f"Could not delete locally downloaded file: {filename}") + print(e) + + if response.status_code == 200: + print(f"Uploaded file: {filename} to dataset {dataset_v2_id}") + return response.json().get("id") + else: + print(f"Failed to upload file: {filename} to dataset {dataset_v2_id}") + + return None def add_file_metadata(file_v1, file_v2_id, headers_v1, headers_v2): # Get metadata from Clowder V1 @@ -1207,9 +1419,12 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): "[Local Account]" in user_v1["identityProvider"] and user_v1["email"] != admin_user["email"] ): - [USER_MAP, DATASET_MAP] = process_user_and_resources_collections( - user_v1, USER_MAP, DATASET_MAP, COLLECTIONS_MAP + [USER_MAP, DATASET_MAP] = process_user_and_resources( + user_v1, USER_MAP, DATASET_MAP ) + # [USER_MAP, DATASET_MAP] = process_user_and_resources_collections( + # user_v1, USER_MAP, DATASET_MAP, COLLECTIONS_MAP + # ) print(f"Migrated user {user_v1['email']} and associated resources.") else: print(f"Skipping user {user_v1['email']} as it is not a local account.") diff --git a/scripts/migration/test.py b/scripts/migration/test.py new file mode 100644 index 000000000..2efc3c1bc --- /dev/null +++ b/scripts/migration/test.py @@ -0,0 +1,72 @@ +import requests +from dotenv import dotenv_values +import os + +path_to_env = os.path.join(os.getcwd(),"scripts","migration", ".env") +path_to_toml = os.path.join(os.getcwd(),"scripts","migration", "config.toml") +config = dotenv_values(dotenv_path=path_to_env) + +CLOWDER_V1 = config["CLOWDER_V1"] +ADMIN_KEY_V1 = config["ADMIN_KEY_V1"] +CLOWDER_V2 = config["CLOWDER_V2"] +ADMIN_KEY_V2 = config["ADMIN_KEY_V2"] + +base_headers_v1 = {"X-API-key": ADMIN_KEY_V1} +base_headers_v2 = {"X-API-key": ADMIN_KEY_V2} + +clowder_headers_v2 = { + **base_headers_v2, + "Content-type": "application/json", + "accept": "application/json", +} + +url = 'http://127.0.0.1:8000/api/v2' + +def get_new_dataset_folders(dataset_id, headers): + endpoint = f"{url}/datasets/{dataset_id}/all_folders" + r = requests.get(endpoint, headers=headers) + foldesr_json = r.json() + print(r.json()) + +def download_and_upload_file_to_folder(file, folder_id, dataset_v2_id, headers_v2): + """Download a file from Clowder v1 and upload it to Clowder v2.""" + + + # Download the file from Clowder v1 + filename = 'test.txt' + + + + # Upload the file to Clowder v2 + dataset_file_upload_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/files" + if folder_id is not None: + # add folder if it is not None + # folder_id = folder["id"] + dataset_file_upload_endpoint += f"Multiple?folder_id={folder_id}" + file_data = {"file": open(filename, "rb")} + response = requests.post( + dataset_file_upload_endpoint, + headers=headers_v2, + files=file_data, + ) + if response.status_code == 200: + print(f"Uploaded file: {filename} to dataset {dataset_v2_id}") + return response.json().get("id") + else: + print(f"Failed to upload file: {filename} to dataset {dataset_v2_id}") + + return None + + + + +test_file = 'july-2018-temperature-precip.csv' +if os.path.exists(test_file): + print('it exists') + +test_folder_id = '68b206b0fb9e6c77930beaab' +test_dataset_id = '68b206a4fb9e6c77930beaa8' + +download_and_upload_file_to_folder(test_file, None, test_dataset_id, clowder_headers_v2) + +# new_folders = get_new_dataset_folders('68b080ee03137d5052c0872c', headers=clowder_headers_v2) \ No newline at end of file From c76e75e33753269c5b21e0597163fbf6dbca1b0b Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 2 Sep 2025 15:58:38 -0500 Subject: [PATCH 76/91] messy, but works, can post the data --- scripts/migration/migrate.py | 140 ++++++++++++++++++++++++++++------- 1 file changed, 113 insertions(+), 27 deletions(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 7df66d528..416ae189c 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -188,7 +188,7 @@ def get_clowder_v1_user_collections_top_level(headers, user_v1): top_level_collections.append(col) return top_level_collections -def process_collection_descendants(collection, headers_v1,headers_v2, v2_parent_id, v2_parent_type, v2_dataset_id): +def process_collection_descendants(collection, headers_v1, base_headers_v2, headers_v2, v2_parent_id, v2_parent_type, v2_dataset_id): child_collections_endpoint = f"{CLOWDER_V1}/api/collections/{collection['id']}/getChildCollections" datasets_endpoint = f"{CLOWDER_V1}/api/collections/{collection['id']}/datasets" @@ -203,13 +203,13 @@ def process_collection_descendants(collection, headers_v1,headers_v2, v2_parent_ print(f"Add folder to the dataset") folder_name = child["name"] new_folder = create_folder_if_not_exists_or_get(folder_name, None, v2_dataset_id, headers_v2) - process_collection_descendants(child, headers_v1,headers_v2, new_folder['id'], 'folder', v2_dataset_id ) + process_collection_descendants(child, headers_v1, base_headers_v2, headers_v2, new_folder['id'], 'folder', v2_dataset_id ) else: print(f"parent was a folder") print(f"Add folder to the dataset") folder_name = child["name"] new_folder = create_folder_if_not_exists_or_get(folder_name, v2_parent_id, v2_dataset_id, headers_v2) - process_collection_descendants(child, headers_v1, headers_v2, new_folder['id'], 'folder', v2_dataset_id) + process_collection_descendants(child, headers_v1, base_headers_v2, headers_v2, new_folder['id'], 'folder', v2_dataset_id) for dataset in dataset_json: if v2_parent_type == "dataset": @@ -218,13 +218,13 @@ def process_collection_descendants(collection, headers_v1,headers_v2, v2_parent_ print(f"Now we need to add the sub folders of this dataset") # TODO get DATASET FOLDERS HERE FROM v1 process_dataset_folders(dataset, headers_v1, headers_v2, new_folder['id'], v2_dataset_id) - process_dataset_files(dataset, headers_v1, headers_v2, 'folder', new_folder['id'], v2_dataset_id) + process_dataset_files(dataset, headers_v1, base_headers_v2, 'folder', new_folder['id'], v2_dataset_id) else: print(f"Parent is a folder") new_folder = create_folder_if_not_exists_or_get(dataset["name"], v2_parent_id, v2_dataset_id, headers_v2) # TODO GET DATASET FOLDERS HERE FROM v1 process_dataset_folders(dataset, headers_v1, headers_v2, new_folder['id'], v2_dataset_id) - process_dataset_files(dataset, headers_v1, headers_v2, 'folder', new_folder['id'], v2_dataset_id) + process_dataset_files(dataset, headers_v1, base_headers_v2, 'folder', new_folder['id'], v2_dataset_id) @@ -265,7 +265,7 @@ def process_dataset_files(dataset, headers_v1, headers_v2, parent_type, parent_i if folder_v2['name'] == file['folders']['name']: print(f"Upload this file to a folder") matching_folder = folder_v2 - download_and_upload_file_to_folder_id(file, matching_folder, dataset_v2_id, headers_v2) + download_and_upload_file_to_matching_folder(file, dataset_v2_id, base_headers_v2, matching_folder) else: print(f"This file is not in a folder") # TODO upload it to the folder @@ -277,7 +277,7 @@ def process_dataset_files(dataset, headers_v1, headers_v2, parent_type, parent_i -def create_v2_dataset_from_collection(collection, user_v1, headers_v1, headers_v2): +def create_v2_dataset_from_collection(collection, user_v1, headers_v1, headers_v2, base_headers_v2): # create the dataset collection_name = collection["name"] collection_description = collection["description"] @@ -295,7 +295,7 @@ def create_v2_dataset_from_collection(collection, user_v1, headers_v1, headers_v new_dataset_json = response.json() v2_dataset_id = new_dataset_json["id"] - process_collection_descendants(collection, headers_v1, headers_v2, new_dataset_json["id"], "dataset", v2_dataset_id) + process_collection_descendants(collection, headers_v1, base_headers_v2, headers_v2, new_dataset_json["id"], "dataset", v2_dataset_id) return response.json()["id"] @@ -685,20 +685,26 @@ def download_and_upload_file_to_folder_id(file, folder_v2, dataset_v2_id, header file_exists = os.path.exists(filename) # Upload the file to Clowder v2 dataset_file_upload_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/files" - if folder_v2 is not None: - dataset_file_upload_endpoint += f"?folder_id={folder_v2['id']}" - response = requests.post( - dataset_file_upload_endpoint, - headers=headers_v2, - files={"file": open(filename, "rb")}, - ) + if folder_v2: + dataset_file_upload_endpoint += f"Multiple?folder_id={folder_v2['id']}" + response = requests.post( + dataset_file_upload_endpoint, + headers=headers_v2, + files=[("files", (filename, open(filename, "rb")))], + ) + else: + response = requests.post( + dataset_file_upload_endpoint, + headers=headers_v2, + files={"file": open(filename, "rb")}, + ) # Clean up the local file after upload - # try: - # os.remove(filename) - # except Exception as e: - # print(f"Could not delete locally downloaded file: {filename}") - # print(e) + try: + os.remove(filename) + except Exception as e: + print(f"Could not delete locally downloaded file: {filename}") + print(e) if response.status_code == 200: print(f"Uploaded file: {filename} to dataset {dataset_v2_id}") @@ -796,6 +802,56 @@ def download_and_upload_file_to_folder_id(file, folder_v2, dataset_v2_id, header # return None +def download_and_upload_file_to_matching_folder(file, dataset_v2_id, headers_v2, matching_folder = None): + """Download a file from Clowder v1 and upload it to Clowder v2.""" + filename = file["filename"] + file_id = file["id"] + + # Download the file from Clowder v1 + v1_download_url = f"{CLOWDER_V1}/api/files/{file_id}?superAdmin=true" + print(f"Downloading file: {filename}") + download_response = requests.get(v1_download_url, headers=clowder_headers_v1) + + with open(filename, "wb") as f: + f.write(download_response.content) + + # Upload the file to Clowder v2 + dataset_file_upload_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/files" + if matching_folder: + dataset_file_upload_endpoint += f"Multiple?folder_id={matching_folder['id']}" + + # DEBUG: Print the exact request details + print(f"DEBUG: URL: {dataset_file_upload_endpoint}") + print(f"DEBUG: Headers: {headers_v2}") + print(f"DEBUG: Folder ID: {matching_folder['id']}") + + response = requests.post( + dataset_file_upload_endpoint, + headers=headers_v2, + files=[("files", (filename, open(filename, "rb")))], + ) + else: + response = requests.post( + dataset_file_upload_endpoint, + headers=headers_v2, + files={"file": open(filename, "rb")}, + ) + + # Clean up the local file after upload + try: + os.remove(filename) + except Exception as e: + print(f"Could not delete locally downloaded file: {filename}") + print(e) + + if response.status_code == 200: + print(f"Uploaded file: {filename} to dataset {dataset_v2_id}") + return response.json().get("id") + else: + print(f"Failed to upload file: {filename} to dataset {dataset_v2_id}") + + return None + def download_and_upload_file(file, all_dataset_folders, dataset_v2_id, headers_v2): """Download a file from Clowder v1 and upload it to Clowder v2.""" @@ -1194,7 +1250,7 @@ def process_user_and_resources_collections(user_v1, USER_MAP, DATASET_MAP, COLLE print(f"Got {len(user_v1_collections)} user collections in the top level") for top_level_col in user_v1_collections: - dataset_v2 = create_v2_dataset_from_collection(top_level_col, user_v1, clowder_headers_v1, user_headers_v2) + dataset_v2 = create_v2_dataset_from_collection(top_level_col, user_v1, clowder_headers_v1 ,user_headers_v2, base_headers_v2) print('did this') for dataset in user_v1_datasets: @@ -1245,6 +1301,21 @@ def process_user_and_resources_collections(user_v1, USER_MAP, DATASET_MAP, COLLE files_result = files_response.json() for file in files_result: + file_folder = file.get("folders", None) + matching_folder = None + if file_folder: + matching_folder = next( + ( + folder + for folder in all_dataset_folders + if folder["name"] == file_folder["name"] + ), + None, + ) + print('did we get matching folder?') + file_v2_id = download_and_upload_file_to_matching_folder( + file, dataset_v2_id, base_user_headers_v2, matching_folder + ) file_v2_id = download_and_upload_file( file, all_dataset_folders, dataset_v2_id, base_user_headers_v2 ) @@ -1344,8 +1415,23 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): dataset_files_endpoint, headers=clowder_headers_v1, verify=False ) files_result = files_response.json() - + # TODO test folde rher for file in files_result: + file_folder = file.get("folders", None) + matching_folder = None + if file_folder: + matching_folder = next( + ( + folder + for folder in all_dataset_folders + if folder["name"] == file_folder["name"] + ), + None, + ) + print('did we get matching folder?') + file_v2_id = download_and_upload_file_to_matching_folder( + file, dataset_v2_id, base_user_headers_v2, matching_folder + ) file_v2_id = download_and_upload_file( file, all_dataset_folders, dataset_v2_id, base_user_headers_v2 ) @@ -1419,12 +1505,12 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): "[Local Account]" in user_v1["identityProvider"] and user_v1["email"] != admin_user["email"] ): - [USER_MAP, DATASET_MAP] = process_user_and_resources( - user_v1, USER_MAP, DATASET_MAP - ) - # [USER_MAP, DATASET_MAP] = process_user_and_resources_collections( - # user_v1, USER_MAP, DATASET_MAP, COLLECTIONS_MAP + # [USER_MAP, DATASET_MAP] = process_user_and_resources( + # user_v1, USER_MAP, DATASET_MAP # ) + [USER_MAP, DATASET_MAP] = process_user_and_resources_collections( + user_v1, USER_MAP, DATASET_MAP, COLLECTIONS_MAP + ) print(f"Migrated user {user_v1['email']} and associated resources.") else: print(f"Skipping user {user_v1['email']} as it is not a local account.") From c45d8f055f084f6fb99bd0df57d0f58f511f800f Mon Sep 17 00:00:00 2001 From: toddn Date: Wed, 3 Sep 2025 12:24:18 -0500 Subject: [PATCH 77/91] worked for one collection with datasets and folders --- scripts/migration/migrate.py | 77 ++++++++++++++++++++++-------------- 1 file changed, 47 insertions(+), 30 deletions(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 416ae189c..96d415221 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -197,7 +197,7 @@ def process_collection_descendants(collection, headers_v1, base_headers_v2, head child_col_json = child_col_response.json() dataset_json = dataset_response.json() - print(f"Got child collections and datasets") + # the below handles creating folders for child collections for child in child_col_json: if v2_parent_type == "dataset": print(f"Add folder to the dataset") @@ -211,20 +211,14 @@ def process_collection_descendants(collection, headers_v1, base_headers_v2, head new_folder = create_folder_if_not_exists_or_get(folder_name, v2_parent_id, v2_dataset_id, headers_v2) process_collection_descendants(child, headers_v1, base_headers_v2, headers_v2, new_folder['id'], 'folder', v2_dataset_id) + # this handles uploading the datasets of the collection as folders for dataset in dataset_json: if v2_parent_type == "dataset": - print(f"Parent is a dataset") - new_folder = create_folder_if_not_exists_or_get(dataset["name"], v2_parent_id, v2_dataset_id, headers_v2) - print(f"Now we need to add the sub folders of this dataset") - # TODO get DATASET FOLDERS HERE FROM v1 - process_dataset_folders(dataset, headers_v1, headers_v2, new_folder['id'], v2_dataset_id) - process_dataset_files(dataset, headers_v1, base_headers_v2, 'folder', new_folder['id'], v2_dataset_id) + new_folder = create_folder_if_not_exists_or_get(dataset["name"], v2_parent_id, v2_parent_type, v2_dataset_id, headers_v2) + process_dataset_files_and_folders(dataset, headers_v1, base_headers_v2, 'folder', new_folder['id'], v2_dataset_id, new_folder) else: - print(f"Parent is a folder") - new_folder = create_folder_if_not_exists_or_get(dataset["name"], v2_parent_id, v2_dataset_id, headers_v2) - # TODO GET DATASET FOLDERS HERE FROM v1 - process_dataset_folders(dataset, headers_v1, headers_v2, new_folder['id'], v2_dataset_id) - process_dataset_files(dataset, headers_v1, base_headers_v2, 'folder', new_folder['id'], v2_dataset_id) + new_folder = create_folder_if_not_exists_or_get(dataset["name"], v2_parent_id, v2_parent_type, v2_dataset_id, headers_v2) + process_dataset_files_and_folders(dataset, headers_v1, base_headers_v2, 'folder', new_folder['id'], v2_dataset_id, new_folder) @@ -240,7 +234,8 @@ def get_v1_dataset_folders(dataset, headers_v1, headers_v2, parent_type, parent_ folder_json = folder_response.json() return folder_json -def process_dataset_files(dataset, headers_v1, headers_v2, parent_type, parent_id, dataset_v2_id): +# processes a dataset adds folders and +def process_dataset_files_and_folders(dataset, headers_v1, headers_v2, parent_type, parent_id, dataset_v2_id, dataset_v2_folder): dataset_v1_folders = get_v1_dataset_folders(dataset, headers_v1, headers_v2, parent_type, parent_id) for folder_v1 in dataset_v1_folders: @@ -255,24 +250,19 @@ def process_dataset_files(dataset, headers_v1, headers_v2, parent_type, parent_i files_endpoint = f"{CLOWDER_V1}/api/datasets/{dataset['id']}/files" files_response = requests.get(files_endpoint, headers=headers_v1) files_json = files_response.json() - # TODO WORK HERE + # go through files and upload them to the correct folder if they have one for file in files_json: if 'folders' in file: - print(f"This file is in a folder") - current_file_folder_name = file['folders']['name'] - matching_folder = None for folder_v2 in all_v2_dataset_folders: if folder_v2['name'] == file['folders']['name']: print(f"Upload this file to a folder") matching_folder = folder_v2 download_and_upload_file_to_matching_folder(file, dataset_v2_id, base_headers_v2, matching_folder) else: - print(f"This file is not in a folder") - # TODO upload it to the folder if parent_type == "dataset": print(f"Upload to a dataset") if parent_type == "folder": - print(f"Upload to a folder") + download_and_upload_file_to_matching_folder(file, dataset_v2_id, base_headers_v2, dataset_v2_folder) print(f"Got dataset files") @@ -295,7 +285,10 @@ def create_v2_dataset_from_collection(collection, user_v1, headers_v1, headers_v new_dataset_json = response.json() v2_dataset_id = new_dataset_json["id"] - process_collection_descendants(collection, headers_v1, base_headers_v2, headers_v2, new_dataset_json["id"], "dataset", v2_dataset_id) + process_collection_descendants(collection=collection, headers_v1=headers_v1, + base_headers_v2=base_headers_v2, headers_v2= headers_v2, + v2_parent_id=new_dataset_json["id"], + v2_parent_type="dataset", v2_dataset_id=v2_dataset_id) return response.json()["id"] @@ -560,7 +553,7 @@ def add_folder_hierarchy_to_migration_folder(folder_hierarchy, dataset_v2, folde current_parent = folder_id_v2 for part in hierarchy_parts: result = create_folder_if_not_exists_or_get( - part, current_parent, dataset_v2, headers + part, current_parent, 'folder', dataset_v2, headers ) if result: current_parent = result["id"] @@ -579,13 +572,16 @@ def add_folder_hierarchy(folder_hierarchy, dataset_v2, headers): current_parent = result["id"] -def create_folder_if_not_exists_or_get(folder, parent, dataset_v2, headers): +def create_folder_if_not_exists_or_get(folder, parent, parent_type, dataset_v2, headers): """Create a folder if it does not exist or return the existing folder.""" # current_folders = get_folder_and_subfolders(dataset_v2, headers) current_all_folders = get_all_folder_and_subfolders(dataset_v2, headers) - folder_data = ( - {"name": folder, "parent_folder": parent} if parent else {"name": folder} - ) + if parent_type == 'folder': + folder_data = ( + {"name": folder, "parent_folder": parent} if parent else {"name": folder} + ) + else: + folder_data = {"name": folder} for existing_folder in current_all_folders: if existing_folder["name"] == folder: @@ -838,6 +834,7 @@ def download_and_upload_file_to_matching_folder(file, dataset_v2_id, headers_v2, ) # Clean up the local file after upload + print(f"Type response {type(response)}") try: os.remove(filename) except Exception as e: @@ -846,7 +843,11 @@ def download_and_upload_file_to_matching_folder(file, dataset_v2_id, headers_v2, if response.status_code == 200: print(f"Uploaded file: {filename} to dataset {dataset_v2_id}") - return response.json().get("id") + response_json = response.json() + if type(response_json) == dict: + return response.json().get("id") + elif type(response_json) == list: + return response_json[0].get("id") else: print(f"Failed to upload file: {filename} to dataset {dataset_v2_id}") @@ -1249,9 +1250,25 @@ def process_user_and_resources_collections(user_v1, USER_MAP, DATASET_MAP, COLLE print(f"Got {len(user_v1_collections)} user collections in the top level") - for top_level_col in user_v1_collections: - dataset_v2 = create_v2_dataset_from_collection(top_level_col, user_v1, clowder_headers_v1 ,user_headers_v2, base_headers_v2) - print('did this') + # filter the collections by space + migrate_top_level_collections = [] + for col in user_v1_collections: + collection_spaces = col["spaces"] + collection_spaces = collection_spaces.lstrip('List(') + collection_spaces = collection_spaces.rstrip(')') + collection_spaces = collection_spaces.split(',') + for space in collection_spaces: + if space in toml_space_ids: + migrate_top_level_collections.append(col) + break + + # create datasets from the top level collections + for top_level_col in migrate_top_level_collections: + dataset_v2 = create_v2_dataset_from_collection(collection=top_level_col, user_v1=user_v1, + headers_v1=clowder_headers_v1 ,headers_v2=user_headers_v2, + base_headers_v2=base_headers_v2) + print(f"Created dataset in v2 from collection: {top_level_col['id']} - {top_level_col['name']}") + COLLETIONS_MAP[top_level_col["id"]] = dataset_v2 for dataset in user_v1_datasets: print(f"Creating dataset in v2: {dataset['id']} - {dataset['name']}") From 9ca88503d55a73b8a7628885675a7cd2bf022a35 Mon Sep 17 00:00:00 2001 From: toddn Date: Wed, 3 Sep 2025 12:27:35 -0500 Subject: [PATCH 78/91] other collection also works now --- scripts/migration/migrate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 96d415221..09bd4f962 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -202,13 +202,13 @@ def process_collection_descendants(collection, headers_v1, base_headers_v2, head if v2_parent_type == "dataset": print(f"Add folder to the dataset") folder_name = child["name"] - new_folder = create_folder_if_not_exists_or_get(folder_name, None, v2_dataset_id, headers_v2) + new_folder = create_folder_if_not_exists_or_get(folder_name, None, v2_parent_type, v2_dataset_id, headers_v2) process_collection_descendants(child, headers_v1, base_headers_v2, headers_v2, new_folder['id'], 'folder', v2_dataset_id ) else: print(f"parent was a folder") print(f"Add folder to the dataset") folder_name = child["name"] - new_folder = create_folder_if_not_exists_or_get(folder_name, v2_parent_id, v2_dataset_id, headers_v2) + new_folder = create_folder_if_not_exists_or_get(folder_name, v2_parent_id,v2_parent_type, v2_dataset_id, headers_v2) process_collection_descendants(child, headers_v1, base_headers_v2, headers_v2, new_folder['id'], 'folder', v2_dataset_id) # this handles uploading the datasets of the collection as folders From e2be15b8af8cd1c12c1fada74d33ca9eba21a89f Mon Sep 17 00:00:00 2001 From: toddn Date: Wed, 3 Sep 2025 13:58:26 -0500 Subject: [PATCH 79/91] get list of datasets in collections, we need this for migration --- scripts/migration/dataset_collection_json.py | 5 +++++ scripts/migration/migrate.py | 2 ++ 2 files changed, 7 insertions(+) diff --git a/scripts/migration/dataset_collection_json.py b/scripts/migration/dataset_collection_json.py index 74230dc0c..5b05e998e 100644 --- a/scripts/migration/dataset_collection_json.py +++ b/scripts/migration/dataset_collection_json.py @@ -35,3 +35,8 @@ def get_dataset_collections_map(): dataset_to_collection[dataset] = current_value return dataset_to_collection +def get_datasets_in_collections(): + map = get_dataset_collections_map() + datasets_in_collections = list(map.keys()) + return datasets_in_collections + diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 09bd4f962..09ef2a6cb 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -1248,6 +1248,8 @@ def process_user_and_resources_collections(user_v1, USER_MAP, DATASET_MAP, COLLE headers=clowder_headers_v1, user_v1=user_v1 ) + all_collection = get_clowder_v1_user_collections(headers=clowder_headers_v1, user_v1=user_v1) + print(f"Got {len(user_v1_collections)} user collections in the top level") # filter the collections by space From c94dd6d9b1865914ca34b05201562f6edb101a18 Mon Sep 17 00:00:00 2001 From: toddn Date: Wed, 3 Sep 2025 14:06:55 -0500 Subject: [PATCH 80/91] this should work for datasets now as well --- scripts/migration/migrate.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 09ef2a6cb..f04f125b9 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -16,7 +16,7 @@ post_metadata_definition, ) -from scripts.migration.dataset_collection_json import get_dataset_collections_map +from scripts.migration.dataset_collection_json import get_dataset_collections_map, get_datasets_in_collections DATASET_COLLECTIONS_MAP = get_dataset_collections_map() @@ -1248,8 +1248,6 @@ def process_user_and_resources_collections(user_v1, USER_MAP, DATASET_MAP, COLLE headers=clowder_headers_v1, user_v1=user_v1 ) - all_collection = get_clowder_v1_user_collections(headers=clowder_headers_v1, user_v1=user_v1) - print(f"Got {len(user_v1_collections)} user collections in the top level") # filter the collections by space @@ -1272,6 +1270,8 @@ def process_user_and_resources_collections(user_v1, USER_MAP, DATASET_MAP, COLLE print(f"Created dataset in v2 from collection: {top_level_col['id']} - {top_level_col['name']}") COLLETIONS_MAP[top_level_col["id"]] = dataset_v2 + datasets_in_collections_v1 = get_datasets_in_collections() + for dataset in user_v1_datasets: print(f"Creating dataset in v2: {dataset['id']} - {dataset['name']}") dataset_v1_id = dataset["id"] @@ -1281,6 +1281,9 @@ def process_user_and_resources_collections(user_v1, USER_MAP, DATASET_MAP, COLLE print(toml_space_ids) print(toml_exclude_space_ids) # Check if dataset is in the excluded dataset list + if dataset_v1_id in datasets_in_collections_v1: + print(f"Skipping dataset {dataset_v1_id} as it is was in a collection, already migrated") + MIGRATE_DATASET = False if dataset_v1_id in toml_exclude_dataset_ids: print(f"Skipping dataset {dataset_v1_id} as it is in the exclude list.") MIGRATE_DATASET = False @@ -1335,9 +1338,6 @@ def process_user_and_resources_collections(user_v1, USER_MAP, DATASET_MAP, COLLE file_v2_id = download_and_upload_file_to_matching_folder( file, dataset_v2_id, base_user_headers_v2, matching_folder ) - file_v2_id = download_and_upload_file( - file, all_dataset_folders, dataset_v2_id, base_user_headers_v2 - ) if file_v2_id is not None: add_file_metadata(file, file_v2_id, clowder_headers_v1, user_headers_v2) # posting the collection hierarchy as metadata From 88e25348d02af8ce4d367830f575e21e71e3382a Mon Sep 17 00:00:00 2001 From: toddn Date: Wed, 3 Sep 2025 14:21:16 -0500 Subject: [PATCH 81/91] single dataset works, except metadata --- scripts/migration/migrate.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index f04f125b9..8b1eb73eb 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -565,12 +565,31 @@ def add_folder_hierarchy(folder_hierarchy, dataset_v2, headers): hierarchy_parts = hierarchy_parts[1:] current_parent = None for part in hierarchy_parts: - result = create_folder_if_not_exists_or_get( + result = create_dataset_folder_if_not_exists_or_get( part, current_parent, dataset_v2, headers ) if result: current_parent = result["id"] +# for creating a folder for a dataset +def create_dataset_folder_if_not_exists_or_get(folder, parent, dataset_v2, headers): + """Create a folder if it does not exist or return the existing folder.""" + # current_folders = get_folder_and_subfolders(dataset_v2, headers) + current_all_folders = get_all_folder_and_subfolders(dataset_v2, headers) + folder_data = ( + {"name": folder, "parent_folder": parent} if parent else {"name": folder} + ) + + for existing_folder in current_all_folders: + if existing_folder["name"] == folder: + return existing_folder + + response = requests.post( + f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2}/folders", + json=folder_data, + headers=headers, + ) + return response.json() def create_folder_if_not_exists_or_get(folder, parent, parent_type, dataset_v2, headers): """Create a folder if it does not exist or return the existing folder.""" @@ -1307,6 +1326,7 @@ def process_user_and_resources_collections(user_v1, USER_MAP, DATASET_MAP, COLLE if MIGRATE_DATASET: dataset_v2_id = create_v2_dataset(dataset, user_headers_v2) DATASET_MAP[dataset["id"]] = dataset_v2_id + # add_dataset_metadata(dataset, dataset_v2_id, base_headers_v1, user_headers_v2) add_dataset_folders(dataset, dataset_v2_id, user_headers_v2) print("Created folders in the new dataset") From 7e139cbc4bb2ab4f121991a5cb69cf38cb22079b Mon Sep 17 00:00:00 2001 From: toddn Date: Wed, 3 Sep 2025 14:40:02 -0500 Subject: [PATCH 82/91] removing unused methods --- scripts/migration/migrate.py | 264 +---------------------------------- 1 file changed, 3 insertions(+), 261 deletions(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 8b1eb73eb..64cf8f932 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -221,13 +221,6 @@ def process_collection_descendants(collection, headers_v1, base_headers_v2, head process_dataset_files_and_folders(dataset, headers_v1, base_headers_v2, 'folder', new_folder['id'], v2_dataset_id, new_folder) - -def process_dataset_folders(dataset, headers_v1, headers_v2, parent_type, parent_id): - folder_endpoint = f"{CLOWDER_V1}/api/datasets/{dataset['id']}/folders" - folder_response = requests.get(folder_endpoint, headers=headers_v1) - folder_json = folder_response.json() - print(f"Got dataset folders") - def get_v1_dataset_folders(dataset, headers_v1, headers_v2, parent_type, parent_id): folder_endpoint = f"{CLOWDER_V1}/api/datasets/{dataset['id']}/folders" folder_response = requests.get(folder_endpoint, headers=headers_v1) @@ -293,9 +286,6 @@ def create_v2_dataset_from_collection(collection, user_v1, headers_v1, headers_v return response.json()["id"] - # go through sub collections creating folders - print("Creating v2-dataset from collection") - # TODO this is too slow, we need to optimize it def get_clowder_v1_dataset_collections(headers, user_v1, dataset_id): @@ -544,7 +534,7 @@ def create_v2_group(space, headers): response = requests.post(group_in_v2_endpoint, json=group, headers=headers) return response.json()["id"] -# TODO try this + def add_folder_hierarchy_to_migration_folder(folder_hierarchy, dataset_v2, folder_id_v2, headers): """Add folder hierarchy to a dataset in Clowder v2.""" hierarchy_parts = folder_hierarchy.split("/") @@ -571,7 +561,7 @@ def add_folder_hierarchy(folder_hierarchy, dataset_v2, headers): if result: current_parent = result["id"] -# for creating a folder for a dataset +# for creating a folder for a dataset that is migrated to a dataset def create_dataset_folder_if_not_exists_or_get(folder, parent, dataset_v2, headers): """Create a folder if it does not exist or return the existing folder.""" # current_folders = get_folder_and_subfolders(dataset_v2, headers) @@ -591,6 +581,7 @@ def create_dataset_folder_if_not_exists_or_get(folder, parent, dataset_v2, heade ) return response.json() +# used for creating folders and subfolders when a collection is migrated to a dataset def create_folder_if_not_exists_or_get(folder, parent, parent_type, dataset_v2, headers): """Create a folder if it does not exist or return the existing folder.""" # current_folders = get_folder_and_subfolders(dataset_v2, headers) @@ -641,182 +632,6 @@ def add_dataset_folders(dataset_v1, dataset_v2, headers): add_folder_hierarchy(folder["name"], dataset_v2, headers) -def download_and_upload_file_to_folder(file, folder, dataset_v2_id, headers_v2): - """Download a file from Clowder v1 and upload it to Clowder v2.""" - filename = file["filename"] - file_id = file["id"] - file_folder = folder - - # Download the file from Clowder v1 - v1_download_url = f"{CLOWDER_V1}/api/files/{file_id}?superAdmin=true" - print(f"Downloading file: {filename}") - download_response = requests.get(v1_download_url, headers=clowder_headers_v1) - - with open(filename, "wb") as f: - f.write(download_response.content) - - # Upload the file to Clowder v2 - dataset_file_upload_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/files" - if folder is not None: - # add folder if it is not None - folder_id = folder["id"] - dataset_file_upload_endpoint += f"Multiple?folder_id={folder_id}" - files = [ - ("files", open(filename, "rb")), - ] - response = requests.post( - dataset_file_upload_endpoint, - headers=headers_v2, - files=files, - ) - if response.status_code == 200: - print(f"Uploaded file: {filename} to dataset {dataset_v2_id}") - return response.json().get("id") - else: - print(f"Failed to upload file: {filename} to dataset {dataset_v2_id}") - - # Clean up the local file after upload - try: - os.remove(filename) - except Exception as e: - print(f"Could not delete locally downloaded file: {filename}") - print(e) - return None - -def download_and_upload_file_to_folder_id(file, folder_v2, dataset_v2_id, headers_v2): - """Download a file from Clowder v1 and upload it to Clowder v2.""" - filename = file["filename"] - file_id = file["id"] - file_folder = file.get("folders", None) - - # Download the file from Clowder v1 - v1_download_url = f"{CLOWDER_V1}/api/files/{file_id}?superAdmin=true" - print(f"Downloading file: {filename}") - download_response = requests.get(v1_download_url, headers=clowder_headers_v1) - - with open(filename, "wb") as f: - f.write(download_response.content) - - file_exists = os.path.exists(filename) - # Upload the file to Clowder v2 - dataset_file_upload_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/files" - if folder_v2: - dataset_file_upload_endpoint += f"Multiple?folder_id={folder_v2['id']}" - response = requests.post( - dataset_file_upload_endpoint, - headers=headers_v2, - files=[("files", (filename, open(filename, "rb")))], - ) - else: - response = requests.post( - dataset_file_upload_endpoint, - headers=headers_v2, - files={"file": open(filename, "rb")}, - ) - - # Clean up the local file after upload - try: - os.remove(filename) - except Exception as e: - print(f"Could not delete locally downloaded file: {filename}") - print(e) - - if response.status_code == 200: - print(f"Uploaded file: {filename} to dataset {dataset_v2_id}") - return response.json().get("id") - else: - print(f"Failed to upload file: {filename} to dataset {dataset_v2_id}") - - return None - -# def download_and_upload_file_to_folder_id(file, folder_v2, dataset_v2_id, headers_v2): -# """Download a file from Clowder v1 and upload it to Clowder v2.""" -# filename = file["filename"] -# file_id = file["id"] -# -# # DEBUG: Print all inputs -# print(f"=== DEBUG START ===") -# print(f"File: {file}") -# print(f"Folder_v2: {folder_v2}") -# print(f"Dataset_v2_id: {dataset_v2_id}") -# print(f"Headers_v2 keys: {list(headers_v2.keys()) if headers_v2 else 'None'}") -# -# # Download the file from Clowder v1 -# v1_download_url = f"{CLOWDER_V1}/api/files/{file_id}?superAdmin=true" -# print(f"Downloading file: {filename} from {v1_download_url}") -# download_response = requests.get(v1_download_url, headers=clowder_headers_v1) -# print(f"Download status: {download_response.status_code}") -# -# with open(filename, "wb") as f: -# f.write(download_response.content) -# -# # Check file exists and has content -# file_size = os.path.getsize(filename) -# print(f"Local file size: {file_size} bytes") -# -# # Upload the file to Clowder v2 -# dataset_file_upload_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/files" -# -# if folder_v2 is not None: -# folder_id = folder_v2['id'] if isinstance(folder_v2, dict) else folder_v2.id -# dataset_file_upload_endpoint += f"Multiple?folder_id={folder_id}" -# -# print(f"Upload endpoint: {dataset_file_upload_endpoint}") -# -# # Read file content to verify it's not corrupted -# with open(filename, "rb") as f: -# file_content = f.read() -# print(f"File content length: {len(file_content)}") -# print(f"File content starts with: {file_content[:100]}...") -# -# # Make the upload request with detailed debugging -# with open(filename, "rb") as file_obj: -# files = {"file": (filename, file_obj)} -# -# print(f"Final files dict: {files}") -# # Create headers without content-type for file uploads -# upload_headers = headers_v2.copy() -# upload_headers.pop('content-type', None) -# print(f"Final headers: {upload_headers}") -# -# # Use a session to see raw request -# session = requests.Session() -# prepared_request = requests.Request( -# 'POST', -# dataset_file_upload_endpoint, -# headers=upload_headers, -# files=files -# ).prepare() -# -# print(f"Prepared request URL: {prepared_request.url}") -# print(f"Prepared request headers: {dict(prepared_request.headers)}") -# # Don't print body as it's binary, but we can check content-type -# print(f"Content-Type header: {prepared_request.headers.get('Content-Type')}") -# -# response = session.send(prepared_request) -# -# # DEBUG: Full response analysis -# print(f"Response status: {response.status_code}") -# print(f"Response headers: {dict(response.headers)}") -# print(f"Response text: {response.text}") -# print(f"=== DEBUG END ===") -# -# # Clean up the local file after upload -# try: -# os.remove(filename) -# except Exception as e: -# print(f"Could not delete locally downloaded file: {filename}") -# print(e) -# -# if response.status_code == 200: -# print(f"Uploaded file: {filename} to dataset {dataset_v2_id}") -# return response.json().get("id") -# else: -# print(f"Failed to upload file: {filename} to dataset {dataset_v2_id}") -# -# return None - - def download_and_upload_file_to_matching_folder(file, dataset_v2_id, headers_v2, matching_folder = None): """Download a file from Clowder v1 and upload it to Clowder v2.""" filename = file["filename"] @@ -930,79 +745,6 @@ def download_and_upload_file(file, all_dataset_folders, dataset_v2_id, headers_v return None -def download_and_upload_file_1(file, all_dataset_folders, dataset_v2_id, headers_v2): - """Download a file from Clowder v1 and upload it to Clowder v2.""" - filename = file["filename"] - file_id = file["id"] - file_folder = file.get("folders", None) - - # Download the file from Clowder v1 - v1_download_url = f"{CLOWDER_V1}/api/files/{file_id}?superAdmin=true" - print(f"Downloading file: {filename}") - download_response = requests.get(v1_download_url, headers=clowder_headers_v1) - - with open(filename, "wb") as f: - f.write(download_response.content) - - # Determine the correct folder in Clowder v2 for the upload - matching_folder = None - if file_folder: - matching_folder = next( - ( - folder - for folder in all_dataset_folders - if folder["name"] == file_folder["name"] - ), - None, - ) - - # Upload the file to Clowder v2 - dataset_file_upload_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/files" - if matching_folder: - dataset_file_upload_endpoint += f"Multiple?folder_id={matching_folder['id']}" - - # DEBUG: Add the same debugging as the new method - print(f"=== WORKING METHOD DEBUG ===") - print(f"Upload endpoint: {dataset_file_upload_endpoint}") - print(f"Headers: {headers_v2}") - - with open(filename, "rb") as file_obj: - files = {"file": (filename, file_obj)} - - # Use a session to see raw request - session = requests.Session() - prepared_request = requests.Request( - 'POST', - dataset_file_upload_endpoint, - headers=headers_v2, - files=files - ).prepare() - - print(f"Prepared request URL: {prepared_request.url}") - print(f"Prepared request headers: {dict(prepared_request.headers)}") - print(f"Content-Type header: {prepared_request.headers.get('Content-Type')}") - - response = session.send(prepared_request) - - print(f"Response status: {response.status_code}") - print(f"Response text: {response.text}") - print(f"=== WORKING METHOD DEBUG END ===") - - # Clean up the local file after upload - try: - os.remove(filename) - except Exception as e: - print(f"Could not delete locally downloaded file: {filename}") - print(e) - - if response.status_code == 200: - print(f"Uploaded file: {filename} to dataset {dataset_v2_id}") - return response.json().get("id") - else: - print(f"Failed to upload file: {filename} to dataset {dataset_v2_id}") - - return None - def add_file_metadata(file_v1, file_v2_id, headers_v1, headers_v2): # Get metadata from Clowder V1 endpoint = f"{CLOWDER_V1}/api/files/{file_v1['id']}/metadata.jsonld?superAdmin=true" From 5b2562dbfbb5c02cb6a4e9690c91e82b0efdf82b Mon Sep 17 00:00:00 2001 From: toddn Date: Wed, 3 Sep 2025 14:42:17 -0500 Subject: [PATCH 83/91] adding todo with dataset metadata to a folder --- scripts/migration/migrate.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 64cf8f932..93b2443cd 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -216,9 +216,11 @@ def process_collection_descendants(collection, headers_v1, base_headers_v2, head if v2_parent_type == "dataset": new_folder = create_folder_if_not_exists_or_get(dataset["name"], v2_parent_id, v2_parent_type, v2_dataset_id, headers_v2) process_dataset_files_and_folders(dataset, headers_v1, base_headers_v2, 'folder', new_folder['id'], v2_dataset_id, new_folder) + # TODO add dataset metadata to the folder else: new_folder = create_folder_if_not_exists_or_get(dataset["name"], v2_parent_id, v2_parent_type, v2_dataset_id, headers_v2) process_dataset_files_and_folders(dataset, headers_v1, base_headers_v2, 'folder', new_folder['id'], v2_dataset_id, new_folder) + # TODO add dataset metadata to the folder def get_v1_dataset_folders(dataset, headers_v1, headers_v2, parent_type, parent_id): From 0a8e764a034d7e9e007cdf4854929308c797e648 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 8 Sep 2025 12:26:02 -0500 Subject: [PATCH 84/91] migrate now adds a metadata file to a folder --- scripts/migration/dataset_collection_json.py | 43 +++++++++++++++++++ scripts/migration/migrate.py | 45 +++++++++++++++++++- 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/scripts/migration/dataset_collection_json.py b/scripts/migration/dataset_collection_json.py index 5b05e998e..88c848a96 100644 --- a/scripts/migration/dataset_collection_json.py +++ b/scripts/migration/dataset_collection_json.py @@ -9,7 +9,22 @@ except ImportError: import tomli as tomllib +path_to_env = os.path.join(os.getcwd(),"scripts","migration", ".env") +config = dotenv_values(dotenv_path=path_to_env) +CLOWDER_V1 = config["CLOWDER_V1"] +ADMIN_KEY_V1 = config["ADMIN_KEY_V1"] +CLOWDER_V2 = config["CLOWDER_V2"] +ADMIN_KEY_V2 = config["ADMIN_KEY_V2"] + +base_headers_v1 = {"X-API-key": ADMIN_KEY_V1} +base_headers_v2 = {"X-API-key": ADMIN_KEY_V2} + +clowder_headers_v1 = { + **base_headers_v1, + "Content-type": "application/json", + "accept": "application/json", +} DEFAULT_PASSWORD = "Password123&" @@ -17,6 +32,23 @@ timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") COLLECTIONS_FILE = "collections_datasets.json" +def get_all_datasets(header): + endpoint = f"{CLOWDER_V1}/api/datasets?superAdmin=true&limit=0" + datasets = requests.get(endpoint, headers=header).json() + return datasets + +def get_dataset_metadata(dataset_v1_id, headers_v1): + # Get metadata from Clowder V1 + endpoint = ( + f"{CLOWDER_V1}/api/datasets/{dataset_v1_id}/metadata.jsonld?superAdmin=true" + ) + metadata_v1 = requests.get(endpoint, headers=headers_v1).json() + if len(metadata_v1) > 0: + print('we got some metadata') + with open('datasets_with_metadata.txt', 'a') as f: + f.write(dataset_v1_id + '\n') + return metadata_v1 + def get_dataset_collections_map(): print("Getting collections and datasets from Clowder v1...") @@ -38,5 +70,16 @@ def get_dataset_collections_map(): def get_datasets_in_collections(): map = get_dataset_collections_map() datasets_in_collections = list(map.keys()) + datasets_with_metadata = [] + for i in range(0, len(datasets_in_collections)): + current_dataset = datasets_in_collections[i] + dataset_metadata = get_dataset_metadata(current_dataset, base_headers_v1, datasets_with_metadata) return datasets_in_collections +if __name__ == "__main__": + all_datasets = get_all_datasets(base_headers_v1) + for i in range(0, len(all_datasets)): + current_dataset = all_datasets[i] + get_dataset_metadata(current_dataset['id'], base_headers_v1) + get_datasets_in_collections() + diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 93b2443cd..95f3f8f61 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -1,6 +1,6 @@ import os from datetime import datetime - +import json import requests from dotenv import dotenv_values @@ -217,9 +217,11 @@ def process_collection_descendants(collection, headers_v1, base_headers_v2, head new_folder = create_folder_if_not_exists_or_get(dataset["name"], v2_parent_id, v2_parent_type, v2_dataset_id, headers_v2) process_dataset_files_and_folders(dataset, headers_v1, base_headers_v2, 'folder', new_folder['id'], v2_dataset_id, new_folder) # TODO add dataset metadata to the folder + add_dataset_metadata_to_folder(dataset, v2_dataset_id, new_folder['id'], headers_v1, base_headers_v2) else: new_folder = create_folder_if_not_exists_or_get(dataset["name"], v2_parent_id, v2_parent_type, v2_dataset_id, headers_v2) process_dataset_files_and_folders(dataset, headers_v1, base_headers_v2, 'folder', new_folder['id'], v2_dataset_id, new_folder) + add_dataset_metadata_to_folder(dataset, v2_dataset_id, new_folder['id'], headers_v1, base_headers_v2) # TODO add dataset metadata to the folder @@ -798,7 +800,6 @@ def add_file_metadata(file_v1, file_v2_id, headers_v1, headers_v2): print("Successfully posted file machine metadata to V2") break # machine metadata no need to iterate through all the keys - def add_dataset_metadata(dataset_v1, dataset_v2_id, headers_v1, headers_v2): # Get metadata from Clowder V1 endpoint = ( @@ -853,6 +854,46 @@ def add_dataset_metadata(dataset_v1, dataset_v2_id, headers_v1, headers_v2): break # machine metadata no need to iterate through all the keys +def add_dataset_metadata_to_folder(dataset_v1, dataset_v2_id, folder_v2_id, headers_v1, headers_v2): + # Get metadata from Clowder V1 + endpoint = ( + f"{CLOWDER_V1}/api/datasets/{dataset_v1['id']}/metadata.jsonld?superAdmin=true" + ) + dataset_name = dataset_v1['name'] + metadata_file_name = dataset_name + '_metadata.json' + metadata_v1 = requests.get(endpoint, headers=headers_v1).json() + with open(metadata_file_name, "w") as metadata_file: + json.dump(metadata_v1, metadata_file) + + # upload the file to the folder in v2 + dataset_file_upload_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/filesMultiple?folder_id={folder_v2_id}" + + response = requests.post( + dataset_file_upload_endpoint, + headers=headers_v2, + files=[("files", (metadata_file_name, open(metadata_file_name, "rb")))], + ) + + # Clean up the local file after upload + print(f"Type response {type(response)}") + try: + os.remove(metadata_file_name) + except Exception as e: + print(f"Could not delete locally created metadata file: {metadata_file_name}") + print(e) + + if response.status_code == 200: + print(f"Uploaded file: {metadata_file_name} to dataset {dataset_v2_id} and folder {folder_v2_id}") + response_json = response.json() + if type(response_json) == dict: + return response.json().get("id") + elif type(response_json) == list: + return response_json[0].get("id") + else: + print(f"Failed to upload file: {metadata_file} to dataset {dataset_v2_id} and folder {folder_v2_id}") + return None + + def register_migration_extractor(): """Register the migration extractor in Clowder v2.""" migration_extractor = { From 19b0f029f4d9426da4f39cc5cb413d3629b46056 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 8 Sep 2025 12:31:39 -0500 Subject: [PATCH 85/91] running pipenv run black --- scripts/migration/dataset_collection_json.py | 20 +- scripts/migration/get_collections.py | 25 +- scripts/migration/get_collections_datasets.py | 15 +- scripts/migration/migrate.py | 316 ++++++++++++------ .../migration/migrate_metadata_definitions.py | 2 +- scripts/migration/test.py | 25 +- 6 files changed, 269 insertions(+), 134 deletions(-) diff --git a/scripts/migration/dataset_collection_json.py b/scripts/migration/dataset_collection_json.py index 88c848a96..fd05c6a99 100644 --- a/scripts/migration/dataset_collection_json.py +++ b/scripts/migration/dataset_collection_json.py @@ -9,7 +9,7 @@ except ImportError: import tomli as tomllib -path_to_env = os.path.join(os.getcwd(),"scripts","migration", ".env") +path_to_env = os.path.join(os.getcwd(), "scripts", "migration", ".env") config = dotenv_values(dotenv_path=path_to_env) CLOWDER_V1 = config["CLOWDER_V1"] @@ -32,11 +32,13 @@ timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") COLLECTIONS_FILE = "collections_datasets.json" + def get_all_datasets(header): endpoint = f"{CLOWDER_V1}/api/datasets?superAdmin=true&limit=0" datasets = requests.get(endpoint, headers=header).json() return datasets + def get_dataset_metadata(dataset_v1_id, headers_v1): # Get metadata from Clowder V1 endpoint = ( @@ -44,11 +46,12 @@ def get_dataset_metadata(dataset_v1_id, headers_v1): ) metadata_v1 = requests.get(endpoint, headers=headers_v1).json() if len(metadata_v1) > 0: - print('we got some metadata') - with open('datasets_with_metadata.txt', 'a') as f: - f.write(dataset_v1_id + '\n') + print("we got some metadata") + with open("datasets_with_metadata.txt", "a") as f: + f.write(dataset_v1_id + "\n") return metadata_v1 + def get_dataset_collections_map(): print("Getting collections and datasets from Clowder v1...") @@ -67,19 +70,22 @@ def get_dataset_collections_map(): dataset_to_collection[dataset] = current_value return dataset_to_collection + def get_datasets_in_collections(): map = get_dataset_collections_map() datasets_in_collections = list(map.keys()) datasets_with_metadata = [] for i in range(0, len(datasets_in_collections)): current_dataset = datasets_in_collections[i] - dataset_metadata = get_dataset_metadata(current_dataset, base_headers_v1, datasets_with_metadata) + dataset_metadata = get_dataset_metadata( + current_dataset, base_headers_v1, datasets_with_metadata + ) return datasets_in_collections + if __name__ == "__main__": all_datasets = get_all_datasets(base_headers_v1) for i in range(0, len(all_datasets)): current_dataset = all_datasets[i] - get_dataset_metadata(current_dataset['id'], base_headers_v1) + get_dataset_metadata(current_dataset["id"], base_headers_v1) get_datasets_in_collections() - diff --git a/scripts/migration/get_collections.py b/scripts/migration/get_collections.py index 5656164dd..199f286b2 100644 --- a/scripts/migration/get_collections.py +++ b/scripts/migration/get_collections.py @@ -24,7 +24,7 @@ OUTPUT_FILE = "collections_ids.txt" # Load environment variables -path_to_env = os.path.join(os.getcwd(),"scripts","migration", ".env") +path_to_env = os.path.join(os.getcwd(), "scripts", "migration", ".env") config = dotenv_values(dotenv_path=path_to_env) @@ -60,12 +60,14 @@ "last_name": "admin", } + def get_clowder_v1_top_level_collections(headers): endpoint = f"{CLOWDER_V1}/api/collections/topLevelCollections?superAdmin=true" response = requests.get(endpoint, headers=headers) user_collections = response.json() return user_collections + def get_collection_v1_descendants(headers, collection_id): descendant_ids = [] @@ -75,17 +77,20 @@ def get_collection_v1_descendants(headers, collection_id): print(collection_json["child_collection_ids"]) if int(collection_json["childCollectionsCount"]) > 0: child_collections_ids = collection_json["child_collection_ids"] - descendant_ids = child_collections_ids[5:-1].split(', ') + descendant_ids = child_collections_ids[5:-1].split(", ") for i in range(0, len(descendant_ids)): id = descendant_ids[i] descendent_endpoint = f"{CLOWDER_V1}/api/collections/{id}" - descendent_response = requests.get(descendent_endpoint, headers=headers, verify=False) + descendent_response = requests.get( + descendent_endpoint, headers=headers, verify=False + ) descendent_json = descendent_response.json() if int(descendent_json["childCollectionsCount"]) > 0: sub_descendants = get_collection_v1_descendants(headers, id) descendant_ids.extend(sub_descendants) return descendant_ids + def get_dataset_ids_in_v1_collection(headers, collection_id): dataset_ids = [] collection_endpoint = f"{CLOWDER_V1}/api/collections/{collection_id}/datasets" @@ -95,17 +100,23 @@ def get_dataset_ids_in_v1_collection(headers, collection_id): dataset_ids.append(dataset["id"]) return dataset_ids + if __name__ == "__main__": top_level_collections = get_clowder_v1_top_level_collections(clowder_headers_v1) all_v1_collections = [] for collection in top_level_collections: - print(f"Getting descendents for collection {collection['name']} ({collection['id']})") + print( + f"Getting descendents for collection {collection['name']} ({collection['id']})" + ) all_v1_collections.append(collection["id"]) if int(collection["childCollectionsCount"]) > 0: - descendant_ids = get_collection_v1_descendants(clowder_headers_v1, collection["id"]) + descendant_ids = get_collection_v1_descendants( + clowder_headers_v1, collection["id"] + ) all_v1_collections.extend(descendant_ids) - print(f"Added descendents for collection {collection['name']} ({collection['id']})") - + print( + f"Added descendents for collection {collection['name']} ({collection['id']})" + ) print(f"TOTAL V1 COLLECTIONS TO MIGRATE: {len(all_v1_collections)}") diff --git a/scripts/migration/get_collections_datasets.py b/scripts/migration/get_collections_datasets.py index 436dc5bad..deaeaa166 100644 --- a/scripts/migration/get_collections_datasets.py +++ b/scripts/migration/get_collections_datasets.py @@ -10,7 +10,6 @@ import tomli as tomllib - DEFAULT_PASSWORD = "Password123&" # Get the current timestamp @@ -18,9 +17,8 @@ COLLECTIONS_FILE = "collections_ids.txt" - # Load environment variables -path_to_env = os.path.join(os.getcwd(),"scripts","migration", ".env") +path_to_env = os.path.join(os.getcwd(), "scripts", "migration", ".env") config = dotenv_values(dotenv_path=path_to_env) @@ -56,6 +54,7 @@ "last_name": "admin", } + def get_collections_datasets(headers, collection_id): collection_dataset_endpoint = ( f"{CLOWDER_V1}/api/collections/{collection_id}/datasets?superAdmin=true" @@ -70,16 +69,16 @@ def get_collections_datasets(headers, collection_id): if __name__ == "__main__": print("Getting collections and datasets from Clowder v1...") - collection_ids =[] + collection_ids = [] if os.path.exists(COLLECTIONS_FILE): - print('exists') + print("exists") else: - print('does not exist') + print("does not exist") with open(COLLECTIONS_FILE, "r") as outfile: lines = outfile.readlines() for line in lines: - collection_ids.append(line.rstrip('\n')) + collection_ids.append(line.rstrip("\n")) print(f"Found {len(collection_ids)} collections in {COLLECTIONS_FILE}") collection_dataset_dict = dict() for id in collection_ids: @@ -94,4 +93,4 @@ def get_collections_datasets(headers, collection_id): json_file = "collections_datasets.json" with open(json_file, "w") as jf: json.dump(collection_dataset_dict, jf) - print("dumped to a file") \ No newline at end of file + print("dumped to a file") diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 95f3f8f61..2ed9dd6ed 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -16,7 +16,10 @@ post_metadata_definition, ) -from scripts.migration.dataset_collection_json import get_dataset_collections_map, get_datasets_in_collections +from scripts.migration.dataset_collection_json import ( + get_dataset_collections_map, + get_datasets_in_collections, +) DATASET_COLLECTIONS_MAP = get_dataset_collections_map() @@ -28,15 +31,15 @@ OUTPUT_FILE = f"migrated_new_users_{timestamp}.log" # Load environment variables -path_to_env = os.path.join(os.getcwd(),"scripts","migration", ".env") -path_to_toml = os.path.join(os.getcwd(),"scripts","migration", "config.toml") +path_to_env = os.path.join(os.getcwd(), "scripts", "migration", ".env") +path_to_toml = os.path.join(os.getcwd(), "scripts", "migration", "config.toml") config = dotenv_values(dotenv_path=path_to_env) -toml_space_ids=None -toml_exclude_space_ids=None -toml_users=None -toml_exclude_users=None -toml_exclude_dataset_ids=None +toml_space_ids = None +toml_exclude_space_ids = None +toml_users = None +toml_exclude_users = None +toml_exclude_dataset_ids = None # Load configuration from toml file if os.path.exists(path_to_toml): @@ -158,16 +161,19 @@ def get_collection_v1_descendants(headers, collection_id): print(collection_json["child_collection_ids"]) if int(collection_json["childCollectionsCount"]) > 0: child_collections_ids = collection_json["child_collection_ids"] - descendant_ids = child_collections_ids[5:-1].split(', ') + descendant_ids = child_collections_ids[5:-1].split(", ") for id in descendant_ids: descendent_endpoint = f"{CLOWDER_V1}/api/collections/{id}" - descendent_response = requests.get(descendent_endpoint, headers=headers, verify=False) + descendent_response = requests.get( + descendent_endpoint, headers=headers, verify=False + ) descendent_json = descendent_response.json() if int(descendent_json["childCollectionsCount"]) > 0: sub_descendants = get_collection_v1_descendants(headers, id) descendant_ids.extend(sub_descendants) return descendant_ids + def get_clowder_v1_user_collections(headers, user_v1): endpoint = f"{CLOWDER_V1}/api/collections" response = requests.get(endpoint, headers=headers) @@ -182,14 +188,25 @@ def get_clowder_v1_user_collections_top_level(headers, user_v1): response_json = response.json() for col in response_json: author = col["author"] - author_id = author.lstrip('MiniUser(') - author_id = author_id[:author_id.index(',')] + author_id = author.lstrip("MiniUser(") + author_id = author_id[: author_id.index(",")] if author_id == user_v1["id"]: - top_level_collections.append(col) + top_level_collections.append(col) return top_level_collections -def process_collection_descendants(collection, headers_v1, base_headers_v2, headers_v2, v2_parent_id, v2_parent_type, v2_dataset_id): - child_collections_endpoint = f"{CLOWDER_V1}/api/collections/{collection['id']}/getChildCollections" + +def process_collection_descendants( + collection, + headers_v1, + base_headers_v2, + headers_v2, + v2_parent_id, + v2_parent_type, + v2_dataset_id, +): + child_collections_endpoint = ( + f"{CLOWDER_V1}/api/collections/{collection['id']}/getChildCollections" + ) datasets_endpoint = f"{CLOWDER_V1}/api/collections/{collection['id']}/datasets" child_col_response = requests.get(child_collections_endpoint, headers=headers_v1) @@ -202,26 +219,70 @@ def process_collection_descendants(collection, headers_v1, base_headers_v2, head if v2_parent_type == "dataset": print(f"Add folder to the dataset") folder_name = child["name"] - new_folder = create_folder_if_not_exists_or_get(folder_name, None, v2_parent_type, v2_dataset_id, headers_v2) - process_collection_descendants(child, headers_v1, base_headers_v2, headers_v2, new_folder['id'], 'folder', v2_dataset_id ) + new_folder = create_folder_if_not_exists_or_get( + folder_name, None, v2_parent_type, v2_dataset_id, headers_v2 + ) + process_collection_descendants( + child, + headers_v1, + base_headers_v2, + headers_v2, + new_folder["id"], + "folder", + v2_dataset_id, + ) else: print(f"parent was a folder") print(f"Add folder to the dataset") folder_name = child["name"] - new_folder = create_folder_if_not_exists_or_get(folder_name, v2_parent_id,v2_parent_type, v2_dataset_id, headers_v2) - process_collection_descendants(child, headers_v1, base_headers_v2, headers_v2, new_folder['id'], 'folder', v2_dataset_id) + new_folder = create_folder_if_not_exists_or_get( + folder_name, v2_parent_id, v2_parent_type, v2_dataset_id, headers_v2 + ) + process_collection_descendants( + child, + headers_v1, + base_headers_v2, + headers_v2, + new_folder["id"], + "folder", + v2_dataset_id, + ) # this handles uploading the datasets of the collection as folders for dataset in dataset_json: if v2_parent_type == "dataset": - new_folder = create_folder_if_not_exists_or_get(dataset["name"], v2_parent_id, v2_parent_type, v2_dataset_id, headers_v2) - process_dataset_files_and_folders(dataset, headers_v1, base_headers_v2, 'folder', new_folder['id'], v2_dataset_id, new_folder) + new_folder = create_folder_if_not_exists_or_get( + dataset["name"], v2_parent_id, v2_parent_type, v2_dataset_id, headers_v2 + ) + process_dataset_files_and_folders( + dataset, + headers_v1, + base_headers_v2, + "folder", + new_folder["id"], + v2_dataset_id, + new_folder, + ) # TODO add dataset metadata to the folder - add_dataset_metadata_to_folder(dataset, v2_dataset_id, new_folder['id'], headers_v1, base_headers_v2) + add_dataset_metadata_to_folder( + dataset, v2_dataset_id, new_folder["id"], headers_v1, base_headers_v2 + ) else: - new_folder = create_folder_if_not_exists_or_get(dataset["name"], v2_parent_id, v2_parent_type, v2_dataset_id, headers_v2) - process_dataset_files_and_folders(dataset, headers_v1, base_headers_v2, 'folder', new_folder['id'], v2_dataset_id, new_folder) - add_dataset_metadata_to_folder(dataset, v2_dataset_id, new_folder['id'], headers_v1, base_headers_v2) + new_folder = create_folder_if_not_exists_or_get( + dataset["name"], v2_parent_id, v2_parent_type, v2_dataset_id, headers_v2 + ) + process_dataset_files_and_folders( + dataset, + headers_v1, + base_headers_v2, + "folder", + new_folder["id"], + v2_dataset_id, + new_folder, + ) + add_dataset_metadata_to_folder( + dataset, v2_dataset_id, new_folder["id"], headers_v1, base_headers_v2 + ) # TODO add dataset metadata to the folder @@ -231,17 +292,29 @@ def get_v1_dataset_folders(dataset, headers_v1, headers_v2, parent_type, parent_ folder_json = folder_response.json() return folder_json + # processes a dataset adds folders and -def process_dataset_files_and_folders(dataset, headers_v1, headers_v2, parent_type, parent_id, dataset_v2_id, dataset_v2_folder): - dataset_v1_folders = get_v1_dataset_folders(dataset, headers_v1, headers_v2, parent_type, parent_id) +def process_dataset_files_and_folders( + dataset, + headers_v1, + headers_v2, + parent_type, + parent_id, + dataset_v2_id, + dataset_v2_folder, +): + dataset_v1_folders = get_v1_dataset_folders( + dataset, headers_v1, headers_v2, parent_type, parent_id + ) for folder_v1 in dataset_v1_folders: - current_folder_hierarchy = folder_v1['name'] - add_folder_hierarchy_to_migration_folder(folder_hierarchy=current_folder_hierarchy, - dataset_v2=dataset_v2_id, - folder_id_v2=parent_id, - headers=headers_v2 - ) + current_folder_hierarchy = folder_v1["name"] + add_folder_hierarchy_to_migration_folder( + folder_hierarchy=current_folder_hierarchy, + dataset_v2=dataset_v2_id, + folder_id_v2=parent_id, + headers=headers_v2, + ) all_v2_dataset_folders = get_all_folder_and_subfolders(dataset_v2_id, headers_v2) files_endpoint = f"{CLOWDER_V1}/api/datasets/{dataset['id']}/files" @@ -249,32 +322,34 @@ def process_dataset_files_and_folders(dataset, headers_v1, headers_v2, parent_ty files_json = files_response.json() # go through files and upload them to the correct folder if they have one for file in files_json: - if 'folders' in file: + if "folders" in file: for folder_v2 in all_v2_dataset_folders: - if folder_v2['name'] == file['folders']['name']: + if folder_v2["name"] == file["folders"]["name"]: print(f"Upload this file to a folder") matching_folder = folder_v2 - download_and_upload_file_to_matching_folder(file, dataset_v2_id, base_headers_v2, matching_folder) + download_and_upload_file_to_matching_folder( + file, dataset_v2_id, base_headers_v2, matching_folder + ) else: if parent_type == "dataset": print(f"Upload to a dataset") if parent_type == "folder": - download_and_upload_file_to_matching_folder(file, dataset_v2_id, base_headers_v2, dataset_v2_folder) + download_and_upload_file_to_matching_folder( + file, dataset_v2_id, base_headers_v2, dataset_v2_folder + ) print(f"Got dataset files") - -def create_v2_dataset_from_collection(collection, user_v1, headers_v1, headers_v2, base_headers_v2): +def create_v2_dataset_from_collection( + collection, user_v1, headers_v1, headers_v2, base_headers_v2 +): # create the dataset collection_name = collection["name"] collection_description = collection["description"] v2_license_id = "CC BY" dataset_in_v2_endpoint = f"{CLOWDER_V2}/api/v2/datasets?license_id={v2_license_id}" - dataset_example = { - "name":collection_name, - "description": collection_description - } + dataset_example = {"name": collection_name, "description": collection_description} response = requests.post( dataset_in_v2_endpoint, headers=headers_v2, json=dataset_example ) @@ -282,15 +357,19 @@ def create_v2_dataset_from_collection(collection, user_v1, headers_v1, headers_v new_dataset_json = response.json() v2_dataset_id = new_dataset_json["id"] - process_collection_descendants(collection=collection, headers_v1=headers_v1, - base_headers_v2=base_headers_v2, headers_v2= headers_v2, - v2_parent_id=new_dataset_json["id"], - v2_parent_type="dataset", v2_dataset_id=v2_dataset_id) + process_collection_descendants( + collection=collection, + headers_v1=headers_v1, + base_headers_v2=base_headers_v2, + headers_v2=headers_v2, + v2_parent_id=new_dataset_json["id"], + v2_parent_type="dataset", + v2_dataset_id=v2_dataset_id, + ) return response.json()["id"] - # TODO this is too slow, we need to optimize it def get_clowder_v1_dataset_collections(headers, user_v1, dataset_id): matching_collections = [] @@ -315,17 +394,17 @@ def get_clowder_v1_dataset_collections(headers, user_v1, dataset_id): except Exception as e: print("Exception", e) if int(collection["childCollectionsCount"]) > 0: - collection_descendants = get_collection_v1_descendants(headers, collection_id) + collection_descendants = get_collection_v1_descendants( + headers, collection_id + ) for descendant in collection_descendants: - collection_dataset_endpoint = ( - f"{CLOWDER_V1}/api/collections/{descendant}/datasets?superAdmin=true" - ) + collection_dataset_endpoint = f"{CLOWDER_V1}/api/collections/{descendant}/datasets?superAdmin=true" collection_dataset_response = requests.get( collection_dataset_endpoint, headers=headers ) collection_dataset_json = collection_dataset_response.json() for ds in collection_dataset_json: - if ds['id'] == dataset_id: + if ds["id"] == dataset_id: if descendant not in matching_collections: matching_collections.append(descendant) return matching_collections @@ -539,23 +618,26 @@ def create_v2_group(space, headers): return response.json()["id"] -def add_folder_hierarchy_to_migration_folder(folder_hierarchy, dataset_v2, folder_id_v2, headers): +def add_folder_hierarchy_to_migration_folder( + folder_hierarchy, dataset_v2, folder_id_v2, headers +): """Add folder hierarchy to a dataset in Clowder v2.""" hierarchy_parts = folder_hierarchy.split("/") - if hierarchy_parts[0] == '': + if hierarchy_parts[0] == "": hierarchy_parts = hierarchy_parts[1:] current_parent = folder_id_v2 for part in hierarchy_parts: result = create_folder_if_not_exists_or_get( - part, current_parent, 'folder', dataset_v2, headers + part, current_parent, "folder", dataset_v2, headers ) if result: current_parent = result["id"] + def add_folder_hierarchy(folder_hierarchy, dataset_v2, headers): """Add folder hierarchy to a dataset in Clowder v2.""" hierarchy_parts = folder_hierarchy.split("/") - if hierarchy_parts[0] == '': + if hierarchy_parts[0] == "": hierarchy_parts = hierarchy_parts[1:] current_parent = None for part in hierarchy_parts: @@ -565,6 +647,7 @@ def add_folder_hierarchy(folder_hierarchy, dataset_v2, headers): if result: current_parent = result["id"] + # for creating a folder for a dataset that is migrated to a dataset def create_dataset_folder_if_not_exists_or_get(folder, parent, dataset_v2, headers): """Create a folder if it does not exist or return the existing folder.""" @@ -585,12 +668,15 @@ def create_dataset_folder_if_not_exists_or_get(folder, parent, dataset_v2, heade ) return response.json() + # used for creating folders and subfolders when a collection is migrated to a dataset -def create_folder_if_not_exists_or_get(folder, parent, parent_type, dataset_v2, headers): +def create_folder_if_not_exists_or_get( + folder, parent, parent_type, dataset_v2, headers +): """Create a folder if it does not exist or return the existing folder.""" # current_folders = get_folder_and_subfolders(dataset_v2, headers) current_all_folders = get_all_folder_and_subfolders(dataset_v2, headers) - if parent_type == 'folder': + if parent_type == "folder": folder_data = ( {"name": folder, "parent_folder": parent} if parent else {"name": folder} ) @@ -608,6 +694,7 @@ def create_folder_if_not_exists_or_get(folder, parent, parent_type, dataset_v2, ) return response.json() + def get_all_folder_and_subfolders(dataset_id, headers): """Retrieve all folders and subfolders in a dataset.""" endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_id}/all_folders" @@ -636,7 +723,9 @@ def add_dataset_folders(dataset_v1, dataset_v2, headers): add_folder_hierarchy(folder["name"], dataset_v2, headers) -def download_and_upload_file_to_matching_folder(file, dataset_v2_id, headers_v2, matching_folder = None): +def download_and_upload_file_to_matching_folder( + file, dataset_v2_id, headers_v2, matching_folder=None +): """Download a file from Clowder v1 and upload it to Clowder v2.""" filename = file["filename"] file_id = file["id"] @@ -749,6 +838,7 @@ def download_and_upload_file(file, all_dataset_folders, dataset_v2_id, headers_v return None + def add_file_metadata(file_v1, file_v2_id, headers_v1, headers_v2): # Get metadata from Clowder V1 endpoint = f"{CLOWDER_V1}/api/files/{file_v1['id']}/metadata.jsonld?superAdmin=true" @@ -800,6 +890,7 @@ def add_file_metadata(file_v1, file_v2_id, headers_v1, headers_v2): print("Successfully posted file machine metadata to V2") break # machine metadata no need to iterate through all the keys + def add_dataset_metadata(dataset_v1, dataset_v2_id, headers_v1, headers_v2): # Get metadata from Clowder V1 endpoint = ( @@ -854,13 +945,15 @@ def add_dataset_metadata(dataset_v1, dataset_v2_id, headers_v1, headers_v2): break # machine metadata no need to iterate through all the keys -def add_dataset_metadata_to_folder(dataset_v1, dataset_v2_id, folder_v2_id, headers_v1, headers_v2): +def add_dataset_metadata_to_folder( + dataset_v1, dataset_v2_id, folder_v2_id, headers_v1, headers_v2 +): # Get metadata from Clowder V1 endpoint = ( f"{CLOWDER_V1}/api/datasets/{dataset_v1['id']}/metadata.jsonld?superAdmin=true" ) - dataset_name = dataset_v1['name'] - metadata_file_name = dataset_name + '_metadata.json' + dataset_name = dataset_v1["name"] + metadata_file_name = dataset_name + "_metadata.json" metadata_v1 = requests.get(endpoint, headers=headers_v1).json() with open(metadata_file_name, "w") as metadata_file: json.dump(metadata_v1, metadata_file) @@ -883,14 +976,18 @@ def add_dataset_metadata_to_folder(dataset_v1, dataset_v2_id, folder_v2_id, head print(e) if response.status_code == 200: - print(f"Uploaded file: {metadata_file_name} to dataset {dataset_v2_id} and folder {folder_v2_id}") + print( + f"Uploaded file: {metadata_file_name} to dataset {dataset_v2_id} and folder {folder_v2_id}" + ) response_json = response.json() if type(response_json) == dict: return response.json().get("id") elif type(response_json) == list: return response_json[0].get("id") else: - print(f"Failed to upload file: {metadata_file} to dataset {dataset_v2_id} and folder {folder_v2_id}") + print( + f"Failed to upload file: {metadata_file} to dataset {dataset_v2_id} and folder {folder_v2_id}" + ) return None @@ -1032,7 +1129,10 @@ def build_collection_space_metadata_for_v1_dataset(dataset, user_v1, headers): print(f"Got space and collection metadata from dataset {dataset_id}") return metadata -def process_user_and_resources_collections(user_v1, USER_MAP, DATASET_MAP, COLLETIONS_MAP): + +def process_user_and_resources_collections( + user_v1, USER_MAP, DATASET_MAP, COLLETIONS_MAP +): """Process user resources from Clowder v1 to Clowder v2.""" # get collections of the user @@ -1058,9 +1158,9 @@ def process_user_and_resources_collections(user_v1, USER_MAP, DATASET_MAP, COLLE migrate_top_level_collections = [] for col in user_v1_collections: collection_spaces = col["spaces"] - collection_spaces = collection_spaces.lstrip('List(') - collection_spaces = collection_spaces.rstrip(')') - collection_spaces = collection_spaces.split(',') + collection_spaces = collection_spaces.lstrip("List(") + collection_spaces = collection_spaces.rstrip(")") + collection_spaces = collection_spaces.split(",") for space in collection_spaces: if space in toml_space_ids: migrate_top_level_collections.append(col) @@ -1068,10 +1168,16 @@ def process_user_and_resources_collections(user_v1, USER_MAP, DATASET_MAP, COLLE # create datasets from the top level collections for top_level_col in migrate_top_level_collections: - dataset_v2 = create_v2_dataset_from_collection(collection=top_level_col, user_v1=user_v1, - headers_v1=clowder_headers_v1 ,headers_v2=user_headers_v2, - base_headers_v2=base_headers_v2) - print(f"Created dataset in v2 from collection: {top_level_col['id']} - {top_level_col['name']}") + dataset_v2 = create_v2_dataset_from_collection( + collection=top_level_col, + user_v1=user_v1, + headers_v1=clowder_headers_v1, + headers_v2=user_headers_v2, + base_headers_v2=base_headers_v2, + ) + print( + f"Created dataset in v2 from collection: {top_level_col['id']} - {top_level_col['name']}" + ) COLLETIONS_MAP[top_level_col["id"]] = dataset_v2 datasets_in_collections_v1 = get_datasets_in_collections() @@ -1086,16 +1192,16 @@ def process_user_and_resources_collections(user_v1, USER_MAP, DATASET_MAP, COLLE print(toml_exclude_space_ids) # Check if dataset is in the excluded dataset list if dataset_v1_id in datasets_in_collections_v1: - print(f"Skipping dataset {dataset_v1_id} as it is was in a collection, already migrated") + print( + f"Skipping dataset {dataset_v1_id} as it is was in a collection, already migrated" + ) MIGRATE_DATASET = False if dataset_v1_id in toml_exclude_dataset_ids: print(f"Skipping dataset {dataset_v1_id} as it is in the exclude list.") MIGRATE_DATASET = False # Check if dataset is in the specified space list if toml_space_ids is not None and len(toml_space_ids) > 0: - if not any( - space_id in dataset_v1_spaces for space_id in toml_space_ids - ): + if not any(space_id in dataset_v1_spaces for space_id in toml_space_ids): print( f"Skipping dataset {dataset_v1_id} as it is not in the specified spaces." ) @@ -1112,11 +1218,15 @@ def process_user_and_resources_collections(user_v1, USER_MAP, DATASET_MAP, COLLE dataset_v2_id = create_v2_dataset(dataset, user_headers_v2) DATASET_MAP[dataset["id"]] = dataset_v2_id # - add_dataset_metadata(dataset, dataset_v2_id, base_headers_v1, user_headers_v2) + add_dataset_metadata( + dataset, dataset_v2_id, base_headers_v1, user_headers_v2 + ) add_dataset_folders(dataset, dataset_v2_id, user_headers_v2) print("Created folders in the new dataset") - all_dataset_folders = get_folder_and_subfolders(dataset_v2_id, user_headers_v2) + all_dataset_folders = get_folder_and_subfolders( + dataset_v2_id, user_headers_v2 + ) # Retrieve files for the dataset in Clowder v1 dataset_files_endpoint = ( @@ -1139,16 +1249,20 @@ def process_user_and_resources_collections(user_v1, USER_MAP, DATASET_MAP, COLLE ), None, ) - print('did we get matching folder?') + print("did we get matching folder?") file_v2_id = download_and_upload_file_to_matching_folder( file, dataset_v2_id, base_user_headers_v2, matching_folder ) if file_v2_id is not None: - add_file_metadata(file, file_v2_id, clowder_headers_v1, user_headers_v2) + add_file_metadata( + file, file_v2_id, clowder_headers_v1, user_headers_v2 + ) # posting the collection hierarchy as metadata try: - collection_space_metadata_dict = build_collection_space_metadata_for_v1_dataset( - dataset=dataset, user_v1=user_v1, headers=clowder_headers_v1 + collection_space_metadata_dict = ( + build_collection_space_metadata_for_v1_dataset( + dataset=dataset, user_v1=user_v1, headers=clowder_headers_v1 + ) ) except Exception as e: print(e) @@ -1162,7 +1276,9 @@ def process_user_and_resources_collections(user_v1, USER_MAP, DATASET_MAP, COLLE "content": collection_space_metadata_dict, "contents": collection_space_metadata_dict, } - v2_metadata_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/metadata" + v2_metadata_endpoint = ( + f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/metadata" + ) response = requests.post( v2_metadata_endpoint, json=migration_extractor_collection_metadata, @@ -1207,9 +1323,7 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): MIGRATE_DATASET = False # Check if dataset is in the specified space list if toml_space_ids is not None and len(toml_space_ids) > 0: - if not any( - space_id in dataset_v1_spaces for space_id in toml_space_ids - ): + if not any(space_id in dataset_v1_spaces for space_id in toml_space_ids): print( f"Skipping dataset {dataset_v1_id} as it is not in the specified spaces." ) @@ -1225,11 +1339,15 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): if MIGRATE_DATASET: dataset_v2_id = create_v2_dataset(dataset, user_headers_v2) DATASET_MAP[dataset["id"]] = dataset_v2_id - add_dataset_metadata(dataset, dataset_v2_id, base_headers_v1, user_headers_v2) + add_dataset_metadata( + dataset, dataset_v2_id, base_headers_v1, user_headers_v2 + ) add_dataset_folders(dataset, dataset_v2_id, user_headers_v2) print("Created folders in the new dataset") - all_dataset_folders = get_folder_and_subfolders(dataset_v2_id, user_headers_v2) + all_dataset_folders = get_folder_and_subfolders( + dataset_v2_id, user_headers_v2 + ) # Retrieve files for the dataset in Clowder v1 dataset_files_endpoint = ( @@ -1252,7 +1370,7 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): ), None, ) - print('did we get matching folder?') + print("did we get matching folder?") file_v2_id = download_and_upload_file_to_matching_folder( file, dataset_v2_id, base_user_headers_v2, matching_folder ) @@ -1260,11 +1378,15 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): file, all_dataset_folders, dataset_v2_id, base_user_headers_v2 ) if file_v2_id is not None: - add_file_metadata(file, file_v2_id, clowder_headers_v1, user_headers_v2) + add_file_metadata( + file, file_v2_id, clowder_headers_v1, user_headers_v2 + ) # posting the collection hierarchy as metadata try: - collection_space_metadata_dict = build_collection_space_metadata_for_v1_dataset( - dataset=dataset, user_v1=user_v1, headers=clowder_headers_v1 + collection_space_metadata_dict = ( + build_collection_space_metadata_for_v1_dataset( + dataset=dataset, user_v1=user_v1, headers=clowder_headers_v1 + ) ) except Exception as e: print(e) @@ -1278,7 +1400,9 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): "content": collection_space_metadata_dict, "contents": collection_space_metadata_dict, } - v2_metadata_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/metadata" + v2_metadata_endpoint = ( + f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/metadata" + ) response = requests.post( v2_metadata_endpoint, json=migration_extractor_collection_metadata, @@ -1319,9 +1443,7 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): if toml_users is not None and len(toml_users) > 0: print(f"Using spaces from config.toml: {toml_users}") - users_v1 = [ - user for user in users_v1 if user["email"] in toml_users - ] + users_v1 = [user for user in users_v1 if user["email"] in toml_users] else: print("No spaces specified in config.toml, migrating all users.") for user_v1 in users_v1: diff --git a/scripts/migration/migrate_metadata_definitions.py b/scripts/migration/migrate_metadata_definitions.py index a31cbfc32..bf9139706 100644 --- a/scripts/migration/migrate_metadata_definitions.py +++ b/scripts/migration/migrate_metadata_definitions.py @@ -4,7 +4,7 @@ import requests from dotenv import dotenv_values -path_to_env = os.path.join(os.getcwd(),"scripts","migration", ".env") +path_to_env = os.path.join(os.getcwd(), "scripts", "migration", ".env") if not os.path.exists(path_to_env): raise FileNotFoundError( f"Environment file not found at {path_to_env}. Please ensure it exists." diff --git a/scripts/migration/test.py b/scripts/migration/test.py index 2efc3c1bc..967a28d9d 100644 --- a/scripts/migration/test.py +++ b/scripts/migration/test.py @@ -2,8 +2,8 @@ from dotenv import dotenv_values import os -path_to_env = os.path.join(os.getcwd(),"scripts","migration", ".env") -path_to_toml = os.path.join(os.getcwd(),"scripts","migration", "config.toml") +path_to_env = os.path.join(os.getcwd(), "scripts", "migration", ".env") +path_to_toml = os.path.join(os.getcwd(), "scripts", "migration", "config.toml") config = dotenv_values(dotenv_path=path_to_env) CLOWDER_V1 = config["CLOWDER_V1"] @@ -20,7 +20,8 @@ "accept": "application/json", } -url = 'http://127.0.0.1:8000/api/v2' +url = "http://127.0.0.1:8000/api/v2" + def get_new_dataset_folders(dataset_id, headers): endpoint = f"{url}/datasets/{dataset_id}/all_folders" @@ -28,14 +29,12 @@ def get_new_dataset_folders(dataset_id, headers): foldesr_json = r.json() print(r.json()) + def download_and_upload_file_to_folder(file, folder_id, dataset_v2_id, headers_v2): """Download a file from Clowder v1 and upload it to Clowder v2.""" - # Download the file from Clowder v1 - filename = 'test.txt' - - + filename = "test.txt" # Upload the file to Clowder v2 dataset_file_upload_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/files" @@ -58,15 +57,13 @@ def download_and_upload_file_to_folder(file, folder_id, dataset_v2_id, headers_v return None - - -test_file = 'july-2018-temperature-precip.csv' +test_file = "july-2018-temperature-precip.csv" if os.path.exists(test_file): - print('it exists') + print("it exists") -test_folder_id = '68b206b0fb9e6c77930beaab' -test_dataset_id = '68b206a4fb9e6c77930beaa8' +test_folder_id = "68b206b0fb9e6c77930beaab" +test_dataset_id = "68b206a4fb9e6c77930beaa8" download_and_upload_file_to_folder(test_file, None, test_dataset_id, clowder_headers_v2) -# new_folders = get_new_dataset_folders('68b080ee03137d5052c0872c', headers=clowder_headers_v2) \ No newline at end of file +# new_folders = get_new_dataset_folders('68b080ee03137d5052c0872c', headers=clowder_headers_v2) From 0fa7465efd9d2b7c4cdd1c088f9da868f6fb9cae Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 8 Sep 2025 12:32:49 -0500 Subject: [PATCH 86/91] running pipenv run black for app --- backend/app/routers/datasets.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/backend/app/routers/datasets.py b/backend/app/routers/datasets.py index 558973322..e9fdfa97b 100644 --- a/backend/app/routers/datasets.py +++ b/backend/app/routers/datasets.py @@ -837,6 +837,7 @@ async def get_dataset_folders_and_files( return page.dict() raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found") + @router.get("/{dataset_id}/all_folders") async def get_dataset_folders_all( dataset_id: str, @@ -866,7 +867,9 @@ async def get_dataset_folders_all( FolderFileViewList.dataset_id == PydanticObjectId(dataset_id), ] - folders = (await FolderFileViewList.find(*query).aggregate( + folders = ( + await FolderFileViewList.find(*query) + .aggregate( [ _get_page_query( skip, @@ -879,15 +882,14 @@ async def get_dataset_folders_all( }, ) ], - ).to_list()) + ) + .to_list() + ) page_metadata = _construct_page_metadata(folders, skip, limit) page = Paged( metadata=page_metadata, - data=[ - FolderOut(id=item.pop("_id"), **item) - for item in folders[0]["data"] - ], + data=[FolderOut(id=item.pop("_id"), **item) for item in folders[0]["data"]], ) return page.dict() From 80ae7cfc71113166527d72ccc0a0654b9dbbf909 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 8 Sep 2025 12:40:29 -0500 Subject: [PATCH 87/91] fix typo that got in --- backend/app/routers/datasets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/app/routers/datasets.py b/backend/app/routers/datasets.py index e9fdfa97b..cac646642 100644 --- a/backend/app/routers/datasets.py +++ b/backend/app/routers/datasets.py @@ -663,7 +663,7 @@ async def delete_freeze_dataset_version( if ( frozen_dataset := await DatasetFreezeDB.find_one( DatasetFreezeDB.origin_id == PydanticObjectId(dataset_id), - DatasetFreezeDxB.frozen_version_num == frozen_version_num, + DatasetFreezeDB.frozen_version_num == frozen_version_num, ) ) is not None: return await _delete_frozen_dataset(frozen_dataset, fs, hard_delete=False) From a034ca55112870f5bdb3dffc4ae335c860b1209d Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 8 Sep 2025 12:51:01 -0500 Subject: [PATCH 88/91] ran codegen? --- frontend/src/openapi/v1/core/ApiError.ts | 28 +- .../src/openapi/v1/core/ApiRequestOptions.ts | 29 +- frontend/src/openapi/v1/core/ApiResult.ts | 12 +- .../src/openapi/v1/core/CancelablePromise.ts | 205 +- frontend/src/openapi/v1/core/OpenAPI.ts | 42 +- frontend/src/openapi/v1/core/request.ts | 454 ++- frontend/src/openapi/v1/index.ts | 64 +- frontend/src/openapi/v1/models/Collection.ts | 10 +- frontend/src/openapi/v1/models/Comment.ts | 4 +- frontend/src/openapi/v1/models/Dataset.ts | 47 +- .../src/openapi/v1/models/ExtractorsLabel.ts | 10 +- frontend/src/openapi/v1/models/JSONLD.ts | 3 +- frontend/src/openapi/v1/models/License.ts | 31 +- frontend/src/openapi/v1/models/MiniUser.ts | 3 +- frontend/src/openapi/v1/models/Space.ts | 6 +- frontend/src/openapi/v1/models/Tags.ts | 6 +- frontend/src/openapi/v1/models/UUID.ts | 2 +- frontend/src/openapi/v1/models/binary.ts | 96 +- .../src/openapi/v1/services/AdminService.ts | 266 +- .../openapi/v1/services/CollectionsService.ts | 1502 ++++----- .../openapi/v1/services/CurationsService.ts | 212 +- .../openapi/v1/services/DTexturesService.ts | 42 +- .../openapi/v1/services/DatasetsService.ts | 2936 +++++++++-------- .../openapi/v1/services/ExtractionsService.ts | 582 ++-- .../openapi/v1/services/ExtractorsService.ts | 200 +- .../src/openapi/v1/services/FilesService.ts | 2630 ++++++++------- .../src/openapi/v1/services/FoldersService.ts | 138 +- .../src/openapi/v1/services/LogosService.ts | 299 +- .../openapi/v1/services/MetadataService.ts | 386 +-- .../openapi/v1/services/PreviewsService.ts | 40 +- .../src/openapi/v1/services/ProxyService.ts | 356 +- .../openapi/v1/services/ReportingService.ts | 238 +- .../src/openapi/v1/services/SearchService.ts | 122 +- .../src/openapi/v1/services/SectionService.ts | 370 ++- .../src/openapi/v1/services/ServiceService.ts | 102 +- .../src/openapi/v1/services/SpacesService.ts | 808 ++--- 36 files changed, 6288 insertions(+), 5993 deletions(-) diff --git a/frontend/src/openapi/v1/core/ApiError.ts b/frontend/src/openapi/v1/core/ApiError.ts index 737148c5d..214d69df3 100644 --- a/frontend/src/openapi/v1/core/ApiError.ts +++ b/frontend/src/openapi/v1/core/ApiError.ts @@ -1,21 +1,21 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ -import type { ApiResult } from "./ApiResult"; +import type { ApiResult } from './ApiResult'; export class ApiError extends Error { - public readonly url: string; - public readonly status: number; - public readonly statusText: string; - public readonly body: any; + public readonly url: string; + public readonly status: number; + public readonly statusText: string; + public readonly body: any; - constructor(response: ApiResult, message: string) { - super(message); + constructor(response: ApiResult, message: string) { + super(message); - this.name = "ApiError"; - this.url = response.url; - this.status = response.status; - this.statusText = response.statusText; - this.body = response.body; - } -} + this.name = 'ApiError'; + this.url = response.url; + this.status = response.status; + this.statusText = response.statusText; + this.body = response.body; + } +} \ No newline at end of file diff --git a/frontend/src/openapi/v1/core/ApiRequestOptions.ts b/frontend/src/openapi/v1/core/ApiRequestOptions.ts index a68e3b8e5..1a1ed384f 100644 --- a/frontend/src/openapi/v1/core/ApiRequestOptions.ts +++ b/frontend/src/openapi/v1/core/ApiRequestOptions.ts @@ -2,21 +2,14 @@ /* tslint:disable */ /* eslint-disable */ export type ApiRequestOptions = { - readonly method: - | "GET" - | "PUT" - | "POST" - | "DELETE" - | "OPTIONS" - | "HEAD" - | "PATCH"; - readonly path: string; - readonly cookies?: Record; - readonly headers?: Record; - readonly query?: Record; - readonly formData?: Record; - readonly body?: any; - readonly mediaType?: string; - readonly responseHeader?: string; - readonly errors?: Record; -}; + readonly method: 'GET' | 'PUT' | 'POST' | 'DELETE' | 'OPTIONS' | 'HEAD' | 'PATCH'; + readonly path: string; + readonly cookies?: Record; + readonly headers?: Record; + readonly query?: Record; + readonly formData?: Record; + readonly body?: any; + readonly mediaType?: string; + readonly responseHeader?: string; + readonly errors?: Record; +} \ No newline at end of file diff --git a/frontend/src/openapi/v1/core/ApiResult.ts b/frontend/src/openapi/v1/core/ApiResult.ts index 0d0afb8d7..7363469d7 100644 --- a/frontend/src/openapi/v1/core/ApiResult.ts +++ b/frontend/src/openapi/v1/core/ApiResult.ts @@ -2,9 +2,9 @@ /* tslint:disable */ /* eslint-disable */ export type ApiResult = { - readonly url: string; - readonly ok: boolean; - readonly status: number; - readonly statusText: string; - readonly body: any; -}; + readonly url: string; + readonly ok: boolean; + readonly status: number; + readonly statusText: string; + readonly body: any; +} \ No newline at end of file diff --git a/frontend/src/openapi/v1/core/CancelablePromise.ts b/frontend/src/openapi/v1/core/CancelablePromise.ts index 3b36b12fa..e34e9dcb4 100644 --- a/frontend/src/openapi/v1/core/CancelablePromise.ts +++ b/frontend/src/openapi/v1/core/CancelablePromise.ts @@ -2,112 +2,113 @@ /* tslint:disable */ /* eslint-disable */ export class CancelError extends Error { - constructor(reason: string = "Promise was canceled") { - super(reason); - this.name = "CancelError"; - } - - public get isCancelled(): boolean { - return true; - } + + constructor(reason: string = 'Promise was canceled') { + super(reason); + this.name = 'CancelError'; + } + + public get isCancelled(): boolean { + return true; + } } export interface OnCancel { - readonly isPending: boolean; - readonly isCancelled: boolean; + readonly isPending: boolean; + readonly isCancelled: boolean; - (cancelHandler: () => void): void; + (cancelHandler: () => void): void; } export class CancelablePromise implements Promise { - readonly [Symbol.toStringTag]: string; - - #isPending: boolean; - #isCancelled: boolean; - readonly #cancelHandlers: (() => void)[]; - readonly #promise: Promise; - #resolve?: (value: T | PromiseLike) => void; - #reject?: (reason?: any) => void; - - constructor( - executor: ( - resolve: (value: T | PromiseLike) => void, - reject: (reason?: any) => void, - onCancel: OnCancel - ) => void - ) { - this.#isPending = true; - this.#isCancelled = false; - this.#cancelHandlers = []; - this.#promise = new Promise((resolve, reject) => { - this.#resolve = resolve; - this.#reject = reject; - - const onResolve = (value: T | PromiseLike): void => { - if (!this.#isCancelled) { - this.#isPending = false; - this.#resolve?.(value); - } - }; - - const onReject = (reason?: any): void => { - this.#isPending = false; - this.#reject?.(reason); - }; - - const onCancel = (cancelHandler: () => void): void => { - if (this.#isPending) { - this.#cancelHandlers.push(cancelHandler); - } - }; - - Object.defineProperty(onCancel, "isPending", { - get: (): boolean => this.#isPending, - }); - - Object.defineProperty(onCancel, "isCancelled", { - get: (): boolean => this.#isCancelled, - }); - - return executor(onResolve, onReject, onCancel as OnCancel); - }); - } - - public then( - onFulfilled?: ((value: T) => TResult1 | PromiseLike) | null, - onRejected?: ((reason: any) => TResult2 | PromiseLike) | null - ): Promise { - return this.#promise.then(onFulfilled, onRejected); - } - - public catch( - onRejected?: ((reason: any) => TResult | PromiseLike) | null - ): Promise { - return this.#promise.catch(onRejected); - } - - public finally(onFinally?: (() => void) | null): Promise { - return this.#promise.finally(onFinally); - } - - public cancel(): void { - if (!this.#isPending || this.#isCancelled) { - return; - } - this.#isCancelled = true; - if (this.#cancelHandlers.length) { - try { - for (const cancelHandler of this.#cancelHandlers) { - cancelHandler(); - } - } catch (error) { - this.#reject?.(error); - return; - } - } - } - - public get isCancelled(): boolean { - return this.#isCancelled; - } -} + readonly [Symbol.toStringTag]: string; + + #isPending: boolean; + #isCancelled: boolean; + readonly #cancelHandlers: (() => void)[]; + readonly #promise: Promise; + #resolve?: (value: T | PromiseLike) => void; + #reject?: (reason?: any) => void; + + constructor( + executor: ( + resolve: (value: T | PromiseLike) => void, + reject: (reason?: any) => void, + onCancel: OnCancel + ) => void + ) { + this.#isPending = true; + this.#isCancelled = false; + this.#cancelHandlers = []; + this.#promise = new Promise((resolve, reject) => { + this.#resolve = resolve; + this.#reject = reject; + + const onResolve = (value: T | PromiseLike): void => { + if (!this.#isCancelled) { + this.#isPending = false; + this.#resolve?.(value); + } + }; + + const onReject = (reason?: any): void => { + this.#isPending = false; + this.#reject?.(reason); + }; + + const onCancel = (cancelHandler: () => void): void => { + if (this.#isPending) { + this.#cancelHandlers.push(cancelHandler); + } + }; + + Object.defineProperty(onCancel, 'isPending', { + get: (): boolean => this.#isPending, + }); + + Object.defineProperty(onCancel, 'isCancelled', { + get: (): boolean => this.#isCancelled, + }); + + return executor(onResolve, onReject, onCancel as OnCancel); + }); + } + + public then( + onFulfilled?: ((value: T) => TResult1 | PromiseLike) | null, + onRejected?: ((reason: any) => TResult2 | PromiseLike) | null + ): Promise { + return this.#promise.then(onFulfilled, onRejected); + } + + public catch( + onRejected?: ((reason: any) => TResult | PromiseLike) | null + ): Promise { + return this.#promise.catch(onRejected); + } + + public finally(onFinally?: (() => void) | null): Promise { + return this.#promise.finally(onFinally); + } + + public cancel(): void { + if (!this.#isPending || this.#isCancelled) { + return; + } + this.#isCancelled = true; + if (this.#cancelHandlers.length) { + try { + for (const cancelHandler of this.#cancelHandlers) { + cancelHandler(); + } + } catch (error) { + this.#reject?.(error); + return; + } + } + } + + public get isCancelled(): boolean { + return this.#isCancelled; + } +} \ No newline at end of file diff --git a/frontend/src/openapi/v1/core/OpenAPI.ts b/frontend/src/openapi/v1/core/OpenAPI.ts index 7cc33d980..2765a5ca7 100644 --- a/frontend/src/openapi/v1/core/OpenAPI.ts +++ b/frontend/src/openapi/v1/core/OpenAPI.ts @@ -1,31 +1,31 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ -import type { ApiRequestOptions } from "./ApiRequestOptions"; +import type { ApiRequestOptions } from './ApiRequestOptions'; type Resolver = (options: ApiRequestOptions) => Promise; type Headers = Record; type Config = { - BASE: string; - VERSION: string; - WITH_CREDENTIALS: boolean; - CREDENTIALS: "include" | "omit" | "same-origin"; - TOKEN?: string | Resolver; - USERNAME?: string | Resolver; - PASSWORD?: string | Resolver; - HEADERS?: Headers | Resolver; - ENCODE_PATH?: (path: string) => string; -}; + BASE: string; + VERSION: string; + WITH_CREDENTIALS: boolean; + CREDENTIALS: 'include' | 'omit' | 'same-origin'; + TOKEN?: string | Resolver; + USERNAME?: string | Resolver; + PASSWORD?: string | Resolver; + HEADERS?: Headers | Resolver; + ENCODE_PATH?: (path: string) => string; +} export const OpenAPI: Config = { - BASE: "https://clowder.ncsa.illinois.edu/clowder/api", - VERSION: "1.21.0", - WITH_CREDENTIALS: false, - CREDENTIALS: "include", - TOKEN: undefined, - USERNAME: undefined, - PASSWORD: undefined, - HEADERS: undefined, - ENCODE_PATH: undefined, -}; + BASE: 'https://clowder.ncsa.illinois.edu/clowder/api', + VERSION: '1.23.0', + WITH_CREDENTIALS: false, + CREDENTIALS: 'include', + TOKEN: undefined, + USERNAME: undefined, + PASSWORD: undefined, + HEADERS: undefined, + ENCODE_PATH: undefined, +}; \ No newline at end of file diff --git a/frontend/src/openapi/v1/core/request.ts b/frontend/src/openapi/v1/core/request.ts index d683578b1..4c6e69dad 100644 --- a/frontend/src/openapi/v1/core/request.ts +++ b/frontend/src/openapi/v1/core/request.ts @@ -1,254 +1,238 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ -import { ApiError } from "./ApiError"; -import type { ApiRequestOptions } from "./ApiRequestOptions"; -import type { ApiResult } from "./ApiResult"; -import { CancelablePromise } from "./CancelablePromise"; -import type { OnCancel } from "./CancelablePromise"; -import { OpenAPI } from "./OpenAPI"; - -function isDefined( - value: T | null | undefined -): value is Exclude { - return value !== undefined && value !== null; +import { ApiError } from './ApiError'; +import type { ApiRequestOptions } from './ApiRequestOptions'; +import type { ApiResult } from './ApiResult'; +import { CancelablePromise } from './CancelablePromise'; +import type { OnCancel } from './CancelablePromise'; +import { OpenAPI } from './OpenAPI'; + +function isDefined(value: T | null | undefined): value is Exclude { + return value !== undefined && value !== null; } function isString(value: any): value is string { - return typeof value === "string"; + return typeof value === 'string'; } function isStringWithValue(value: any): value is string { - return isString(value) && value !== ""; + return isString(value) && value !== ''; } function isBlob(value: any): value is Blob { - return value instanceof Blob; + return value instanceof Blob; } function base64(str: string): string { - try { - return btoa(str); - } catch (err) { - return Buffer.from(str).toString("base64"); - } + try { + return btoa(str); + } catch (err) { + return Buffer.from(str).toString('base64'); + } } function getQueryString(params: Record): string { - const qs: string[] = []; - - const append = (key: string, value: any) => { - qs.push(`${encodeURIComponent(key)}=${encodeURIComponent(String(value))}`); - }; - - Object.entries(params) - .filter(([_, value]) => isDefined(value)) - .forEach(([key, value]) => { - if (Array.isArray(value)) { - value.forEach((v) => append(key, v)); - } else { - append(key, value); - } - }); - - if (qs.length > 0) { - return `?${qs.join("&")}`; - } - - return ""; + const qs: string[] = []; + + const append = (key: string, value: any) => { + qs.push(`${encodeURIComponent(key)}=${encodeURIComponent(String(value))}`); + }; + + Object.entries(params) + .filter(([_, value]) => isDefined(value)) + .forEach(([key, value]) => { + if (Array.isArray(value)) { + value.forEach(v => append(key, v)); + } else { + append(key, value); + } + }); + + if (qs.length > 0) { + return `?${qs.join('&')}`; + } + + return ''; } function getUrl(options: ApiRequestOptions): string { - const path = OpenAPI.ENCODE_PATH - ? OpenAPI.ENCODE_PATH(options.path) - : options.path; - const url = `${OpenAPI.BASE}${path}`; - if (options.query) { - return `${url}${getQueryString(options.query)}`; - } - - return url; + const path = OpenAPI.ENCODE_PATH ? OpenAPI.ENCODE_PATH(options.path) : options.path; + const url = `${OpenAPI.BASE}${path}`; + if (options.query) { + return `${url}${getQueryString(options.query)}`; + } + + return url; } function getFormData(options: ApiRequestOptions): FormData | undefined { - if (options.formData) { - const formData = new FormData(); - - const append = (key: string, value: any) => { - if (isString(value) || isBlob(value)) { - formData.append(key, value); - } else { - formData.append(key, JSON.stringify(value)); - } - }; - - Object.entries(options.formData) - .filter(([_, value]) => isDefined(value)) - .forEach(([key, value]) => { - if (Array.isArray(value)) { - value.forEach((v) => append(key, v)); - } else { - append(key, value); - } - }); - - return formData; - } - return; + if (options.formData) { + const formData = new FormData(); + + const append = (key: string, value: any) => { + if (isString(value) || isBlob(value)) { + formData.append(key, value); + } else { + formData.append(key, JSON.stringify(value)); + } + }; + + Object.entries(options.formData) + .filter(([_, value]) => isDefined(value)) + .forEach(([key, value]) => { + if (Array.isArray(value)) { + value.forEach(v => append(key, v)); + } else { + append(key, value); + } + }); + + return formData; + } + return; } type Resolver = (options: ApiRequestOptions) => Promise; -async function resolve( - options: ApiRequestOptions, - resolver?: T | Resolver -): Promise { - if (typeof resolver === "function") { - return (resolver as Resolver)(options); - } - return resolver; +async function resolve(options: ApiRequestOptions, resolver?: T | Resolver): Promise { + if (typeof resolver === 'function') { + return (resolver as Resolver)(options); + } + return resolver; } async function getHeaders(options: ApiRequestOptions): Promise { - const token = await resolve(options, OpenAPI.TOKEN); - const username = await resolve(options, OpenAPI.USERNAME); - const password = await resolve(options, OpenAPI.PASSWORD); - const additionalHeaders = await resolve(options, OpenAPI.HEADERS); - - const defaultHeaders = Object.entries({ - Accept: "application/json", - ...additionalHeaders, - ...options.headers, - }) - .filter(([_, value]) => isDefined(value)) - .reduce( - (headers, [key, value]) => ({ - ...headers, - [key]: String(value), - }), - {} as Record - ); - - const headers = new Headers(defaultHeaders); - - if (isStringWithValue(token)) { - headers.append("Authorization", `Bearer ${token}`); - } - - if (isStringWithValue(username) && isStringWithValue(password)) { - const credentials = base64(`${username}:${password}`); - headers.append("Authorization", `Basic ${credentials}`); - } - - if (options.body) { - if (options.mediaType) { - headers.append("Content-Type", options.mediaType); - } else if (isBlob(options.body)) { - headers.append( - "Content-Type", - options.body.type || "application/octet-stream" - ); - } else if (isString(options.body)) { - headers.append("Content-Type", "text/plain"); - } else { - headers.append("Content-Type", "application/json"); - } - } - - return headers; + const token = await resolve(options, OpenAPI.TOKEN); + const username = await resolve(options, OpenAPI.USERNAME); + const password = await resolve(options, OpenAPI.PASSWORD); + const additionalHeaders = await resolve(options, OpenAPI.HEADERS); + + const defaultHeaders = Object.entries({ + Accept: 'application/json', + ...additionalHeaders, + ...options.headers, + }) + .filter(([_, value]) => isDefined(value)) + .reduce((headers, [key, value]) => ({ + ...headers, + [key]: String(value), + }), {} as Record); + + const headers = new Headers(defaultHeaders); + + if (isStringWithValue(token)) { + headers.append('Authorization', `Bearer ${token}`); + } + + if (isStringWithValue(username) && isStringWithValue(password)) { + const credentials = base64(`${username}:${password}`); + headers.append('Authorization', `Basic ${credentials}`); + } + + if (options.body) { + if (options.mediaType) { + headers.append('Content-Type', options.mediaType); + } else if (isBlob(options.body)) { + headers.append('Content-Type', options.body.type || 'application/octet-stream'); + } else if (isString(options.body)) { + headers.append('Content-Type', 'text/plain'); + } else { + headers.append('Content-Type', 'application/json'); + } + } + + return headers; } function getRequestBody(options: ApiRequestOptions): BodyInit | undefined { - if (options.body) { - if (options.mediaType?.includes("/json")) { - return JSON.stringify(options.body); - } else if (isString(options.body) || isBlob(options.body)) { - return options.body; - } else { - return JSON.stringify(options.body); - } - } - return; + if (options.body) { + if (options.mediaType?.includes('/json')) { + return JSON.stringify(options.body) + } else if (isString(options.body) || isBlob(options.body)) { + return options.body; + } else { + return JSON.stringify(options.body); + } + } + return; } async function sendRequest( - options: ApiRequestOptions, - url: string, - formData: FormData | undefined, - body: BodyInit | undefined, - headers: Headers, - onCancel: OnCancel + options: ApiRequestOptions, + url: string, + formData: FormData | undefined, + body: BodyInit | undefined, + headers: Headers, + onCancel: OnCancel ): Promise { - const controller = new AbortController(); + const controller = new AbortController(); - const request: RequestInit = { - headers, - body: body || formData, - method: options.method, - signal: controller.signal, - }; + const request: RequestInit = { + headers, + body: body || formData, + method: options.method, + signal: controller.signal, + }; - if (OpenAPI.WITH_CREDENTIALS) { - request.credentials = OpenAPI.CREDENTIALS; - } + if (OpenAPI.WITH_CREDENTIALS) { + request.credentials = OpenAPI.CREDENTIALS; + } - onCancel(() => controller.abort()); + onCancel(() => controller.abort()); - return await fetch(url, request); + return await fetch(url, request); } -function getResponseHeader( - response: Response, - responseHeader?: string -): string | undefined { - if (responseHeader) { - const content = response.headers.get(responseHeader); - if (isString(content)) { - return content; - } - } - return; +function getResponseHeader(response: Response, responseHeader?: string): string | undefined { + if (responseHeader) { + const content = response.headers.get(responseHeader); + if (isString(content)) { + return content; + } + } + return; } async function getResponseBody(response: Response): Promise { - if (response.status !== 204) { - try { - const contentType = response.headers.get("Content-Type"); - if (contentType) { - const isJSON = contentType.toLowerCase().startsWith("application/json"); - if (isJSON) { - return await response.json(); - } else { - return await response.text(); - } - } - } catch (error) { - console.error(error); - } - } - return; + if (response.status !== 204) { + try { + const contentType = response.headers.get('Content-Type'); + if (contentType) { + const isJSON = contentType.toLowerCase().startsWith('application/json'); + if (isJSON) { + return await response.json(); + } else { + return await response.text(); + } + } + } catch (error) { + console.error(error); + } + } + return; } function catchErrors(options: ApiRequestOptions, result: ApiResult): void { - const errors: Record = { - 400: "Bad Request", - 401: "Unauthorized", - 403: "Forbidden", - 404: "Not Found", - 500: "Internal Server Error", - 502: "Bad Gateway", - 503: "Service Unavailable", - ...options.errors, - }; - - const error = errors[result.status]; - if (error) { - throw new ApiError(result, error); - } - - if (!result.ok) { - throw new ApiError(result, "Generic Error"); - } + const errors: Record = { + 400: 'Bad Request', + 401: 'Unauthorized', + 403: 'Forbidden', + 404: 'Not Found', + 500: 'Internal Server Error', + 502: 'Bad Gateway', + 503: 'Service Unavailable', + ...options.errors, + } + + const error = errors[result.status]; + if (error) { + throw new ApiError(result, error); + } + + if (!result.ok) { + throw new ApiError(result, 'Generic Error'); + } } /** @@ -258,42 +242,32 @@ function catchErrors(options: ApiRequestOptions, result: ApiResult): void { * @throws ApiError */ export function request(options: ApiRequestOptions): CancelablePromise { - return new CancelablePromise(async (resolve, reject, onCancel) => { - try { - const url = getUrl(options); - const formData = getFormData(options); - const body = getRequestBody(options); - const headers = await getHeaders(options); - - if (!onCancel.isCancelled) { - const response = await sendRequest( - options, - url, - formData, - body, - headers, - onCancel - ); - const responseBody = await getResponseBody(response); - const responseHeader = getResponseHeader( - response, - options.responseHeader - ); - - const result: ApiResult = { - url, - ok: response.ok, - status: response.status, - statusText: response.statusText, - body: responseHeader || responseBody, - }; - - catchErrors(options, result); - - resolve(result.body); - } - } catch (error) { - reject(error); - } - }); -} + return new CancelablePromise(async (resolve, reject, onCancel) => { + try { + const url = getUrl(options); + const formData = getFormData(options); + const body = getRequestBody(options); + const headers = await getHeaders(options); + + if (!onCancel.isCancelled) { + const response = await sendRequest(options, url, formData, body, headers, onCancel); + const responseBody = await getResponseBody(response); + const responseHeader = getResponseHeader(response, options.responseHeader); + + const result: ApiResult = { + url, + ok: response.ok, + status: response.status, + statusText: response.statusText, + body: responseHeader || responseBody, + }; + + catchErrors(options, result); + + resolve(result.body); + } + } catch (error) { + reject(error); + } + }); +} \ No newline at end of file diff --git a/frontend/src/openapi/v1/index.ts b/frontend/src/openapi/v1/index.ts index e04c19a8e..b2e017584 100644 --- a/frontend/src/openapi/v1/index.ts +++ b/frontend/src/openapi/v1/index.ts @@ -1,37 +1,37 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ -export { ApiError } from "./core/ApiError"; -export { CancelablePromise } from "./core/CancelablePromise"; -export { OpenAPI } from "./core/OpenAPI"; +export { ApiError } from './core/ApiError'; +export { CancelablePromise } from './core/CancelablePromise'; +export { OpenAPI } from './core/OpenAPI'; -export type { binary } from "./models/binary"; -export type { Collection } from "./models/Collection"; -export type { Comment } from "./models/Comment"; -export { Dataset } from "./models/Dataset"; -export type { ExtractorsLabel } from "./models/ExtractorsLabel"; -export type { JSONLD } from "./models/JSONLD"; -export { License } from "./models/License"; -export type { MiniUser } from "./models/MiniUser"; -export type { Space } from "./models/Space"; -export type { Tags } from "./models/Tags"; -export type { UUID } from "./models/UUID"; +export type { binary } from './models/binary'; +export type { Collection } from './models/Collection'; +export type { Comment } from './models/Comment'; +export { Dataset } from './models/Dataset'; +export type { ExtractorsLabel } from './models/ExtractorsLabel'; +export type { JSONLD } from './models/JSONLD'; +export { License } from './models/License'; +export type { MiniUser } from './models/MiniUser'; +export type { Space } from './models/Space'; +export type { Tags } from './models/Tags'; +export type { UUID } from './models/UUID'; -export { AdminService } from "./services/AdminService"; -export { CollectionsService } from "./services/CollectionsService"; -export { CurationsService } from "./services/CurationsService"; -export { DatasetsService } from "./services/DatasetsService"; -export { DTexturesService } from "./services/DTexturesService"; -export { ExtractionsService } from "./services/ExtractionsService"; -export { ExtractorsService } from "./services/ExtractorsService"; -export { FilesService } from "./services/FilesService"; -export { FoldersService } from "./services/FoldersService"; -export { LogosService } from "./services/LogosService"; -export { MetadataService } from "./services/MetadataService"; -export { PreviewsService } from "./services/PreviewsService"; -export { ProxyService } from "./services/ProxyService"; -export { ReportingService } from "./services/ReportingService"; -export { SearchService } from "./services/SearchService"; -export { SectionService } from "./services/SectionService"; -export { ServiceService } from "./services/ServiceService"; -export { SpacesService } from "./services/SpacesService"; +export { AdminService } from './services/AdminService'; +export { CollectionsService } from './services/CollectionsService'; +export { CurationsService } from './services/CurationsService'; +export { DatasetsService } from './services/DatasetsService'; +export { DTexturesService } from './services/DTexturesService'; +export { ExtractionsService } from './services/ExtractionsService'; +export { ExtractorsService } from './services/ExtractorsService'; +export { FilesService } from './services/FilesService'; +export { FoldersService } from './services/FoldersService'; +export { LogosService } from './services/LogosService'; +export { MetadataService } from './services/MetadataService'; +export { PreviewsService } from './services/PreviewsService'; +export { ProxyService } from './services/ProxyService'; +export { ReportingService } from './services/ReportingService'; +export { SearchService } from './services/SearchService'; +export { SectionService } from './services/SectionService'; +export { ServiceService } from './services/ServiceService'; +export { SpacesService } from './services/SpacesService'; diff --git a/frontend/src/openapi/v1/models/Collection.ts b/frontend/src/openapi/v1/models/Collection.ts index 0e24020e3..3f644e633 100644 --- a/frontend/src/openapi/v1/models/Collection.ts +++ b/frontend/src/openapi/v1/models/Collection.ts @@ -3,8 +3,8 @@ /* eslint-disable */ export type Collection = { - name: string; - description?: string; - space?: string; - googleAnalytics?: string; -}; + name: string; + description?: string; + space?: string; + googleAnalytics?: string; +} diff --git a/frontend/src/openapi/v1/models/Comment.ts b/frontend/src/openapi/v1/models/Comment.ts index 6cdcdc043..630c8e267 100644 --- a/frontend/src/openapi/v1/models/Comment.ts +++ b/frontend/src/openapi/v1/models/Comment.ts @@ -3,5 +3,5 @@ /* eslint-disable */ export type Comment = { - text: string; -}; + text: string; +} diff --git a/frontend/src/openapi/v1/models/Dataset.ts b/frontend/src/openapi/v1/models/Dataset.ts index a14b84b13..fa4c38cca 100644 --- a/frontend/src/openapi/v1/models/Dataset.ts +++ b/frontend/src/openapi/v1/models/Dataset.ts @@ -3,28 +3,31 @@ /* eslint-disable */ export type Dataset = { - name: string; - description?: string; - access?: Dataset.access; - /** - * if space is not set or set as default, this dataset will not be add to any Spaces. - */ - space?: Array; - /** - * if collection is not set or set as default, this dataset will not be add to any Collections. - */ - collection?: Array; - /** - * the list of file ids, seperated with comma - */ - existingfiles?: string; -}; + name: string; + description?: string; + access?: Dataset.access; + /** + * if space is not set or set as default, this dataset will not be add to any Spaces. + */ + space?: Array; + /** + * if collection is not set or set as default, this dataset will not be add to any Collections. + */ + collection?: Array; + /** + * the list of file ids, seperated with comma + */ + existingfiles?: string; +} export namespace Dataset { - export enum access { - PUBLIC = "PUBLIC", - PRIVATE = "PRIVATE", - DEFAULT = "DEFAULT", - TRIAL = "TRIAL", - } + + export enum access { + PUBLIC = 'PUBLIC', + PRIVATE = 'PRIVATE', + DEFAULT = 'DEFAULT', + TRIAL = 'TRIAL', + } + + } diff --git a/frontend/src/openapi/v1/models/ExtractorsLabel.ts b/frontend/src/openapi/v1/models/ExtractorsLabel.ts index 9de56a16d..2a568e741 100644 --- a/frontend/src/openapi/v1/models/ExtractorsLabel.ts +++ b/frontend/src/openapi/v1/models/ExtractorsLabel.ts @@ -6,8 +6,8 @@ * An arbitrary text label on which users can filter the Extractor Catalog */ export type ExtractorsLabel = { - id?: string; - name?: string; - category?: string; - extractors?: Array; -}; + id?: string; + name?: string; + category?: string; + extractors?: Array; +} diff --git a/frontend/src/openapi/v1/models/JSONLD.ts b/frontend/src/openapi/v1/models/JSONLD.ts index 74df702da..d64b8e011 100644 --- a/frontend/src/openapi/v1/models/JSONLD.ts +++ b/frontend/src/openapi/v1/models/JSONLD.ts @@ -2,4 +2,5 @@ /* tslint:disable */ /* eslint-disable */ -export type JSONLD = {}; +export type JSONLD = { +} diff --git a/frontend/src/openapi/v1/models/License.ts b/frontend/src/openapi/v1/models/License.ts index c46c784ef..85cfc9603 100644 --- a/frontend/src/openapi/v1/models/License.ts +++ b/frontend/src/openapi/v1/models/License.ts @@ -3,20 +3,23 @@ /* eslint-disable */ export type License = { - licenseType: string; - rightsHolder: string; - licenseUrl: string; - licenseText: License.licenseText; - allowDownload: string; -}; + licenseType: string; + rightsHolder: string; + licenseUrl: string; + licenseText: License.licenseText; + allowDownload: string; +} export namespace License { - export enum licenseText { - BY_NC_ND = "by-nc-nd", - BY_ND = "by-nd", - BY_NC = "by-nc", - BY_NC_SA = "by-nc-sa", - BY_SA = "by-sa", - BY = "by", - } + + export enum licenseText { + BY_NC_ND = 'by-nc-nd', + BY_ND = 'by-nd', + BY_NC = 'by-nc', + BY_NC_SA = 'by-nc-sa', + BY_SA = 'by-sa', + BY = 'by', + } + + } diff --git a/frontend/src/openapi/v1/models/MiniUser.ts b/frontend/src/openapi/v1/models/MiniUser.ts index 2e3591a0e..4fa8f14b2 100644 --- a/frontend/src/openapi/v1/models/MiniUser.ts +++ b/frontend/src/openapi/v1/models/MiniUser.ts @@ -2,4 +2,5 @@ /* tslint:disable */ /* eslint-disable */ -export type MiniUser = {}; +export type MiniUser = { +} diff --git a/frontend/src/openapi/v1/models/Space.ts b/frontend/src/openapi/v1/models/Space.ts index d534398cb..f86d895d3 100644 --- a/frontend/src/openapi/v1/models/Space.ts +++ b/frontend/src/openapi/v1/models/Space.ts @@ -3,6 +3,6 @@ /* eslint-disable */ export type Space = { - name: string; - description: string; -}; + name: string; + description: string; +} diff --git a/frontend/src/openapi/v1/models/Tags.ts b/frontend/src/openapi/v1/models/Tags.ts index 2a5b9271c..5ffe06318 100644 --- a/frontend/src/openapi/v1/models/Tags.ts +++ b/frontend/src/openapi/v1/models/Tags.ts @@ -3,6 +3,6 @@ /* eslint-disable */ export type Tags = { - tags?: Array; - extractor_id?: string; -}; + tags?: Array; + extractor_id?: string; +} diff --git a/frontend/src/openapi/v1/models/UUID.ts b/frontend/src/openapi/v1/models/UUID.ts index b0ff1ba2f..c1bd6c8b7 100644 --- a/frontend/src/openapi/v1/models/UUID.ts +++ b/frontend/src/openapi/v1/models/UUID.ts @@ -5,4 +5,4 @@ /** * ID of an object */ -export type UUID = string; +export type UUID = string; \ No newline at end of file diff --git a/frontend/src/openapi/v1/models/binary.ts b/frontend/src/openapi/v1/models/binary.ts index 90c4b05ee..7e3f457d4 100644 --- a/frontend/src/openapi/v1/models/binary.ts +++ b/frontend/src/openapi/v1/models/binary.ts @@ -6,51 +6,51 @@ * holds all information about a file */ export type binary = { - id?: string; - filename?: string; - /** - * only specified if user is serveradmin, and storage is - * DiskByteStorageService - * - */ - filepath?: string; - /** - * only specified if user is serveradmin, and storage is - * S3ByteStorageService - * - */ - "service-endpoint"?: string; - /** - * only specified if user is serveradmin, and storage is - * S3ByteStorageService - * - */ - "bucket-name"?: string; - /** - * only specified if user is serveradmin, and storage is - * S3ByteStorageService - * - */ - "object-key"?: string; - filedescription?: string; - "content-type"?: string; - "date-created"?: string; - size?: string; - /** - * id of user who uploaded the file - */ - authorId?: string; - /** - * optinal id of the thumbnail image of this file - * - */ - thumbnail?: string; - /** - * status of the file, this can be one of the following: - * - CREATED file is created, but blob is not in final place yes - * - PROCESSING blob is in final place, final processing is done such as sending messagess to RabbitMQ - * - PROCESSED file is fully procesed by clowder - * - */ - status?: string; -}; + id?: string; + filename?: string; + /** + * only specified if user is serveradmin, and storage is + * DiskByteStorageService + * + */ + filepath?: string; + /** + * only specified if user is serveradmin, and storage is + * S3ByteStorageService + * + */ + 'service-endpoint'?: string; + /** + * only specified if user is serveradmin, and storage is + * S3ByteStorageService + * + */ + 'bucket-name'?: string; + /** + * only specified if user is serveradmin, and storage is + * S3ByteStorageService + * + */ + 'object-key'?: string; + filedescription?: string; + 'content-type'?: string; + 'date-created'?: string; + size?: string; + /** + * id of user who uploaded the file + */ + authorId?: string; + /** + * optinal id of the thumbnail image of this file + * + */ + thumbnail?: string; + /** + * status of the file, this can be one of the following: + * - CREATED file is created, but blob is not in final place yes + * - PROCESSING blob is in final place, final processing is done such as sending messagess to RabbitMQ + * - PROCESSED file is fully procesed by clowder + * + */ + status?: string; +} diff --git a/frontend/src/openapi/v1/services/AdminService.ts b/frontend/src/openapi/v1/services/AdminService.ts index 5f9d44f50..3b5fdbdf0 100644 --- a/frontend/src/openapi/v1/services/AdminService.ts +++ b/frontend/src/openapi/v1/services/AdminService.ts @@ -1,137 +1,147 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ -import type { CancelablePromise } from "../core/CancelablePromise"; -import { request as __request } from "../core/request"; +import type { CancelablePromise } from '../core/CancelablePromise'; +import { request as __request } from '../core/request'; export class AdminService { - /** - * Force a reindex of all resources in Elasticsearch. - * Force a reindex of all resources in Elasticsearch. - * Must be a server admin. - * - * @returns void - * @throws ApiError - */ - public static postAdmin(): CancelablePromise { - return __request({ - method: "POST", - path: `/reindex`, - errors: { - 401: `Not authorized`, - 403: `Forbidden`, - }, - }); - } - /** - * Update system configuration. - * Update system configuration. - * Must be a server admin. - * - * @param requestBody The body of the POST request. - * @returns void - * @throws ApiError - */ - public static postAdmin1(requestBody: { - theme?: string; - displayName?: string; - welcomeMessage?: string; - googleAnalytics?: string; - sensors?: string; - sensor?: string; - parameters?: string; - parameter?: string; - tosText?: string; - tosHtml?: string; - }): CancelablePromise { - return __request({ - method: "POST", - path: `/admin/configuration`, - body: requestBody, - mediaType: "application/json", - errors: { - 401: `Not authorized`, - 403: `Forbidden`, - }, - }); - } + /** + * Force a reindex of all resources in Elasticsearch. + * Force a reindex of all resources in Elasticsearch. + * Must be a server admin. + * + * @returns void + * @throws ApiError + */ + public static postAdmin(): CancelablePromise { + return __request({ + method: 'POST', + path: `/reindex`, + errors: { + 401: `Not authorized`, + 403: `Forbidden`, + }, + }); + } - /** - * Send email to admins. - * Send email to admins. - * Must be a server admin. - * - * @param requestBody The body of the POST request. - * @returns void - * @throws ApiError - */ - public static postAdmin2(requestBody: { - body?: string; - subject?: string; - }): CancelablePromise { - return __request({ - method: "POST", - path: `/admin/mail`, - body: requestBody, - mediaType: "application/json", - errors: { - 401: `Not authorized`, - 403: `Forbidden`, - }, - }); - } + /** + * Update system configuration. + * Update system configuration. + * Must be a server admin. + * + * @param requestBody The body of the POST request. + * @returns void + * @throws ApiError + */ + public static postAdmin1( + requestBody: { + theme?: string; + displayName?: string; + welcomeMessage?: string; + googleAnalytics?: string; + sensors?: string; + sensor?: string; + parameters?: string; + parameter?: string; + tosText?: string; + tosHtml?: string; + }, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/admin/configuration`, + body: requestBody, + mediaType: 'application/json', + errors: { + 401: `Not authorized`, + 403: `Forbidden`, + }, + }); + } - /** - * Update all users status, set a list of users as active, inactive, admin or normal user. - * Update all users status, set a list of users as active, inactive, admin or normal user. - * Must be a server admin. - * - * @param requestBody The body of the POST request. - * @returns void - * @throws ApiError - */ - public static postAdmin3(requestBody: { - active?: Array; - inactive?: Array; - admin?: Array; - unadmin?: Array; - }): CancelablePromise { - return __request({ - method: "POST", - path: `/admin/users`, - body: requestBody, - mediaType: "application/json", - errors: { - 401: `Not authorized`, - 403: `Forbidden`, - }, - }); - } + /** + * Send email to admins. + * Send email to admins. + * Must be a server admin. + * + * @param requestBody The body of the POST request. + * @returns void + * @throws ApiError + */ + public static postAdmin2( + requestBody: { + body?: string; + subject?: string; + }, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/admin/mail`, + body: requestBody, + mediaType: 'application/json', + errors: { + 401: `Not authorized`, + 403: `Forbidden`, + }, + }); + } - /** - * Change themes of the clowder instance. - * Change themes of the clowder instance. - * Must be a server admin. - * - * @param requestBody The body of the POST request. - * @returns void - * @throws ApiError - */ - public static postAdmin4(requestBody: { - theme?: string; - displayName?: string; - welcomeMessage?: string; - googleAnalytics?: string; - }): CancelablePromise { - return __request({ - method: "POST", - path: `/changeAppearance`, - body: requestBody, - mediaType: "application/json", - errors: { - 401: `Not authorized`, - }, - }); - } -} + /** + * Update all users status, set a list of users as active, inactive, admin or normal user. + * Update all users status, set a list of users as active, inactive, admin or normal user. + * Must be a server admin. + * + * @param requestBody The body of the POST request. + * @returns void + * @throws ApiError + */ + public static postAdmin3( + requestBody: { + active?: Array; + inactive?: Array; + admin?: Array; + unadmin?: Array; + }, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/admin/users`, + body: requestBody, + mediaType: 'application/json', + errors: { + 401: `Not authorized`, + 403: `Forbidden`, + }, + }); + } + + /** + * Change themes of the clowder instance. + * Change themes of the clowder instance. + * Must be a server admin. + * + * @param requestBody The body of the POST request. + * @returns void + * @throws ApiError + */ + public static postAdmin4( + requestBody: { + theme?: string; + displayName?: string; + welcomeMessage?: string; + googleAnalytics?: string; + }, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/changeAppearance`, + body: requestBody, + mediaType: 'application/json', + errors: { + 401: `Not authorized`, + }, + }); + } + +} \ No newline at end of file diff --git a/frontend/src/openapi/v1/services/CollectionsService.ts b/frontend/src/openapi/v1/services/CollectionsService.ts index 8bc6b5a2e..839316508 100644 --- a/frontend/src/openapi/v1/services/CollectionsService.ts +++ b/frontend/src/openapi/v1/services/CollectionsService.ts @@ -1,740 +1,772 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ -import type { Collection } from "../models/Collection"; -import type { CancelablePromise } from "../core/CancelablePromise"; -import { request as __request } from "../core/request"; +import type { Collection } from '../models/Collection'; +import type { CancelablePromise } from '../core/CancelablePromise'; +import { request as __request } from '../core/request'; export class CollectionsService { - /** - * Return a list of collections that fit the query standard and user has view permission - * Collections are groupings of datasets - * @param when - * @param title The title/ name of colletions - * @param date The date collection is created - * @param limit The number of collections returns, default as 12. - * @param exact - * @returns void - * @throws ApiError - */ - public static getCollections( - when?: string, - title?: string, - date?: string, - limit?: number, - exact?: boolean - ): CancelablePromise { - return __request({ - method: "GET", - path: `/collections`, - query: { - when: when, - title: title, - date: date, - limit: limit, - exact: exact, - }, - errors: { - 401: `Not authorized`, - }, - }); - } - - /** - * Create a collection - * Collections are groupings of datasets - * @param requestBody The body of the POST request to create a collection. - * @returns any OK - * @throws ApiError - */ - public static postCollections( - requestBody?: Collection - ): CancelablePromise { - return __request({ - method: "POST", - path: `/collections`, - body: requestBody, - mediaType: "application/json", - }); - } - - /** - * Get a specific collection - * Get a specific collection - * @param collId - * @returns any OK - * @throws ApiError - */ - public static getCollections1(collId: string): CancelablePromise { - return __request({ - method: "GET", - path: `/collections/${collId}`, - }); - } - - /** - * Remove collection - * Does not delete the individual datasets in the collection. - * @param collId - * @returns any OK - * @throws ApiError - */ - public static deleteCollections(collId: string): CancelablePromise { - return __request({ - method: "DELETE", - path: `/collections/${collId}`, - }); - } - - /** - * Get parent collections for collection - * Collections are groupings of datasets - * @param collId - * @returns any OK - * @throws ApiError - */ - public static getCollections2(collId: string): CancelablePromise { - return __request({ - method: "GET", - path: `/collections/${collId}/getParentCollections`, - }); - } - - /** - * Get all collections - * Collections are groupings of datasets - * @param limit - * @param showAll - * @returns any OK - * @throws ApiError - */ - public static getCollections3( - limit?: number, - showAll?: boolean - ): CancelablePromise { - return __request({ - method: "GET", - path: `/collections/allCollections`, - query: { - limit: limit, - showAll: showAll, - }, - }); - } - - /** - * Update a collection name - * Takes one argument, a UUID of the collection. Request body takes a - * key-value pair for the name - * - * @param collId - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static putCollections( - collId: string, - requestBody: { - name: string; - } - ): CancelablePromise { - return __request({ - method: "PUT", - path: `/collections/${collId}/title`, - body: requestBody, - mediaType: "application/json", - }); - } - - /** - * List all collections the user can edit except itself and its parent - * collections - * - * This will check for Permission.AddResourceToCollection and - * Permission.EditCollection - * - * @param currentCollectionId - * @param when - * @param title - * @param date - * @param limit - * @param exact - * @returns any OK - * @throws ApiError - */ - public static getCollections4( - currentCollectionId: string, - when?: string, - title?: string, - date?: string, - limit?: number, - exact?: boolean - ): CancelablePromise { - return __request({ - method: "GET", - path: `/collections/possibleParents`, - query: { - when: when, - currentCollectionId: currentCollectionId, - title: title, - date: date, - limit: limit, - exact: exact, - }, - }); - } - - /** - * Unfollow collection. - * Remove user from collection followers and remove collection from user - * followed collections. - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static postCollections1(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/collections/${id}/unfollow`, - }); - } - - /** - * Get all root collections or collections that do not have a parent - * Collections are groupings of datasets - * @returns any OK - * @throws ApiError - */ - public static getCollections5(): CancelablePromise { - return __request({ - method: "GET", - path: `/collections/topLevelCollections`, - }); - } - - /** - * Get all root collections - * Collections are groupings of datasets - * @returns any OK - * @throws ApiError - */ - public static getCollections6(): CancelablePromise { - return __request({ - method: "GET", - path: `/collections/rootCollections`, - }); - } - - /** - * Remove subcollection from collection - * Collections are groupings of datasets - * @param collId - * @param subCollId - * @returns any OK - * @throws ApiError - */ - public static postCollections2( - collId: string, - subCollId: string - ): CancelablePromise { - return __request({ - method: "POST", - path: `/collections/${collId}/removeSubCollection/${subCollId}`, - }); - } - - /** - * List all collections the user can edit - * This will check for Permission.AddResourceToCollection and - * Permission.EditCollection - * - * @param when - * @param title - * @param date - * @param limit - * @param exact - * @returns any OK - * @throws ApiError - */ - public static getCollections7( - when?: string, - title?: string, - date?: string, - limit?: number, - exact?: boolean - ): CancelablePromise { - return __request({ - method: "GET", - path: `/collections/canEdit`, - query: { - when: when, - title: title, - date: date, - limit: limit, - exact: exact, - }, - }); - } - - /** - * Deleted all collections in trash older than days specified - * Server admin action. - * @param days - * @returns any OK - * @throws ApiError - */ - public static deleteCollections1(days?: number): CancelablePromise { - return __request({ - method: "DELETE", - path: `/collections/clearOldCollectionsTrash`, - query: { - days: days, - }, - }); - } - - /** - * Emptying trash collections - * @returns any OK - * @throws ApiError - */ - public static deleteCollections2(): CancelablePromise { - return __request({ - method: "DELETE", - path: `/collections/emptyTrash`, - }); - } - - /** - * list trash collections - * @param limit - * @returns any OK - * @throws ApiError - */ - public static getCollections8(limit?: number): CancelablePromise { - return __request({ - method: "GET", - path: `/collections/listTrash`, - query: { - limit: limit, - }, - }); - } - - /** - * restore a trash collection - * This will check for Permission.DeleteCollection - * - * @param collId - * @returns any OK - * @throws ApiError - */ - public static putCollections1(collId: string): CancelablePromise { - return __request({ - method: "PUT", - path: `/collections/restore/${collId}`, - }); - } - - /** - * Add subcollection to collection - * Collections are groupings of datasets - * @param collId - * @param subCollId - * @returns any OK - * @throws ApiError - */ - public static postCollections3( - collId: string, - subCollId: string - ): CancelablePromise { - return __request({ - method: "POST", - path: `/collections/${collId}/addSubCollection/${subCollId}`, - }); - } - - /** - * Get child collections in collection - * Collections are groupings of datasets - * @param collId - * @returns any OK - * @throws ApiError - */ - public static getCollections9(collId: string): CancelablePromise { - return __request({ - method: "GET", - path: `/collections/${collId}/getChildCollections`, - }); - } - - /** - * list all datasets in the collection - * Collections are groupings of datasets - * @param collId - * @returns any OK - * @throws ApiError - */ - public static getCollections10(collId: string): CancelablePromise { - return __request({ - method: "GET", - path: `/collections/${collId}/datasets`, - }); - } - - /** - * If dataset is in a space, list all collections can be the parent of - * the dataset in this space, otherwise list all possiable collections - * - * Collections are groupings of datasets - * @param collId - * @param dsId - * @param when - * @param title - * @param date - * @param limit default as 12 - * @param exact default as false - * @returns any OK - * @throws ApiError - */ - public static getCollections11( - collId: string, - dsId: string, - when?: string, - title?: string, - date?: string, - limit?: number, - exact?: boolean - ): CancelablePromise { - return __request({ - method: "GET", - path: `/collections/${collId}/datasetPossibleParents/${dsId}`, - query: { - when: when, - title: title, - date: date, - limit: limit, - exact: exact, - }, - }); - } - - /** - * Add dataset to collection - * Collections are groupings of datasets - * @param collId - * @param dsId - * @returns any OK - * @throws ApiError - */ - public static postCollections4( - collId: string, - dsId: string - ): CancelablePromise { - return __request({ - method: "POST", - path: `/collections/${collId}/datasets/${dsId}`, - }); - } - - /** - * Detach a dataset from collection - * Collections are groupings of datasets - * @param collId - * @param dsId - * @param ignoreNotFound default as True - * @returns any OK - * @throws ApiError - */ - public static deleteCollections3( - collId: string, - dsId: string, - ignoreNotFound?: boolean - ): CancelablePromise { - return __request({ - method: "DELETE", - path: `/collections/${collId}/datasets/${dsId}`, - query: { - ignoreNotFound: ignoreNotFound, - }, - }); - } - - /** - * Attach existing preview to collection - * Collections are groupings of datasets - * @param cId - * @param pId - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postCollections5( - cId: string, - pId: string, - requestBody?: { - extractor_id?: string; - preview_type?: string; - } - ): CancelablePromise { - return __request({ - method: "POST", - path: `/collections/${cId}/previews/${pId}`, - body: requestBody, - mediaType: "application/json", - }); - } - - /** - * Follow collection. - * Add user to collection followers and add collection to user followed - * collections. - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static postCollections6(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/collections/${id}/follow`, - }); - } - - /** - * Get parent collection ids of collection - * Collections are groupings of datasets - * @param collId - * @returns any OK - * @throws ApiError - */ - public static getCollections12(collId: string): CancelablePromise { - return __request({ - method: "GET", - path: `/collections/${collId}/getParentCollectionIds`, - }); - } - - /** - * Reindex a collection - * Reindex the existing collection, if recursive is set to true it will - * also reindex all datasets and files. - * - * @param collId - * @returns any OK - * @throws ApiError - */ - public static postCollections7(collId: string): CancelablePromise { - return __request({ - method: "POST", - path: `/collections/${collId}/reindex`, - }); - } - - /** - * Removes root flag from a collection in a space - * Collections are groupings of datasets - * @param collId - * @param spaceId - * @returns any OK - * @throws ApiError - */ - public static postCollections8( - collId: string, - spaceId: string - ): CancelablePromise { - return __request({ - method: "POST", - path: `/collections/${collId}/unsetRootFlag/${spaceId}`, - }); - } - - /** - * Checks if we can remove a collection from a space - * This will check if the collection has parent collection in this space. - * If not, we can remove the collection from the space - * - * @param collId - * @param spaceId - * @returns any OK - * @throws ApiError - */ - public static getCollections13( - collId: string, - spaceId: string - ): CancelablePromise { - return __request({ - method: "GET", - path: `/collections/${collId}/removeFromSpaceAllowed/${spaceId}`, - }); - } - - /** - * Get child collection ids in collection - * Collections are groupings of datasets - * @param collId - * @returns any OK - * @throws ApiError - */ - public static getCollections14(collId: string): CancelablePromise { - return __request({ - method: "GET", - path: `/collections/${collId}/getChildCollectionIds`, - }); - } - - /** - * Create a collection with parent - * Collections are groupings of datasets - * @param requestBody The body of the POST request to create a collection. - * @returns any OK - * @throws ApiError - */ - public static postCollections9(requestBody?: { - name: string; - description?: string; - space?: string; - googleAnalytics?: string; - parentId?: string; - }): CancelablePromise { - return __request({ - method: "POST", - path: `/collections/newCollectionWithParent`, - body: requestBody, - mediaType: "application/json", - }); - } - - /** - * Update collection description - * Takes one argument, a UUID of the collection. Request body takes - * key-value pair for the description - * - * @param collId - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static putCollections2( - collId: string, - requestBody: { - description: string; - } - ): CancelablePromise { - return __request({ - method: "PUT", - path: `/collections/${collId}/description`, - body: requestBody, - mediaType: "application/json", - errors: { - 404: `Not Found`, - }, - }); - } - - /** - * Add root flags for a collection in space - * Collections are groupings of datasets - * @param collId - * @param spaceId - * @returns any OK - * @throws ApiError - */ - public static postCollections10( - collId: string, - spaceId: string - ): CancelablePromise { - return __request({ - method: "POST", - path: `/collections/${collId}/rootFlag/${spaceId}`, - }); - } - - /** - * Download collection - * Downloads all child collections, datasets and files in a collection. - * @param id - * @param compression default as -1 - * @returns any OK - * @throws ApiError - */ - public static getCollections15( - id: string, - compression?: number - ): CancelablePromise { - return __request({ - method: "GET", - path: `/collections/${id}/download`, - query: { - compression: compression, - }, - }); - } - - /** - * @deprecated - * List all collections the user can view - * This will check for Permission.ViewCollection - * @param when - * @param title - * @param date - * @param limit - * @param exact - * @returns any OK - * @throws ApiError - */ - public static getCollections16( - when?: string, - title?: string, - date?: string, - limit?: number, - exact?: boolean - ): CancelablePromise { - return __request({ - method: "GET", - path: `/collections/list`, - query: { - when: when, - title: title, - date: date, - limit: limit, - exact: exact, - }, - }); - } - - /** - * @deprecated - * Remove a dataset from a collection. - * @param collId - * @param dsId - * @param ignoreNotFound - * @returns any OK - * @throws ApiError - */ - public static postCollections11( - collId: string, - dsId: string, - ignoreNotFound: string - ): CancelablePromise { - return __request({ - method: "POST", - path: `/collections/${collId}/datasetsRemove/${dsId}/${ignoreNotFound}`, - }); - } - - /** - * @deprecated - * remove a collection - * @param collId - * @returns any OK - * @throws ApiError - */ - public static postCollections12(collId: string): CancelablePromise { - return __request({ - method: "POST", - path: `/collections/${collId}/remove`, - }); - } -} + + /** + * Return a list of collections that fit the query standard and user has view permission + * Collections are groupings of datasets + * @param when + * @param title The title/ name of colletions + * @param date The date collection is created + * @param limit The number of collections returns, default as 12. + * @param exact + * @returns void + * @throws ApiError + */ + public static getCollections( + when?: string, + title?: string, + date?: string, + limit?: number, + exact?: boolean, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/collections`, + query: { + 'when': when, + 'title': title, + 'date': date, + 'limit': limit, + 'exact': exact, + }, + errors: { + 401: `Not authorized`, + }, + }); + } + + /** + * Create a collection + * Collections are groupings of datasets + * @param requestBody The body of the POST request to create a collection. + * @returns any OK + * @throws ApiError + */ + public static postCollections( + requestBody?: Collection, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/collections`, + body: requestBody, + mediaType: 'application/json', + }); + } + + /** + * Get a specific collection + * Get a specific collection + * @param collId + * @returns any OK + * @throws ApiError + */ + public static getCollections1( + collId: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/collections/${collId}`, + }); + } + + /** + * Remove collection + * Does not delete the individual datasets in the collection. + * @param collId + * @returns any OK + * @throws ApiError + */ + public static deleteCollections( + collId: string, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/collections/${collId}`, + }); + } + + /** + * Get parent collections for collection + * Collections are groupings of datasets + * @param collId + * @returns any OK + * @throws ApiError + */ + public static getCollections2( + collId: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/collections/${collId}/getParentCollections`, + }); + } + + /** + * Get all collections + * Collections are groupings of datasets + * @param limit + * @param showAll + * @returns any OK + * @throws ApiError + */ + public static getCollections3( + limit?: number, + showAll?: boolean, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/collections/allCollections`, + query: { + 'limit': limit, + 'showAll': showAll, + }, + }); + } + + /** + * Update a collection name + * Takes one argument, a UUID of the collection. Request body takes a + * key-value pair for the name + * + * @param collId + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static putCollections( + collId: string, + requestBody: { + name: string; + }, + ): CancelablePromise { + return __request({ + method: 'PUT', + path: `/collections/${collId}/title`, + body: requestBody, + mediaType: 'application/json', + }); + } + + /** + * List all collections the user can edit except itself and its parent + * collections + * + * This will check for Permission.AddResourceToCollection and + * Permission.EditCollection + * + * @param currentCollectionId + * @param when + * @param title + * @param date + * @param limit + * @param exact + * @returns any OK + * @throws ApiError + */ + public static getCollections4( + currentCollectionId: string, + when?: string, + title?: string, + date?: string, + limit?: number, + exact?: boolean, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/collections/possibleParents`, + query: { + 'when': when, + 'currentCollectionId': currentCollectionId, + 'title': title, + 'date': date, + 'limit': limit, + 'exact': exact, + }, + }); + } + + /** + * Unfollow collection. + * Remove user from collection followers and remove collection from user + * followed collections. + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static postCollections1( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/collections/${id}/unfollow`, + }); + } + + /** + * Get all root collections or collections that do not have a parent + * Collections are groupings of datasets + * @returns any OK + * @throws ApiError + */ + public static getCollections5(): CancelablePromise { + return __request({ + method: 'GET', + path: `/collections/topLevelCollections`, + }); + } + + /** + * Get all root collections + * Collections are groupings of datasets + * @returns any OK + * @throws ApiError + */ + public static getCollections6(): CancelablePromise { + return __request({ + method: 'GET', + path: `/collections/rootCollections`, + }); + } + + /** + * Remove subcollection from collection + * Collections are groupings of datasets + * @param collId + * @param subCollId + * @returns any OK + * @throws ApiError + */ + public static postCollections2( + collId: string, + subCollId: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/collections/${collId}/removeSubCollection/${subCollId}`, + }); + } + + /** + * List all collections the user can edit + * This will check for Permission.AddResourceToCollection and + * Permission.EditCollection + * + * @param when + * @param title + * @param date + * @param limit + * @param exact + * @returns any OK + * @throws ApiError + */ + public static getCollections7( + when?: string, + title?: string, + date?: string, + limit?: number, + exact?: boolean, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/collections/canEdit`, + query: { + 'when': when, + 'title': title, + 'date': date, + 'limit': limit, + 'exact': exact, + }, + }); + } + + /** + * Deleted all collections in trash older than days specified + * Server admin action. + * @param days + * @returns any OK + * @throws ApiError + */ + public static deleteCollections1( + days?: number, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/collections/clearOldCollectionsTrash`, + query: { + 'days': days, + }, + }); + } + + /** + * Emptying trash collections + * @returns any OK + * @throws ApiError + */ + public static deleteCollections2(): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/collections/emptyTrash`, + }); + } + + /** + * list trash collections + * @param limit + * @returns any OK + * @throws ApiError + */ + public static getCollections8( + limit?: number, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/collections/listTrash`, + query: { + 'limit': limit, + }, + }); + } + + /** + * restore a trash collection + * This will check for Permission.DeleteCollection + * + * @param collId + * @returns any OK + * @throws ApiError + */ + public static putCollections1( + collId: string, + ): CancelablePromise { + return __request({ + method: 'PUT', + path: `/collections/restore/${collId}`, + }); + } + + /** + * Add subcollection to collection + * Collections are groupings of datasets + * @param collId + * @param subCollId + * @returns any OK + * @throws ApiError + */ + public static postCollections3( + collId: string, + subCollId: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/collections/${collId}/addSubCollection/${subCollId}`, + }); + } + + /** + * Get child collections in collection + * Collections are groupings of datasets + * @param collId + * @returns any OK + * @throws ApiError + */ + public static getCollections9( + collId: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/collections/${collId}/getChildCollections`, + }); + } + + /** + * list all datasets in the collection + * Collections are groupings of datasets + * @param collId + * @returns any OK + * @throws ApiError + */ + public static getCollections10( + collId: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/collections/${collId}/datasets`, + }); + } + + /** + * If dataset is in a space, list all collections can be the parent of + * the dataset in this space, otherwise list all possiable collections + * + * Collections are groupings of datasets + * @param collId + * @param dsId + * @param when + * @param title + * @param date + * @param limit default as 12 + * @param exact default as false + * @returns any OK + * @throws ApiError + */ + public static getCollections11( + collId: string, + dsId: string, + when?: string, + title?: string, + date?: string, + limit?: number, + exact?: boolean, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/collections/${collId}/datasetPossibleParents/${dsId}`, + query: { + 'when': when, + 'title': title, + 'date': date, + 'limit': limit, + 'exact': exact, + }, + }); + } + + /** + * Add dataset to collection + * Collections are groupings of datasets + * @param collId + * @param dsId + * @returns any OK + * @throws ApiError + */ + public static postCollections4( + collId: string, + dsId: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/collections/${collId}/datasets/${dsId}`, + }); + } + + /** + * Detach a dataset from collection + * Collections are groupings of datasets + * @param collId + * @param dsId + * @param ignoreNotFound default as True + * @returns any OK + * @throws ApiError + */ + public static deleteCollections3( + collId: string, + dsId: string, + ignoreNotFound?: boolean, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/collections/${collId}/datasets/${dsId}`, + query: { + 'ignoreNotFound': ignoreNotFound, + }, + }); + } + + /** + * Attach existing preview to collection + * Collections are groupings of datasets + * @param cId + * @param pId + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postCollections5( + cId: string, + pId: string, + requestBody?: { + extractor_id?: string; + preview_type?: string; + }, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/collections/${cId}/previews/${pId}`, + body: requestBody, + mediaType: 'application/json', + }); + } + + /** + * Follow collection. + * Add user to collection followers and add collection to user followed + * collections. + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static postCollections6( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/collections/${id}/follow`, + }); + } + + /** + * Get parent collection ids of collection + * Collections are groupings of datasets + * @param collId + * @returns any OK + * @throws ApiError + */ + public static getCollections12( + collId: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/collections/${collId}/getParentCollectionIds`, + }); + } + + /** + * Reindex a collection + * Reindex the existing collection, if recursive is set to true it will + * also reindex all datasets and files. + * + * @param collId + * @returns any OK + * @throws ApiError + */ + public static postCollections7( + collId: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/collections/${collId}/reindex`, + }); + } + + /** + * Removes root flag from a collection in a space + * Collections are groupings of datasets + * @param collId + * @param spaceId + * @returns any OK + * @throws ApiError + */ + public static postCollections8( + collId: string, + spaceId: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/collections/${collId}/unsetRootFlag/${spaceId}`, + }); + } + + /** + * Checks if we can remove a collection from a space + * This will check if the collection has parent collection in this space. + * If not, we can remove the collection from the space + * + * @param collId + * @param spaceId + * @returns any OK + * @throws ApiError + */ + public static getCollections13( + collId: string, + spaceId: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/collections/${collId}/removeFromSpaceAllowed/${spaceId}`, + }); + } + + /** + * Get child collection ids in collection + * Collections are groupings of datasets + * @param collId + * @returns any OK + * @throws ApiError + */ + public static getCollections14( + collId: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/collections/${collId}/getChildCollectionIds`, + }); + } + + /** + * Create a collection with parent + * Collections are groupings of datasets + * @param requestBody The body of the POST request to create a collection. + * @returns any OK + * @throws ApiError + */ + public static postCollections9( + requestBody?: { + name: string; + description?: string; + space?: string; + googleAnalytics?: string; + parentId?: string; + }, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/collections/newCollectionWithParent`, + body: requestBody, + mediaType: 'application/json', + }); + } + + /** + * Update collection description + * Takes one argument, a UUID of the collection. Request body takes + * key-value pair for the description + * + * @param collId + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static putCollections2( + collId: string, + requestBody: { + description: string; + }, + ): CancelablePromise { + return __request({ + method: 'PUT', + path: `/collections/${collId}/description`, + body: requestBody, + mediaType: 'application/json', + errors: { + 404: `Not Found`, + }, + }); + } + + /** + * Add root flags for a collection in space + * Collections are groupings of datasets + * @param collId + * @param spaceId + * @returns any OK + * @throws ApiError + */ + public static postCollections10( + collId: string, + spaceId: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/collections/${collId}/rootFlag/${spaceId}`, + }); + } + + /** + * Download collection + * Downloads all child collections, datasets and files in a collection. + * @param id + * @param compression default as -1 + * @returns any OK + * @throws ApiError + */ + public static getCollections15( + id: string, + compression?: number, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/collections/${id}/download`, + query: { + 'compression': compression, + }, + }); + } + + /** + * @deprecated + * List all collections the user can view + * This will check for Permission.ViewCollection + * @param when + * @param title + * @param date + * @param limit + * @param exact + * @returns any OK + * @throws ApiError + */ + public static getCollections16( + when?: string, + title?: string, + date?: string, + limit?: number, + exact?: boolean, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/collections/list`, + query: { + 'when': when, + 'title': title, + 'date': date, + 'limit': limit, + 'exact': exact, + }, + }); + } + + /** + * @deprecated + * Remove a dataset from a collection. + * @param collId + * @param dsId + * @param ignoreNotFound + * @returns any OK + * @throws ApiError + */ + public static postCollections11( + collId: string, + dsId: string, + ignoreNotFound: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/collections/${collId}/datasetsRemove/${dsId}/${ignoreNotFound}`, + }); + } + + /** + * @deprecated + * remove a collection + * @param collId + * @returns any OK + * @throws ApiError + */ + public static postCollections12( + collId: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/collections/${collId}/remove`, + }); + } + +} \ No newline at end of file diff --git a/frontend/src/openapi/v1/services/CurationsService.ts b/frontend/src/openapi/v1/services/CurationsService.ts index 3c334d12d..881d397f9 100644 --- a/frontend/src/openapi/v1/services/CurationsService.ts +++ b/frontend/src/openapi/v1/services/CurationsService.ts @@ -1,111 +1,121 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ -import type { CancelablePromise } from "../core/CancelablePromise"; -import { request as __request } from "../core/request"; +import type { CancelablePromise } from '../core/CancelablePromise'; +import { request as __request } from '../core/request'; export class CurationsService { - /** - * Update the user repository preferences and call the matchmaker - * A curation object is a request for publication that captures the state - * of a dataset ready for publication - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static postCurations(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/spaces/curations/${id}/matchmaker`, - }); - } - /** - * Retract the publication request - * A curation object is a request for publication that captures the state - * of a dataset ready for publication - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static deleteCurations(id: string): CancelablePromise { - return __request({ - method: "DELETE", - path: `/spaces/curations/retract/${id}`, - }); - } + /** + * Update the user repository preferences and call the matchmaker + * A curation object is a request for publication that captures the state + * of a dataset ready for publication + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static postCurations( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/spaces/curations/${id}/matchmaker`, + }); + } - /** - * Delete a folder from a publication request - * A curation object is a request for publication that captures the state - * of a dataset ready for publication - * - * @param id - * @param curationFolderId - * @returns any OK - * @throws ApiError - */ - public static deleteCurations1( - id: string, - curationFolderId: string - ): CancelablePromise { - return __request({ - method: "DELETE", - path: `/spaces/curations/${id}/folders/${curationFolderId}`, - }); - } + /** + * Retract the publication request + * A curation object is a request for publication that captures the state + * of a dataset ready for publication + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static deleteCurations( + id: string, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/spaces/curations/retract/${id}`, + }); + } - /** - * Get files in publication request - * A curation object is a request for publication that captures the state - * of a dataset ready for publication - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static getCurations(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/spaces/curations/${id}/curationFile`, - }); - } + /** + * Delete a folder from a publication request + * A curation object is a request for publication that captures the state + * of a dataset ready for publication + * + * @param id + * @param curationFolderId + * @returns any OK + * @throws ApiError + */ + public static deleteCurations1( + id: string, + curationFolderId: string, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/spaces/curations/${id}/folders/${curationFolderId}`, + }); + } - /** - * Delete a file from a publication request - * A curation object is a request for publication that captures the state - * of a dataset ready for publication - * - * @param id - * @param curationFileId - * @returns any OK - * @throws ApiError - */ - public static deleteCurations2( - id: string, - curationFileId: string - ): CancelablePromise { - return __request({ - method: "DELETE", - path: `/spaces/curations/${id}/files/${curationFileId}`, - }); - } + /** + * Get files in publication request + * A curation object is a request for publication that captures the state + * of a dataset ready for publication + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static getCurations( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/spaces/curations/${id}/curationFile`, + }); + } - /** - * Get the ORE map for the proposed publication - * A curation object is a request for publication that captures the state - * of a dataset ready for publication - * - * @param curationId - * @returns any OK - * @throws ApiError - */ - public static getCurations1(curationId: string): CancelablePromise { - return __request({ - method: "GET", - path: `/curations/${curationId}/ore`, - }); - } -} + /** + * Delete a file from a publication request + * A curation object is a request for publication that captures the state + * of a dataset ready for publication + * + * @param id + * @param curationFileId + * @returns any OK + * @throws ApiError + */ + public static deleteCurations2( + id: string, + curationFileId: string, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/spaces/curations/${id}/files/${curationFileId}`, + }); + } + + /** + * Get the ORE map for the proposed publication + * A curation object is a request for publication that captures the state + * of a dataset ready for publication + * + * @param curationId + * @returns any OK + * @throws ApiError + */ + public static getCurations1( + curationId: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/curations/${curationId}/ore`, + }); + } + +} \ No newline at end of file diff --git a/frontend/src/openapi/v1/services/DTexturesService.ts b/frontend/src/openapi/v1/services/DTexturesService.ts index 12036a22c..766184f9b 100644 --- a/frontend/src/openapi/v1/services/DTexturesService.ts +++ b/frontend/src/openapi/v1/services/DTexturesService.ts @@ -1,25 +1,27 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ -import type { CancelablePromise } from "../core/CancelablePromise"; -import { request as __request } from "../core/request"; +import type { CancelablePromise } from '../core/CancelablePromise'; +import { request as __request } from '../core/request'; export class DTexturesService { - /** - * @deprecated - * Upload a 3D texture - * Upload a 3D texture file to Clowder. - * - * @returns void - * @throws ApiError - */ - public static postDTextures(): CancelablePromise { - return __request({ - method: "POST", - path: `/3dTextures`, - errors: { - 401: `Not authorized`, - }, - }); - } -} + + /** + * @deprecated + * Upload a 3D texture + * Upload a 3D texture file to Clowder. + * + * @returns void + * @throws ApiError + */ + public static postDTextures(): CancelablePromise { + return __request({ + method: 'POST', + path: `/3dTextures`, + errors: { + 401: `Not authorized`, + }, + }); + } + +} \ No newline at end of file diff --git a/frontend/src/openapi/v1/services/DatasetsService.ts b/frontend/src/openapi/v1/services/DatasetsService.ts index deb15bd68..57c4b50f7 100644 --- a/frontend/src/openapi/v1/services/DatasetsService.ts +++ b/frontend/src/openapi/v1/services/DatasetsService.ts @@ -1,1438 +1,1508 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ -import type { Comment } from "../models/Comment"; -import type { Dataset } from "../models/Dataset"; -import type { JSONLD } from "../models/JSONLD"; -import type { License } from "../models/License"; -import type { Tags } from "../models/Tags"; -import type { CancelablePromise } from "../core/CancelablePromise"; -import { request as __request } from "../core/request"; +import type { Comment } from '../models/Comment'; +import type { Dataset } from '../models/Dataset'; +import type { JSONLD } from '../models/JSONLD'; +import type { License } from '../models/License'; +import type { Tags } from '../models/Tags'; +import type { CancelablePromise } from '../core/CancelablePromise'; +import { request as __request } from '../core/request'; export class DatasetsService { - /** - * Upload a file to a specific dataset - * Uploads the file, then links it with the dataset. Returns file id as - * JSON object, or ids with filenames if multiple files are sent. ID can be - * used to work on the file using the API. Uploaded file can be an XML - * metadata file to be added to the dataset. If the optional Boolean - * parameter extract is set to false, it does not send the file for - * extraction. By default, Boolean parameter extract is set to true. - * - * @param id the dataset id - * @param formData - * @param showPreviews default as "DatasetLevel" - * @param originalZipFile the UUID string of original zip file - * @param flags flags for previews - * @param extract - * @param folderId - * @returns any OK - * @throws ApiError - */ - public static postDatasets( - id: string, - formData: any, - showPreviews?: string, - originalZipFile?: string, - flags?: string, - extract?: boolean, - folderId?: string - ): CancelablePromise { - return __request({ - method: "POST", - path: `/uploadToDataset/${id}`, - query: { - showPreviews: showPreviews, - originalZipFile: originalZipFile, - flags: flags, - extract: extract, - folder_id: folderId, - }, - formData: formData, - mediaType: "multipart/form-data", - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * List all datasets the user can view - * This will check for Permission.ViewDataset - * @param when - * @param date - * @param title - * @param limit The number of collections returns, default as 12. - * @param exact - * @returns any OK - * @throws ApiError - */ - public static getDatasets( - when?: string, - date?: string, - title?: string, - limit?: number, - exact?: boolean - ): CancelablePromise { - return __request({ - method: "GET", - path: `/datasets`, - query: { - when: when, - date: date, - title: title, - limit: limit, - exact: exact, - }, - }); - } - - /** - * Create new dataset - * New dataset containing one existing file, based on values of fields in - * attached JSON. Returns dataset id as JSON object. - * - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postDatasets1(requestBody: { - name: string; - description?: string; - space?: string; - file_id: string; - }): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets`, - body: requestBody, - mediaType: "application/json", - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Get a specific dataset - * This will return a specific dataset requested - * @param id - * @returns any OK - * @throws ApiError - */ - public static getDatasets1(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/datasets/${id}`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Delete dataset - * Cascading action (deletes all previews and metadata of the dataset and - * all files existing only in the deleted dataset). - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static deleteDatasets(id: string): CancelablePromise { - return __request({ - method: "DELETE", - path: `/datasets/${id}`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Copy the dataset, as well as the folders and files within the dataset - * into a space, return the new dataset id - * - * Check AddResourceToSpace permission. - * @param dsId - * @param spaceId - * @returns any OK - * @throws ApiError - */ - public static postDatasets2( - dsId: string, - spaceId: string - ): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${dsId}/copyDatasetToSpace/${spaceId}`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Attach existing file to dataset - * If the file is an XML metadata file, the metadata are added to the - * dataset. - * - * @param dsId - * @param fileId - * @returns any OK - * @throws ApiError - */ - public static postDatasets3( - dsId: string, - fileId: string - ): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${dsId}/files/${fileId}`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Move existing file to a new folder within the same dataset - * @param dsId - * @param folderId destination folder id - * @param fileId - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postDatasets4( - dsId: string, - folderId: string, - fileId: string, - requestBody: { - /** - * old folder id - */ - folderId?: string; - } - ): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${dsId}/moveFile/${folderId}/${fileId}`, - body: requestBody, - mediaType: "application/json", - }); - } - - /** - * Move existing file from the old folder to a new dataset - * @param dsId - * @param folderId old folder id - * @param fileId - * @returns any OK - * @throws ApiError - */ - public static postDatasets5( - dsId: string, - folderId: string, - fileId: string - ): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${dsId}/moveToDataset/${folderId}/${fileId}`, - }); - } - - /** - * Detach existing file from a dataset - * Check CreateDataset permission - * @param dsId - * @param fileId - * @param ignoreNotFound - * @returns any OK - * @throws ApiError - */ - public static postDatasets6( - dsId: string, - fileId: string, - ignoreNotFound: string - ): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${dsId}/filesRemove/${fileId}/${ignoreNotFound}`, - }); - } - - /** - * list all folders in the dataset - * @param dsId - * @returns any OK - * @throws ApiError - */ - public static getDatasets2(dsId: string): CancelablePromise { - return __request({ - method: "GET", - path: `/datasets/${dsId}/folders`, - }); - } - - /** - * Remove all tags of dataset - * Forcefully remove all tags for this dataset. It is mainly intended for - * testing. - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static postDatasets7(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${id}/tags/remove_all`, - }); - } - - /** - * Get the tags associated with this dataset - * Returns a JSON object of multiple fields - * @param id - * @returns any OK - * @throws ApiError - */ - public static getDatasets3(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/datasets/${id}/tags`, - }); - } - - /** - * Add tags to dataset - * Requires that the request body contains a 'tags' field of List[String] - * type. - * - * @param id - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postDatasets8( - id: string, - requestBody: Tags - ): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${id}/tags`, - body: requestBody, - mediaType: "application/json", - }); - } - - /** - * Remove tags of dataset - * Requires that the request body contains a 'tags' field of List[String] - * type. - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static deleteDatasets1(id: string): CancelablePromise { - return __request({ - method: "DELETE", - path: `/datasets/${id}/tags`, - }); - } - - /** - * Insert add_file Event - * Insert an Event into the Events Collection - * @param id - * @param inFolder if the file is add directly to the dataset or add to a folder - * @param fileCount - * @returns any OK - * @throws ApiError - */ - public static postDatasets9( - id: string, - inFolder: boolean, - fileCount: number - ): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${id}/addFileEvent`, - query: { - inFolder: inFolder, - fileCount: fileCount, - }, - }); - } - - /** - * Is being processed - * Return whether a dataset is currently being processed by a preprocessor. - * @param id - * @returns any OK - * @throws ApiError - */ - public static getDatasets4(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/datasets/${id}/isBeingProcessed`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Create new dataset with no file - * New dataset requiring zero files based on values of fields in attached - * JSON. Returns dataset id as JSON object. Requires name, description, and - * space. Optional list of existing file ids to add. - * - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postDatasets10(requestBody: Dataset): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/createempty`, - body: requestBody, - mediaType: "application/json", - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Update dataset description. - * Takes one argument, a UUID of the dataset. Request body takes key-value - * pair for description. - * - * @param id - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static putDatasets( - id: string, - requestBody: { - description: string; - } - ): CancelablePromise { - return __request({ - method: "PUT", - path: `/datasets/${id}/description`, - body: requestBody, - mediaType: "application/json", - }); - } - - /** - * Add user-generated metadata to dataset - * A dataset is a container for files and metadata - * @param id - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postDatasets11( - id: string, - requestBody?: any - ): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${id}/usermetadata`, - body: requestBody, - mediaType: "application/json", - }); - } - - /** - * List extractors generated metadata of a dataset - * A dataset is a container for files and metadata - * @param id - * @returns any OK - * @throws ApiError - */ - public static getDatasets5(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/datasets/${id}/technicalmetadatajson`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Get community-generated metadata of the resource described by the dataset - * A dataset is a container for files and metadata - * @param id - * @returns any OK - * @throws ApiError - */ - public static getDatasets6(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/datasets/${id}/usermetadatajson`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Get metadata of the resource described by the dataset that were input as - * XML - * - * A dataset is a container for files and metadata - * @param id - * @returns any OK - * @throws ApiError - */ - public static getDatasets7(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/datasets/${id}/xmlmetadatajson`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Detach file from dataset - * File is not deleted, only separated from the selected dataset. If the - * file is an XML metadata file, the metadata are removed from the dataset. - * - * @param dsId - * @param fileId - * @returns any OK - * @throws ApiError - */ - public static deleteDatasets2( - dsId: string, - fileId: string - ): CancelablePromise { - return __request({ - method: "DELETE", - path: `/datasets/${dsId}/${fileId}`, - }); - } - - /** - * Add comment to dataset - * A dataset is a container for files and metadata - * @param id - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postDatasets12( - id: string, - requestBody: Comment - ): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${id}/comment`, - body: requestBody, - mediaType: "application/json", - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * List all users in the spaces that contain this dataset in json-ld format - * A dataset is a container for files and metadata - * @param id - * @returns any OK - * @throws ApiError - */ - public static getDatasets8(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/datasets/${id}/user`, - }); - } - - /** - * Retrieve available metadata definitions for a dataset. It is an - * aggregation of the metadata that a space belongs to. - * - * A dataset is a container for files and metadata - * @param id - * @returns any OK - * @throws ApiError - */ - public static getDatasets9(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/datasets/${id}/metadata`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Add metadata to dataset - * The extractor is set as "http://clowder.ncsa.illinois.edu/extractors/deprecatedapi", - * contextURL and contextID as None. - * Returns success or failure. - * - * @param id - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postDatasets13( - id: string, - requestBody: any - ): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${id}/metadata`, - body: requestBody, - mediaType: "application/json", - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Get URLs of dataset's RDF metadata exports - * URLs of metadata exported as RDF from XML files contained in the - * dataset, as well as the URL used to export the dataset's user-generated - * metadata as RDF. - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static getDatasets10(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/datasets/getRDFURLsForDataset/${id}`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Get the user-generated metadata of the selected dataset in an RDF file - * A dataset is a container for files and metadata - * @param id - * @returns any OK - * @throws ApiError - */ - public static getDatasets11(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/datasets/rdfUserMetadata/${id}`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Unfollow dataset. - * Remove user from dataset followers and remove dataset from user followed - * datasets. - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static postDatasets14(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${id}/unfollow`, - }); - } - - /** - * Add a creator to the Dataset's list of Creators. - * Takes one argument, a UUID of the dataset. Request body takes key-value - * pair for creator. - * - * @param id - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postDatasets15( - id: string, - requestBody: { - creator?: string; - } - ): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${id}/creator`, - body: requestBody, - mediaType: "application/json", - }); - } - - /** - * Remove a creator from the Dataset's list of Creators. - * Takes the UUID of the dataset and the entry to delete (a String). - * @param id - * @param creator - * @returns any OK - * @throws ApiError - */ - public static deleteDatasets3( - id: string, - creator: string - ): CancelablePromise { - return __request({ - method: "DELETE", - path: `/datasets/${id}/creator/remove`, - query: { - creator: creator, - }, - }); - } - - /** - * Move a creator in a Dataset's list of creators. - * Takes the UUID of the dataset, the creator to move (a String) and the - * new position of the creator in the overall list of creators. - * - * @param id - * @param creator - * @param newPos - * @returns any OK - * @throws ApiError - */ - public static putDatasets1( - id: string, - creator: string, - newPos: number - ): CancelablePromise { - return __request({ - method: "PUT", - path: `/datasets/${id}/creator/reorder`, - query: { - creator: creator, - newPos: newPos, - }, - }); - } - - /** - * Detach and delete dataset - * Detaches all files before proceeding to perform the stanadard delete on - * the dataset. - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static postDatasets16(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${id}/detachdelete`, - }); - } - - /** - * change the access of dataset - * Downloads all files contained in a dataset. - * @param id - * @param access default as PRIVATE - * @returns any OK - * @throws ApiError - */ - public static putDatasets2( - id: string, - access?: string - ): CancelablePromise { - return __request({ - method: "PUT", - path: `/datasets/${id}/access`, - query: { - access: access, - }, - }); - } - - /** - * List all datasets the user can edit - * This will check for Permission.AddResourceToDataset and - * Permission.EditDataset - * - * @param when - * @param date - * @param title - * @param limit The number of collections returns, default as 12. - * @param exact - * @returns any OK - * @throws ApiError - */ - public static getDatasets12( - when?: string, - date?: string, - title?: string, - limit?: number, - exact?: boolean - ): CancelablePromise { - return __request({ - method: "GET", - path: `/datasets/canEdit`, - query: { - when: when, - date: date, - title: title, - limit: limit, - exact: exact, - }, - }); - } - - /** - * Update dataset name - * Takes one argument, a UUID of the dataset. Request body takes key-value - * pair for name. - * - * @param id - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static putDatasets3( - id: string, - requestBody: { - name: string; - } - ): CancelablePromise { - return __request({ - method: "PUT", - path: `/datasets/${id}/title`, - body: requestBody, - mediaType: "application/json", - }); - } - - /** - * Queue up all files in a dataset to be marked as ARCHIVED - * Queue up all files in a dataset to be marked as ARCHIVED - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static postDatasets17(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${id}/queueArchival`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Queue up all files in a dataset to be marked as PROCESSED - * Queue up all files in a dataset to be marked as PROCESSED - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static postDatasets18(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${id}/queueUnarchival`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * @deprecated - * List all datasets in a collection - * Returns list of datasets and descriptions. - * @param collId - * @returns any OK - * @throws ApiError - */ - public static getDatasets13(collId: string): CancelablePromise { - return __request({ - method: "GET", - path: `/collections/${collId}/getDatasets`, - }); - } - - /** - * List all datasets outside a collection - * Returns list of datasets and descriptions. - * @param collId - * @returns any OK - * @throws ApiError - */ - public static getDatasets14(collId: string): CancelablePromise { - return __request({ - method: "GET", - path: `/datasets/listOutsideCollection/${collId}`, - }); - } - - /** - * Update dataset administrative information - * Takes one argument, a UUID of the dataset. Request body takes key-value - * pairs for name and description. - * - * @param id - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postDatasets19( - id: string, - requestBody: { - name: string; - description: string; - } - ): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${id}/editing`, - body: requestBody, - mediaType: "application/json", - }); - } - - /** - * Get dataset previews - * Return the currently existing previews of the selected dataset (full - * description, including paths to preview files, previewer names etc). - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static getDatasets15(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/datasets/${id}/getPreviews`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * List datasets satisfying a general metadata search tree - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postDatasets20(requestBody: any): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/searchmetadata`, - body: requestBody, - mediaType: "application/json", - }); - } - - /** - * List datasets satisfying a user metadata search tree - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postDatasets21(requestBody: any): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/searchusermetadata`, - body: requestBody, - mediaType: "application/json", - }); - } - - /** - * Retrieve metadata as JSON-LD - * Get metadata of the dataset object as JSON-LD. - * @param id - * @param extractor - * @returns any OK - * @throws ApiError - */ - public static getDatasets16( - id: string, - extractor?: string - ): CancelablePromise { - return __request({ - method: "GET", - path: `/datasets/${id}/metadata.jsonld`, - query: { - extractor: extractor, - }, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Add JSON-LD metadata to the database. - * Metadata in attached JSON-LD object will be added to metadata Mongo db - * collection. - * - * @param id - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postDatasets22( - id: string, - requestBody: JSONLD - ): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${id}/metadata.jsonld`, - body: requestBody, - mediaType: "application/json", - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Remove JSON-LD metadata, filtered by extractor if necessary - * Remove JSON-LD metadata from dataset object - * @param id - * @param extractor - * @returns any OK - * @throws ApiError - */ - public static deleteDatasets4( - id: string, - extractor?: string - ): CancelablePromise { - return __request({ - method: "DELETE", - path: `/datasets/${id}/metadata.jsonld`, - query: { - extractor: extractor, - }, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Update license information of a dataset - * Takes four arguments, all Strings. licenseType, rightsHolder, - * licenseText, licenseUrl - * - * @param id - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postDatasets23( - id: string, - requestBody: License - ): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${id}/license`, - body: requestBody, - mediaType: "application/json", - }); - } - - /** - * Follow dataset - * Add user to dataset followers and add dataset to user followed datasets. - * @param id - * @returns any OK - * @throws ApiError - */ - public static postDatasets24(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${id}/follow`, - }); - } - - /** - * Attach multiple files to an existing dataset - * Add multiple files, by ID, to a dataset that is already in the system. - * Requires file ids and dataset id. - * - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postDatasets25(requestBody: { - datasetid: string; - /** - * file ids seperated by comma - */ - existingfiles: string; - }): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/attachmultiple`, - body: requestBody, - mediaType: "application/json", - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Reindex a dataset - * Reindex the existing dataset, if recursive is set to true if will also - * reindex all files in that dataset. - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static postDatasets26(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${id}/reindex`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * List files in dataset, including those within folders - * Datasets and descriptions. - * @param id - * @param max - * @returns any OK - * @throws ApiError - */ - public static getDatasets17( - id: string, - max?: number - ): CancelablePromise { - return __request({ - method: "GET", - path: `/datasets/${id}/files`, - query: { - max: max, - }, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Upload files and attach to given dataset - * This will take a list of url or path objects that point to files that - * will be ingested and added to this dataset. - * - * @param id - * @param formData - * @returns any OK - * @throws ApiError - */ - public static postDatasets27( - id: string, - formData: any - ): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${id}/files`, - formData: formData, - mediaType: "multipart/form-data", - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Upload files and attach to given dataset - * This will take an URL of file object that are added to this dataset, - * the file source will be added as metadata. Request body takes key-value - * pairs for file url, the key can be fileurl, weburl or url. - * This can also add metadata at the same time. - * - * @param id - * @param requestBody the key can be fileurl, weburl or url - * @returns any OK - * @throws ApiError - */ - public static postDatasets28( - id: string, - requestBody?: any - ): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${id}/urls`, - body: requestBody, - mediaType: "application/json", - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * List all datasets in the space the user can edit and thus move the file - * to - * - * This will check for Permission.AddResourceToDataset and - * Permission.EditDataset - * - * @returns any OK - * @throws ApiError - */ - public static getDatasets18(): CancelablePromise { - return __request({ - method: "GET", - path: `/datasets/moveFileToDataset`, - }); - } - - /** - * List files in dataset not in any folders - * Datasets and descriptions. - * @param id - * @returns any OK - * @throws ApiError - */ - public static getDatasets19(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/datasets/${id}/listFiles`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * List all file within a dataset - * @param dsId - * @returns any OK - * @throws ApiError - */ - public static getDatasets20(dsId: string): CancelablePromise { - return __request({ - method: "GET", - path: `/datasets/${dsId}/listAllFiles`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Remove tag of dataset - * A dataset is a container for files and metadata - * @param id - * @param requestBody The body of the POST request. - * @returns any OK - * @throws ApiError - */ - public static postDatasets29( - id: string, - requestBody: { - tagId: string; - } - ): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${id}/removeTag`, - body: requestBody, - mediaType: "application/json", - }); - } - - /** - * Attach existing file to a new dataset and delete it from the old one - * If the file is an XML metadata file, the metadata are added to the - * dataset. - * - * @param datasetId - * @param toDataset - * @param fileId - * @returns any OK - * @throws ApiError - */ - public static postDatasets30( - datasetId: string, - toDataset: string, - fileId: string - ): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${datasetId}/moveBetweenDatasets/${toDataset}/${fileId}`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Download dataset - * Downloads all files contained in a dataset. - * @param id - * @param compression default as -1 - * @param tracking default as true - * @returns any OK - * @throws ApiError - */ - public static getDatasets21( - id: string, - compression?: number, - tracking?: boolean - ): CancelablePromise { - return __request({ - method: "GET", - path: `/datasets/${id}/download`, - query: { - compression: compression, - tracking: tracking, - }, - }); - } - - /** - * Download a subset of files from a dataset - * Takes dataset ID and a JSON-string representing a list of file IDs - * @param id - * @param fileList ID of the files to download from the dataset - * @returns any OK - * @throws ApiError - */ - public static getDatasets22( - id: string, - fileList: string - ): CancelablePromise { - return __request({ - method: "GET", - path: `/datasets/${id}/downloadPartial`, - query: { - fileList: fileList, - }, - }); - } - - /** - * Download a folder from a dataset - * Takes dataset ID and a folder ID in that dataset and streams just that folder and sub-folders as a zip - * @param id - * @param folderId ID of the folder to download - * @returns any OK - * @throws ApiError - */ - public static getDatasets23( - id: string, - folderId: string - ): CancelablePromise { - return __request({ - method: "GET", - path: `/datasets/${id}/downloadFolder`, - query: { - folderId: folderId, - }, - }); - } - - /** - * Deleted all datasets in trash older than days specified - * Server admin action. - * @param days - * @returns any OK - * @throws ApiError - */ - public static deleteDatasets5(days?: number): CancelablePromise { - return __request({ - method: "DELETE", - path: `/datasets/clearOldDatasetsTrash`, - query: { - days: days, - }, - }); - } - - /** - * Emptying trash datasets - * @returns any OK - * @throws ApiError - */ - public static deleteDatasets6(): CancelablePromise { - return __request({ - method: "DELETE", - path: `/datasets/emptyTrash`, - }); - } - - /** - * list trash datasets - * @param limit - * @returns any OK - * @throws ApiError - */ - public static getDatasets24(limit?: number): CancelablePromise { - return __request({ - method: "GET", - path: `/datasets/listTrash`, - query: { - limit: limit, - }, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * restore a trash dataset - * This will check for Permission.DeleteDataset - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static putDatasets4(id: string): CancelablePromise { - return __request({ - method: "PUT", - path: `/datasets/restore/${id}`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * @deprecated - * remove a tag from a dataset - * @param id - * @returns any OK - * @throws ApiError - */ - public static postDatasets31(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${id}/tags/remove`, - }); - } -} + + /** + * Upload a file to a specific dataset + * Uploads the file, then links it with the dataset. Returns file id as + * JSON object, or ids with filenames if multiple files are sent. ID can be + * used to work on the file using the API. Uploaded file can be an XML + * metadata file to be added to the dataset. If the optional Boolean + * parameter extract is set to false, it does not send the file for + * extraction. By default, Boolean parameter extract is set to true. + * + * @param id the dataset id + * @param formData + * @param showPreviews default as "DatasetLevel" + * @param originalZipFile the UUID string of original zip file + * @param flags flags for previews + * @param extract + * @param folderId + * @returns any OK + * @throws ApiError + */ + public static postDatasets( + id: string, + formData: any, + showPreviews?: string, + originalZipFile?: string, + flags?: string, + extract?: boolean, + folderId?: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/uploadToDataset/${id}`, + query: { + 'showPreviews': showPreviews, + 'originalZipFile': originalZipFile, + 'flags': flags, + 'extract': extract, + 'folder_id': folderId, + }, + formData: formData, + mediaType: 'multipart/form-data', + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * List all datasets the user can view + * This will check for Permission.ViewDataset + * @param when + * @param date + * @param title + * @param limit The number of collections returns, default as 12. + * @param exact + * @returns any OK + * @throws ApiError + */ + public static getDatasets( + when?: string, + date?: string, + title?: string, + limit?: number, + exact?: boolean, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/datasets`, + query: { + 'when': when, + 'date': date, + 'title': title, + 'limit': limit, + 'exact': exact, + }, + }); + } + + /** + * Create new dataset + * New dataset containing one existing file, based on values of fields in + * attached JSON. Returns dataset id as JSON object. + * + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postDatasets1( + requestBody: { + name: string; + description?: string; + space?: string; + file_id: string; + }, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets`, + body: requestBody, + mediaType: 'application/json', + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Get a specific dataset + * This will return a specific dataset requested + * @param id + * @returns any OK + * @throws ApiError + */ + public static getDatasets1( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/datasets/${id}`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Delete dataset + * Cascading action (deletes all previews and metadata of the dataset and + * all files existing only in the deleted dataset). + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static deleteDatasets( + id: string, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/datasets/${id}`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Copy the dataset, as well as the folders and files within the dataset + * into a space, return the new dataset id + * + * Check AddResourceToSpace permission. + * @param dsId + * @param spaceId + * @returns any OK + * @throws ApiError + */ + public static postDatasets2( + dsId: string, + spaceId: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${dsId}/copyDatasetToSpace/${spaceId}`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Attach existing file to dataset + * If the file is an XML metadata file, the metadata are added to the + * dataset. + * + * @param dsId + * @param fileId + * @returns any OK + * @throws ApiError + */ + public static postDatasets3( + dsId: string, + fileId: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${dsId}/files/${fileId}`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Move existing file to a new folder within the same dataset + * @param dsId + * @param folderId destination folder id + * @param fileId + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postDatasets4( + dsId: string, + folderId: string, + fileId: string, + requestBody: { + /** + * old folder id + */ + folderId?: string; + }, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${dsId}/moveFile/${folderId}/${fileId}`, + body: requestBody, + mediaType: 'application/json', + }); + } + + /** + * Move existing file from the old folder to a new dataset + * @param dsId + * @param folderId old folder id + * @param fileId + * @returns any OK + * @throws ApiError + */ + public static postDatasets5( + dsId: string, + folderId: string, + fileId: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${dsId}/moveToDataset/${folderId}/${fileId}`, + }); + } + + /** + * Detach existing file from a dataset + * Check CreateDataset permission + * @param dsId + * @param fileId + * @param ignoreNotFound + * @returns any OK + * @throws ApiError + */ + public static postDatasets6( + dsId: string, + fileId: string, + ignoreNotFound: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${dsId}/filesRemove/${fileId}/${ignoreNotFound}`, + }); + } + + /** + * list all folders in the dataset + * @param dsId + * @returns any OK + * @throws ApiError + */ + public static getDatasets2( + dsId: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/datasets/${dsId}/folders`, + }); + } + + /** + * Remove all tags of dataset + * Forcefully remove all tags for this dataset. It is mainly intended for + * testing. + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static postDatasets7( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${id}/tags/remove_all`, + }); + } + + /** + * Get the tags associated with this dataset + * Returns a JSON object of multiple fields + * @param id + * @returns any OK + * @throws ApiError + */ + public static getDatasets3( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/datasets/${id}/tags`, + }); + } + + /** + * Add tags to dataset + * Requires that the request body contains a 'tags' field of List[String] + * type. + * + * @param id + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postDatasets8( + id: string, + requestBody: Tags, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${id}/tags`, + body: requestBody, + mediaType: 'application/json', + }); + } + + /** + * Remove tags of dataset + * Requires that the request body contains a 'tags' field of List[String] + * type. + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static deleteDatasets1( + id: string, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/datasets/${id}/tags`, + }); + } + + /** + * Insert add_file Event + * Insert an Event into the Events Collection + * @param id + * @param inFolder if the file is add directly to the dataset or add to a folder + * @param fileCount + * @returns any OK + * @throws ApiError + */ + public static postDatasets9( + id: string, + inFolder: boolean, + fileCount: number, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${id}/addFileEvent`, + query: { + 'inFolder': inFolder, + 'fileCount': fileCount, + }, + }); + } + + /** + * Is being processed + * Return whether a dataset is currently being processed by a preprocessor. + * @param id + * @returns any OK + * @throws ApiError + */ + public static getDatasets4( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/datasets/${id}/isBeingProcessed`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Create new dataset with no file + * New dataset requiring zero files based on values of fields in attached + * JSON. Returns dataset id as JSON object. Requires name, description, and + * space. Optional list of existing file ids to add. + * + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postDatasets10( + requestBody: Dataset, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/createempty`, + body: requestBody, + mediaType: 'application/json', + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Update dataset description. + * Takes one argument, a UUID of the dataset. Request body takes key-value + * pair for description. + * + * @param id + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static putDatasets( + id: string, + requestBody: { + description: string; + }, + ): CancelablePromise { + return __request({ + method: 'PUT', + path: `/datasets/${id}/description`, + body: requestBody, + mediaType: 'application/json', + }); + } + + /** + * Add user-generated metadata to dataset + * A dataset is a container for files and metadata + * @param id + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postDatasets11( + id: string, + requestBody?: any, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${id}/usermetadata`, + body: requestBody, + mediaType: 'application/json', + }); + } + + /** + * List extractors generated metadata of a dataset + * A dataset is a container for files and metadata + * @param id + * @returns any OK + * @throws ApiError + */ + public static getDatasets5( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/datasets/${id}/technicalmetadatajson`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Get community-generated metadata of the resource described by the dataset + * A dataset is a container for files and metadata + * @param id + * @returns any OK + * @throws ApiError + */ + public static getDatasets6( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/datasets/${id}/usermetadatajson`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Get metadata of the resource described by the dataset that were input as + * XML + * + * A dataset is a container for files and metadata + * @param id + * @returns any OK + * @throws ApiError + */ + public static getDatasets7( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/datasets/${id}/xmlmetadatajson`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Detach file from dataset + * File is not deleted, only separated from the selected dataset. If the + * file is an XML metadata file, the metadata are removed from the dataset. + * + * @param dsId + * @param fileId + * @returns any OK + * @throws ApiError + */ + public static deleteDatasets2( + dsId: string, + fileId: string, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/datasets/${dsId}/${fileId}`, + }); + } + + /** + * Add comment to dataset + * A dataset is a container for files and metadata + * @param id + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postDatasets12( + id: string, + requestBody: Comment, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${id}/comment`, + body: requestBody, + mediaType: 'application/json', + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * List all users in the spaces that contain this dataset in json-ld format + * A dataset is a container for files and metadata + * @param id + * @returns any OK + * @throws ApiError + */ + public static getDatasets8( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/datasets/${id}/user`, + }); + } + + /** + * Retrieve available metadata definitions for a dataset. It is an + * aggregation of the metadata that a space belongs to. + * + * A dataset is a container for files and metadata + * @param id + * @returns any OK + * @throws ApiError + */ + public static getDatasets9( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/datasets/${id}/metadata`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Add metadata to dataset + * The extractor is set as "http://clowder.ncsa.illinois.edu/extractors/deprecatedapi", + * contextURL and contextID as None. + * Returns success or failure. + * + * @param id + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postDatasets13( + id: string, + requestBody: any, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${id}/metadata`, + body: requestBody, + mediaType: 'application/json', + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Get URLs of dataset's RDF metadata exports + * URLs of metadata exported as RDF from XML files contained in the + * dataset, as well as the URL used to export the dataset's user-generated + * metadata as RDF. + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static getDatasets10( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/datasets/getRDFURLsForDataset/${id}`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Get the user-generated metadata of the selected dataset in an RDF file + * A dataset is a container for files and metadata + * @param id + * @returns any OK + * @throws ApiError + */ + public static getDatasets11( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/datasets/rdfUserMetadata/${id}`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Unfollow dataset. + * Remove user from dataset followers and remove dataset from user followed + * datasets. + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static postDatasets14( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${id}/unfollow`, + }); + } + + /** + * Add a creator to the Dataset's list of Creators. + * Takes one argument, a UUID of the dataset. Request body takes key-value + * pair for creator. + * + * @param id + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postDatasets15( + id: string, + requestBody: { + creator?: string; + }, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${id}/creator`, + body: requestBody, + mediaType: 'application/json', + }); + } + + /** + * Remove a creator from the Dataset's list of Creators. + * Takes the UUID of the dataset and the entry to delete (a String). + * @param id + * @param creator + * @returns any OK + * @throws ApiError + */ + public static deleteDatasets3( + id: string, + creator: string, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/datasets/${id}/creator/remove`, + query: { + 'creator': creator, + }, + }); + } + + /** + * Move a creator in a Dataset's list of creators. + * Takes the UUID of the dataset, the creator to move (a String) and the + * new position of the creator in the overall list of creators. + * + * @param id + * @param creator + * @param newPos + * @returns any OK + * @throws ApiError + */ + public static putDatasets1( + id: string, + creator: string, + newPos: number, + ): CancelablePromise { + return __request({ + method: 'PUT', + path: `/datasets/${id}/creator/reorder`, + query: { + 'creator': creator, + 'newPos': newPos, + }, + }); + } + + /** + * Detach and delete dataset + * Detaches all files before proceeding to perform the stanadard delete on + * the dataset. + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static postDatasets16( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${id}/detachdelete`, + }); + } + + /** + * change the access of dataset + * Downloads all files contained in a dataset. + * @param id + * @param access default as PRIVATE + * @returns any OK + * @throws ApiError + */ + public static putDatasets2( + id: string, + access?: string, + ): CancelablePromise { + return __request({ + method: 'PUT', + path: `/datasets/${id}/access`, + query: { + 'access': access, + }, + }); + } + + /** + * List all datasets the user can edit + * This will check for Permission.AddResourceToDataset and + * Permission.EditDataset + * + * @param when + * @param date + * @param title + * @param limit The number of collections returns, default as 12. + * @param exact + * @returns any OK + * @throws ApiError + */ + public static getDatasets12( + when?: string, + date?: string, + title?: string, + limit?: number, + exact?: boolean, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/datasets/canEdit`, + query: { + 'when': when, + 'date': date, + 'title': title, + 'limit': limit, + 'exact': exact, + }, + }); + } + + /** + * Update dataset name + * Takes one argument, a UUID of the dataset. Request body takes key-value + * pair for name. + * + * @param id + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static putDatasets3( + id: string, + requestBody: { + name: string; + }, + ): CancelablePromise { + return __request({ + method: 'PUT', + path: `/datasets/${id}/title`, + body: requestBody, + mediaType: 'application/json', + }); + } + + /** + * Queue up all files in a dataset to be marked as ARCHIVED + * Queue up all files in a dataset to be marked as ARCHIVED + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static postDatasets17( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${id}/queueArchival`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Queue up all files in a dataset to be marked as PROCESSED + * Queue up all files in a dataset to be marked as PROCESSED + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static postDatasets18( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${id}/queueUnarchival`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * @deprecated + * List all datasets in a collection + * Returns list of datasets and descriptions. + * @param collId + * @returns any OK + * @throws ApiError + */ + public static getDatasets13( + collId: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/collections/${collId}/getDatasets`, + }); + } + + /** + * List all datasets outside a collection + * Returns list of datasets and descriptions. + * @param collId + * @returns any OK + * @throws ApiError + */ + public static getDatasets14( + collId: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/datasets/listOutsideCollection/${collId}`, + }); + } + + /** + * Update dataset administrative information + * Takes one argument, a UUID of the dataset. Request body takes key-value + * pairs for name and description. + * + * @param id + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postDatasets19( + id: string, + requestBody: { + name: string; + description: string; + }, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${id}/editing`, + body: requestBody, + mediaType: 'application/json', + }); + } + + /** + * Get dataset previews + * Return the currently existing previews of the selected dataset (full + * description, including paths to preview files, previewer names etc). + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static getDatasets15( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/datasets/${id}/getPreviews`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * List datasets satisfying a general metadata search tree + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postDatasets20( + requestBody: any, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/searchmetadata`, + body: requestBody, + mediaType: 'application/json', + }); + } + + /** + * List datasets satisfying a user metadata search tree + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postDatasets21( + requestBody: any, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/searchusermetadata`, + body: requestBody, + mediaType: 'application/json', + }); + } + + /** + * Retrieve metadata as JSON-LD + * Get metadata of the dataset object as JSON-LD. + * @param id + * @param extractor + * @returns any OK + * @throws ApiError + */ + public static getDatasets16( + id: string, + extractor?: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/datasets/${id}/metadata.jsonld`, + query: { + 'extractor': extractor, + }, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Add JSON-LD metadata to the database. + * Metadata in attached JSON-LD object will be added to metadata Mongo db + * collection. + * + * @param id + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postDatasets22( + id: string, + requestBody: JSONLD, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${id}/metadata.jsonld`, + body: requestBody, + mediaType: 'application/json', + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Remove JSON-LD metadata, filtered by extractor if necessary + * Remove JSON-LD metadata from dataset object + * @param id + * @param extractor + * @returns any OK + * @throws ApiError + */ + public static deleteDatasets4( + id: string, + extractor?: string, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/datasets/${id}/metadata.jsonld`, + query: { + 'extractor': extractor, + }, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Update license information of a dataset + * Takes four arguments, all Strings. licenseType, rightsHolder, + * licenseText, licenseUrl + * + * @param id + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postDatasets23( + id: string, + requestBody: License, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${id}/license`, + body: requestBody, + mediaType: 'application/json', + }); + } + + /** + * Follow dataset + * Add user to dataset followers and add dataset to user followed datasets. + * @param id + * @returns any OK + * @throws ApiError + */ + public static postDatasets24( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${id}/follow`, + }); + } + + /** + * Attach multiple files to an existing dataset + * Add multiple files, by ID, to a dataset that is already in the system. + * Requires file ids and dataset id. + * + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postDatasets25( + requestBody: { + datasetid: string; + /** + * file ids seperated by comma + */ + existingfiles: string; + }, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/attachmultiple`, + body: requestBody, + mediaType: 'application/json', + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Reindex a dataset + * Reindex the existing dataset, if recursive is set to true if will also + * reindex all files in that dataset. + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static postDatasets26( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${id}/reindex`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * List files in dataset, including those within folders + * Datasets and descriptions. + * @param id + * @param max + * @returns any OK + * @throws ApiError + */ + public static getDatasets17( + id: string, + max?: number, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/datasets/${id}/files`, + query: { + 'max': max, + }, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Upload files and attach to given dataset + * This will take a list of url or path objects that point to files that + * will be ingested and added to this dataset. + * + * @param id + * @param formData + * @returns any OK + * @throws ApiError + */ + public static postDatasets27( + id: string, + formData: any, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${id}/files`, + formData: formData, + mediaType: 'multipart/form-data', + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Upload files and attach to given dataset + * This will take an URL of file object that are added to this dataset, + * the file source will be added as metadata. Request body takes key-value + * pairs for file url, the key can be fileurl, weburl or url. + * This can also add metadata at the same time. + * + * @param id + * @param requestBody the key can be fileurl, weburl or url + * @returns any OK + * @throws ApiError + */ + public static postDatasets28( + id: string, + requestBody?: any, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${id}/urls`, + body: requestBody, + mediaType: 'application/json', + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * List all datasets in the space the user can edit and thus move the file + * to + * + * This will check for Permission.AddResourceToDataset and + * Permission.EditDataset + * + * @returns any OK + * @throws ApiError + */ + public static getDatasets18(): CancelablePromise { + return __request({ + method: 'GET', + path: `/datasets/moveFileToDataset`, + }); + } + + /** + * List files in dataset not in any folders + * Datasets and descriptions. + * @param id + * @returns any OK + * @throws ApiError + */ + public static getDatasets19( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/datasets/${id}/listFiles`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * List all file within a dataset + * @param dsId + * @returns any OK + * @throws ApiError + */ + public static getDatasets20( + dsId: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/datasets/${dsId}/listAllFiles`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Remove tag of dataset + * A dataset is a container for files and metadata + * @param id + * @param requestBody The body of the POST request. + * @returns any OK + * @throws ApiError + */ + public static postDatasets29( + id: string, + requestBody: { + tagId: string; + }, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${id}/removeTag`, + body: requestBody, + mediaType: 'application/json', + }); + } + + /** + * Attach existing file to a new dataset and delete it from the old one + * If the file is an XML metadata file, the metadata are added to the + * dataset. + * + * @param datasetId + * @param toDataset + * @param fileId + * @returns any OK + * @throws ApiError + */ + public static postDatasets30( + datasetId: string, + toDataset: string, + fileId: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${datasetId}/moveBetweenDatasets/${toDataset}/${fileId}`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Download dataset + * Downloads all files contained in a dataset. + * @param id + * @param compression default as -1 + * @param tracking default as true + * @returns any OK + * @throws ApiError + */ + public static getDatasets21( + id: string, + compression?: number, + tracking?: boolean, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/datasets/${id}/download`, + query: { + 'compression': compression, + 'tracking': tracking, + }, + }); + } + + /** + * Download a subset of files from a dataset + * Takes dataset ID and a JSON-string representing a list of file IDs + * @param id + * @param fileList ID of the files to download from the dataset + * @returns any OK + * @throws ApiError + */ + public static getDatasets22( + id: string, + fileList: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/datasets/${id}/downloadPartial`, + query: { + 'fileList': fileList, + }, + }); + } + + /** + * Download a folder from a dataset + * Takes dataset ID and a folder ID in that dataset and streams just that folder and sub-folders as a zip + * @param id + * @param folderId ID of the folder to download + * @returns any OK + * @throws ApiError + */ + public static getDatasets23( + id: string, + folderId: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/datasets/${id}/downloadFolder`, + query: { + 'folderId': folderId, + }, + }); + } + + /** + * Deleted all datasets in trash older than days specified + * Server admin action. + * @param days + * @returns any OK + * @throws ApiError + */ + public static deleteDatasets5( + days?: number, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/datasets/clearOldDatasetsTrash`, + query: { + 'days': days, + }, + }); + } + + /** + * Emptying trash datasets + * @returns any OK + * @throws ApiError + */ + public static deleteDatasets6(): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/datasets/emptyTrash`, + }); + } + + /** + * list trash datasets + * @param limit + * @returns any OK + * @throws ApiError + */ + public static getDatasets24( + limit?: number, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/datasets/listTrash`, + query: { + 'limit': limit, + }, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * restore a trash dataset + * This will check for Permission.DeleteDataset + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static putDatasets4( + id: string, + ): CancelablePromise { + return __request({ + method: 'PUT', + path: `/datasets/restore/${id}`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * @deprecated + * remove a tag from a dataset + * @param id + * @returns any OK + * @throws ApiError + */ + public static postDatasets31( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${id}/tags/remove`, + }); + } + +} \ No newline at end of file diff --git a/frontend/src/openapi/v1/services/ExtractionsService.ts b/frontend/src/openapi/v1/services/ExtractionsService.ts index cc7905db9..1f9f66268 100644 --- a/frontend/src/openapi/v1/services/ExtractionsService.ts +++ b/frontend/src/openapi/v1/services/ExtractionsService.ts @@ -1,307 +1,315 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ -import type { CancelablePromise } from "../core/CancelablePromise"; -import { request as __request } from "../core/request"; +import type { CancelablePromise } from '../core/CancelablePromise'; +import { request as __request } from '../core/request'; export class ExtractionsService { - /** - * Submit file for extraction by a specific extractor - * Extractions for Files. - * @param id - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postExtractions( - id: string, - requestBody: { - parameters?: Array; - /** - * the extractor Id - */ - extractor?: string; - } - ): CancelablePromise { - return __request({ - method: "POST", - path: `/files/${id}/extractions`, - body: requestBody, - mediaType: "application/json", - }); - } - /** - * Cancel a submitted file extraction. - * Extractions for file. - * @param id - * @param msgId - * @returns any OK - * @throws ApiError - */ - public static deleteExtractions( - id: string, - msgId: string - ): CancelablePromise { - return __request({ - method: "DELETE", - path: `/files/${id}/extractions/${msgId}`, - }); - } + /** + * Submit file for extraction by a specific extractor + * Extractions for Files. + * @param id + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postExtractions( + id: string, + requestBody: { + parameters?: Array; + /** + * the extractor Id + */ + extractor?: string; + }, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/files/${id}/extractions`, + body: requestBody, + mediaType: 'application/json', + }); + } - /** - * Submit dataset for extraction by a specific extractor - * Extractions for dataset. - * @param dsId - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postExtractions1( - dsId: string, - requestBody: { - parameters?: Array; - /** - * the extractor Id - */ - extractor?: string; - } - ): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${dsId}/extractions`, - body: requestBody, - mediaType: "application/json", - }); - } + /** + * Cancel a submitted file extraction. + * Extractions for file. + * @param id + * @param msgId + * @returns any OK + * @throws ApiError + */ + public static deleteExtractions( + id: string, + msgId: string, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/files/${id}/extractions/${msgId}`, + }); + } - /** - * Cancel a submitted dataset extraction. - * Extractions for dataset. - * @param dsId - * @param msgId - * @returns any OK - * @throws ApiError - */ - public static deleteExtractions1( - dsId: string, - msgId: string - ): CancelablePromise { - return __request({ - method: "DELETE", - path: `/datasets/${dsId}/extractions/${msgId}`, - }); - } + /** + * Submit dataset for extraction by a specific extractor + * Extractions for dataset. + * @param dsId + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postExtractions1( + dsId: string, + requestBody: { + parameters?: Array; + /** + * the extractor Id + */ + extractor?: string; + }, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${dsId}/extractions`, + body: requestBody, + mediaType: 'application/json', + }); + } - /** - * Lists servers IPs running the extractors - * Extractions for Files. - * @returns any OK - * @throws ApiError - */ - public static getExtractions(): CancelablePromise { - return __request({ - method: "GET", - path: `/extractions/servers_ips`, - }); - } + /** + * Cancel a submitted dataset extraction. + * Extractions for dataset. + * @param dsId + * @param msgId + * @returns any OK + * @throws ApiError + */ + public static deleteExtractions1( + dsId: string, + msgId: string, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/datasets/${dsId}/extractions/${msgId}`, + }); + } - /** - * Lists the currently running extractors - * Extractions for Files. - * @returns any OK - * @throws ApiError - */ - public static getExtractions1(): CancelablePromise { - return __request({ - method: "GET", - path: `/extractions/extractors_names`, - }); - } + /** + * Lists servers IPs running the extractors + * Extractions for Files. + * @returns any OK + * @throws ApiError + */ + public static getExtractions(): CancelablePromise { + return __request({ + method: 'GET', + path: `/extractions/servers_ips`, + }); + } - /** - * Uploads a file for extraction using the file's URL - * Saves the uploaded file and sends it for extraction. If the optional URL - * parameter extract is set to false, it does not send the file for - * extraction. Does not index the file. - * - * @param extract default as true - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postExtractions2( - extract?: boolean, - requestBody?: Array - ): CancelablePromise { - return __request({ - method: "POST", - path: `/extractions/upload_url`, - query: { - extract: extract, - }, - body: requestBody, - mediaType: "application/json", - }); - } + /** + * Lists the currently running extractors + * Extractions for Files. + * @returns any OK + * @throws ApiError + */ + public static getExtractions1(): CancelablePromise { + return __request({ + method: 'GET', + path: `/extractions/extractors_names`, + }); + } - /** - * Uploads a file for extraction of metadata and returns file id - * Saves the uploaded file and sends it for extraction to Rabbitmq. If the - * optional URL parameter extract is set to false, it does not send the - * file for extraction. Does not index the file. Same as upload() except - * for upload() - * - * @param formData - * @param extract default as true - * @param showPreviews default as DatasetLevel - * @returns any OK - * @throws ApiError - */ - public static postExtractions3( - formData: any, - extract?: boolean, - showPreviews?: string - ): CancelablePromise { - return __request({ - method: "POST", - path: `/extractions/upload_file`, - query: { - extract: extract, - showPreviews: showPreviews, - }, - formData: formData, - mediaType: "multipart/form-data", - }); - } + /** + * Uploads a file for extraction using the file's URL + * Saves the uploaded file and sends it for extraction. If the optional URL + * parameter extract is set to false, it does not send the file for + * extraction. Does not index the file. + * + * @param extract default as true + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postExtractions2( + extract?: boolean, + requestBody?: Array, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/extractions/upload_url`, + query: { + 'extract': extract, + }, + body: requestBody, + mediaType: 'application/json', + }); + } - /** - * Lists the input file format supported by currenlty running extractors - * Extractions for Files. - * @returns any OK - * @throws ApiError - */ - public static getExtractions2(): CancelablePromise { - return __request({ - method: "GET", - path: `/extractions/supported_input_types`, - }); - } + /** + * Uploads a file for extraction of metadata and returns file id + * Saves the uploaded file and sends it for extraction to Rabbitmq. If the + * optional URL parameter extract is set to false, it does not send the + * file for extraction. Does not index the file. Same as upload() except + * for upload() + * + * @param formData + * @param extract default as true + * @param showPreviews default as DatasetLevel + * @returns any OK + * @throws ApiError + */ + public static postExtractions3( + formData: any, + extract?: boolean, + showPreviews?: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/extractions/upload_file`, + query: { + 'extract': extract, + 'showPreviews': showPreviews, + }, + formData: formData, + mediaType: 'multipart/form-data', + }); + } - /** - * Uploads files for a given list of files' URLs - * Saves the uploaded files and sends it for extraction. Does not index the - * files. Returns id for the web resource - * - * @returns any OK - * @throws ApiError - */ - public static postExtractions4(): CancelablePromise { - return __request({ - method: "POST", - path: `/extractions/multiple_uploadby_url`, - }); - } + /** + * Lists the input file format supported by currenlty running extractors + * Extractions for Files. + * @returns any OK + * @throws ApiError + */ + public static getExtractions2(): CancelablePromise { + return __request({ + method: 'GET', + path: `/extractions/supported_input_types`, + }); + } - /** - * Checks for the extraction statuses of all files - * Returns a list (file id, status of extractions) - * @param id - * @returns any OK - * @throws ApiError - */ - public static getExtractions3(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/extractions/${id}/statuses`, - }); - } + /** + * Uploads files for a given list of files' URLs + * Saves the uploaded files and sends it for extraction. Does not index the + * files. Returns id for the web resource + * + * @returns any OK + * @throws ApiError + */ + public static postExtractions4(): CancelablePromise { + return __request({ + method: 'POST', + path: `/extractions/multiple_uploadby_url`, + }); + } - /** - * Lists dts extraction requests information - * Extractions for Files. - * @returns any OK - * @throws ApiError - */ - public static getExtractions4(): CancelablePromise { - return __request({ - method: "GET", - path: `/extractions/requests`, - }); - } + /** + * Checks for the extraction statuses of all files + * Returns a list (file id, status of extractions) + * @param id + * @returns any OK + * @throws ApiError + */ + public static getExtractions3( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/extractions/${id}/statuses`, + }); + } - /** - * Provides the metadata extracted from the file - * Retruns Status of extractions and metadata extracted so far - * @param id - * @returns any OK - * @throws ApiError - */ - public static getExtractions5(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/extractions/${id}/metadata`, - }); - } + /** + * Lists dts extraction requests information + * Extractions for Files. + * @returns any OK + * @throws ApiError + */ + public static getExtractions4(): CancelablePromise { + return __request({ + method: 'GET', + path: `/extractions/requests`, + }); + } - /** - * Lists the currenlty details running extractors - * Extractions for Files. - * @returns any OK - * @throws ApiError - */ - public static getExtractions6(): CancelablePromise { - return __request({ - method: "GET", - path: `/extractions/extractors_details`, - }); - } + /** + * Provides the metadata extracted from the file + * Retruns Status of extractions and metadata extracted so far + * @param id + * @returns any OK + * @throws ApiError + */ + public static getExtractions5( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/extractions/${id}/metadata`, + }); + } - /** - * Checks for the status of all extractors processing the file with id - * A list of status of all extractors responsible for extractions on the - * file and the final status of extraction job - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static getExtractions7(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/extractions/${id}/status`, - }); - } + /** + * Lists the currenlty details running extractors + * Extractions for Files. + * @returns any OK + * @throws ApiError + */ + public static getExtractions6(): CancelablePromise { + return __request({ + method: 'GET', + path: `/extractions/extractors_details`, + }); + } - /** - * Submit all selected files for extraction - * Submit all selected files for extraction. - * - * @param dsId - * @param fileIds - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postExtractions5( - dsId: string, - fileIds: string, - requestBody: { - parameters?: Array; - /** - * the extractor Id - */ - extractor?: string; - } - ): CancelablePromise { - return __request({ - method: "POST", - path: `/selected/submit/${dsId}/${fileIds}`, - body: requestBody, - mediaType: "application/json", - errors: { - 401: `Not authorized`, - }, - }); - } -} + /** + * Checks for the status of all extractors processing the file with id + * A list of status of all extractors responsible for extractions on the + * file and the final status of extraction job + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static getExtractions7( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/extractions/${id}/status`, + }); + } + + /** + * Submit all selected files for extraction + * Submit all selected files for extraction. + * + * @param dsId + * @param fileIds + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postExtractions5( + dsId: string, + fileIds: string, + requestBody: { + parameters?: Array; + /** + * the extractor Id + */ + extractor?: string; + }, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/selected/submit/${dsId}/${fileIds}`, + body: requestBody, + mediaType: 'application/json', + errors: { + 401: `Not authorized`, + }, + }); + } + +} \ No newline at end of file diff --git a/frontend/src/openapi/v1/services/ExtractorsService.ts b/frontend/src/openapi/v1/services/ExtractorsService.ts index f47a4679d..79625cc6f 100644 --- a/frontend/src/openapi/v1/services/ExtractorsService.ts +++ b/frontend/src/openapi/v1/services/ExtractorsService.ts @@ -1,107 +1,113 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ -import type { ExtractorsLabel } from "../models/ExtractorsLabel"; -import type { CancelablePromise } from "../core/CancelablePromise"; -import { request as __request } from "../core/request"; +import type { ExtractorsLabel } from '../models/ExtractorsLabel'; +import type { CancelablePromise } from '../core/CancelablePromise'; +import { request as __request } from '../core/request'; export class ExtractorsService { - /** - * Lists information about all known extractors - * Extractions for Files. - * @returns any OK - * @throws ApiError - */ - public static getExtractors(): CancelablePromise { - return __request({ - method: "GET", - path: `/extractors`, - }); - } - /** - * Register information about an extractor - * Register information about an extractor. - * Used when an extractor starts up. - * - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postExtractors(requestBody: any): CancelablePromise { - return __request({ - method: "POST", - path: `/extractors`, - body: requestBody, - mediaType: "application/json", - }); - } + /** + * Lists information about all known extractors + * Extractions for Files. + * @returns any OK + * @throws ApiError + */ + public static getExtractors(): CancelablePromise { + return __request({ + method: 'GET', + path: `/extractors`, + }); + } - /** - * Lists information about a specific extractor - * Extractions for Files. - * @param name - * @returns any OK - * @throws ApiError - */ - public static getExtractors1(name: string): CancelablePromise { - return __request({ - method: "GET", - path: `/extractors/${name}`, - errors: { - 404: `Not found`, - }, - }); - } + /** + * Register information about an extractor + * Register information about an extractor. + * Used when an extractor starts up. + * + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postExtractors( + requestBody: any, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/extractors`, + body: requestBody, + mediaType: 'application/json', + }); + } - /** - * Create a new extractor label in the database - * @param requestBody - * @returns ExtractorsLabel OK - * @throws ApiError - */ - public static postExtractors1( - requestBody?: ExtractorsLabel - ): CancelablePromise { - return __request({ - method: "POST", - path: `/extractors/labels`, - body: requestBody, - mediaType: "application/json", - }); - } + /** + * Lists information about a specific extractor + * Extractions for Files. + * @param name + * @returns any OK + * @throws ApiError + */ + public static getExtractors1( + name: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/extractors/${name}`, + errors: { + 404: `Not found`, + }, + }); + } - /** - * Updates an extractor label in the database - * @param id - * @param requestBody - * @returns ExtractorsLabel OK - * @throws ApiError - */ - public static putExtractors( - id: string, - requestBody?: ExtractorsLabel - ): CancelablePromise { - return __request({ - method: "PUT", - path: `/extractors/labels/${id}`, - body: requestBody, - mediaType: "application/json", - }); - } + /** + * Create a new extractor label in the database + * @param requestBody + * @returns ExtractorsLabel OK + * @throws ApiError + */ + public static postExtractors1( + requestBody?: ExtractorsLabel, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/extractors/labels`, + body: requestBody, + mediaType: 'application/json', + }); + } - /** - * Deletes an extractor label from the database - * @param id - * @returns ExtractorsLabel OK - * @throws ApiError - */ - public static deleteExtractors( - id: string - ): CancelablePromise { - return __request({ - method: "DELETE", - path: `/extractors/labels/${id}`, - }); - } -} + /** + * Updates an extractor label in the database + * @param id + * @param requestBody + * @returns ExtractorsLabel OK + * @throws ApiError + */ + public static putExtractors( + id: string, + requestBody?: ExtractorsLabel, + ): CancelablePromise { + return __request({ + method: 'PUT', + path: `/extractors/labels/${id}`, + body: requestBody, + mediaType: 'application/json', + }); + } + + /** + * Deletes an extractor label from the database + * @param id + * @returns ExtractorsLabel OK + * @throws ApiError + */ + public static deleteExtractors( + id: string, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/extractors/labels/${id}`, + }); + } + +} \ No newline at end of file diff --git a/frontend/src/openapi/v1/services/FilesService.ts b/frontend/src/openapi/v1/services/FilesService.ts index 3fc7a1358..1dcfaacfa 100644 --- a/frontend/src/openapi/v1/services/FilesService.ts +++ b/frontend/src/openapi/v1/services/FilesService.ts @@ -1,1297 +1,1387 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ -import type { Comment } from "../models/Comment"; -import type { File } from "../models/File"; -import type { JSONLD } from "../models/JSONLD"; -import type { License } from "../models/License"; -import type { Tags } from "../models/Tags"; -import type { UUID } from "../models/UUID"; -import type { CancelablePromise } from "../core/CancelablePromise"; -import { request as __request } from "../core/request"; +import type { Comment } from '../models/Comment'; +import type { File } from '../models/File'; +import type { JSONLD } from '../models/JSONLD'; +import type { License } from '../models/License'; +import type { Tags } from '../models/Tags'; +import type { UUID } from '../models/UUID'; +import type { CancelablePromise } from '../core/CancelablePromise'; +import { request as __request } from '../core/request'; export class FilesService { - /** - * List all files - * Returns list of files and descriptions. - * @returns File Successfully returns a list of files - * @throws ApiError - */ - public static getFiles(): CancelablePromise> { - return __request({ - method: "GET", - path: `/files`, - errors: { - 401: `Access to this resource is disabled + + /** + * List all files + * Returns list of files and descriptions. + * @returns File Successfully returns a list of files + * @throws ApiError + */ + public static getFiles(): CancelablePromise> { + return __request({ + method: 'GET', + path: `/files`, + errors: { + 401: `Access to this resource is disabled * `, - }, - }); - } - - /** - * Download file - * Can use Chunked transfer encoding if the HTTP header RANGE is set. - * This function will be reused to actually download the metadata of - * the file, please use /files/{id}/blob to get the actual bytes. - * - * @param id ID of file object - * @returns binary The requested data in binary form - * @throws ApiError - */ - public static getFiles1(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/files/${id}`, - errors: { - 400: `The server could not process your request, this happens for example when + }, + }); + } + + /** + * Download file + * Can use Chunked transfer encoding if the HTTP header RANGE is set. + * This function will be reused to actually download the metadata of + * the file, please use /files/{id}/blob to get the actual bytes. + * + * @param id ID of file object + * @returns binary The requested data in binary form + * @throws ApiError + */ + public static getFiles1( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/files/${id}`, + errors: { + 400: `The server could not process your request, this happens for example when * the id specified is not of the correct form. See the message field for * more information. * `, - 401: `The request has not been applied because it lacks valid authentication + 401: `The request has not been applied because it lacks valid authentication * credentials for the target resource. * `, - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Delete file - * Cascading action (removes file from any datasets containing it and - * deletes its previews, metadata and thumbnail). - * - * @param id ID of file object - * @returns any OK - * @throws ApiError - */ - public static deleteFiles(id: string): CancelablePromise { - return __request({ - method: "DELETE", - path: `/files/${id}`, - errors: { - 400: `The server could not process your request, this happens for example when + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Delete file + * Cascading action (removes file from any datasets containing it and + * deletes its previews, metadata and thumbnail). + * + * @param id ID of file object + * @returns any OK + * @throws ApiError + */ + public static deleteFiles( + id: string, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/files/${id}`, + errors: { + 400: `The server could not process your request, this happens for example when * the id specified is not of the correct form. See the message field for * more information. * `, - 401: `The request has not been applied because it lacks valid authentication + 401: `The request has not been applied because it lacks valid authentication * credentials for the target resource. * `, - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Download file - * Can use Chunked transfer encoding if the HTTP header RANGE is set. - * @param id ID of file object - * @returns binary The requested data in binary form - * @throws ApiError - */ - public static getFiles2(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/files/${id}/blob`, - errors: { - 400: `The server could not process your request, this happens for example when + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Download file + * Can use Chunked transfer encoding if the HTTP header RANGE is set. + * @param id ID of file object + * @returns binary The requested data in binary form + * @throws ApiError + */ + public static getFiles2( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/files/${id}/blob`, + errors: { + 400: `The server could not process your request, this happens for example when * the id specified is not of the correct form. See the message field for * more information. * `, - 401: `The request has not been applied because it lacks valid authentication + 401: `The request has not been applied because it lacks valid authentication * credentials for the target resource. * `, - 404: `The requested resource was not found`, - }, - }); - } - - /** - * @deprecated - * Delete file - * use DELETE /files/{id} instead - * @param id ID of file object - * @returns any OK - * @throws ApiError - */ - public static postFiles(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/files/${id}/remove`, - }); - } - - /** - * Deletes files - * Deletes a list of files by fileIds - * @returns UUID Returns Status Success - * @throws ApiError - */ - public static postFiles1(): CancelablePromise> { - return __request({ - method: "POST", - path: `/files/bulkRemove`, - errors: { - 401: `The request has not been applied because it lacks valid authentication + 404: `The requested resource was not found`, + }, + }); + } + + /** + * @deprecated + * Delete file + * use DELETE /files/{id} instead + * @param id ID of file object + * @returns any OK + * @throws ApiError + */ + public static postFiles( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/files/${id}/remove`, + }); + } + + /** + * Deletes files + * Deletes a list of files by fileIds + * @returns UUID Returns Status Success + * @throws ApiError + */ + public static postFiles1(): CancelablePromise> { + return __request({ + method: 'POST', + path: `/files/bulkRemove`, + errors: { + 401: `The request has not been applied because it lacks valid authentication * credentials for the target resource. * `, - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Get file previews - * Return the currently existing previews of the selected file (full - * description, including paths to preview files, previewer names etc). - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static getFiles3(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/files/${id}/getPreviews`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Retrieve physical file object metadata - * Get information of the file object (not the resource it describes) as JSON. - * For example, size of file, date created, content type, filename. - * - * @param id - * @returns File Successfully returns a list of files - * @throws ApiError - */ - public static getFiles4(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/files/${id}/metadata`, - errors: { - 401: `Access to this resource is disabled + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Get file previews + * Return the currently existing previews of the selected file (full + * description, including paths to preview files, previewer names etc). + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static getFiles3( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/files/${id}/getPreviews`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Retrieve physical file object metadata + * Get information of the file object (not the resource it describes) as JSON. + * For example, size of file, date created, content type, filename. + * + * @param id + * @returns File Successfully returns a list of files + * @throws ApiError + */ + public static getFiles4( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/files/${id}/metadata`, + errors: { + 401: `Access to this resource is disabled * `, - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Add technical metadata to file - * Metadata in attached JSON object will describe the file's described - * resource, not the file object itself. - * - * @param id - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postFiles2( - id: string, - requestBody?: any - ): CancelablePromise { - return __request({ - method: "POST", - path: `/files/${id}/metadata`, - body: requestBody, - mediaType: "application/json", - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Get path to file in dataset - * Return the path from the dataset down to the folder - * containing this file id. - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static getFiles5(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/files/${id}/paths`, - }); - } - - /** - * Submit this file to be archived - * Submit this file to the queue to be archived. - * This requires RabbitMQ and a compatible archival extractor to be running. - * See https://github.com/clowder-framework/extractors-archival - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static postFiles3(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/files/${id}/sendArchiveRequest`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Changes a file's status to ARCHIVED - * Callback that will change a file's status to ARCHIVED. - * This is used by archival extractors and is not typically used by clients. - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static postFiles4(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/files/${id}/archive`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Submit this file to be unarchived - * Submit this file to the queue to be unarchived. - * This requires RabbitMQ and a compatible archival extractor to be running. - * See https://github.com/clowder-framework/extractors-archival - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static postFiles5(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/files/${id}/sendUnarchiveRequest`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Changes a file's status back to PROCESSED - * Callback that will change a file's status back to PROCESSED. - * This is used by archival extractors and is not typically used by clients. - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static postFiles6(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/files/${id}/unarchive`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Submit file for extraction by a specific extractor - * Extractions for Files. - * @param id - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postFiles7( - id: string, - requestBody: { - parameters?: Array; - /** - * the extractor Id - */ - extractor?: string; - } - ): CancelablePromise { - return __request({ - method: "POST", - path: `/files/${id}/extractions`, - body: requestBody, - mediaType: "application/json", - }); - } - - /** - * Cancel a submitted file extraction. - * Extractions for file. - * @param id - * @param msgId - * @returns any OK - * @throws ApiError - */ - public static deleteFiles1( - id: string, - msgId: string - ): CancelablePromise { - return __request({ - method: "DELETE", - path: `/files/${id}/extractions/${msgId}`, - }); - } - - /** - * Upload a file to a specific dataset - * Uploads the file, then links it with the dataset. Returns file id as - * JSON object, or ids with filenames if multiple files are sent. ID can be - * used to work on the file using the API. Uploaded file can be an XML - * metadata file to be added to the dataset. If the optional Boolean - * parameter extract is set to false, it does not send the file for - * extraction. By default, Boolean parameter extract is set to true. - * - * @param id the dataset id - * @param formData - * @param showPreviews default as "DatasetLevel" - * @param originalZipFile the UUID string of original zip file - * @param flags flags for previews - * @param extract - * @param folderId - * @returns any OK - * @throws ApiError - */ - public static postFiles8( - id: string, - formData: any, - showPreviews?: string, - originalZipFile?: string, - flags?: string, - extract?: boolean, - folderId?: string - ): CancelablePromise { - return __request({ - method: "POST", - path: `/uploadToDataset/${id}`, - query: { - showPreviews: showPreviews, - originalZipFile: originalZipFile, - flags: flags, - extract: extract, - folder_id: folderId, - }, - formData: formData, - mediaType: "multipart/form-data", - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Unfollow file - * Remove user from file followers and remove file from user followed - * files. - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static postFiles9(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/files/${id}/unfollow`, - }); - } - - /** - * List file previews - * Return the currently existing previews' basic characteristics (id, - * filename, content type) of the selected file. - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static getFiles6(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/files/${id}/listpreviews`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Upload file - * Upload the attached file using multipart form enconding. Returns file id - * as JSON object, or ids with filenames if multiple files are sent. ID can - * be used to work on the file using the API. Uploaded file can be an XML - * metadata file. - * - * @param flags - * @returns any OK - * @throws ApiError - */ - public static postFiles10(flags: string): CancelablePromise { - return __request({ - method: "POST", - path: `/files/withFlags/${flags}`, - }); - } - - /** - * Add user-generated metadata to file - * Metadata in attached JSON object will describe the file's described - * resource, not the file object itself. - * - * @param id - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postFiles11( - id: string, - requestBody: any - ): CancelablePromise { - return __request({ - method: "POST", - path: `/files/${id}/usermetadata`, - body: requestBody, - mediaType: "application/json", - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Get URLs of file's RDF metadata exports. - * URLs of metadata files exported from XML (if the file was an XML - * metadata file) as well as the URL used to export the file's - * user-generated metadata as RDF. - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static getFiles7(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/files/getRDFURLsForFile/${id}`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Upload a preview - * Upload a preview - * @param formData - * @returns any OK - * @throws ApiError - */ - public static postFiles12(formData?: any): CancelablePromise { - return __request({ - method: "POST", - path: `/previews`, - formData: formData, - mediaType: "multipart/form-data", - }); - } - - /** - * Attach existing preview to file - * A file is the raw bytes plus metadata. - * @param id - * @param pId - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postFiles13( - id: string, - pId: string, - requestBody?: { - extractor_id?: string; - } - ): CancelablePromise { - return __request({ - method: "POST", - path: `/files/${id}/previews/${pId}`, - body: requestBody, - mediaType: "application/json", - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Is being processed - * Return whether a file is currently being processed by a preprocessor. - * @param id - * @returns any OK - * @throws ApiError - */ - public static getFiles8(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/files/${id}/isBeingProcessed`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Reindex a file - * Reindex the existing file. - * @param id - * @returns any OK - * @throws ApiError - */ - public static postFiles14(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/files/${id}/reindex`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Update a file name - * Takes one argument, a UUID of the file. Request body takes a key-value - * pair for the name - * - * @param id - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static putFiles( - id: string, - requestBody: { - name?: string; - } - ): CancelablePromise { - return __request({ - method: "PUT", - path: `/files/${id}/filename`, - body: requestBody, - mediaType: "application/json", - }); - } - - /** - * Update technical metadata of a file generated by a specific extractor - * Metadata in attached JSON object will describe the file's described - * resource, not the file object itself. The method will search the entire - * techincal metadata array for the metadata generated by a specific - * extractor (using extractor_id provided as an argument) and if a match is - * found, it will update the corresponding metadata element. - * - * @param id - * @param requestBody json body that can be parsed as DBObject - * @param extractorId - * @returns any OK - * @throws ApiError - */ - public static postFiles15( - id: string, - requestBody: any, - extractorId?: string - ): CancelablePromise { - return __request({ - method: "POST", - path: `/files/${id}/updateMetadata`, - query: { - extractor_id: extractorId, - }, - body: requestBody, - mediaType: "application/json", - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Follow file - * Add user to file followers and add file to user followed files. - * @param id - * @returns any OK - * @throws ApiError - */ - public static postFiles16(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/files/${id}/follow`, - }); - } - - /** - * Retrieve metadata as JSON-LD - * Get metadata of the file object as JSON-LD. - * @param id - * @param extractor - * @returns any OK - * @throws ApiError - */ - public static getFiles9( - id: string, - extractor?: string - ): CancelablePromise { - return __request({ - method: "GET", - path: `/files/${id}/metadata.jsonld`, - query: { - extractor: extractor, - }, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Add JSON-LD metadata to the database. - * Metadata in attached JSON-LD object will be added to metadata Mongo db - * collection. - * - * @param id - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postFiles17( - id: string, - requestBody: JSONLD - ): CancelablePromise { - return __request({ - method: "POST", - path: `/files/${id}/metadata.jsonld`, - body: requestBody, - mediaType: "application/json", - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Remove JSON-LD metadata, filtered by extractor if necessary - * Remove JSON-LD metadata from file object - * @param id - * @param extractor - * @returns any OK - * @throws ApiError - */ - public static deleteFiles2( - id: string, - extractor?: string - ): CancelablePromise { - return __request({ - method: "DELETE", - path: `/files/${id}/metadata.jsonld`, - query: { - extractor: extractor, - }, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Retrieve metadata as JSON-LD for multiple files at once - * Use ?id=123&id=456&... to retrieve metadata for multiple files at once. - * @param id - * @returns any OK - * @throws ApiError - */ - public static getFiles10(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/files/metadata.jsonld`, - query: { - id: id, - }, - }); - } - - /** - * Add JSON-LD metadata to multiple files at once. - * JSON object in post should have a list of file IDs under "files" key and metadata under "metadata" key. - * Metadata will be added to each file in metadata Mongo db collection. - * - * @param requestBody the metadata to add and the file IDs to add it to - * @returns any OK - * @throws ApiError - */ - public static postFiles18(requestBody?: { - files: Array; - metadata: any; - }): CancelablePromise { - return __request({ - method: "POST", - path: `/files/metadata.jsonld`, - body: requestBody, - mediaType: "application/json", - }); - } - - /** - * Gets tags of a file - * Returns a list of strings, List[String]. - * @param id - * @returns any OK - * @throws ApiError - */ - public static getFiles11(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/files/${id}/tags`, - }); - } - - /** - * Adds tags to a file - * Tag's (name, userId, extractor_id) tuple is used as a unique key. In - * other words, the same tag names but diff userId or extractor_id are - * considered as diff tags, so will be added. The tags are expected as a - * list of strings: List[String]. An example is:
curl -H - * 'Content-Type: application/json' -d '{"tags":["namo", "amitabha"], - * "extractor_id": "curl"}' - * "http://localhost:9000/api/files/533c2389e4b02a14f0943356/tags?key=theKey" - * - * @param id - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postFiles19( - id: string, - requestBody: Tags - ): CancelablePromise { - return __request({ - method: "POST", - path: `/files/${id}/tags`, - body: requestBody, - mediaType: "application/json", - }); - } - - /** - * Removes tags of a file - * Tag's (name, userId, extractor_id) tuple is unique key. Same tag names - * but diff userId or extractor_id are considered diff tags. Tags can only - * be removed by the same user or extractor. The tags are expected as a - * list of strings: List[String]. - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static deleteFiles3(id: string): CancelablePromise { - return __request({ - method: "DELETE", - path: `/files/${id}/tags`, - }); - } - - /** - * Update License information of a file - * Takes four arguments, all Strings. licenseType, rightsHolder, - * licenseText, licenseUrl - * - * @param id - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postFiles20( - id: string, - requestBody: License - ): CancelablePromise { - return __request({ - method: "POST", - path: `/files/${id}/license`, - body: requestBody, - mediaType: "application/json", - }); - } - - /** - * Get Versus metadata of the resource described by the file - * A file is the raw bytes plus metadata. - * @param id - * @returns any OK - * @throws ApiError - */ - public static getFiles12(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/files/${id}/versus_metadata`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Update file description - * Takes one argument, a UUID of the file. Request body takes key-value - * pair for the description - * - * @param id - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static putFiles1( - id: string, - requestBody: { - description: string; - } - ): CancelablePromise { - return __request({ - method: "PUT", - path: `/files/${id}/updateDescription`, - body: requestBody, - mediaType: "application/json", - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Get technical metadata of the resource described by the file - * A file is the raw bytes plus metadata. - * @param id - * @returns any OK - * @throws ApiError - */ - public static getFiles13(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/files/${id}/technicalmetadatajson`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * (Re)send preprocessing job for file - * Force Clowder to (re)send preprocessing job for selected file, - * processing the file as a file of the selected MIME type. Returns file id - * on success. In the requested file type, replace / with __ (two - * underscores). - * - * @param fileId - * @param fileType - * @returns any OK - * @throws ApiError - */ - public static postFiles21( - fileId: string, - fileType: string - ): CancelablePromise { - return __request({ - method: "POST", - path: `/files/sendJob/${fileId}/${fileType}`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Get metadata of the resource described by the file that were input as - * XML - * - * A file is the raw bytes plus metadata. - * @param id - * @returns any OK - * @throws ApiError - */ - public static getFiles14(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/files/${id}/xmlmetadatajson`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Provides metadata extracted for a file - * A file is the raw bytes plus metadata. - * @param id - * @returns any OK - * @throws ApiError - */ - public static getFiles15(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/files/${id}/extracted_metadata`, - }); - } - - /** - * Add comment to file - * A file is the raw bytes plus metadata. - * @param id - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postFiles22( - id: string, - requestBody: Comment - ): CancelablePromise { - return __request({ - method: "POST", - path: `/files/${id}/comment`, - body: requestBody, - mediaType: "application/json", - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * @deprecated - * Removes a tag from a file - * @param id - * @returns any OK - * @throws ApiError - */ - public static postFiles23(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/files/${id}/tags/remove`, - }); - } - - /** - * Removes all tags of a file - * This is a big hammer -- it does not check the userId or extractor_id and - * forcefully remove all tags for this file. It is mainly intended for - * testing. - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static postFiles24(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/files/${id}/tags/remove_all`, - }); - } - - /** - * Upload a thumbnail - * Upload a thumbnail. - * @returns any OK - * @throws ApiError - */ - public static postFiles25(): CancelablePromise { - return __request({ - method: "POST", - path: `/fileThumbnail`, - errors: { - 400: `The server could not process your request, this happens for example when + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Add technical metadata to file + * Metadata in attached JSON object will describe the file's described + * resource, not the file object itself. + * + * @param id + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postFiles2( + id: string, + requestBody?: any, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/files/${id}/metadata`, + body: requestBody, + mediaType: 'application/json', + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Get the sections of a file + * Get the sections of a file. + * + * @param id + * @returns UUID OK + * @throws ApiError + */ + public static getFiles5( + id: string, + ): CancelablePromise> { + return __request({ + method: 'GET', + path: `/files/${id}/sections`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Get path to file in dataset + * Return the path from the dataset down to the folder + * containing this file id. + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static getFiles6( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/files/${id}/paths`, + }); + } + + /** + * Submit this file to be archived + * Submit this file to the queue to be archived. + * This requires RabbitMQ and a compatible archival extractor to be running. + * See https://github.com/clowder-framework/extractors-archival + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static postFiles3( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/files/${id}/sendArchiveRequest`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Changes a file's status to ARCHIVED + * Callback that will change a file's status to ARCHIVED. + * This is used by archival extractors and is not typically used by clients. + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static postFiles4( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/files/${id}/archive`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Submit this file to be unarchived + * Submit this file to the queue to be unarchived. + * This requires RabbitMQ and a compatible archival extractor to be running. + * See https://github.com/clowder-framework/extractors-archival + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static postFiles5( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/files/${id}/sendUnarchiveRequest`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Changes a file's status back to PROCESSED + * Callback that will change a file's status back to PROCESSED. + * This is used by archival extractors and is not typically used by clients. + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static postFiles6( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/files/${id}/unarchive`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Submit file for extraction by a specific extractor + * Extractions for Files. + * @param id + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postFiles7( + id: string, + requestBody: { + parameters?: Array; + /** + * the extractor Id + */ + extractor?: string; + }, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/files/${id}/extractions`, + body: requestBody, + mediaType: 'application/json', + }); + } + + /** + * Cancel a submitted file extraction. + * Extractions for file. + * @param id + * @param msgId + * @returns any OK + * @throws ApiError + */ + public static deleteFiles1( + id: string, + msgId: string, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/files/${id}/extractions/${msgId}`, + }); + } + + /** + * Upload a file to a specific dataset + * Uploads the file, then links it with the dataset. Returns file id as + * JSON object, or ids with filenames if multiple files are sent. ID can be + * used to work on the file using the API. Uploaded file can be an XML + * metadata file to be added to the dataset. If the optional Boolean + * parameter extract is set to false, it does not send the file for + * extraction. By default, Boolean parameter extract is set to true. + * + * @param id the dataset id + * @param formData + * @param showPreviews default as "DatasetLevel" + * @param originalZipFile the UUID string of original zip file + * @param flags flags for previews + * @param extract + * @param folderId + * @returns any OK + * @throws ApiError + */ + public static postFiles8( + id: string, + formData: any, + showPreviews?: string, + originalZipFile?: string, + flags?: string, + extract?: boolean, + folderId?: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/uploadToDataset/${id}`, + query: { + 'showPreviews': showPreviews, + 'originalZipFile': originalZipFile, + 'flags': flags, + 'extract': extract, + 'folder_id': folderId, + }, + formData: formData, + mediaType: 'multipart/form-data', + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Unfollow file + * Remove user from file followers and remove file from user followed + * files. + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static postFiles9( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/files/${id}/unfollow`, + }); + } + + /** + * List file previews + * Return the currently existing previews' basic characteristics (id, + * filename, content type) of the selected file. + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static getFiles7( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/files/${id}/listpreviews`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Upload file + * Upload the attached file using multipart form enconding. Returns file id + * as JSON object, or ids with filenames if multiple files are sent. ID can + * be used to work on the file using the API. Uploaded file can be an XML + * metadata file. + * + * @param flags + * @returns any OK + * @throws ApiError + */ + public static postFiles10( + flags: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/files/withFlags/${flags}`, + }); + } + + /** + * Add user-generated metadata to file + * Metadata in attached JSON object will describe the file's described + * resource, not the file object itself. + * + * @param id + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postFiles11( + id: string, + requestBody: any, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/files/${id}/usermetadata`, + body: requestBody, + mediaType: 'application/json', + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Get URLs of file's RDF metadata exports. + * URLs of metadata files exported from XML (if the file was an XML + * metadata file) as well as the URL used to export the file's + * user-generated metadata as RDF. + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static getFiles8( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/files/getRDFURLsForFile/${id}`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Upload a preview + * Upload a preview + * @param formData + * @returns any OK + * @throws ApiError + */ + public static postFiles12( + formData?: any, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/previews`, + formData: formData, + mediaType: 'multipart/form-data', + }); + } + + /** + * Attach existing preview to file + * A file is the raw bytes plus metadata. + * @param id + * @param pId + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postFiles13( + id: string, + pId: string, + requestBody?: { + extractor_id?: string; + }, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/files/${id}/previews/${pId}`, + body: requestBody, + mediaType: 'application/json', + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Is being processed + * Return whether a file is currently being processed by a preprocessor. + * @param id + * @returns any OK + * @throws ApiError + */ + public static getFiles9( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/files/${id}/isBeingProcessed`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Reindex a file + * Reindex the existing file. + * @param id + * @returns any OK + * @throws ApiError + */ + public static postFiles14( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/files/${id}/reindex`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Update a file name + * Takes one argument, a UUID of the file. Request body takes a key-value + * pair for the name + * + * @param id + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static putFiles( + id: string, + requestBody: { + name?: string; + }, + ): CancelablePromise { + return __request({ + method: 'PUT', + path: `/files/${id}/filename`, + body: requestBody, + mediaType: 'application/json', + }); + } + + /** + * Update technical metadata of a file generated by a specific extractor + * Metadata in attached JSON object will describe the file's described + * resource, not the file object itself. The method will search the entire + * techincal metadata array for the metadata generated by a specific + * extractor (using extractor_id provided as an argument) and if a match is + * found, it will update the corresponding metadata element. + * + * @param id + * @param requestBody json body that can be parsed as DBObject + * @param extractorId + * @returns any OK + * @throws ApiError + */ + public static postFiles15( + id: string, + requestBody: any, + extractorId?: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/files/${id}/updateMetadata`, + query: { + 'extractor_id': extractorId, + }, + body: requestBody, + mediaType: 'application/json', + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Follow file + * Add user to file followers and add file to user followed files. + * @param id + * @returns any OK + * @throws ApiError + */ + public static postFiles16( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/files/${id}/follow`, + }); + } + + /** + * Retrieve metadata as JSON-LD + * Get metadata of the file object as JSON-LD. + * @param id + * @param extractor + * @returns any OK + * @throws ApiError + */ + public static getFiles10( + id: string, + extractor?: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/files/${id}/metadata.jsonld`, + query: { + 'extractor': extractor, + }, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Add JSON-LD metadata to the database. + * Metadata in attached JSON-LD object will be added to metadata Mongo db + * collection. + * + * @param id + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postFiles17( + id: string, + requestBody: JSONLD, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/files/${id}/metadata.jsonld`, + body: requestBody, + mediaType: 'application/json', + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Remove JSON-LD metadata, filtered by extractor if necessary + * Remove JSON-LD metadata from file object + * @param id + * @param extractor + * @returns any OK + * @throws ApiError + */ + public static deleteFiles2( + id: string, + extractor?: string, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/files/${id}/metadata.jsonld`, + query: { + 'extractor': extractor, + }, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Retrieve metadata as JSON-LD for multiple files at once + * Use ?id=123&id=456&... to retrieve metadata for multiple files at once. + * @param id + * @returns any OK + * @throws ApiError + */ + public static getFiles11( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/files/metadata.jsonld`, + query: { + 'id': id, + }, + }); + } + + /** + * Add JSON-LD metadata to multiple files at once. + * JSON object in post should have a list of file IDs under "files" key and metadata under "metadata" key. + * Metadata will be added to each file in metadata Mongo db collection. + * + * @param requestBody the metadata to add and the file IDs to add it to + * @returns any OK + * @throws ApiError + */ + public static postFiles18( + requestBody?: { + files: Array; + metadata: any; + }, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/files/metadata.jsonld`, + body: requestBody, + mediaType: 'application/json', + }); + } + + /** + * Gets tags of a file + * Returns a list of strings, List[String]. + * @param id + * @returns any OK + * @throws ApiError + */ + public static getFiles12( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/files/${id}/tags`, + }); + } + + /** + * Adds tags to a file + * Tag's (name, userId, extractor_id) tuple is used as a unique key. In + * other words, the same tag names but diff userId or extractor_id are + * considered as diff tags, so will be added. The tags are expected as a + * list of strings: List[String]. An example is:
curl -H + * 'Content-Type: application/json' -d '{"tags":["namo", "amitabha"], + * "extractor_id": "curl"}' + * "http://localhost:9000/api/files/533c2389e4b02a14f0943356/tags?key=theKey" + * + * @param id + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postFiles19( + id: string, + requestBody: Tags, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/files/${id}/tags`, + body: requestBody, + mediaType: 'application/json', + }); + } + + /** + * Removes tags of a file + * Tag's (name, userId, extractor_id) tuple is unique key. Same tag names + * but diff userId or extractor_id are considered diff tags. Tags can only + * be removed by the same user or extractor. The tags are expected as a + * list of strings: List[String]. + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static deleteFiles3( + id: string, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/files/${id}/tags`, + }); + } + + /** + * Update License information of a file + * Takes four arguments, all Strings. licenseType, rightsHolder, + * licenseText, licenseUrl + * + * @param id + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postFiles20( + id: string, + requestBody: License, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/files/${id}/license`, + body: requestBody, + mediaType: 'application/json', + }); + } + + /** + * Get Versus metadata of the resource described by the file + * A file is the raw bytes plus metadata. + * @param id + * @returns any OK + * @throws ApiError + */ + public static getFiles13( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/files/${id}/versus_metadata`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Update file description + * Takes one argument, a UUID of the file. Request body takes key-value + * pair for the description + * + * @param id + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static putFiles1( + id: string, + requestBody: { + description: string; + }, + ): CancelablePromise { + return __request({ + method: 'PUT', + path: `/files/${id}/updateDescription`, + body: requestBody, + mediaType: 'application/json', + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Get technical metadata of the resource described by the file + * A file is the raw bytes plus metadata. + * @param id + * @returns any OK + * @throws ApiError + */ + public static getFiles14( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/files/${id}/technicalmetadatajson`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * (Re)send preprocessing job for file + * Force Clowder to (re)send preprocessing job for selected file, + * processing the file as a file of the selected MIME type. Returns file id + * on success. In the requested file type, replace / with __ (two + * underscores). + * + * @param fileId + * @param fileType + * @returns any OK + * @throws ApiError + */ + public static postFiles21( + fileId: string, + fileType: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/files/sendJob/${fileId}/${fileType}`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Get metadata of the resource described by the file that were input as + * XML + * + * A file is the raw bytes plus metadata. + * @param id + * @returns any OK + * @throws ApiError + */ + public static getFiles15( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/files/${id}/xmlmetadatajson`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Provides metadata extracted for a file + * A file is the raw bytes plus metadata. + * @param id + * @returns any OK + * @throws ApiError + */ + public static getFiles16( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/files/${id}/extracted_metadata`, + }); + } + + /** + * Add comment to file + * A file is the raw bytes plus metadata. + * @param id + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postFiles22( + id: string, + requestBody: Comment, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/files/${id}/comment`, + body: requestBody, + mediaType: 'application/json', + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * @deprecated + * Removes a tag from a file + * @param id + * @returns any OK + * @throws ApiError + */ + public static postFiles23( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/files/${id}/tags/remove`, + }); + } + + /** + * Removes all tags of a file + * This is a big hammer -- it does not check the userId or extractor_id and + * forcefully remove all tags for this file. It is mainly intended for + * testing. + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static postFiles24( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/files/${id}/tags/remove_all`, + }); + } + + /** + * Upload a thumbnail + * Upload a thumbnail. + * @returns any OK + * @throws ApiError + */ + public static postFiles25(): CancelablePromise { + return __request({ + method: 'POST', + path: `/fileThumbnail`, + errors: { + 400: `The server could not process your request, this happens for example when * the id specified is not of the correct form. See the message field for * more information. * `, - }, - }); - } - - /** - * Download a thumbnail - * Download a thumbnail. - * @param id - * @returns any OK - * @throws ApiError - */ - public static postFiles26(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/fileThumbnail/${id}/blob`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Add thumbnail to file - * Attaches an already-existing thumbnail to a file. - * @param id - * @param thumbnailId - * @returns any OK - * @throws ApiError - */ - public static postFiles27( - id: string, - thumbnailId: string - ): CancelablePromise { - return __request({ - method: "POST", - path: `/files/${id}/thumbnails/${thumbnailId}`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Get metadata definitions available for a file - * The metadata definitions come from the spaces that the dataset the file - * is part of. Directly or within a folder - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static getFiles16(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/files/${id}/metadataDefinitions`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Get the user-generated metadata of the selected file in an RDF file - * A file is the raw bytes plus metadata. - * @param id - * @returns any OK - * @throws ApiError - */ - public static getFiles17(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/files/rdfUserMetadata/${id}`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Get community-generated metadata of the resource described by the file - * A file is the raw bytes plus metadata. - * @param id - * @returns any OK - * @throws ApiError - */ - public static getFiles18(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/files/${id}/usermetadatajson`, - }); - } - - /** - * Fetches and downloads a particular query - * Fetches and downloads a particular query. - * @param id - * @returns any OK - * @throws ApiError - */ - public static getFiles19(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/queries/${id}`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Add thumbnail to a query image - * Attaches an already-existing thumbnail to a query image. - * @param fileId - * @param thumbnailId - * @returns any OK - * @throws ApiError - */ - public static postFiles28( - fileId: string, - thumbnailId: string - ): CancelablePromise { - return __request({ - method: "POST", - path: `/queries/${fileId}/thumbnails/${thumbnailId}`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } - - /** - * Get the list of user-selected files - * Get the list of user-selected files. - * - * @returns any OK - * @throws ApiError - */ - public static getFiles20(): CancelablePromise { - return __request({ - method: "GET", - path: `/selected`, - errors: { - 401: `Not authorized`, - }, - }); - } - - /** - * Select a specific file - * Select a specific file. - * - * @returns any OK - * @throws ApiError - */ - public static postFiles29(): CancelablePromise { - return __request({ - method: "POST", - path: `/selected`, - errors: { - 401: `Not authorized`, - }, - }); - } - - /** - * De-select a specific file - * De-select a specific file. - * - * @returns any OK - * @throws ApiError - */ - public static postFiles30(): CancelablePromise { - return __request({ - method: "POST", - path: `/selected/remove`, - errors: { - 401: `Not authorized`, - }, - }); - } - - /** - * Download all selected files - * Download all selected files. - * - * @returns any OK - * @throws ApiError - */ - public static getFiles21(): CancelablePromise { - return __request({ - method: "GET", - path: `/selected/files`, - errors: { - 401: `Not authorized`, - }, - }); - } - - /** - * Delete all selected files - * Delete all selected files. - * - * @returns any OK - * @throws ApiError - */ - public static deleteFiles4(): CancelablePromise { - return __request({ - method: "DELETE", - path: `/selected/files`, - errors: { - 401: `Not authorized`, - }, - }); - } - - /** - * De-select all files - * De-select all files. - * - * @returns any OK - * @throws ApiError - */ - public static postFiles31(): CancelablePromise { - return __request({ - method: "POST", - path: `/selected/clear`, - errors: { - 401: `Not authorized`, - }, - }); - } - - /** - * Apply a tag to all selected files - * Apply a tag to all selected files. - * - * @returns any OK - * @throws ApiError - */ - public static postFiles32(): CancelablePromise { - return __request({ - method: "POST", - path: `/selected/tag`, - errors: { - 401: `Not authorized`, - }, - }); - } - - /** - * Submit all selected files for extraction - * Submit all selected files for extraction. - * - * @param dsId - * @param fileIds - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postFiles33( - dsId: string, - fileIds: string, - requestBody: { - parameters?: Array; - /** - * the extractor Id - */ - extractor?: string; - } - ): CancelablePromise { - return __request({ - method: "POST", - path: `/selected/submit/${dsId}/${fileIds}`, - body: requestBody, - mediaType: "application/json", - errors: { - 401: `Not authorized`, - }, - }); - } -} + }, + }); + } + + /** + * Download a thumbnail + * Download a thumbnail. + * @param id + * @returns any OK + * @throws ApiError + */ + public static postFiles26( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/fileThumbnail/${id}/blob`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Add thumbnail to file + * Attaches an already-existing thumbnail to a file. + * @param id + * @param thumbnailId + * @returns any OK + * @throws ApiError + */ + public static postFiles27( + id: string, + thumbnailId: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/files/${id}/thumbnails/${thumbnailId}`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Get metadata definitions available for a file + * The metadata definitions come from the spaces that the dataset the file + * is part of. Directly or within a folder + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static getFiles17( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/files/${id}/metadataDefinitions`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Get the user-generated metadata of the selected file in an RDF file + * A file is the raw bytes plus metadata. + * @param id + * @returns any OK + * @throws ApiError + */ + public static getFiles18( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/files/rdfUserMetadata/${id}`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Get community-generated metadata of the resource described by the file + * A file is the raw bytes plus metadata. + * @param id + * @returns any OK + * @throws ApiError + */ + public static getFiles19( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/files/${id}/usermetadatajson`, + }); + } + + /** + * Fetches and downloads a particular query + * Fetches and downloads a particular query. + * @param id + * @returns any OK + * @throws ApiError + */ + public static getFiles20( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/queries/${id}`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Add thumbnail to a query image + * Attaches an already-existing thumbnail to a query image. + * @param fileId + * @param thumbnailId + * @returns any OK + * @throws ApiError + */ + public static postFiles28( + fileId: string, + thumbnailId: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/queries/${fileId}/thumbnails/${thumbnailId}`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } + + /** + * Get the list of user-selected files + * Get the list of user-selected files. + * + * @returns any OK + * @throws ApiError + */ + public static getFiles21(): CancelablePromise { + return __request({ + method: 'GET', + path: `/selected`, + errors: { + 401: `Not authorized`, + }, + }); + } + + /** + * Select a specific file + * Select a specific file. + * + * @returns any OK + * @throws ApiError + */ + public static postFiles29(): CancelablePromise { + return __request({ + method: 'POST', + path: `/selected`, + errors: { + 401: `Not authorized`, + }, + }); + } + + /** + * De-select a specific file + * De-select a specific file. + * + * @returns any OK + * @throws ApiError + */ + public static postFiles30(): CancelablePromise { + return __request({ + method: 'POST', + path: `/selected/remove`, + errors: { + 401: `Not authorized`, + }, + }); + } + + /** + * Download all selected files + * Download all selected files. + * + * @returns any OK + * @throws ApiError + */ + public static getFiles22(): CancelablePromise { + return __request({ + method: 'GET', + path: `/selected/files`, + errors: { + 401: `Not authorized`, + }, + }); + } + + /** + * Delete all selected files + * Delete all selected files. + * + * @returns any OK + * @throws ApiError + */ + public static deleteFiles4(): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/selected/files`, + errors: { + 401: `Not authorized`, + }, + }); + } + + /** + * De-select all files + * De-select all files. + * + * @returns any OK + * @throws ApiError + */ + public static postFiles31(): CancelablePromise { + return __request({ + method: 'POST', + path: `/selected/clear`, + errors: { + 401: `Not authorized`, + }, + }); + } + + /** + * Apply a tag to all selected files + * Apply a tag to all selected files. + * + * @returns any OK + * @throws ApiError + */ + public static postFiles32(): CancelablePromise { + return __request({ + method: 'POST', + path: `/selected/tag`, + errors: { + 401: `Not authorized`, + }, + }); + } + + /** + * Submit all selected files for extraction + * Submit all selected files for extraction. + * + * @param dsId + * @param fileIds + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postFiles33( + dsId: string, + fileIds: string, + requestBody: { + parameters?: Array; + /** + * the extractor Id + */ + extractor?: string; + }, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/selected/submit/${dsId}/${fileIds}`, + body: requestBody, + mediaType: 'application/json', + errors: { + 401: `Not authorized`, + }, + }); + } + +} \ No newline at end of file diff --git a/frontend/src/openapi/v1/services/FoldersService.ts b/frontend/src/openapi/v1/services/FoldersService.ts index 61af83e6b..a4016f829 100644 --- a/frontend/src/openapi/v1/services/FoldersService.ts +++ b/frontend/src/openapi/v1/services/FoldersService.ts @@ -1,75 +1,77 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ -import type { CancelablePromise } from "../core/CancelablePromise"; -import { request as __request } from "../core/request"; +import type { CancelablePromise } from '../core/CancelablePromise'; +import { request as __request } from '../core/request'; export class FoldersService { - /** - * Create a Folder - * New empty folder. Requires the ID of the dataset within the folder will - * be created, and the parent information - * - * @param parentDatasetId - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postFolders( - parentDatasetId: string, - requestBody: { - name?: string; - parentId?: string; - parentType?: "dataset" | "folder"; - } - ): CancelablePromise { - return __request({ - method: "POST", - path: `/datasets/${parentDatasetId}/newFolder`, - body: requestBody, - mediaType: "application/json", - }); - } - /** - * Update folder name - * Updates a folder's name - * @param parentDatasetId - * @param folderId - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static putFolders( - parentDatasetId: string, - folderId: string, - requestBody: { - name?: string; - } - ): CancelablePromise { - return __request({ - method: "PUT", - path: `/datasets/${parentDatasetId}/updateName/${folderId}`, - body: requestBody, - mediaType: "application/json", - }); - } + /** + * Create a Folder + * New empty folder. Requires the ID of the dataset within the folder will + * be created, and the parent information + * + * @param parentDatasetId + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postFolders( + parentDatasetId: string, + requestBody: { + name?: string; + parentId?: string; + parentType?: 'dataset' | 'folder'; + }, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/datasets/${parentDatasetId}/newFolder`, + body: requestBody, + mediaType: 'application/json', + }); + } - /** - * Delete a folder - * Deletes all the files and folder within a folder and then deletes itself - * @param parentDatasetId - * @param folderId - * @returns any OK - * @throws ApiError - */ - public static deleteFolders( - parentDatasetId: string, - folderId: string - ): CancelablePromise { - return __request({ - method: "DELETE", - path: `/datasets/${parentDatasetId}/deleteFolder/${folderId}`, - }); - } -} + /** + * Update folder name + * Updates a folder's name + * @param parentDatasetId + * @param folderId + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static putFolders( + parentDatasetId: string, + folderId: string, + requestBody: { + name?: string; + }, + ): CancelablePromise { + return __request({ + method: 'PUT', + path: `/datasets/${parentDatasetId}/updateName/${folderId}`, + body: requestBody, + mediaType: 'application/json', + }); + } + + /** + * Delete a folder + * Deletes all the files and folder within a folder and then deletes itself + * @param parentDatasetId + * @param folderId + * @returns any OK + * @throws ApiError + */ + public static deleteFolders( + parentDatasetId: string, + folderId: string, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/datasets/${parentDatasetId}/deleteFolder/${folderId}`, + }); + } + +} \ No newline at end of file diff --git a/frontend/src/openapi/v1/services/LogosService.ts b/frontend/src/openapi/v1/services/LogosService.ts index a29699b5a..86a4ef858 100644 --- a/frontend/src/openapi/v1/services/LogosService.ts +++ b/frontend/src/openapi/v1/services/LogosService.ts @@ -1,154 +1,173 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ -import type { CancelablePromise } from "../core/CancelablePromise"; -import { request as __request } from "../core/request"; +import type { CancelablePromise } from '../core/CancelablePromise'; +import { request as __request } from '../core/request'; export class LogosService { - /** - * List logos - * List logos known to system - * @returns any OK - * @throws ApiError - */ - public static getLogos(): CancelablePromise { - return __request({ - method: "GET", - path: `/logos`, - }); - } - /** - * Upload file - * Files uploaded to this endpoint will be marked as special files, such as - * favicon.png, logo.png. The file needs to be specified with image. - * - * @returns any OK - * @throws ApiError - */ - public static postLogos(): CancelablePromise { - return __request({ - method: "POST", - path: `/logos`, - }); - } + /** + * List logos + * List logos known to system + * @returns any OK + * @throws ApiError + */ + public static getLogos(): CancelablePromise { + return __request({ + method: 'GET', + path: `/logos`, + }); + } - /** - * Get logo - * Return logo information - * @param id - * @returns any OK - * @throws ApiError - */ - public static getLogos1(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/logos/${id}`, - }); - } + /** + * Upload file + * Files uploaded to this endpoint will be marked as special files, such as + * favicon.png, logo.png. The file needs to be specified with image. + * + * @returns any OK + * @throws ApiError + */ + public static postLogos(): CancelablePromise { + return __request({ + method: 'POST', + path: `/logos`, + }); + } - /** - * Set logo - * Updates logo information - * @param id - * @returns any OK - * @throws ApiError - */ - public static putLogos(id: string): CancelablePromise { - return __request({ - method: "PUT", - path: `/logos/${id}`, - }); - } + /** + * Get logo + * Return logo information + * @param id + * @returns any OK + * @throws ApiError + */ + public static getLogos1( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/logos/${id}`, + }); + } - /** - * Delete file - * Delete a static file - * @param id - * @returns any OK - * @throws ApiError - */ - public static deleteLogos(id: string): CancelablePromise { - return __request({ - method: "DELETE", - path: `/logos/${id}`, - }); - } + /** + * Set logo + * Updates logo information + * @param id + * @returns any OK + * @throws ApiError + */ + public static putLogos( + id: string, + ): CancelablePromise { + return __request({ + method: 'PUT', + path: `/logos/${id}`, + }); + } - /** - * Download file - * Download a static file, or the alternate file - * @param id - * @returns any OK - * @throws ApiError - */ - public static getLogos2(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/logos/${id}/blob`, - }); - } + /** + * Delete file + * Delete a static file + * @param id + * @returns any OK + * @throws ApiError + */ + public static deleteLogos( + id: string, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/logos/${id}`, + }); + } - /** - * Get logo - * Return logo information - * @param path - * @param name - * @returns any OK - * @throws ApiError - */ - public static getLogos3(path: string, name: string): CancelablePromise { - return __request({ - method: "GET", - path: `/logos/${path}/${name}`, - }); - } + /** + * Download file + * Download a static file, or the alternate file + * @param id + * @returns any OK + * @throws ApiError + */ + public static getLogos2( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/logos/${id}/blob`, + }); + } - /** - * Set logo - * Updates logo information - * @param path - * @param name - * @returns any OK - * @throws ApiError - */ - public static putLogos1(path: string, name: string): CancelablePromise { - return __request({ - method: "PUT", - path: `/logos/${path}/${name}`, - }); - } + /** + * Get logo + * Return logo information + * @param path + * @param name + * @returns any OK + * @throws ApiError + */ + public static getLogos3( + path: string, + name: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/logos/${path}/${name}`, + }); + } - /** - * Delete file - * Delete a static file - * @param path - * @param name - * @returns any OK - * @throws ApiError - */ - public static deleteLogos1( - path: string, - name: string - ): CancelablePromise { - return __request({ - method: "DELETE", - path: `/logos/${path}/${name}`, - }); - } + /** + * Set logo + * Updates logo information + * @param path + * @param name + * @returns any OK + * @throws ApiError + */ + public static putLogos1( + path: string, + name: string, + ): CancelablePromise { + return __request({ + method: 'PUT', + path: `/logos/${path}/${name}`, + }); + } - /** - * Download file - * Download a static file, or the alternate file - * @param path - * @param name - * @returns any OK - * @throws ApiError - */ - public static getLogos4(path: string, name: string): CancelablePromise { - return __request({ - method: "GET", - path: `/logos/${path}/${name}/blob`, - }); - } -} + /** + * Delete file + * Delete a static file + * @param path + * @param name + * @returns any OK + * @throws ApiError + */ + public static deleteLogos1( + path: string, + name: string, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/logos/${path}/${name}`, + }); + } + + /** + * Download file + * Download a static file, or the alternate file + * @param path + * @param name + * @returns any OK + * @throws ApiError + */ + public static getLogos4( + path: string, + name: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/logos/${path}/${name}/blob`, + }); + } + +} \ No newline at end of file diff --git a/frontend/src/openapi/v1/services/MetadataService.ts b/frontend/src/openapi/v1/services/MetadataService.ts index 0c618b255..cc3865f98 100644 --- a/frontend/src/openapi/v1/services/MetadataService.ts +++ b/frontend/src/openapi/v1/services/MetadataService.ts @@ -1,201 +1,215 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ -import type { CancelablePromise } from "../core/CancelablePromise"; -import { request as __request } from "../core/request"; +import type { CancelablePromise } from '../core/CancelablePromise'; +import { request as __request } from '../core/request'; export class MetadataService { - /** - * Delete the metadata by id. - * @param id - * @returns any OK - * @throws ApiError - */ - public static deleteMetadata(id: string): CancelablePromise { - return __request({ - method: "DELETE", - path: `/metadata.jsonld/${id}`, - errors: { - 400: `Invalid Metadata`, - }, - }); - } - /** - * Get set of metadata fields containing filter substring for autocomplete. - * @param filter - * @returns any OK - * @throws ApiError - */ - public static getMetadata(filter: string): CancelablePromise { - return __request({ - method: "GET", - path: `/metadata/autocompletenames`, - query: { - filter: filter, - }, - }); - } + /** + * Delete the metadata by id. + * @param id + * @returns any OK + * @throws ApiError + */ + public static deleteMetadata( + id: string, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/metadata.jsonld/${id}`, + errors: { + 400: `Invalid Metadata`, + }, + }); + } - /** - * Get Vocabulary definitions with distinct names. - * @returns any OK - * @throws ApiError - */ - public static getMetadata1(): CancelablePromise { - return __request({ - method: "GET", - path: `/metadata/distinctdefinitions`, - }); - } + /** + * Get set of metadata fields containing filter substring for autocomplete. + * @param filter + * @returns any OK + * @throws ApiError + */ + public static getMetadata( + filter: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/metadata/autocompletenames`, + query: { + 'filter': filter, + }, + }); + } - /** - * Get Vocabulary definitions. - * @returns any OK - * @throws ApiError - */ - public static getMetadata2(): CancelablePromise { - return __request({ - method: "GET", - path: `/metadata/definitions`, - }); - } + /** + * Get Vocabulary definitions with distinct names. + * @returns any OK + * @throws ApiError + */ + public static getMetadata1(): CancelablePromise { + return __request({ + method: 'GET', + path: `/metadata/distinctdefinitions`, + }); + } - /** - * Get Vocabulary definitions by id. - * @param id - * @returns any OK - * @throws ApiError - */ - public static getMetadata3(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/metadata/definitions/${id}`, - }); - } + /** + * Get Vocabulary definitions. + * @returns any OK + * @throws ApiError + */ + public static getMetadata2(): CancelablePromise { + return __request({ + method: 'GET', + path: `/metadata/definitions`, + }); + } - /** - * List all Standard Vocabularies - * Retrieve all Standard Vocabulary objects. - * - * @returns any OK - * @throws ApiError - */ - public static getMetadata4(): CancelablePromise { - return __request({ - method: "GET", - path: `/standardvocab`, - errors: { - 400: `Bad request`, - 401: `Not authorized`, - 403: `Forbidden`, - }, - }); - } + /** + * Get Vocabulary definitions by id. + * @param id + * @returns any OK + * @throws ApiError + */ + public static getMetadata3( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/metadata/definitions/${id}`, + }); + } - /** - * Create a new Standard Vocabulary - * Requires that the request body contains a List[String] of terms. - * - * @param requestBody The list of terms to add to this vocabulary. - * @returns any OK - * @throws ApiError - */ - public static postMetadata( - requestBody: Array - ): CancelablePromise { - return __request({ - method: "POST", - path: `/standardvocab`, - body: requestBody, - mediaType: "application/json", - errors: { - 400: `Bad request`, - 401: `Not authorized`, - 403: `Forbidden`, - }, - }); - } + /** + * List all Standard Vocabularies + * Retrieve all Standard Vocabulary objects. + * + * @returns any OK + * @throws ApiError + */ + public static getMetadata4(): CancelablePromise { + return __request({ + method: 'GET', + path: `/standardvocab`, + errors: { + 400: `Bad request`, + 401: `Not authorized`, + 403: `Forbidden`, + }, + }); + } - /** - * Get a specific Standard Vocabulary - * Lookup and return an existing Standard Vocabulary using its ID. - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static getMetadata5(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/standardvocab/${id}`, - errors: { - 400: `Bad request`, - 401: `Not authorized`, - 403: `Forbidden`, - }, - }); - } + /** + * Create a new Standard Vocabulary + * Requires that the request body contains a List[String] of terms. + * + * @param requestBody The list of terms to add to this vocabulary. + * @returns any OK + * @throws ApiError + */ + public static postMetadata( + requestBody: Array, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/standardvocab`, + body: requestBody, + mediaType: 'application/json', + errors: { + 400: `Bad request`, + 401: `Not authorized`, + 403: `Forbidden`, + }, + }); + } - /** - * Update a Standard Vocabulary - * Requires that the request body contains a List[String] of terms. - * - * @param id - * @param requestBody The list of terms to add to this vocabulary. - * @returns any OK - * @throws ApiError - */ - public static putMetadata( - id: string, - requestBody: Array - ): CancelablePromise { - return __request({ - method: "PUT", - path: `/standardvocab/${id}`, - body: requestBody, - mediaType: "application/json", - errors: { - 400: `Bad request`, - 401: `Not authorized`, - 403: `Forbidden`, - }, - }); - } + /** + * Get a specific Standard Vocabulary + * Lookup and return an existing Standard Vocabulary using its ID. + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static getMetadata5( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/standardvocab/${id}`, + errors: { + 400: `Bad request`, + 401: `Not authorized`, + 403: `Forbidden`, + }, + }); + } - /** - * Delete a Standard Vocabulary - * Lookup and remove a Standard Vocabulary using its ID. - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static deleteMetadata1(id: string): CancelablePromise { - return __request({ - method: "DELETE", - path: `/standardvocab/${id}`, - }); - } + /** + * Update a Standard Vocabulary + * Requires that the request body contains a List[String] of terms. + * + * @param id + * @param requestBody The list of terms to add to this vocabulary. + * @returns any OK + * @throws ApiError + */ + public static putMetadata( + id: string, + requestBody: Array, + ): CancelablePromise { + return __request({ + method: 'PUT', + path: `/standardvocab/${id}`, + body: requestBody, + mediaType: 'application/json', + errors: { + 400: `Bad request`, + 401: `Not authorized`, + 403: `Forbidden`, + }, + }); + } - /** - * Get the terms list of a specific Standard Vocabulary - * Lookup and return the term list of an existing Standard Vocabulary - * using its ID. - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static getMetadata6(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/standardvocab/${id}/terms`, - errors: { - 400: `Bad request`, - 401: `Not authorized`, - 403: `Forbidden`, - }, - }); - } -} + /** + * Delete a Standard Vocabulary + * Lookup and remove a Standard Vocabulary using its ID. + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static deleteMetadata1( + id: string, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/standardvocab/${id}`, + }); + } + + /** + * Get the terms list of a specific Standard Vocabulary + * Lookup and return the term list of an existing Standard Vocabulary + * using its ID. + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static getMetadata6( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/standardvocab/${id}/terms`, + errors: { + 400: `Bad request`, + 401: `Not authorized`, + 403: `Forbidden`, + }, + }); + } + +} \ No newline at end of file diff --git a/frontend/src/openapi/v1/services/PreviewsService.ts b/frontend/src/openapi/v1/services/PreviewsService.ts index ec4c2ff21..9ff128c2a 100644 --- a/frontend/src/openapi/v1/services/PreviewsService.ts +++ b/frontend/src/openapi/v1/services/PreviewsService.ts @@ -1,23 +1,27 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ -import type { CancelablePromise } from "../core/CancelablePromise"; -import { request as __request } from "../core/request"; +import type { CancelablePromise } from '../core/CancelablePromise'; +import { request as __request } from '../core/request'; export class PreviewsService { - /** - * Upload a preview - * Upload a preview - * @param formData - * @returns any OK - * @throws ApiError - */ - public static postPreviews(formData?: any): CancelablePromise { - return __request({ - method: "POST", - path: `/previews`, - formData: formData, - mediaType: "multipart/form-data", - }); - } -} + + /** + * Upload a preview + * Upload a preview + * @param formData + * @returns any OK + * @throws ApiError + */ + public static postPreviews( + formData?: any, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/previews`, + formData: formData, + mediaType: 'multipart/form-data', + }); + } + +} \ No newline at end of file diff --git a/frontend/src/openapi/v1/services/ProxyService.ts b/frontend/src/openapi/v1/services/ProxyService.ts index 3c381aef0..a67d0bb43 100644 --- a/frontend/src/openapi/v1/services/ProxyService.ts +++ b/frontend/src/openapi/v1/services/ProxyService.ts @@ -1,187 +1,193 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ -import type { CancelablePromise } from "../core/CancelablePromise"; -import { request as __request } from "../core/request"; +import type { CancelablePromise } from '../core/CancelablePromise'; +import { request as __request } from '../core/request'; export class ProxyService { - /** - * Proxy a GET request to an endpoint specified in the proxy's configuration. - * Access to the endpoint will be restricted by Clowder authentication. - * @param endpointKey - * @returns void - * @throws ApiError - */ - public static getProxy(endpointKey: string): CancelablePromise { - return __request({ - method: "GET", - path: `/proxy/${endpointKey}`, - errors: { - 401: `Not authorized`, - 404: `Endpoint not configured`, - }, - }); - } - /** - * Proxy a PUT request to an endpoint specified in the proxy's configuration. - * Access to the endpoint will be restricted by Clowder authentication. - * @param endpointKey - * @param requestBody The body of the PUT request to proxy. - * @returns void - * @throws ApiError - */ - public static putProxy( - endpointKey: string, - requestBody?: any - ): CancelablePromise { - return __request({ - method: "PUT", - path: `/proxy/${endpointKey}`, - body: requestBody, - mediaType: "application/json", - errors: { - 401: `Not authorized`, - 404: `Endpoint not configured`, - }, - }); - } + /** + * Proxy a GET request to an endpoint specified in the proxy's configuration. + * Access to the endpoint will be restricted by Clowder authentication. + * @param endpointKey + * @returns void + * @throws ApiError + */ + public static getProxy( + endpointKey: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/proxy/${endpointKey}`, + errors: { + 401: `Not authorized`, + 404: `Endpoint not configured`, + }, + }); + } - /** - * Proxy a POST request to an endpoint specified in the proxy's configuration. - * Access to the endpoint will be restricted by Clowder authentication. - * @param endpointKey - * @param requestBody The body of the POST request to proxy. - * @returns void - * @throws ApiError - */ - public static postProxy( - endpointKey: string, - requestBody?: any - ): CancelablePromise { - return __request({ - method: "POST", - path: `/proxy/${endpointKey}`, - body: requestBody, - mediaType: "application/json", - errors: { - 401: `Not authorized`, - 404: `Endpoint not configured`, - }, - }); - } + /** + * Proxy a PUT request to an endpoint specified in the proxy's configuration. + * Access to the endpoint will be restricted by Clowder authentication. + * @param endpointKey + * @param requestBody The body of the PUT request to proxy. + * @returns void + * @throws ApiError + */ + public static putProxy( + endpointKey: string, + requestBody?: any, + ): CancelablePromise { + return __request({ + method: 'PUT', + path: `/proxy/${endpointKey}`, + body: requestBody, + mediaType: 'application/json', + errors: { + 401: `Not authorized`, + 404: `Endpoint not configured`, + }, + }); + } - /** - * Proxy a DELETE request to an endpoint specified in the proxy's configuration. - * Access to the endpoint will be restricted by Clowder authentication. - * @param endpointKey - * @returns void - * @throws ApiError - */ - public static deleteProxy(endpointKey: string): CancelablePromise { - return __request({ - method: "DELETE", - path: `/proxy/${endpointKey}`, - errors: { - 401: `Not authorized`, - 404: `Endpoint not configured`, - }, - }); - } + /** + * Proxy a POST request to an endpoint specified in the proxy's configuration. + * Access to the endpoint will be restricted by Clowder authentication. + * @param endpointKey + * @param requestBody The body of the POST request to proxy. + * @returns void + * @throws ApiError + */ + public static postProxy( + endpointKey: string, + requestBody?: any, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/proxy/${endpointKey}`, + body: requestBody, + mediaType: 'application/json', + errors: { + 401: `Not authorized`, + 404: `Endpoint not configured`, + }, + }); + } - /** - * Proxy a GET request to an endpoint specified in the proxy's configuration. - * Access to the endpoint will be restricted by Clowder authentication. - * @param endpointKey - * @param pathSuffix - * @returns void - * @throws ApiError - */ - public static getProxy1( - endpointKey: string, - pathSuffix: string - ): CancelablePromise { - return __request({ - method: "GET", - path: `/proxy/${endpointKey}/${pathSuffix}`, - errors: { - 401: `Not authorized`, - 404: `Endpoint not configured`, - }, - }); - } + /** + * Proxy a DELETE request to an endpoint specified in the proxy's configuration. + * Access to the endpoint will be restricted by Clowder authentication. + * @param endpointKey + * @returns void + * @throws ApiError + */ + public static deleteProxy( + endpointKey: string, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/proxy/${endpointKey}`, + errors: { + 401: `Not authorized`, + 404: `Endpoint not configured`, + }, + }); + } - /** - * Proxy a PUT request to an endpoint specified in the proxy's configuration. - * Access to the endpoint will be restricted by Clowder authentication. - * @param endpointKey - * @param pathSuffix - * @param requestBody The body of the PUT request to proxy. - * @returns void - * @throws ApiError - */ - public static putProxy1( - endpointKey: string, - pathSuffix: string, - requestBody?: any - ): CancelablePromise { - return __request({ - method: "PUT", - path: `/proxy/${endpointKey}/${pathSuffix}`, - body: requestBody, - mediaType: "application/json", - errors: { - 401: `Not authorized`, - 404: `Endpoint not configured`, - }, - }); - } + /** + * Proxy a GET request to an endpoint specified in the proxy's configuration. + * Access to the endpoint will be restricted by Clowder authentication. + * @param endpointKey + * @param pathSuffix + * @returns void + * @throws ApiError + */ + public static getProxy1( + endpointKey: string, + pathSuffix: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/proxy/${endpointKey}/${pathSuffix}`, + errors: { + 401: `Not authorized`, + 404: `Endpoint not configured`, + }, + }); + } - /** - * Proxy a POST request to an endpoint specified in the proxy's configuration. - * Access to the endpoint will be restricted by Clowder authentication. - * @param endpointKey - * @param pathSuffix - * @param requestBody The body of the POST request to proxy. - * @returns void - * @throws ApiError - */ - public static postProxy1( - endpointKey: string, - pathSuffix: string, - requestBody?: any - ): CancelablePromise { - return __request({ - method: "POST", - path: `/proxy/${endpointKey}/${pathSuffix}`, - body: requestBody, - mediaType: "application/json", - errors: { - 401: `Not authorized`, - 404: `Endpoint not configured`, - }, - }); - } + /** + * Proxy a PUT request to an endpoint specified in the proxy's configuration. + * Access to the endpoint will be restricted by Clowder authentication. + * @param endpointKey + * @param pathSuffix + * @param requestBody The body of the PUT request to proxy. + * @returns void + * @throws ApiError + */ + public static putProxy1( + endpointKey: string, + pathSuffix: string, + requestBody?: any, + ): CancelablePromise { + return __request({ + method: 'PUT', + path: `/proxy/${endpointKey}/${pathSuffix}`, + body: requestBody, + mediaType: 'application/json', + errors: { + 401: `Not authorized`, + 404: `Endpoint not configured`, + }, + }); + } - /** - * Proxy a DELETE request to an endpoint specified in the proxy's configuration. - * Access to the endpoint will be restricted by Clowder authentication. - * @param endpointKey - * @param pathSuffix - * @returns void - * @throws ApiError - */ - public static deleteProxy1( - endpointKey: string, - pathSuffix: string - ): CancelablePromise { - return __request({ - method: "DELETE", - path: `/proxy/${endpointKey}/${pathSuffix}`, - errors: { - 401: `Not authorized`, - 404: `Endpoint not configured`, - }, - }); - } -} + /** + * Proxy a POST request to an endpoint specified in the proxy's configuration. + * Access to the endpoint will be restricted by Clowder authentication. + * @param endpointKey + * @param pathSuffix + * @param requestBody The body of the POST request to proxy. + * @returns void + * @throws ApiError + */ + public static postProxy1( + endpointKey: string, + pathSuffix: string, + requestBody?: any, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/proxy/${endpointKey}/${pathSuffix}`, + body: requestBody, + mediaType: 'application/json', + errors: { + 401: `Not authorized`, + 404: `Endpoint not configured`, + }, + }); + } + + /** + * Proxy a DELETE request to an endpoint specified in the proxy's configuration. + * Access to the endpoint will be restricted by Clowder authentication. + * @param endpointKey + * @param pathSuffix + * @returns void + * @throws ApiError + */ + public static deleteProxy1( + endpointKey: string, + pathSuffix: string, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/proxy/${endpointKey}/${pathSuffix}`, + errors: { + 401: `Not authorized`, + 404: `Endpoint not configured`, + }, + }); + } + +} \ No newline at end of file diff --git a/frontend/src/openapi/v1/services/ReportingService.ts b/frontend/src/openapi/v1/services/ReportingService.ts index 495a5520a..ac861e204 100644 --- a/frontend/src/openapi/v1/services/ReportingService.ts +++ b/frontend/src/openapi/v1/services/ReportingService.ts @@ -1,127 +1,129 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ -import type { CancelablePromise } from "../core/CancelablePromise"; -import { request as __request } from "../core/request"; +import type { CancelablePromise } from '../core/CancelablePromise'; +import { request as __request } from '../core/request'; export class ReportingService { - /** - * Download usage metrics report for files, datasets and collections as CSV. - * Download usage metrics report for files, datasets and collections as a CSV file. - * Must be a server admin to access the report. - * - * @returns binary A CSV with usage metrics - * @throws ApiError - */ - public static getReporting(): CancelablePromise { - return __request({ - method: "GET", - path: `/reports/metrics`, - errors: { - 401: `Not authorized`, - }, - }); - } - /** - * Download usage metrics report for files as CSV. - * Download usage metrics report for files as a CSV file. - * Must be a server admin to access the report. Supports ISO8601 date range filters. - * - * @param since - * @param until - * @returns binary A CSV with usage metrics - * @throws ApiError - */ - public static getReporting1( - since?: string, - until?: string - ): CancelablePromise { - return __request({ - method: "GET", - path: `/reports/metrics/files`, - query: { - since: since, - until: until, - }, - errors: { - 401: `Not authorized`, - }, - }); - } + /** + * Download usage metrics report for files, datasets and collections as CSV. + * Download usage metrics report for files, datasets and collections as a CSV file. + * Must be a server admin to access the report. + * + * @returns binary A CSV with usage metrics + * @throws ApiError + */ + public static getReporting(): CancelablePromise { + return __request({ + method: 'GET', + path: `/reports/metrics`, + errors: { + 401: `Not authorized`, + }, + }); + } - /** - * Download usage metrics report for datasets as CSV. - * Download usage metrics report for datasets as a CSV file. - * Must be a server admin to access the report. Supports ISO8601 date range filters. - * - * @param since - * @param until - * @returns binary A CSV with usage metrics - * @throws ApiError - */ - public static getReporting2( - since?: string, - until?: string - ): CancelablePromise { - return __request({ - method: "GET", - path: `/reports/metrics/datasets`, - query: { - since: since, - until: until, - }, - errors: { - 401: `Not authorized`, - }, - }); - } + /** + * Download usage metrics report for files as CSV. + * Download usage metrics report for files as a CSV file. + * Must be a server admin to access the report. Supports ISO8601 date range filters. + * + * @param since + * @param until + * @returns binary A CSV with usage metrics + * @throws ApiError + */ + public static getReporting1( + since?: string, + until?: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/reports/metrics/files`, + query: { + 'since': since, + 'until': until, + }, + errors: { + 401: `Not authorized`, + }, + }); + } - /** - * Download usage metrics report for collections as CSV. - * Download usage metrics report for collections as a CSV file. - * Must be a server admin to access the report. - * - * @returns binary A CSV with usage metrics - * @throws ApiError - */ - public static getReporting3(): CancelablePromise { - return __request({ - method: "GET", - path: `/reports/metrics/collections`, - errors: { - 401: `Not authorized`, - }, - }); - } + /** + * Download usage metrics report for datasets as CSV. + * Download usage metrics report for datasets as a CSV file. + * Must be a server admin to access the report. Supports ISO8601 date range filters. + * + * @param since + * @param until + * @returns binary A CSV with usage metrics + * @throws ApiError + */ + public static getReporting2( + since?: string, + until?: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/reports/metrics/datasets`, + query: { + 'since': since, + 'until': until, + }, + errors: { + 401: `Not authorized`, + }, + }); + } - /** - * Download storage metrics report for spaces as CSV. - * Download storage metrics report for files in a space as a CSV file. Space id is optional parameter. - * Must be a server admin to access the report. Supports ISO8601 date range filters. - * - * @param space - * @param since - * @param until - * @returns binary A CSV with usage metrics - * @throws ApiError - */ - public static getReporting4( - space?: string, - since?: string, - until?: string - ): CancelablePromise { - return __request({ - method: "GET", - path: `/reports/storage/spaces`, - query: { - space: space, - since: since, - until: until, - }, - errors: { - 401: `Not authorized`, - }, - }); - } -} + /** + * Download usage metrics report for collections as CSV. + * Download usage metrics report for collections as a CSV file. + * Must be a server admin to access the report. + * + * @returns binary A CSV with usage metrics + * @throws ApiError + */ + public static getReporting3(): CancelablePromise { + return __request({ + method: 'GET', + path: `/reports/metrics/collections`, + errors: { + 401: `Not authorized`, + }, + }); + } + + /** + * Download storage metrics report for spaces as CSV. + * Download storage metrics report for files in a space as a CSV file. Space id is optional parameter. + * Must be a server admin to access the report. Supports ISO8601 date range filters. + * + * @param space + * @param since + * @param until + * @returns binary A CSV with usage metrics + * @throws ApiError + */ + public static getReporting4( + space?: string, + since?: string, + until?: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/reports/storage/spaces`, + query: { + 'space': space, + 'since': since, + 'until': until, + }, + errors: { + 401: `Not authorized`, + }, + }); + } + +} \ No newline at end of file diff --git a/frontend/src/openapi/v1/services/SearchService.ts b/frontend/src/openapi/v1/services/SearchService.ts index a64ea44d8..86d8bc8c7 100644 --- a/frontend/src/openapi/v1/services/SearchService.ts +++ b/frontend/src/openapi/v1/services/SearchService.ts @@ -1,65 +1,67 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ -import type { CancelablePromise } from "../core/CancelablePromise"; -import { request as __request } from "../core/request"; +import type { CancelablePromise } from '../core/CancelablePromise'; +import { request as __request } from '../core/request'; export class SearchService { - /** - * Perform a basic search - * Return files, datasets and/or collections matching query criteria. Search - * can be filtered by a specific resource ID, limited to tags or specific - * metadata fields. - * - * @param query String term to search for. Can include regular expressions using Elasticsearch syntax. - * @param resourceType Restrict search results to "file", "dataset", or "collection". - * @param datasetid Return only resources belonging to a specific dataset. - * @param collectionid Return only resources belonging to a specific collection. - * @param spaceid Return only resources belonging to a specific space. - * @param folderid Return only resources belonging to a specific folder. - * @param field Restrict search to a specific metadata field, e.g. "Alternative Title". - * @param tag Search for resources with a specific tag. - * @param from Starting index of first result; useful for pagination. - * @param size Number of search results to include; useful for pagination. - * @param page An alternative to "from" for pagination. Return the Nth page assuming "size" items per page. - * @param sort A date or numeric field to sort by. If order is given but no field specified, created date is used. - * @param order Whether to scored in asc (ascending) or desc (descending) order. If a field is given without an order, asc is used. - * @returns any OK - * @throws ApiError - */ - public static getSearch( - query?: string, - resourceType?: string, - datasetid?: string, - collectionid?: string, - spaceid?: string, - folderid?: string, - field?: string, - tag?: string, - from?: number, - size?: number, - page?: number, - sort?: string, - order: "asc" | "desc" = "asc" - ): CancelablePromise { - return __request({ - method: "GET", - path: `/search`, - query: { - query: query, - resource_type: resourceType, - datasetid: datasetid, - collectionid: collectionid, - spaceid: spaceid, - folderid: folderid, - field: field, - tag: tag, - from: from, - size: size, - page: page, - sort: sort, - order: order, - }, - }); - } -} + + /** + * Perform a basic search + * Return files, datasets and/or collections matching query criteria. Search + * can be filtered by a specific resource ID, limited to tags or specific + * metadata fields. + * + * @param query String term to search for. Can include regular expressions using Elasticsearch syntax. + * @param resourceType Restrict search results to "file", "dataset", or "collection". + * @param datasetid Return only resources belonging to a specific dataset. + * @param collectionid Return only resources belonging to a specific collection. + * @param spaceid Return only resources belonging to a specific space. + * @param folderid Return only resources belonging to a specific folder. + * @param field Restrict search to a specific metadata field, e.g. "Alternative Title". + * @param tag Search for resources with a specific tag. + * @param from Starting index of first result; useful for pagination. + * @param size Number of search results to include; useful for pagination. + * @param page An alternative to "from" for pagination. Return the Nth page assuming "size" items per page. + * @param sort A date or numeric field to sort by. If order is given but no field specified, created date is used. + * @param order Whether to scored in asc (ascending) or desc (descending) order. If a field is given without an order, asc is used. + * @returns any OK + * @throws ApiError + */ + public static getSearch( + query?: string, + resourceType?: string, + datasetid?: string, + collectionid?: string, + spaceid?: string, + folderid?: string, + field?: string, + tag?: string, + from?: number, + size?: number, + page?: number, + sort?: string, + order: 'asc' | 'desc' = 'asc', + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/search`, + query: { + 'query': query, + 'resource_type': resourceType, + 'datasetid': datasetid, + 'collectionid': collectionid, + 'spaceid': spaceid, + 'folderid': folderid, + 'field': field, + 'tag': tag, + 'from': from, + 'size': size, + 'page': page, + 'sort': sort, + 'order': order, + }, + }); + } + +} \ No newline at end of file diff --git a/frontend/src/openapi/v1/services/SectionService.ts b/frontend/src/openapi/v1/services/SectionService.ts index a2d8d5637..fdcf9e2f0 100644 --- a/frontend/src/openapi/v1/services/SectionService.ts +++ b/frontend/src/openapi/v1/services/SectionService.ts @@ -1,192 +1,208 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ -import type { CancelablePromise } from "../core/CancelablePromise"; -import { request as __request } from "../core/request"; +import type { CancelablePromise } from '../core/CancelablePromise'; +import { request as __request } from '../core/request'; export class SectionService { - /** - * Add a section. - * @param requestBody The body of the POST request. - * @returns void - * @throws ApiError - */ - public static postSection(requestBody: { - file_id: string; - }): CancelablePromise { - return __request({ - method: "POST", - path: `/sections`, - body: requestBody, - mediaType: "application/json", - errors: { - 400: `No "file_id" specified, request body.`, - }, - }); - } - /** - * Get a section. - * @param id - * @returns any OK - * @throws ApiError - */ - public static getSection(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/sections/${id}`, - }); - } + /** + * Add a section. + * @param requestBody The body of the POST request. + * @returns void + * @throws ApiError + */ + public static postSection( + requestBody: { + file_id: string; + }, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/sections`, + body: requestBody, + mediaType: 'application/json', + errors: { + 400: `No "file_id" specified, request body.`, + }, + }); + } - /** - * Delete a section. - * @param id - * @returns any OK - * @throws ApiError - */ - public static deleteSection(id: string): CancelablePromise { - return __request({ - method: "DELETE", - path: `/sections/${id}`, - }); - } + /** + * Get a section. + * @param id + * @returns any OK + * @throws ApiError + */ + public static getSection( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/sections/${id}`, + }); + } - /** - * Add comment to a section. - * @param id - * @param requestBody The body of the POST request. - * @returns any OK - * @throws ApiError - */ - public static postSection1( - id: string, - requestBody: { - text: string; - } - ): CancelablePromise { - return __request({ - method: "POST", - path: `/sections/${id}/comments`, - body: requestBody, - mediaType: "application/json", - }); - } + /** + * Delete a section. + * @param id + * @returns any OK + * @throws ApiError + */ + public static deleteSection( + id: string, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/sections/${id}`, + }); + } - /** - * Add description to a section. - * @param id - * @param requestBody The body of the POST request. - * @returns any OK - * @throws ApiError - */ - public static postSection2( - id: string, - requestBody: { - description: string; - } - ): CancelablePromise { - return __request({ - method: "POST", - path: `/sections/${id}/description`, - body: requestBody, - mediaType: "application/json", - errors: { - 400: `no section description specified.`, - }, - }); - } + /** + * Add comment to a section. + * @param id + * @param requestBody The body of the POST request. + * @returns any OK + * @throws ApiError + */ + public static postSection1( + id: string, + requestBody: { + text: string; + }, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/sections/${id}/comments`, + body: requestBody, + mediaType: 'application/json', + }); + } - /** - * Add thumbnail to section. - * @param id - * @param thumbnailId - * @returns any OK - * @throws ApiError - */ - public static postSection3( - id: string, - thumbnailId: string - ): CancelablePromise { - return __request({ - method: "POST", - path: `/sections/${id}/thumbnails/${thumbnailId}`, - errors: { - 400: `Thumbnail not found.`, - }, - }); - } + /** + * Add description to a section. + * @param id + * @param requestBody The body of the POST request. + * @returns any OK + * @throws ApiError + */ + public static postSection2( + id: string, + requestBody: { + description: string; + }, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/sections/${id}/description`, + body: requestBody, + mediaType: 'application/json', + errors: { + 400: `no section description specified.`, + }, + }); + } - /** - * Get tags of a section. - * @param id - * @returns any OK - * @throws ApiError - */ - public static getSection1(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/sections/${id}/tags`, - }); - } + /** + * Add thumbnail to section. + * @param id + * @param thumbnailId + * @returns any OK + * @throws ApiError + */ + public static postSection3( + id: string, + thumbnailId: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/sections/${id}/thumbnails/${thumbnailId}`, + errors: { + 400: `Thumbnail not found.`, + }, + }); + } - /** - * Add tags to section. - * @param id - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postSection4( - id: string, - requestBody: { - tags: Array; - } - ): CancelablePromise { - return __request({ - method: "POST", - path: `/sections/${id}/tags`, - body: requestBody, - mediaType: "application/json", - }); - } + /** + * Get tags of a section. + * @param id + * @returns any OK + * @throws ApiError + */ + public static getSection1( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/sections/${id}/tags`, + }); + } - /** - * Delete tags from section. - * @param id - * @returns any OK - * @throws ApiError - */ - public static deleteSection1(id: string): CancelablePromise { - return __request({ - method: "DELETE", - path: `/sections/${id}/tags`, - }); - } + /** + * Add tags to section. + * @param id + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postSection4( + id: string, + requestBody: { + tags: Array; + }, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/sections/${id}/tags`, + body: requestBody, + mediaType: 'application/json', + }); + } - /** - * @deprecated - * remove a tag from a section - * @param id - * @returns any OK - * @throws ApiError - */ - public static postSection5(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/sections/${id}/tags/remove`, - }); - } + /** + * Delete tags from section. + * @param id + * @returns any OK + * @throws ApiError + */ + public static deleteSection1( + id: string, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/sections/${id}/tags`, + }); + } - /** - * Delete all tags from section. - * @param id - * @returns any OK - * @throws ApiError - */ - public static postSection6(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/sections/${id}/tags/remove_all`, - }); - } -} + /** + * @deprecated + * remove a tag from a section + * @param id + * @returns any OK + * @throws ApiError + */ + public static postSection5( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/sections/${id}/tags/remove`, + }); + } + + /** + * Delete all tags from section. + * @param id + * @returns any OK + * @throws ApiError + */ + public static postSection6( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/sections/${id}/tags/remove_all`, + }); + } + +} \ No newline at end of file diff --git a/frontend/src/openapi/v1/services/ServiceService.ts b/frontend/src/openapi/v1/services/ServiceService.ts index 6f6a72420..a0ef53594 100644 --- a/frontend/src/openapi/v1/services/ServiceService.ts +++ b/frontend/src/openapi/v1/services/ServiceService.ts @@ -1,57 +1,59 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ -import type { CancelablePromise } from "../core/CancelablePromise"; -import { request as __request } from "../core/request"; +import type { CancelablePromise } from '../core/CancelablePromise'; +import { request as __request } from '../core/request'; export class ServiceService { - /** - * Fetch current user's info - * Retrieve informatation on the currently logged-in user. - * - * @returns any OK - * @throws ApiError - */ - public static getService(): CancelablePromise { - return __request({ - method: "GET", - path: `/me`, - errors: { - 401: `Not authorized`, - }, - }); - } - /** - * Gets the status of the system - * Returns a nested JSON object that contains the status of Clowder. - * This includes health checks, enabled plugins, as well as helpful debug - * information, such as the current size of the reindex queue. - * - * @returns any OK - * @throws ApiError - */ - public static getService1(): CancelablePromise { - return __request({ - method: "GET", - path: `/status`, - }); - } + /** + * Fetch current user's info + * Retrieve informatation on the currently logged-in user. + * + * @returns any OK + * @throws ApiError + */ + public static getService(): CancelablePromise { + return __request({ + method: 'GET', + path: `/me`, + errors: { + 401: `Not authorized`, + }, + }); + } - /** - * Fetch children of the given node in the file tree - * Fetch children of the given node in the file tree. - * - * @returns any OK - * @throws ApiError - */ - public static getService2(): CancelablePromise { - return __request({ - method: "GET", - path: `/tree/getChildrenOfNode`, - errors: { - 401: `Not authorized`, - }, - }); - } -} + /** + * Gets the status of the system + * Returns a nested JSON object that contains the status of Clowder. + * This includes health checks, enabled plugins, as well as helpful debug + * information, such as the current size of the reindex queue. + * + * @returns any OK + * @throws ApiError + */ + public static getService1(): CancelablePromise { + return __request({ + method: 'GET', + path: `/status`, + }); + } + + /** + * Fetch children of the given node in the file tree + * Fetch children of the given node in the file tree. + * + * @returns any OK + * @throws ApiError + */ + public static getService2(): CancelablePromise { + return __request({ + method: 'GET', + path: `/tree/getChildrenOfNode`, + errors: { + 401: `Not authorized`, + }, + }); + } + +} \ No newline at end of file diff --git a/frontend/src/openapi/v1/services/SpacesService.ts b/frontend/src/openapi/v1/services/SpacesService.ts index 39c3a2e00..0eb99596c 100644 --- a/frontend/src/openapi/v1/services/SpacesService.ts +++ b/frontend/src/openapi/v1/services/SpacesService.ts @@ -1,419 +1,443 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ -import type { Space } from "../models/Space"; -import type { CancelablePromise } from "../core/CancelablePromise"; -import { request as __request } from "../core/request"; +import type { Space } from '../models/Space'; +import type { CancelablePromise } from '../core/CancelablePromise'; +import { request as __request } from '../core/request'; export class SpacesService { - /** - * List spaces the user can view - * Retrieves information about spaces - * @param when - * @param title - * @param date - * @param limit default as 12 - * @returns any OK - * @throws ApiError - */ - public static getSpaces( - when?: string, - title?: string, - date?: string, - limit?: number - ): CancelablePromise { - return __request({ - method: "GET", - path: `/spaces`, - query: { - when: when, - title: title, - date: date, - limit: limit, - }, - }); - } - /** - * Create a space - * Spaces are groupings of collections and datasets. - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postSpaces(requestBody: Space): CancelablePromise { - return __request({ - method: "POST", - path: `/spaces`, - body: requestBody, - mediaType: "application/json", - errors: { - 400: `The server could not process your request, this happens for example when + /** + * List spaces the user can view + * Retrieves information about spaces + * @param when + * @param title + * @param date + * @param limit default as 12 + * @returns any OK + * @throws ApiError + */ + public static getSpaces( + when?: string, + title?: string, + date?: string, + limit?: number, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/spaces`, + query: { + 'when': when, + 'title': title, + 'date': date, + 'limit': limit, + }, + }); + } + + /** + * Create a space + * Spaces are groupings of collections and datasets. + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postSpaces( + requestBody: Space, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/spaces`, + body: requestBody, + mediaType: 'application/json', + errors: { + 400: `The server could not process your request, this happens for example when * the id specified is not of the correct form. See the message field for * more information. * `, - }, - }); - } + }, + }); + } + + /** + * Get a space + * Retrieves information about a space + * @param id + * @returns any OK + * @throws ApiError + */ + public static getSpaces1( + id: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/spaces/${id}`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } - /** - * Get a space - * Retrieves information about a space - * @param id - * @returns any OK - * @throws ApiError - */ - public static getSpaces1(id: string): CancelablePromise { - return __request({ - method: "GET", - path: `/spaces/${id}`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } + /** + * Remove a space + * Does not delete the individual datasets and collections in the space. + * @param id + * @returns any OK + * @throws ApiError + */ + public static deleteSpaces( + id: string, + ): CancelablePromise { + return __request({ + method: 'DELETE', + path: `/spaces/${id}`, + }); + } - /** - * Remove a space - * Does not delete the individual datasets and collections in the space. - * @param id - * @returns any OK - * @throws ApiError - */ - public static deleteSpaces(id: string): CancelablePromise { - return __request({ - method: "DELETE", - path: `/spaces/${id}`, - }); - } + /** + * Remove a dataset from a space + * Spaces are groupings of collections and datasets. + * @param spaceId + * @param datasetId + * @returns any OK + * @throws ApiError + */ + public static postSpaces1( + spaceId: string, + datasetId: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/spaces/${spaceId}/removeDataset/${datasetId}`, + }); + } - /** - * Remove a dataset from a space - * Spaces are groupings of collections and datasets. - * @param spaceId - * @param datasetId - * @returns any OK - * @throws ApiError - */ - public static postSpaces1( - spaceId: string, - datasetId: string - ): CancelablePromise { - return __request({ - method: "POST", - path: `/spaces/${spaceId}/removeDataset/${datasetId}`, - }); - } + /** + * List UUIDs of all collections in a space + * Spaces are groupings of collections and datasets. + * @param id + * @param limit default as 12 + * @returns any OK + * @throws ApiError + */ + public static getSpaces2( + id: string, + limit?: number, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/spaces/${id}/collections`, + query: { + 'limit': limit, + }, + errors: { + 404: `The requested resource was not found`, + }, + }); + } - /** - * List UUIDs of all collections in a space - * Spaces are groupings of collections and datasets. - * @param id - * @param limit default as 12 - * @returns any OK - * @throws ApiError - */ - public static getSpaces2(id: string, limit?: number): CancelablePromise { - return __request({ - method: "GET", - path: `/spaces/${id}/collections`, - query: { - limit: limit, - }, - errors: { - 404: `The requested resource was not found`, - }, - }); - } + /** + * List UUIDs of all datasets in a space + * Spaces are groupings of collections and datasets. + * @param id + * @param limit default as 12 + * @returns any OK + * @throws ApiError + */ + public static getSpaces3( + id: string, + limit?: number, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/spaces/${id}/datasets`, + query: { + 'limit': limit, + }, + errors: { + 404: `The requested resource was not found`, + }, + }); + } - /** - * List UUIDs of all datasets in a space - * Spaces are groupings of collections and datasets. - * @param id - * @param limit default as 12 - * @returns any OK - * @throws ApiError - */ - public static getSpaces3(id: string, limit?: number): CancelablePromise { - return __request({ - method: "GET", - path: `/spaces/${id}/datasets`, - query: { - limit: limit, - }, - errors: { - 404: `The requested resource was not found`, - }, - }); - } + /** + * Set the space as verified + * Server Admin Action + * @param id + * @returns any OK + * @throws ApiError + */ + public static putSpaces( + id: string, + ): CancelablePromise { + return __request({ + method: 'PUT', + path: `/spaces/${id}/verify`, + }); + } - /** - * Set the space as verified - * Server Admin Action - * @param id - * @returns any OK - * @throws ApiError - */ - public static putSpaces(id: string): CancelablePromise { - return __request({ - method: "PUT", - path: `/spaces/${id}/verify`, - }); - } + /** + * Associate a collection to a space + * Spaces are groupings of collections and datasets. + * @param spaceId + * @param collectionId + * @returns any OK + * @throws ApiError + */ + public static postSpaces2( + spaceId: string, + collectionId: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/spaces/${spaceId}/addCollectionToSpace/${collectionId}`, + }); + } - /** - * Associate a collection to a space - * Spaces are groupings of collections and datasets. - * @param spaceId - * @param collectionId - * @returns any OK - * @throws ApiError - */ - public static postSpaces2( - spaceId: string, - collectionId: string - ): CancelablePromise { - return __request({ - method: "POST", - path: `/spaces/${spaceId}/addCollectionToSpace/${collectionId}`, - }); - } + /** + * Reject Request + * Reject user's request to the space, remove the request and send email to + * the request user + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static postSpaces3( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/spaces/${id}/rejectRequest`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } - /** - * Reject Request - * Reject user's request to the space, remove the request and send email to - * the request user - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static postSpaces3(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/spaces/${id}/rejectRequest`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } + /** + * Accept Request + * Accept user's request to the space and assign a specific Role, remove + * the request and send email to the request user + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static postSpaces4( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/spaces/${id}/acceptRequest`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } - /** - * Accept Request - * Accept user's request to the space and assign a specific Role, remove - * the request and send email to the request user - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static postSpaces4(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/spaces/${id}/acceptRequest`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } + /** + * Remove a collection from a space + * Spaces are groupings of collections and datasets. + * @param spaceId + * @param collectionId + * @returns any OK + * @throws ApiError + */ + public static postSpaces5( + spaceId: string, + collectionId: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/spaces/${spaceId}/removeCollection/${collectionId}`, + errors: { + 404: `The requested resource was not found`, + }, + }); + } - /** - * Remove a collection from a space - * Spaces are groupings of collections and datasets. - * @param spaceId - * @param collectionId - * @returns any OK - * @throws ApiError - */ - public static postSpaces5( - spaceId: string, - collectionId: string - ): CancelablePromise { - return __request({ - method: "POST", - path: `/spaces/${spaceId}/removeCollection/${collectionId}`, - errors: { - 404: `The requested resource was not found`, - }, - }); - } + /** + * Update the user information associated with a space + * Spaces are groupings of collections and datasets. + * @param id + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postSpaces6( + id: string, + requestBody: { + /** + * A map that contains a role level as a key and a comma separated String of user IDs as the value + */ + rolesandusers?: any; + }, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/spaces/${id}/updateUsers`, + body: requestBody, + mediaType: 'application/json', + errors: { + 400: `rolesandusers data is missing from the updateUsers call.`, + 404: `The requested resource was not found`, + }, + }); + } - /** - * Update the user information associated with a space - * Spaces are groupings of collections and datasets. - * @param id - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postSpaces6( - id: string, - requestBody: { - /** - * A map that contains a role level as a key and a comma separated String of user IDs as the value - */ - rolesandusers?: any; - } - ): CancelablePromise { - return __request({ - method: "POST", - path: `/spaces/${id}/updateUsers`, - body: requestBody, - mediaType: "application/json", - errors: { - 400: `rolesandusers data is missing from the updateUsers call.`, - 404: `The requested resource was not found`, - }, - }); - } + /** + * List spaces the user can add to + * Retrieves a list of spaces that the user has permission to add to + * @param when + * @param title + * @param date + * @param limit default as 12 + * @returns any OK + * @throws ApiError + */ + public static getSpaces4( + when?: string, + title?: string, + date?: string, + limit?: number, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/spaces/canEdit`, + query: { + 'when': when, + 'title': title, + 'date': date, + 'limit': limit, + }, + }); + } - /** - * List spaces the user can add to - * Retrieves a list of spaces that the user has permission to add to - * @param when - * @param title - * @param date - * @param limit default as 12 - * @returns any OK - * @throws ApiError - */ - public static getSpaces4( - when?: string, - title?: string, - date?: string, - limit?: number - ): CancelablePromise { - return __request({ - method: "GET", - path: `/spaces/canEdit`, - query: { - when: when, - title: title, - date: date, - limit: limit, - }, - }); - } + /** + * Remove a user from a space + * Spaces are groupings of collections and datasets. + * @param id + * @param removeUser the UUID of the user + * @returns any OK + * @throws ApiError + */ + public static postSpaces7( + id: string, + removeUser: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/spaces/${id}/removeUser`, + query: { + 'removeUser': removeUser, + }, + }); + } - /** - * Remove a user from a space - * Spaces are groupings of collections and datasets. - * @param id - * @param removeUser the UUID of the user - * @returns any OK - * @throws ApiError - */ - public static postSpaces7( - id: string, - removeUser: string - ): CancelablePromise { - return __request({ - method: "POST", - path: `/spaces/${id}/removeUser`, - query: { - removeUser: removeUser, - }, - }); - } + /** + * Update the information associated with a space + * Spaces are groupings of collections and datasets. + * @param id + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postSpaces8( + id: string, + requestBody: { + name: string; + description: string; + timetolive: string; + enabled: boolean; + access: string; + }, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/spaces/${id}/update`, + body: requestBody, + mediaType: 'application/json', + }); + } - /** - * Update the information associated with a space - * Spaces are groupings of collections and datasets. - * @param id - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postSpaces8( - id: string, - requestBody: { - name: string; - description: string; - timetolive: string; - enabled: boolean; - access: string; - } - ): CancelablePromise { - return __request({ - method: "POST", - path: `/spaces/${id}/update`, - body: requestBody, - mediaType: "application/json", - }); - } + /** + * Unfollow space + * Remove user from space followers and remove space from user followed + * spaces. + * + * @param id + * @returns any OK + * @throws ApiError + */ + public static postSpaces9( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/spaces/${id}/unfollow`, + }); + } - /** - * Unfollow space - * Remove user from space followers and remove space from user followed - * spaces. - * - * @param id - * @returns any OK - * @throws ApiError - */ - public static postSpaces9(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/spaces/${id}/unfollow`, - }); - } + /** + * Follow space + * Add user to space followers and add space to user followed spaces. + * @param id + * @returns any OK + * @throws ApiError + */ + public static postSpaces10( + id: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/spaces/${id}/follow`, + }); + } - /** - * Follow space - * Add user to space followers and add space to user followed spaces. - * @param id - * @returns any OK - * @throws ApiError - */ - public static postSpaces10(id: string): CancelablePromise { - return __request({ - method: "POST", - path: `/spaces/${id}/follow`, - }); - } + /** + * Associate a dataset to a space + * Spaces are groupings of collections and datasets. + * @param spaceId + * @param datasetId + * @returns any OK + * @throws ApiError + */ + public static postSpaces11( + spaceId: string, + datasetId: string, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/spaces/${spaceId}/addDatasetToSpace/${datasetId}`, + }); + } - /** - * Associate a dataset to a space - * Spaces are groupings of collections and datasets. - * @param spaceId - * @param datasetId - * @returns any OK - * @throws ApiError - */ - public static postSpaces11( - spaceId: string, - datasetId: string - ): CancelablePromise { - return __request({ - method: "POST", - path: `/spaces/${spaceId}/addDatasetToSpace/${datasetId}`, - }); - } + /** + * add metadata definition to a space + * If uri is not defined, the uri will be set as + * "http://clowder.ncsa.illinois.edu/metadata/{uuid}#CamelCase" + * + * @param spaceId + * @param requestBody + * @returns any OK + * @throws ApiError + */ + public static postSpaces12( + spaceId: string, + requestBody: { + label: string; + type: string; + description: string; + uri?: string; + }, + ): CancelablePromise { + return __request({ + method: 'POST', + path: `/spaces/${spaceId}/metadata`, + body: requestBody, + mediaType: 'application/json', + }); + } - /** - * add metadata definition to a space - * If uri is not defined, the uri will be set as - * "http://clowder.ncsa.illinois.edu/metadata/{uuid}#CamelCase" - * - * @param spaceId - * @param requestBody - * @returns any OK - * @throws ApiError - */ - public static postSpaces12( - spaceId: string, - requestBody: { - label: string; - type: string; - description: string; - uri?: string; - } - ): CancelablePromise { - return __request({ - method: "POST", - path: `/spaces/${spaceId}/metadata`, - body: requestBody, - mediaType: "application/json", - }); - } -} +} \ No newline at end of file From 1f3e5ed59c26a183fef98308bd1e294512028a82 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 8 Sep 2025 12:59:17 -0500 Subject: [PATCH 89/91] ran codegen? --- .../openapi/v2/services/DatasetsService.ts | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/frontend/src/openapi/v2/services/DatasetsService.ts b/frontend/src/openapi/v2/services/DatasetsService.ts index 11db1018b..c8c859705 100644 --- a/frontend/src/openapi/v2/services/DatasetsService.ts +++ b/frontend/src/openapi/v2/services/DatasetsService.ts @@ -456,6 +456,35 @@ export class DatasetsService { }); } + /** + * Get Dataset Folders All + * @param datasetId + * @param skip + * @param limit + * @param enableAdmin + * @returns any Successful Response + * @throws ApiError + */ + public static getDatasetFoldersAllApiV2DatasetsDatasetIdAllFoldersGet( + datasetId: string, + skip?: number, + limit: number = 10, + enableAdmin: boolean = false, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/api/v2/datasets/${datasetId}/all_folders`, + query: { + 'skip': skip, + 'limit': limit, + 'enable_admin': enableAdmin, + }, + errors: { + 422: `Validation Error`, + }, + }); + } + /** * Get Folder * @param datasetId From 34c868bc2987a08d55e744f9488cdc18a5dc4fa0 Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 9 Sep 2025 15:59:15 -0500 Subject: [PATCH 90/91] more readme instructions refactoring to make it clear when a v1 dataset is a v2 dataset or when a v1 collection is a v2 dataset --- scripts/migration/README.md | 1 + scripts/migration/migrate.py | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/migration/README.md b/scripts/migration/README.md index 3aa2b5801..bd21aa83b 100644 --- a/scripts/migration/README.md +++ b/scripts/migration/README.md @@ -13,5 +13,6 @@ This file can be used to limit what gets migrated by space or user. 2. add your values to the .env and the config.toml file 3. run the script `get_collections.py` this gets all collections from the v1 instance 4. run the script `get_collection_datasets.py` this gets all the datasets in the collections +5. run the script `dataset_collection_json.py` this creates a json file that shows which datasets are in a collection, and which are not. The datasets that are in a collection will be migrated to folders, while theo nes that are not will be migrated to datasets 5. Now you are ready to run `migrate.py` - it uses the data from the previous scripts to place datasets into the right collections diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 2ed9dd6ed..195c78a0e 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -284,6 +284,7 @@ def process_collection_descendants( dataset, v2_dataset_id, new_folder["id"], headers_v1, base_headers_v2 ) # TODO add dataset metadata to the folder + print('done with collection descendants') def get_v1_dataset_folders(dataset, headers_v1, headers_v2, parent_type, parent_id): @@ -340,7 +341,7 @@ def process_dataset_files_and_folders( print(f"Got dataset files") -def create_v2_dataset_from_collection( +def create_v2_dataset_from_v1_collection( collection, user_v1, headers_v1, headers_v2, base_headers_v2 ): # create the dataset @@ -590,7 +591,7 @@ def add_dataset_license(v1_license, headers): return license_id -def create_v2_dataset(dataset, headers): +def create_v2_dataset_from_v1_dataset(dataset, headers): """Create a dataset in Clowder v2.""" # TODO: GET correct license print("Creating dataset license in Clowder v2.") @@ -1168,7 +1169,7 @@ def process_user_and_resources_collections( # create datasets from the top level collections for top_level_col in migrate_top_level_collections: - dataset_v2 = create_v2_dataset_from_collection( + dataset_v2 = create_v2_dataset_from_v1_collection( collection=top_level_col, user_v1=user_v1, headers_v1=clowder_headers_v1, @@ -1215,7 +1216,7 @@ def process_user_and_resources_collections( ) MIGRATE_DATASET = False if MIGRATE_DATASET: - dataset_v2_id = create_v2_dataset(dataset, user_headers_v2) + dataset_v2_id = create_v2_dataset_from_v1_dataset(dataset, user_headers_v2) DATASET_MAP[dataset["id"]] = dataset_v2_id # add_dataset_metadata( @@ -1337,7 +1338,7 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP): ) MIGRATE_DATASET = False if MIGRATE_DATASET: - dataset_v2_id = create_v2_dataset(dataset, user_headers_v2) + dataset_v2_id = create_v2_dataset_from_v1_dataset(dataset, user_headers_v2) DATASET_MAP[dataset["id"]] = dataset_v2_id add_dataset_metadata( dataset, dataset_v2_id, base_headers_v1, user_headers_v2 From 27a135596a2f8ea3d8b77d7bf41a343c5746d9df Mon Sep 17 00:00:00 2001 From: toddn Date: Wed, 5 Nov 2025 17:13:18 -0600 Subject: [PATCH 91/91] fixing lint --- scripts/migration/migrate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py index 195c78a0e..140a2aa5b 100644 --- a/scripts/migration/migrate.py +++ b/scripts/migration/migrate.py @@ -284,7 +284,7 @@ def process_collection_descendants( dataset, v2_dataset_id, new_folder["id"], headers_v1, base_headers_v2 ) # TODO add dataset metadata to the folder - print('done with collection descendants') + print("done with collection descendants") def get_v1_dataset_folders(dataset, headers_v1, headers_v2, parent_type, parent_id):