From 15638b0144e36f1c0a8bd80e2574a54d5b77dcda Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Fri, 24 May 2024 15:34:07 -0500 Subject: [PATCH 1/7] Prepopulating attempy --- jupyterhub/Clowder_APIs.ipynb | 151 +++++++++++++++++++++++++++++++ jupyterhub/Dockerfile.jupyterlab | 21 +++++ jupyterhub/jupyterhub_config.py | 17 +++- 3 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 jupyterhub/Clowder_APIs.ipynb create mode 100644 jupyterhub/Dockerfile.jupyterlab diff --git a/jupyterhub/Clowder_APIs.ipynb b/jupyterhub/Clowder_APIs.ipynb new file mode 100644 index 000000000..f975f1bc9 --- /dev/null +++ b/jupyterhub/Clowder_APIs.ipynb @@ -0,0 +1,151 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "6dfcb963-cf69-4505-9871-5cc13471f5dd", + "metadata": {}, + "source": [ + "## Clowder APIs" + ] + }, + { + "cell_type": "markdown", + "id": "310051ad-4262-42fc-ac28-911f92842a7e", + "metadata": {}, + "source": [ + "## Import libraries and setup utility function\n", + "\n", + "We start by importing the rquired libraries, data and setting up some utility functions and variables that we will use below." + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "ffec7cb7-1a82-4148-aad1-bb3b1b19915b", + "metadata": {}, + "outputs": [], + "source": [ + "import pyclowder\n", + "import json\n", + "import os\n", + "import pandas as pd\n", + "\n", + "import requests\n", + "\n", + "# Function to download the IRIS dataset\n", + "def download_iris_dataset():\n", + " # URL for the Iris dataset hosted by UCI Machine Learning Repository\n", + " url = \"https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data\"\n", + " \n", + " response = requests.get(url)\n", + " \n", + " if response.status_code == 200:\n", + " with open(\"iris.csv\", \"wb\") as f:\n", + " f.write(response.content)\n", + " print(\"The Iris dataset has been downloaded and saved as iris.csv.\")\n", + " else:\n", + " print(\"Failed to download the dataset. Status code:\", response.status_code)\n", + "\n", + "\n", + "CLOWDER_URL = \"http://localhost:8000\"" + ] + }, + { + "cell_type": "markdown", + "id": "f8d5c0ae-659b-4a52-a30a-0fa8b7051694", + "metadata": {}, + "source": [ + "## Token Generation" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "59fca5f8-a5d6-419d-835a-023a73c5a1d7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI2dUVlQ0xOc1hTQXZUN1VDek1FRVk2VmI4ajJnY1RhWlFESUpnbnFGSHVJIn0.eyJleHAiOjE3MTY1NzcwNTgsImlhdCI6MTcxNjU3Njc1OCwianRpIjoiZjBjYmZlYzYtZjJlMS00NDg0LTg0YWMtNzFjMGMwZWNmNTNlIiwiaXNzIjoiaHR0cDovL2xvY2FsaG9zdDo4MDgwL2tleWNsb2FrL3JlYWxtcy9jbG93ZGVyIiwic3ViIjoiZjg0Y2JjNmQtYzEzZC00MmVmLWFhN2MtMWQ4MmFjYzVhZWViIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiY2xvd2RlcjItYmFja2VuZCIsInNlc3Npb25fc3RhdGUiOiIzOTgyN2EyNy1kMDA5LTRiY2YtOGIzYS1jZTk1MGM3YTI4OWQiLCJhbGxvd2VkLW9yaWdpbnMiOlsiaHR0cDovL2xvY2FsaG9zdDo4MDAwIl0sInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJkZWZhdWx0LXJvbGVzLWNsb3dkZXIiLCJvZmZsaW5lX2FjY2VzcyIsInVtYV9hdXRob3JpemF0aW9uIl19LCJzY29wZSI6Im9wZW5pZCBwcm9maWxlIGVtYWlsIiwic2lkIjoiMzk4MjdhMjctZDAwOS00YmNmLThiM2EtY2U5NTBjN2EyODlkIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJWaXNtYXlhayBNb2hhbmFyYWphbiIsInByZWZlcnJlZF91c2VybmFtZSI6Im1vaGFuYXIyQGlsbGlub2lzLmVkdSIsImdpdmVuX25hbWUiOiJWaXNtYXlhayIsImZhbWlseV9uYW1lIjoiTW9oYW5hcmFqYW4iLCJlbWFpbCI6Im1vaGFuYXIyQGlsbGlub2lzLmVkdSJ9.Z3upy5xkoXfTwILujF7C8vpE_XaL8F32vuRd67V2TVqUK4qEVuN_5yTsyCcXyte9osG2jeZTHlJXreekOrpTvOW4Qbel1JAHnCQfGmlri_io-XFE57aT379HZrLGVlrpiHOGTqKG5mDVxIbaHEqdWhJkAEWY55rDqpkCScHUP7_SoJgAUj9Xx_LSS1hDHLwBCYRvGPFEoMADM40F-P3qxkW5Qsv9gPqP5ChpfG_7KZeBMRhtFXH_xk3M18qjNsSiQ9QKBFC2IrLFnT89bCK_slaUfUWLyiONt6ASPkAw85aajD3jc8RKq_AM42a2-wFVF7vERGPnoMscOpvjp9lBiQ'}\n" + ] + } + ], + "source": [ + "user_login_json = {\n", + " \"email\": \"mohanar2@illinois.edu\",\n", + " \"password\": \"password\"\n", + "}\n", + "login_url = CLOWDER_URL + \"/api/v2/login\"\n", + "response = requests.post(login_url, json = user_login_json)\n", + "token = response.json()[\"token\"]\n", + "print(response.json())" + ] + }, + { + "cell_type": "markdown", + "id": "2b273e10-8efc-46a8-b220-b63354735c49", + "metadata": {}, + "source": [ + "## Creating Dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "91639e11-b553-4b57-a0df-6e91c661662f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'name': 'Flower Dataset', 'description': 'Dataset for Flower Data', 'status': 'PRIVATE', 'id': '6650e202fcd9057e97ae2d8a', 'creator': {'email': 'mohanar2@illinois.edu', 'first_name': 'Vismayak', 'last_name': 'Mohanarajan', 'id': '663a45d5b75ca83d17ac6564', 'admin': True, 'admin_mode': True}, 'created': '2024-05-24T18:52:50.457552', 'modified': '2024-05-24T18:52:50.457556', 'user_views': 0, 'downloads': 0, 'thumbnail_id': None, 'standard_license': True, 'license_id': 'CC BY'}\n" + ] + } + ], + "source": [ + "dataset_json = {\n", + " \"name\": \"Flower Dataset\",\n", + " \"description\": \"Dataset for Flower Data\",\n", + " \"status\": \"PRIVATE\",\n", + "}\n", + "dataset_params = {\n", + " \"license_id\": \"CC BY\"\n", + "}\n", + "\n", + "token = \"eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI2dUVlQ0xOc1hTQXZUN1VDek1FRVk2VmI4ajJnY1RhWlFESUpnbnFGSHVJIn0.eyJleHAiOjE3MTY1NzY5NjcsImlhdCI6MTcxNjU3NjY2NywiYXV0aF90aW1lIjoxNzE2NTc2Mzc1LCJqdGkiOiI5NzU3NTM1Mi04Y2FhLTRhMmUtODZhZC0wM2JkNDcwODhhZjkiLCJpc3MiOiJodHRwOi8vbG9jYWxob3N0OjgwODAva2V5Y2xvYWsvcmVhbG1zL2Nsb3dkZXIiLCJzdWIiOiJmODRjYmM2ZC1jMTNkLTQyZWYtYWE3Yy0xZDgyYWNjNWFlZWIiLCJ0eXAiOiJCZWFyZXIiLCJhenAiOiJjbG93ZGVyMi1iYWNrZW5kIiwic2Vzc2lvbl9zdGF0ZSI6IjdhMjJjZDFkLTQ3NDctNDAyNS1iNDk2LTUyYzNhOWUwZDhiYyIsImFsbG93ZWQtb3JpZ2lucyI6WyJodHRwOi8vbG9jYWxob3N0OjgwMDAiXSwicmVhbG1fYWNjZXNzIjp7InJvbGVzIjpbImRlZmF1bHQtcm9sZXMtY2xvd2RlciIsIm9mZmxpbmVfYWNjZXNzIiwidW1hX2F1dGhvcml6YXRpb24iXX0sInNjb3BlIjoib3BlbmlkIHByb2ZpbGUgZW1haWwiLCJzaWQiOiI3YTIyY2QxZC00NzQ3LTQwMjUtYjQ5Ni01MmMzYTllMGQ4YmMiLCJlbWFpbF92ZXJpZmllZCI6dHJ1ZSwibmFtZSI6IlZpc21heWFrIE1vaGFuYXJhamFuIiwicHJlZmVycmVkX3VzZXJuYW1lIjoibW9oYW5hcjJAaWxsaW5vaXMuZWR1IiwiZ2l2ZW5fbmFtZSI6IlZpc21heWFrIiwiZmFtaWx5X25hbWUiOiJNb2hhbmFyYWphbiIsImVtYWlsIjoibW9oYW5hcjJAaWxsaW5vaXMuZWR1In0.AHqeZ79L8OUW2zTYEaDeUmuhHAeDMcqlgR8CE7ScGBRt-4gj8QFipbkiiMt0LYbMNi97dv0O_-iojrMtnlaX1OubnEwI_mMcoGckpXc5CzCoAQ_Qri2ZnpfnClDa9wPu-G_aGL-Sv4UpAglSTCbpBxsE99EiBubMb0T3NpP8p0k_lJTSvAZPJdMJ2sPpBo4BPUUIZq9JBXAZF7YPL9ZzUOoQKJvpNlA_7fHBHDjdSKRbVjP5cCNldjHJ70D2j1HM4JJUwMirNWZ8SFaJiny5a7NoZ2fa_JQkA3ZRrwGPHxGh6JThhu_F-a-pv2CDSytPCSU5DJxnslNJ4ePLVKDWpQ\"\n", + "headers = {\n", + " 'Authorization': f'Bearer {token}'\n", + "}\n", + "\n", + "\n", + "dataset_url = CLOWDER_URL + \"/api/v2/datasets\"\n", + "response = requests.post(dataset_url, json = dataset_json, headers = headers, params = dataset_params)\n", + "print(response.json())" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/jupyterhub/Dockerfile.jupyterlab b/jupyterhub/Dockerfile.jupyterlab new file mode 100644 index 000000000..ee0bb22f6 --- /dev/null +++ b/jupyterhub/Dockerfile.jupyterlab @@ -0,0 +1,21 @@ +# Base Image +FROM quay.io/jupyter/base-notebook:latest + +# Install additional packages +USER root +RUN apt-get -qq update && apt-get install -y --no-install-recommends \ + curl \ + git \ + zip unzip \ + nano \ + vim-tiny \ + lsof && \ + rm -rf /var/lib/apt/lists/* + +USER $NB_USER + +# Install Python packages +RUN pip install --no-cache-dir \ + requests \ + pyclowder \ + pandas diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py index 136f80962..58105897f 100644 --- a/jupyterhub/jupyterhub_config.py +++ b/jupyterhub/jupyterhub_config.py @@ -3,6 +3,8 @@ # Configuration file for JupyterHub import os +import shutil +import logging from customauthenticator.custom import CustomTokenAuthenticator @@ -61,7 +63,7 @@ # TODO:Change this keycloak_url as required c.CustomTokenAuthenticator.auth_cookie_header = "Authorization" -c.CustomTokenAuthenticator.auth_username_key = "preferred_username" +c.CustomTokenAuthenticator.auth_username_key = "email" c.CustomTokenAuthenticator.auth_uid_number_key = "uid_number" c.CustomTokenAuthenticator.enable_auth_state = True c.CustomTokenAuthenticator.auto_login = True @@ -92,7 +94,20 @@ c.JupyterHub.cookie_secret = os.getenv("JUPYTERHUB_CRYPT_KEY") +# Allow all users to access +c.Authenticator.allow_all = True + # Allowed admins admin = os.environ.get("JUPYTERHUB_ADMIN") if admin: c.Authenticator.admin_users = [admin] + + +# Pre spawn hook +# def pre_spawn_hook(spawner): +# # Git clone +# +# + + +# c.Spawner.post_stop_hook = pre_spawn_hook From f04103ad6ef9ac17a78d9854e27eaf7be78cfb2c Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Sun, 26 May 2024 19:16:59 -0500 Subject: [PATCH 2/7] Not working --- jupyterhub/Dockerfile.jupyterhub | 3 +-- jupyterhub/Dockerfile.jupyterlab | 3 +++ jupyterhub/jupyterhub_config.py | 18 ++++++++++++------ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/jupyterhub/Dockerfile.jupyterhub b/jupyterhub/Dockerfile.jupyterhub index 349782ad7..a27ff8c65 100644 --- a/jupyterhub/Dockerfile.jupyterhub +++ b/jupyterhub/Dockerfile.jupyterhub @@ -1,5 +1,4 @@ -ARG JUPYTERHUB_VERSION -FROM quay.io/jupyterhub/jupyterhub:$JUPYTERHUB_VERSION +FROM quay.io/jupyterhub/jupyterhub:latest # Install dockerspawner, # hadolint ignore=DL3013 diff --git a/jupyterhub/Dockerfile.jupyterlab b/jupyterhub/Dockerfile.jupyterlab index ee0bb22f6..06afd0deb 100644 --- a/jupyterhub/Dockerfile.jupyterlab +++ b/jupyterhub/Dockerfile.jupyterlab @@ -19,3 +19,6 @@ RUN pip install --no-cache-dir \ requests \ pyclowder \ pandas + +# Copy Clowder API notebook +COPY Clowder_APIs.ipynb /etc/jupyter/Clowder_APIs.ipynb diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py index 58105897f..1e4cdb99d 100644 --- a/jupyterhub/jupyterhub_config.py +++ b/jupyterhub/jupyterhub_config.py @@ -1,10 +1,10 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. + # Configuration file for JupyterHub import os import shutil -import logging from customauthenticator.custom import CustomTokenAuthenticator @@ -31,11 +31,13 @@ # We follow the same convention. notebook_dir = os.environ.get("DOCKER_NOTEBOOK_DIR", "/home/jovyan/work") c.DockerSpawner.notebook_dir = notebook_dir +c.Spawner.args = ["--NotebookApp.default_url=/notebooks/Welcome.ipynb"] # Mount the real user's Docker volume on the host to the notebook user's # notebook directory in the container c.DockerSpawner.volumes = {"jupyterhub-user-{username}": notebook_dir} + # Remove containers once they are stopped c.DockerSpawner.remove = True @@ -104,10 +106,14 @@ # Pre spawn hook -# def pre_spawn_hook(spawner): -# # Git clone -# -# +def post_spawn_hook(spawner, auth_state): + username = spawner.user.name + spawner.environment["GREETING"] = f"Hello Master {username}" + + target_file_path = f"/home/jovyan/work/Clowder_APIs.ipynb" + + if not os.path.exists(target_file_path): + shutil.copy2("/etc/jupyter/Clowder_APIs.ipynb", target_file_path) -# c.Spawner.post_stop_hook = pre_spawn_hook +# c.Spawner.auth_state_hook = post_spawn_hook From 6718d5a67cd0635c8374a6a1ed4047ee74c5483e Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Fri, 31 May 2024 09:29:18 -0500 Subject: [PATCH 3/7] not working --- jupyterhub/Dockerfile.jupyterlab | 7 +++-- jupyterhub/jupyterhub_config.py | 48 +++++++++++++++++++++----------- 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/jupyterhub/Dockerfile.jupyterlab b/jupyterhub/Dockerfile.jupyterlab index 06afd0deb..84d08cc7f 100644 --- a/jupyterhub/Dockerfile.jupyterlab +++ b/jupyterhub/Dockerfile.jupyterlab @@ -14,11 +14,12 @@ RUN apt-get -qq update && apt-get install -y --no-install-recommends \ USER $NB_USER +COPY Clowder_APIs.ipynb /home/jovyan/work/ + +COPY Clowder_APIs.ipynb /home/jovyan/ + # Install Python packages RUN pip install --no-cache-dir \ requests \ pyclowder \ pandas - -# Copy Clowder API notebook -COPY Clowder_APIs.ipynb /etc/jupyter/Clowder_APIs.ipynb diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py index 1e4cdb99d..1321a18a8 100644 --- a/jupyterhub/jupyterhub_config.py +++ b/jupyterhub/jupyterhub_config.py @@ -4,7 +4,6 @@ # Configuration file for JupyterHub import os -import shutil from customauthenticator.custom import CustomTokenAuthenticator @@ -14,6 +13,38 @@ # avoid having to rebuild the JupyterHub container every time we change a # configuration parameter. + +def create_dir_hook(spawner): + username = spawner.user.name # get the username + + volume_path = os.path.join("/volumes/jupyterhub/", username) + print(f"Checking if {volume_path} exists…") + + if not os.path.exists(volume_path): + print(f"{volume_path} does not exist. Creating directory...") + # create a directory with umask 0755 + # hub and container user must have the same UID to be writeable + # still readable by other users on the system + try: + # Attempt to create the directory + os.mkdir(volume_path) + print(f"Directory {volume_path} created.") + # Now, do whatever you think your user needs + # ... + except OSError as e: + print(f"Error creating directory {volume_path}: {e}") + + try: + os.chown(volume_path, 1003, 1003) + print("Ownership of changed successfully.") + except OSError as e: + print(f"Error changing ownership of: {e}") + # now do whatever you think your user needs + + +# attach the hook function to the spawner +c.Spawner.pre_spawn_hook = create_dir_hook + # Spawn single-user servers as Docker containers c.JupyterHub.spawner_class = "dockerspawner.DockerSpawner" @@ -37,7 +68,6 @@ # notebook directory in the container c.DockerSpawner.volumes = {"jupyterhub-user-{username}": notebook_dir} - # Remove containers once they are stopped c.DockerSpawner.remove = True @@ -103,17 +133,3 @@ admin = os.environ.get("JUPYTERHUB_ADMIN") if admin: c.Authenticator.admin_users = [admin] - - -# Pre spawn hook -def post_spawn_hook(spawner, auth_state): - username = spawner.user.name - spawner.environment["GREETING"] = f"Hello Master {username}" - - target_file_path = f"/home/jovyan/work/Clowder_APIs.ipynb" - - if not os.path.exists(target_file_path): - shutil.copy2("/etc/jupyter/Clowder_APIs.ipynb", target_file_path) - - -# c.Spawner.auth_state_hook = post_spawn_hook From fcf9d470f29fca1e06c1e9260f14c5d078aa307b Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Fri, 7 Jun 2024 13:00:13 -0500 Subject: [PATCH 4/7] Updating lab dockerfile --- jupyterhub/Dockerfile.jupyterlab | 2 -- 1 file changed, 2 deletions(-) diff --git a/jupyterhub/Dockerfile.jupyterlab b/jupyterhub/Dockerfile.jupyterlab index 84d08cc7f..dd9c2e4f4 100644 --- a/jupyterhub/Dockerfile.jupyterlab +++ b/jupyterhub/Dockerfile.jupyterlab @@ -16,8 +16,6 @@ USER $NB_USER COPY Clowder_APIs.ipynb /home/jovyan/work/ -COPY Clowder_APIs.ipynb /home/jovyan/ - # Install Python packages RUN pip install --no-cache-dir \ requests \ From be74dbe11e15b794c67fab3f73d2177b90ed7446 Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Fri, 7 Jun 2024 13:00:45 -0500 Subject: [PATCH 5/7] Updating config --- jupyterhub/jupyterhub_config.py | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py index 1321a18a8..1cdd33b84 100644 --- a/jupyterhub/jupyterhub_config.py +++ b/jupyterhub/jupyterhub_config.py @@ -13,38 +13,6 @@ # avoid having to rebuild the JupyterHub container every time we change a # configuration parameter. - -def create_dir_hook(spawner): - username = spawner.user.name # get the username - - volume_path = os.path.join("/volumes/jupyterhub/", username) - print(f"Checking if {volume_path} exists…") - - if not os.path.exists(volume_path): - print(f"{volume_path} does not exist. Creating directory...") - # create a directory with umask 0755 - # hub and container user must have the same UID to be writeable - # still readable by other users on the system - try: - # Attempt to create the directory - os.mkdir(volume_path) - print(f"Directory {volume_path} created.") - # Now, do whatever you think your user needs - # ... - except OSError as e: - print(f"Error creating directory {volume_path}: {e}") - - try: - os.chown(volume_path, 1003, 1003) - print("Ownership of changed successfully.") - except OSError as e: - print(f"Error changing ownership of: {e}") - # now do whatever you think your user needs - - -# attach the hook function to the spawner -c.Spawner.pre_spawn_hook = create_dir_hook - # Spawn single-user servers as Docker containers c.JupyterHub.spawner_class = "dockerspawner.DockerSpawner" From 20ca74a3cd31a7eea71d99e7db19571ec39519a5 Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Fri, 28 Jun 2024 17:24:02 -0500 Subject: [PATCH 6/7] reverting changes --- jupyterhub/Dockerfile.jupyterhub | 3 ++- jupyterhub/jupyterhub_config.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/jupyterhub/Dockerfile.jupyterhub b/jupyterhub/Dockerfile.jupyterhub index a27ff8c65..349782ad7 100644 --- a/jupyterhub/Dockerfile.jupyterhub +++ b/jupyterhub/Dockerfile.jupyterhub @@ -1,4 +1,5 @@ -FROM quay.io/jupyterhub/jupyterhub:latest +ARG JUPYTERHUB_VERSION +FROM quay.io/jupyterhub/jupyterhub:$JUPYTERHUB_VERSION # Install dockerspawner, # hadolint ignore=DL3013 diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py index 85c7dbcaf..5f0b19b99 100644 --- a/jupyterhub/jupyterhub_config.py +++ b/jupyterhub/jupyterhub_config.py @@ -72,7 +72,7 @@ # TODO:Change this keycloak_url as required c.CustomTokenAuthenticator.auth_cookie_header = "Authorization" -c.CustomTokenAuthenticator.auth_username_key = "email" +c.CustomTokenAuthenticator.auth_username_key = "preferred_username" c.CustomTokenAuthenticator.auth_uid_number_key = "uid_number" c.CustomTokenAuthenticator.enable_auth_state = True c.CustomTokenAuthenticator.auto_login = True From a1f4161627bb1f451a08f7930d79e0a1df4e12a3 Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Tue, 2 Jul 2024 09:23:21 -0500 Subject: [PATCH 7/7] Preloading is working, working on notebook --- jupyterhub/Clowder_APIs.ipynb | 68 ++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 13 deletions(-) diff --git a/jupyterhub/Clowder_APIs.ipynb b/jupyterhub/Clowder_APIs.ipynb index f975f1bc9..14beefba8 100644 --- a/jupyterhub/Clowder_APIs.ipynb +++ b/jupyterhub/Clowder_APIs.ipynb @@ -20,7 +20,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 43, "id": "ffec7cb7-1a82-4148-aad1-bb3b1b19915b", "metadata": {}, "outputs": [], @@ -40,9 +40,7 @@ " response = requests.get(url)\n", " \n", " if response.status_code == 200:\n", - " with open(\"iris.csv\", \"wb\") as f:\n", - " f.write(response.content)\n", - " print(\"The Iris dataset has been downloaded and saved as iris.csv.\")\n", + " return response.content\n", " else:\n", " print(\"Failed to download the dataset. Status code:\", response.status_code)\n", "\n", @@ -60,7 +58,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 44, "id": "59fca5f8-a5d6-419d-835a-023a73c5a1d7", "metadata": {}, "outputs": [ @@ -68,7 +66,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI2dUVlQ0xOc1hTQXZUN1VDek1FRVk2VmI4ajJnY1RhWlFESUpnbnFGSHVJIn0.eyJleHAiOjE3MTY1NzcwNTgsImlhdCI6MTcxNjU3Njc1OCwianRpIjoiZjBjYmZlYzYtZjJlMS00NDg0LTg0YWMtNzFjMGMwZWNmNTNlIiwiaXNzIjoiaHR0cDovL2xvY2FsaG9zdDo4MDgwL2tleWNsb2FrL3JlYWxtcy9jbG93ZGVyIiwic3ViIjoiZjg0Y2JjNmQtYzEzZC00MmVmLWFhN2MtMWQ4MmFjYzVhZWViIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiY2xvd2RlcjItYmFja2VuZCIsInNlc3Npb25fc3RhdGUiOiIzOTgyN2EyNy1kMDA5LTRiY2YtOGIzYS1jZTk1MGM3YTI4OWQiLCJhbGxvd2VkLW9yaWdpbnMiOlsiaHR0cDovL2xvY2FsaG9zdDo4MDAwIl0sInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJkZWZhdWx0LXJvbGVzLWNsb3dkZXIiLCJvZmZsaW5lX2FjY2VzcyIsInVtYV9hdXRob3JpemF0aW9uIl19LCJzY29wZSI6Im9wZW5pZCBwcm9maWxlIGVtYWlsIiwic2lkIjoiMzk4MjdhMjctZDAwOS00YmNmLThiM2EtY2U5NTBjN2EyODlkIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJWaXNtYXlhayBNb2hhbmFyYWphbiIsInByZWZlcnJlZF91c2VybmFtZSI6Im1vaGFuYXIyQGlsbGlub2lzLmVkdSIsImdpdmVuX25hbWUiOiJWaXNtYXlhayIsImZhbWlseV9uYW1lIjoiTW9oYW5hcmFqYW4iLCJlbWFpbCI6Im1vaGFuYXIyQGlsbGlub2lzLmVkdSJ9.Z3upy5xkoXfTwILujF7C8vpE_XaL8F32vuRd67V2TVqUK4qEVuN_5yTsyCcXyte9osG2jeZTHlJXreekOrpTvOW4Qbel1JAHnCQfGmlri_io-XFE57aT379HZrLGVlrpiHOGTqKG5mDVxIbaHEqdWhJkAEWY55rDqpkCScHUP7_SoJgAUj9Xx_LSS1hDHLwBCYRvGPFEoMADM40F-P3qxkW5Qsv9gPqP5ChpfG_7KZeBMRhtFXH_xk3M18qjNsSiQ9QKBFC2IrLFnT89bCK_slaUfUWLyiONt6ASPkAw85aajD3jc8RKq_AM42a2-wFVF7vERGPnoMscOpvjp9lBiQ'}\n" + "{'token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI2dUVlQ0xOc1hTQXZUN1VDek1FRVk2VmI4ajJnY1RhWlFESUpnbnFGSHVJIn0.eyJleHAiOjE3MTk5MzAzNTMsImlhdCI6MTcxOTkzMDA1MywianRpIjoiYjZhYTAwNjUtZmY1OS00M2QyLWJiOGYtNzliNGI0MTgwNzNmIiwiaXNzIjoiaHR0cDovL2xvY2FsaG9zdDo4MDgwL2tleWNsb2FrL3JlYWxtcy9jbG93ZGVyIiwic3ViIjoiZjg0Y2JjNmQtYzEzZC00MmVmLWFhN2MtMWQ4MmFjYzVhZWViIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiY2xvd2RlcjItYmFja2VuZCIsInNlc3Npb25fc3RhdGUiOiJlZDM0MzA3Ny1lZmNiLTRlOGMtYjM4OS0zN2JlMzk0MmUyNzAiLCJhbGxvd2VkLW9yaWdpbnMiOlsiaHR0cDovL2xvY2FsaG9zdDo4MDAwIl0sInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJkZWZhdWx0LXJvbGVzLWNsb3dkZXIiLCJvZmZsaW5lX2FjY2VzcyIsInVtYV9hdXRob3JpemF0aW9uIl19LCJzY29wZSI6Im9wZW5pZCBwcm9maWxlIGVtYWlsIiwic2lkIjoiZWQzNDMwNzctZWZjYi00ZThjLWIzODktMzdiZTM5NDJlMjcwIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJWaXNtYXlhayBNb2hhbmFyYWphbiIsInByZWZlcnJlZF91c2VybmFtZSI6Im1vaGFuYXIyQGlsbGlub2lzLmVkdSIsImdpdmVuX25hbWUiOiJWaXNtYXlhayIsImZhbWlseV9uYW1lIjoiTW9oYW5hcmFqYW4iLCJlbWFpbCI6Im1vaGFuYXIyQGlsbGlub2lzLmVkdSJ9.NAZs_sDtIdmy02bIjILG59jeRK99bMPdODQqEkyn-jC0YP_949LAPhWMSvxG5-eVdhHOeGILlbPmWgzU9tuw4YOkn-mNdVIBh16EicMJJ6zXeYzVj5RuUfhtZYJ3LfoknjuPBABI44dqo-Ixqh760m6HQKyZUfW62Lg-nGLYTvfoGtlResziGZ7u4L6vdsmmr_05SKGWdXxpVQUzwBNiXI2SO3FbWNk2uwR_qXL8WHyajHKCXvgHYHUbnAKJ1SRKO0xNfYDDlGKXuEn4fZfVmICq9693Z8emwydlcp8BVOaVuFqbsWwoRmEoSnK4bzLAH-CM-PFESPfbTrnu0Scnnw'}\n" ] } ], @@ -80,6 +78,9 @@ "login_url = CLOWDER_URL + \"/api/v2/login\"\n", "response = requests.post(login_url, json = user_login_json)\n", "token = response.json()[\"token\"]\n", + "headers = {\n", + " 'Authorization': f'Bearer {token}'\n", + "}\n", "print(response.json())" ] }, @@ -93,7 +94,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 45, "id": "91639e11-b553-4b57-a0df-6e91c661662f", "metadata": {}, "outputs": [ @@ -101,7 +102,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'name': 'Flower Dataset', 'description': 'Dataset for Flower Data', 'status': 'PRIVATE', 'id': '6650e202fcd9057e97ae2d8a', 'creator': {'email': 'mohanar2@illinois.edu', 'first_name': 'Vismayak', 'last_name': 'Mohanarajan', 'id': '663a45d5b75ca83d17ac6564', 'admin': True, 'admin_mode': True}, 'created': '2024-05-24T18:52:50.457552', 'modified': '2024-05-24T18:52:50.457556', 'user_views': 0, 'downloads': 0, 'thumbnail_id': None, 'standard_license': True, 'license_id': 'CC BY'}\n" + "{'name': 'Flower Dataset', 'description': 'Dataset for Flower Data', 'status': 'PRIVATE', 'id': '66840cc55713c91cd9b89483', 'creator': {'email': 'mohanar2@illinois.edu', 'first_name': 'Vismayak', 'last_name': 'Mohanarajan', 'id': '663a45d5b75ca83d17ac6564', 'admin': True, 'admin_mode': True, 'read_only_user': False}, 'created': '2024-07-02T14:20:53.503934', 'modified': '2024-07-02T14:20:53.503942', 'user_views': 0, 'downloads': 0, 'thumbnail_id': None, 'standard_license': True, 'license_id': 'CC BY'}\n" ] } ], @@ -115,16 +116,57 @@ " \"license_id\": \"CC BY\"\n", "}\n", "\n", - "token = \"eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI2dUVlQ0xOc1hTQXZUN1VDek1FRVk2VmI4ajJnY1RhWlFESUpnbnFGSHVJIn0.eyJleHAiOjE3MTY1NzY5NjcsImlhdCI6MTcxNjU3NjY2NywiYXV0aF90aW1lIjoxNzE2NTc2Mzc1LCJqdGkiOiI5NzU3NTM1Mi04Y2FhLTRhMmUtODZhZC0wM2JkNDcwODhhZjkiLCJpc3MiOiJodHRwOi8vbG9jYWxob3N0OjgwODAva2V5Y2xvYWsvcmVhbG1zL2Nsb3dkZXIiLCJzdWIiOiJmODRjYmM2ZC1jMTNkLTQyZWYtYWE3Yy0xZDgyYWNjNWFlZWIiLCJ0eXAiOiJCZWFyZXIiLCJhenAiOiJjbG93ZGVyMi1iYWNrZW5kIiwic2Vzc2lvbl9zdGF0ZSI6IjdhMjJjZDFkLTQ3NDctNDAyNS1iNDk2LTUyYzNhOWUwZDhiYyIsImFsbG93ZWQtb3JpZ2lucyI6WyJodHRwOi8vbG9jYWxob3N0OjgwMDAiXSwicmVhbG1fYWNjZXNzIjp7InJvbGVzIjpbImRlZmF1bHQtcm9sZXMtY2xvd2RlciIsIm9mZmxpbmVfYWNjZXNzIiwidW1hX2F1dGhvcml6YXRpb24iXX0sInNjb3BlIjoib3BlbmlkIHByb2ZpbGUgZW1haWwiLCJzaWQiOiI3YTIyY2QxZC00NzQ3LTQwMjUtYjQ5Ni01MmMzYTllMGQ4YmMiLCJlbWFpbF92ZXJpZmllZCI6dHJ1ZSwibmFtZSI6IlZpc21heWFrIE1vaGFuYXJhamFuIiwicHJlZmVycmVkX3VzZXJuYW1lIjoibW9oYW5hcjJAaWxsaW5vaXMuZWR1IiwiZ2l2ZW5fbmFtZSI6IlZpc21heWFrIiwiZmFtaWx5X25hbWUiOiJNb2hhbmFyYWphbiIsImVtYWlsIjoibW9oYW5hcjJAaWxsaW5vaXMuZWR1In0.AHqeZ79L8OUW2zTYEaDeUmuhHAeDMcqlgR8CE7ScGBRt-4gj8QFipbkiiMt0LYbMNi97dv0O_-iojrMtnlaX1OubnEwI_mMcoGckpXc5CzCoAQ_Qri2ZnpfnClDa9wPu-G_aGL-Sv4UpAglSTCbpBxsE99EiBubMb0T3NpP8p0k_lJTSvAZPJdMJ2sPpBo4BPUUIZq9JBXAZF7YPL9ZzUOoQKJvpNlA_7fHBHDjdSKRbVjP5cCNldjHJ70D2j1HM4JJUwMirNWZ8SFaJiny5a7NoZ2fa_JQkA3ZRrwGPHxGh6JThhu_F-a-pv2CDSytPCSU5DJxnslNJ4ePLVKDWpQ\"\n", - "headers = {\n", - " 'Authorization': f'Bearer {token}'\n", - "}\n", - "\n", "\n", "dataset_url = CLOWDER_URL + \"/api/v2/datasets\"\n", "response = requests.post(dataset_url, json = dataset_json, headers = headers, params = dataset_params)\n", + "dataset_id = response.json()['id']\n", + "print(response.json())" + ] + }, + { + "cell_type": "markdown", + "id": "51ad8ad2-bcb6-40d4-9587-44f6a1a7c09d", + "metadata": {}, + "source": [ + "## Uploading File" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "id": "ed02e036-edd0-403b-8292-9600946bc156", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'name': 'file', 'status': 'PRIVATE', 'id': '66840ced5713c91cd9b89485', 'creator': {'email': 'mohanar2@illinois.edu', 'first_name': 'Vismayak', 'last_name': 'Mohanarajan', 'id': '663a45d5b75ca83d17ac6564', 'admin': True, 'admin_mode': True, 'read_only_user': False}, 'created': '2024-07-02T14:21:33.733027', 'version_id': '2a4bbb53-8149-4384-9d55-2696c9d587ed', 'version_num': 1, 'dataset_id': '66840cc55713c91cd9b89483', 'folder_id': None, 'views': 0, 'downloads': 0, 'bytes': 4551, 'content_type': {'content_type': 'application/octet-stream', 'main_type': 'application'}, 'thumbnail_id': None, 'storage_type': 'minio', 'storage_path': None, 'object_type': 'file'}\n" + ] + } + ], + "source": [ + "\n", + "file = download_iris_dataset()\n", + "file_json = {\n", + " \"file\": file,\n", + " \"mediaType\": 'multipart/form-data'\n", + "}\n", + "save_file_url = CLOWDER_URL + \"/api/v2/datasets/\" + dataset_id + '/files'\n", + "headers = {\n", + " 'Authorization': f'Bearer {token}'\n", + "}\n", + "response = requests.post(save_file_url, files = file_json, headers = headers)\n", "print(response.json())" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "08c6673d-70d2-4879-948f-8db76eda4cde", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": {