From b5b67621309a441bca9707ba234737214d36740a Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Mon, 22 Jul 2024 16:57:25 -0500 Subject: [PATCH 01/32] release updates --- CHANGELOG.md | 28 +++++++++++++++++++ backend/app/config.py | 2 +- backend/app/main.py | 2 +- .../kubernetes/charts/clowder2/Chart.yaml | 4 +-- .../kubernetes/charts/clowder2/README.md | 4 +-- docker-compose.yml | 4 +-- frontend/package-lock.json | 2 +- frontend/package.json | 2 +- frontend/src/app.config.ts | 2 +- frontend/src/openapi/v2/core/OpenAPI.ts | 2 +- openapi.json | 4 +-- 11 files changed, 42 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a2d26d27..90a2ab48c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,34 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [v2.0.0-beta.3] - 2024-07-29 + +### Added + +- License management +- Release dataset with versions +- Enable and disable user account through Keycloak +- Jupyterhub integration +- Interface for creating and editing matching criteria for triggering extractors +- Interface for editing metadata definitions +- My dataset tab listing all the datasets created by the user +- Drag and drop upload multiple files +- Footer with links to documentation, source code, and contact information +- Documentation through MKDocs + +### Changed + +- Allow public datasets and files to be searchable +- List all the extractors with the ability to enable/disable the extractors +- Filter listeners based on their support for file or dataset +- Helm chart updated to support custom existing secret + +### Fixed + +- Clowder registration link on the top bar +- Case-insensitive search +- Download count immediately increments after download + ## [v2.0.0-beta.2] - 2024-02-16 ### Added diff --git a/backend/app/config.py b/backend/app/config.py index 2f3041853..ae88e43d3 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -9,7 +9,7 @@ class Settings(BaseSettings): API_V2_STR: str = "/api/v2" admin_email: str = "devnull@ncsa.illinois.edu" frontend_url: str = "http://localhost:3000" - version: str = "2.0.0-beta.2" + version: str = "2.0.0-beta.3" # Unique secret for hashing API keys. Generate with `openssl rand -hex 32` local_auth_secret = "clowder_secret_key" diff --git a/backend/app/main.py b/backend/app/main.py index 84f54d38b..0c688080f 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -86,7 +86,7 @@ description="A cloud native data management framework to support any research domain. Clowder was " "developed to help researchers and scientists in data intensive domains manage raw data, complex " "metadata, and automatic data pipelines. ", - version="2.0.0-beta.2", + version="2.0.0-beta.3", contact={"name": "Clowder", "url": "https://clowderframework.org/"}, license_info={ "name": "Apache 2.0", diff --git a/deployments/kubernetes/charts/clowder2/Chart.yaml b/deployments/kubernetes/charts/clowder2/Chart.yaml index dabacef56..fea872e94 100644 --- a/deployments/kubernetes/charts/clowder2/Chart.yaml +++ b/deployments/kubernetes/charts/clowder2/Chart.yaml @@ -13,13 +13,13 @@ description: > # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.9.1 +version: 1.9.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "2.0.0-beta.2" +appVersion: "2.0.0-beta.3" # List of people that maintain this helm chart. maintainers: diff --git a/deployments/kubernetes/charts/clowder2/README.md b/deployments/kubernetes/charts/clowder2/README.md index 76f43513e..0f7c69768 100644 --- a/deployments/kubernetes/charts/clowder2/README.md +++ b/deployments/kubernetes/charts/clowder2/README.md @@ -17,13 +17,13 @@ hostname: { your hostname. E.g. clowder2.software-dev.ncsa.illinois.edu } frontend: image: repository: clowder/clowder2-frontend - tag: 2.0.0-beta.2 + tag: 2.0.0-beta.3 pullPolicy: Always backend: image: repository: clowder/clowder2-backend - tag: 2.0.0-beta.2 + tag: 2.0.0-beta.3 pullPolicy: Always geoserver: diff --git a/docker-compose.yml b/docker-compose.yml index c23660a7c..daf946f41 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,7 +37,7 @@ services: - /var/run/docker.sock:/var/run/docker.sock backend: - image: 'clowder/clowder2-backend:2.0.0-beta.2' + image: 'clowder/clowder2-backend:2.0.0-beta.3' restart: unless-stopped build: context: ./backend @@ -77,7 +77,7 @@ services: - "traefik.http.routers.backend.priority=5" frontend: - image: "clowder/clowder2-frontend:2.0.0-beta.2" + image: "clowder/clowder2-frontend:2.0.0-beta.3" restart: unless-stopped build: context: ./frontend diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 8033d07d8..73f257417 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,6 +1,6 @@ { "name": "clowder-frontend", - "version": "2.0.0-beta.2", + "version": "2.0.0-beta.3", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 2681aa035..6f1938575 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "clowder-frontend", - "version": "2.0.0-beta.2", + "version": "2.0.0-beta.3", "description": "", "engines": { "npm": ">=8.11", diff --git a/frontend/src/app.config.ts b/frontend/src/app.config.ts index 7c325852c..feeac410e 100644 --- a/frontend/src/app.config.ts +++ b/frontend/src/app.config.ts @@ -42,7 +42,7 @@ const hostname = // TODO when add auth piece remove this env const apikey = process.env.APIKEY || ""; -config["appVersion"] = "v2.0.0-beta.2"; +config["appVersion"] = "v2.0.0-beta.3"; config["mailingList"] = "clowder@lists.illinois.edu"; config["slackChannel"] = "https://join.slack.com/t/clowder-software/shared_invite/enQtMzQzOTg0Nzk3OTUzLTYwZDlkZDI0NGI4YmI0ZjE5MTZiYmZhZTIyNWE1YzM0NWMwMzIxODNhZTA1Y2E3MTQzOTg1YThiNzkwOWQwYWE"; diff --git a/frontend/src/openapi/v2/core/OpenAPI.ts b/frontend/src/openapi/v2/core/OpenAPI.ts index cdf38cb7d..473413160 100644 --- a/frontend/src/openapi/v2/core/OpenAPI.ts +++ b/frontend/src/openapi/v2/core/OpenAPI.ts @@ -20,7 +20,7 @@ type Config = { export const OpenAPI: Config = { BASE: '', - VERSION: '2.0.0-beta.2', + VERSION: '2.0.0-beta.3', WITH_CREDENTIALS: false, CREDENTIALS: 'include', TOKEN: undefined, diff --git a/openapi.json b/openapi.json index 74418d8de..6122b2f3e 100644 --- a/openapi.json +++ b/openapi.json @@ -11,7 +11,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0.html" }, - "version": "2.0.0-beta.2" + "version": "2.0.0-beta.3" }, "paths": { "/api/v2/users": { @@ -13972,7 +13972,7 @@ "version": { "title": "Version", "type": "string", - "default": "2.0.0-beta.2" + "default": "2.0.0-beta.3" } } }, From 4d136e86c724413bf4aec066a3c9518150de46f8 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Thu, 25 Jul 2024 09:04:40 -0500 Subject: [PATCH 02/32] 1147 clowder 2 helm chart clean up (#1148) * turn off unecessary ingress for minio and rabbitmq * remove default tls host * slight update the secret read * use the wrong secretname * add more gitignore * take out default extractors in the default values * allow backend env to be overwritten by value files with default * add quote to number and boolean * allow environment variable setting in heartbeat and message extractors * extractor doesn't need to be commented out; add message and heartbeat * use fullname for matching labels * typo * allow use existing secret and secret keys * duplicate rabbitmq pass --- .gitignore | 1 + .../charts/clowder2/templates/NOTES.txt | 2 +- .../charts/clowder2/templates/_helpers.tpl | 2 +- .../templates/backend/deployment.yaml | 46 ++++++++++++------- .../clowder2/templates/backend/ingress.yaml | 1 - .../templates/extractors/deployment.yaml | 9 +++- .../templates/frontend/deployment.yaml | 4 +- .../clowder2/templates/frontend/ingress.yaml | 1 - .../templates/geoserver/deployment.yaml | 16 ++++--- .../clowder2/templates/geoserver/service.yaml | 2 +- .../templates/heartbeat/deployment.yaml | 17 +++++-- .../templates/messages/deployment.yaml | 17 +++++-- .../kubernetes/charts/clowder2/values.yaml | 38 ++++++++++----- 13 files changed, 104 insertions(+), 52 deletions(-) diff --git a/.gitignore b/.gitignore index aa73ebb11..ce36a8619 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,7 @@ scripts/keycloak/data/* # ignore clowder chart deps deployments/kubernetes/charts/clowder2/charts deployments/kubernetes/charts/clowder2/*clowder2-software-dev.yaml +*secret*.yaml # Environments .env diff --git a/deployments/kubernetes/charts/clowder2/templates/NOTES.txt b/deployments/kubernetes/charts/clowder2/templates/NOTES.txt index e0dfa09a9..8ba1c1c8a 100644 --- a/deployments/kubernetes/charts/clowder2/templates/NOTES.txt +++ b/deployments/kubernetes/charts/clowder2/templates/NOTES.txt @@ -15,7 +15,7 @@ export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "clowder2.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") echo http://$SERVICE_IP:{{ .Values.frontend.service.port }} {{- else if contains "ClusterIP" .Values.frontend.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "clowder2.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "clowder2.fullname" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") echo "Visit http://127.0.0.1:8080 to use your application" kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT diff --git a/deployments/kubernetes/charts/clowder2/templates/_helpers.tpl b/deployments/kubernetes/charts/clowder2/templates/_helpers.tpl index da435e845..49cbaa634 100644 --- a/deployments/kubernetes/charts/clowder2/templates/_helpers.tpl +++ b/deployments/kubernetes/charts/clowder2/templates/_helpers.tpl @@ -46,7 +46,7 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} Selector labels */}} {{- define "clowder2.selectorLabels" -}} -app.kubernetes.io/name: {{ include "clowder2.name" . }} +app.kubernetes.io/name: {{ include "clowder2.fullname" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} diff --git a/deployments/kubernetes/charts/clowder2/templates/backend/deployment.yaml b/deployments/kubernetes/charts/clowder2/templates/backend/deployment.yaml index 988fc536c..4daa2a606 100644 --- a/deployments/kubernetes/charts/clowder2/templates/backend/deployment.yaml +++ b/deployments/kubernetes/charts/clowder2/templates/backend/deployment.yaml @@ -10,7 +10,7 @@ spec: {{- end }} selector: matchLabels: - app.kubernetes.io/name: {{ include "clowder2.name" . }}-backend + app.kubernetes.io/name: {{ include "clowder2.fullname" . }}-backend app.kubernetes.io/instance: {{ .Release.Name }} template: metadata: @@ -19,7 +19,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} labels: - app.kubernetes.io/name: {{ include "clowder2.name" . }}-backend + app.kubernetes.io/name: {{ include "clowder2.fullname" . }}-backend app.kubernetes.io/instance: {{ .Release.Name }} spec: {{- with .Values.imagePullSecrets }} @@ -39,25 +39,31 @@ spec: - name: WEB_CONCURRENCY value: "1" - name: MINIO_SERVER_URL - value: {{ include "clowder2.name" . }}-minio:9000 -# value: {{ include "clowder2.name" . }}-minio-headless:9000 + value: {{ .Values.backend.env.MINIO_SERVER_URL }} - name: MINIO_EXTERNAL_SERVER_URL value: minio-api.{{ .Values.hostname }} - name: MINIO_SECURE - value: "true" + value: {{ .Values.backend.env.MINIO_SECURE | quote }} - name: MINIO_BUCKET_NAME - value: clowder + value: {{ .Values.backend.env.MINIO_BUCKET_NAME }} - name: MINIO_ACCESS_KEY value: {{ .Values.minio.auth.rootUser }} - name: MINIO_SECRET_KEY + {{- if .Values.backend.existingSecret }} valueFrom: secretKeyRef: - name: clowder2-minio + name: {{ .Values.backend.existingSecret }} + key: {{ .Values.backend.existingMinioSecretKey | default "root-password" }} + {{- else }} + valueFrom: + secretKeyRef: + name: {{ include "clowder2.fullname" . }}-minio key: root-password + {{- end }} - name: MINIO_UPLOAD_CHUNK_SIZE - value: "10485760" + value: {{ .Values.backend.env.MINIO_UPLOAD_CHUNK_SIZE | quote}} - name: MONGODB_URL - value: mongodb://{{ include "clowder2.name" . }}-mongodb:27017 + value: {{ .Values.backend.env.MONGODB_URL }} - name: MONGO_DATABASE value: {{ .Values.mongodb.database }} - name: CLOWDER2_URL @@ -73,11 +79,11 @@ spec: - name: auth_url value: $(CLOWDER2_URL)/keycloak/realms/clowder/protocol/openid-connect/auth?client_id=clowder2-backend&response_type=code - name: oauth2_scheme_auth_url - value: http://{{ include "clowder2.name" .}}-keycloak-headless:8080/keycloak/realms/clowder/protocol/openid-connect/auth?client_id=clowder2-backend&response_type=code + value: {{ .Values.backend.env.oauth2_scheme_auth_url }} - name: auth_register_url - value: $(CLOWDER2_URL)/keycloak/realms/clowder/protocol/openid-connect/registrations?client_id=clowder2-backend&response_type=code&redirect_uri=$(auth_redirect_uri)&scope=openid%20email + value: $(CLOWDER2_URL){{ .Values.backend.env.auth_register_url }} - name: auth_token_url - value: http://{{ include "clowder2.name" .}}-keycloak-headless:8080/keycloak/realms/clowder/protocol/openid-connect/token + value: {{ .Values.backend.env.auth_token_url }} - name: auth_server_url value: $(CLOWDER2_URL)/keycloak/ - name: keycloak_base @@ -85,7 +91,7 @@ spec: - name: frontend_url value: $(CLOWDER2_URL) - name: elasticsearch_url - value: http://{{ include "clowder2.name" . }}-elasticsearch:9200 + value: {{ .Values.backend.env.elasticsearch_url }} - name: elasticsearch_no_of_shards value: "5" - name: elasticsearch_no_of_replicas @@ -93,17 +99,23 @@ spec: - name: RABBITMQ_USER value: {{ .Values.rabbitmq.auth.username }} - name: RABBITMQ_PASS + {{- if .Values.backend.existingSecret }} + valueFrom: + secretKeyRef: + name: {{ .Values.backend.existingSecret }} + key: {{ .Values.backend.existingRabbitMQSecretKey | default "rabbitmq-password" }} + {{- else }} valueFrom: secretKeyRef: - name: clowder2-rabbitmq + name: {{ include "clowder2.fullname" . }}-rabbitmq key: rabbitmq-password + {{- end }} - name: RABBITMQ_HOST - value: {{ include "clowder2.name" . }}-rabbitmq + value: {{ .Values.backend.env.RABBITMQ_HOST }} - name: HEARTBEAT_EXCHANGE value: "extractors" - name: API_HOST -# value: $(CLOWDER2_URL) - value: http://{{ include "clowder2.name" . }}-backend:{{ .Values.backend.service.port }} + value: {{ .Values.backend.env.API_HOST }} ports: - name: http containerPort: 80 diff --git a/deployments/kubernetes/charts/clowder2/templates/backend/ingress.yaml b/deployments/kubernetes/charts/clowder2/templates/backend/ingress.yaml index 41423cc31..0b7ddec6b 100644 --- a/deployments/kubernetes/charts/clowder2/templates/backend/ingress.yaml +++ b/deployments/kubernetes/charts/clowder2/templates/backend/ingress.yaml @@ -21,7 +21,6 @@ spec: {{- if .Values.ingress.tls }} tls: - hosts: - - {{ .Values.hostname }} {{- range .Values.ingress.tls }} {{- range .hosts }} - {{ . | quote }} diff --git a/deployments/kubernetes/charts/clowder2/templates/extractors/deployment.yaml b/deployments/kubernetes/charts/clowder2/templates/extractors/deployment.yaml index ab055b7c7..7d9c4babb 100644 --- a/deployments/kubernetes/charts/clowder2/templates/extractors/deployment.yaml +++ b/deployments/kubernetes/charts/clowder2/templates/extractors/deployment.yaml @@ -38,10 +38,17 @@ spec: - name: RABBITMQ_USER value: {{ $.Values.rabbitmq.auth.username }} - name: RABBITMQ_PASS + {{- if $val.existingSecret }} valueFrom: secretKeyRef: - name: clowder2-rabbitmq + name: {{ $val.existingSecret }} + key: {{ $val.existingRabbitMQSecretKey | default "rabbitmq-password" }} + {{- else }} + valueFrom: + secretKeyRef: + name: {{ $fullname }}-rabbitmq key: rabbitmq-password + {{- end }} - name: CLOWDER_VERSION value: "2" - name: RABBITMQ_URI diff --git a/deployments/kubernetes/charts/clowder2/templates/frontend/deployment.yaml b/deployments/kubernetes/charts/clowder2/templates/frontend/deployment.yaml index eab44f6be..51cb9ca6c 100644 --- a/deployments/kubernetes/charts/clowder2/templates/frontend/deployment.yaml +++ b/deployments/kubernetes/charts/clowder2/templates/frontend/deployment.yaml @@ -10,7 +10,7 @@ spec: {{- end }} selector: matchLabels: - app.kubernetes.io/name: {{ include "clowder2.name" . }}-frontend + app.kubernetes.io/name: {{ include "clowder2.fullname" . }}-frontend app.kubernetes.io/instance: {{ .Release.Name }} template: metadata: @@ -19,7 +19,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} labels: - app.kubernetes.io/name: {{ include "clowder2.name" . }}-frontend + app.kubernetes.io/name: {{ include "clowder2.fullname" . }}-frontend app.kubernetes.io/instance: {{ .Release.Name }} spec: {{- with .Values.imagePullSecrets }} diff --git a/deployments/kubernetes/charts/clowder2/templates/frontend/ingress.yaml b/deployments/kubernetes/charts/clowder2/templates/frontend/ingress.yaml index dde70f74c..f54044a36 100644 --- a/deployments/kubernetes/charts/clowder2/templates/frontend/ingress.yaml +++ b/deployments/kubernetes/charts/clowder2/templates/frontend/ingress.yaml @@ -21,7 +21,6 @@ spec: {{- if .Values.ingress.tls }} tls: - hosts: - - {{ .Values.hostname }} {{- range .Values.ingress.tls }} {{- range .hosts }} - {{ . | quote }} diff --git a/deployments/kubernetes/charts/clowder2/templates/geoserver/deployment.yaml b/deployments/kubernetes/charts/clowder2/templates/geoserver/deployment.yaml index 2a97c6ac1..7ee3a45da 100644 --- a/deployments/kubernetes/charts/clowder2/templates/geoserver/deployment.yaml +++ b/deployments/kubernetes/charts/clowder2/templates/geoserver/deployment.yaml @@ -9,14 +9,14 @@ spec: replicas: {{ .Values.geoserver.replicaCount }} selector: matchLabels: - app.kubernetes.io/name: {{ include "clowder2.name" . }}-geoserver + app.kubernetes.io/name: {{ include "clowder2.fullname" . }}-geoserver app.kubernetes.io/instance: {{ .Release.Name }} strategy: type: RollingUpdate template: metadata: labels: - app.kubernetes.io/name: {{ include "clowder2.name" . }}-geoserver + app.kubernetes.io/name: {{ include "clowder2.fullname" . }}-geoserver app.kubernetes.io/instance: {{ .Release.Name }} spec: {{- with .Values.imagePullSecrets }} @@ -29,15 +29,19 @@ spec: imagePullPolicy: {{ .Values.geoserver.image.pullPolicy }} env: - name: GEOSERVER_ADMIN_USER + value: {{ .Values.geoserver.username }} + - name: GEOSERVER_ADMIN_PASSWORD + {{- if .Values.geoserver.existingSecret }} valueFrom: secretKeyRef: - name: {{ include "clowder2.fullname" . }}-services - key: GEOSERVER_USER - - name: GEOSERVER_ADMIN_PASSWORD + name: {{.Values.geoserver.existingSecret }} + key: {{.Values.geoserver.existingGeoserverSecretKey | default "GEOSERVER_PW" }} + {{- else }} valueFrom: secretKeyRef: - name: {{ include "clowder2.fullname" . }}-services + name: {{include "clowder2.fullname" . }}-services key: GEOSERVER_PW + {{- end }} ports: - containerPort: 8080 name: geoserver diff --git a/deployments/kubernetes/charts/clowder2/templates/geoserver/service.yaml b/deployments/kubernetes/charts/clowder2/templates/geoserver/service.yaml index 3a967f618..83e341cb0 100644 --- a/deployments/kubernetes/charts/clowder2/templates/geoserver/service.yaml +++ b/deployments/kubernetes/charts/clowder2/templates/geoserver/service.yaml @@ -13,6 +13,6 @@ spec: targetPort: geoserver protocol: TCP selector: - app.kubernetes.io/name: {{ include "clowder2.name" . }}-geoserver + app.kubernetes.io/name: {{ include "clowder2.fullname" . }}-geoserver app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} diff --git a/deployments/kubernetes/charts/clowder2/templates/heartbeat/deployment.yaml b/deployments/kubernetes/charts/clowder2/templates/heartbeat/deployment.yaml index a2f53a69d..44dea9954 100644 --- a/deployments/kubernetes/charts/clowder2/templates/heartbeat/deployment.yaml +++ b/deployments/kubernetes/charts/clowder2/templates/heartbeat/deployment.yaml @@ -10,7 +10,7 @@ spec: {{- end }} selector: matchLabels: - app.kubernetes.io/name: {{ include "clowder2.name" . }}-heartbeat + app.kubernetes.io/name: {{ include "clowder2.fullname" . }}-heartbeat app.kubernetes.io/instance: {{ .Release.Name }} template: metadata: @@ -19,7 +19,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} labels: - app.kubernetes.io/name: {{ include "clowder2.name" . }}-heartbeat + app.kubernetes.io/name: {{ include "clowder2.fullname" . }}-heartbeat app.kubernetes.io/instance: {{ .Release.Name }} spec: {{- with .Values.imagePullSecrets }} @@ -34,16 +34,23 @@ spec: imagePullPolicy: {{ .Values.heartbeat.image.pullPolicy }} env: - name: MONGODB_URL - value: mongodb://{{ include "clowder2.name" . }}-mongodb:27017 + value: {{ .Values.heartbeat.env.MONGODB_URL }} - name: RABBITMQ_USER value: {{ .Values.rabbitmq.auth.username }} - name: RABBITMQ_PASS + {{- if .Values.heartbeat.existingSecret }} valueFrom: secretKeyRef: - name: clowder2-rabbitmq + name: {{.Values.heartbeat.existingSecret }} + key: {{.Values.heartbeat.existingRabbitMQSecretKey | default "rabbitmq-password" }} + {{- else }} + valueFrom: + secretKeyRef: + name: {{include "clowder2.fullname" . }}-rabbitmq key: rabbitmq-password + {{- end }} - name: RABBITMQ_HOST - value: {{ include "clowder2.name" . }}-rabbitmq + value: {{ .Values.heartbeat.env.RABBITMQ_HOST }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/deployments/kubernetes/charts/clowder2/templates/messages/deployment.yaml b/deployments/kubernetes/charts/clowder2/templates/messages/deployment.yaml index 680bd1cd3..d85dfde00 100644 --- a/deployments/kubernetes/charts/clowder2/templates/messages/deployment.yaml +++ b/deployments/kubernetes/charts/clowder2/templates/messages/deployment.yaml @@ -10,7 +10,7 @@ spec: {{- end }} selector: matchLabels: - app.kubernetes.io/name: {{ include "clowder2.name" . }}-messages + app.kubernetes.io/name: {{ include "clowder2.fullname" . }}-messages app.kubernetes.io/instance: {{ .Release.Name }} template: metadata: @@ -19,7 +19,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} labels: - app.kubernetes.io/name: {{ include "clowder2.name" . }}-messages + app.kubernetes.io/name: {{ include "clowder2.fullname" . }}-messages app.kubernetes.io/instance: {{ .Release.Name }} spec: {{- with .Values.imagePullSecrets }} @@ -34,16 +34,23 @@ spec: imagePullPolicy: {{ .Values.message.image.pullPolicy }} env: - name: MONGODB_URL - value: mongodb://{{ include "clowder2.name" . }}-mongodb:27017 + value: {{ .Values.message.env.MONGODB_URL }} - name: RABBITMQ_USER value: {{ .Values.rabbitmq.auth.username }} - name: RABBITMQ_PASS + {{- if .Values.message.existingSecret }} valueFrom: secretKeyRef: - name: clowder2-rabbitmq + name: {{ .Values.message.existingSecret }} + key: {{ .Values.message.existingRabbitMQSecretKey | default "rabbitmq-password" }} + {{- else }} + valueFrom: + secretKeyRef: + name: {{ include "clowder2.fullname" . }}-rabbitmq key: rabbitmq-password + {{- end }} - name: RABBITMQ_HOST - value: {{ include "clowder2.name" . }}-rabbitmq + value: {{ .Values.message.env.RABBITMQ_HOST }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/deployments/kubernetes/charts/clowder2/values.yaml b/deployments/kubernetes/charts/clowder2/values.yaml index 262aea520..bed1ba18b 100644 --- a/deployments/kubernetes/charts/clowder2/values.yaml +++ b/deployments/kubernetes/charts/clowder2/values.yaml @@ -51,7 +51,18 @@ backend: pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. tag: "" - + env: + MINIO_SERVER_URL: clowder2-minio:9000 + MINIO_SECURE: "true" + MINIO_UPLOAD_CHUNK_SIZE: "10485760" + MINIO_BUCKET_NAME: clowder + MONGODB_URL: mongodb://clowder2-mongodb:27017 + oauth2_scheme_auth_url: http://clowder2-keycloak-headless:8080/keycloak/realms/clowder/protocol/openid-connect/auth?client_id=clowder2-backend&response_type=code + auth_register_url: /keycloak/realms/clowder/protocol/openid-connect/registrations?client_id=clowder2-backend&response_type=code&redirect_uri=$(auth_redirect_uri)&scope=openid%20email + auth_token_url: http://clowder2-keycloak-headless:8080/keycloak/realms/clowder/protocol/openid-connect/token + elasticsearch_url: http://clowder2-elasticsearch:9200 + RABBITMQ_HOST: clowder2-rabbitmq + API_HOST: http://clowder2-backend:80 service: type: ClusterIP port: 80 @@ -60,7 +71,7 @@ backend: # GEOSERVER # ---------------------------------------------------------------------- geoserver: - enabled : true + enabled : false username: admin password: admin workspace: clowder2 @@ -97,12 +108,12 @@ minio: # enable ingress for webui ingress: - enabled: true + enabled: false hostname: minio.clowder2.localhost # enable ingress for api apiIngress: - enabled: true + enabled: false hostname: minio-api.clowder2.localhost # enabled distributed mode (or standalone) @@ -165,7 +176,7 @@ rabbitmq: # enable webfrontend ingress: - enabled: true + enabled: false hostname: rabbitmq.clowder2.localhost persistence: @@ -175,28 +186,33 @@ rabbitmq: # ---------------------------------------------------------------------- # DEFAULT EXTRACTOR SERVICES # ---------------------------------------------------------------------- + + +# ---------------------------------------------------------------------- +# EXTRACTORS message: image: repository: clowder/clowder2-messages -# tag: main pullPolicy: Always + env: + MONGODB_URL: mongodb://clowder2-mongodb:27017 + RABBITMQ_HOST: clowder2-rabbitmq heartbeat: image: repository: clowder/clowder2-heartbeat -# tag: main pullPolicy: Always + env: + MONGODB_URL: mongodb://clowder2-mongodb:27017 + RABBITMQ_HOST: clowder2-rabbitmq - -# ---------------------------------------------------------------------- -# EXTRACTORS -# ---------------------------------------------------------------------- ## list of extractors to be installed. Each extractor should have the ## following fields: ## enabled : true/false - should the extractor be installed ## image : dokeer/image:tag - the extractor docker image ## replicaCount : number - the number of this extractor to install ## env : additional environment variables to pass to extractor. + extractors: wordcount: image: clowder/extractors-wordcount:latest From 91ad4e74e9c0de9234ce0ac7177634f937a54b60 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Fri, 26 Jul 2024 11:09:51 -0500 Subject: [PATCH 03/32] turn off default extractors --- deployments/kubernetes/charts/clowder2/values.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/deployments/kubernetes/charts/clowder2/values.yaml b/deployments/kubernetes/charts/clowder2/values.yaml index bed1ba18b..a09fc20f4 100644 --- a/deployments/kubernetes/charts/clowder2/values.yaml +++ b/deployments/kubernetes/charts/clowder2/values.yaml @@ -215,16 +215,22 @@ heartbeat: extractors: wordcount: + enabled: false image: clowder/extractors-wordcount:latest image-preview: + enabled: false image: clowder/extractors-image-preview:latest image-metadata: + enabled: false image: clowder/extractors-image-metadata:latest audio-preview: + enabled: false image: clowder/extractors-audio-preview:latest file-digest: + enabled: false image: clowder/extractors-digest:latest geotiff-preview: + enabled: false image: clowder/extractors-geotiff-preview:latest # ---------------------------------------------------------------------- From b3395df497618b00d8a8f972986f58a42921a4c2 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 29 Jul 2024 13:36:12 -0500 Subject: [PATCH 04/32] initial commit new model --- backend/app/models/project.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 backend/app/models/project.py diff --git a/backend/app/models/project.py b/backend/app/models/project.py new file mode 100644 index 000000000..eab0ce347 --- /dev/null +++ b/backend/app/models/project.py @@ -0,0 +1,22 @@ +from datetime import datetime +from enum import Enum, auto +from typing import List, Optional + +import pymongo +from app.models.authorization import AuthorizationDB, RoleType +from app.models.groups import GroupOut +from app.models.users import UserOut +from beanie import Document, PydanticObjectId, View +from pydantic import BaseModel, Field + + +class Project(BaseModel): + id: PydanticObjectId = Field(default_factory=PydanticObjectId, alias="_id") + name: str + description: Optional[str] = None + created: datetime = Field(default_factory=datetime.utcnow) + modified: datetime = Field(default_factory=datetime.utcnow) + dataset_ids: Optional[List[PydanticObjectId]] = None + folder_ids: Optional[List[PydanticObjectId]] = None + file_ids: Optional[List[PydanticObjectId]] = None + creator: UserOut \ No newline at end of file From 1f73eadfb26adf211eb5ee27891742a1a02f1878 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 29 Jul 2024 13:38:48 -0500 Subject: [PATCH 05/32] adding db, out, in, etc --- backend/app/models/project.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/backend/app/models/project.py b/backend/app/models/project.py index eab0ce347..76bb27bb3 100644 --- a/backend/app/models/project.py +++ b/backend/app/models/project.py @@ -10,7 +10,7 @@ from pydantic import BaseModel, Field -class Project(BaseModel): +class ProjectBase(BaseModel): id: PydanticObjectId = Field(default_factory=PydanticObjectId, alias="_id") name: str description: Optional[str] = None @@ -19,4 +19,15 @@ class Project(BaseModel): dataset_ids: Optional[List[PydanticObjectId]] = None folder_ids: Optional[List[PydanticObjectId]] = None file_ids: Optional[List[PydanticObjectId]] = None - creator: UserOut \ No newline at end of file + creator: UserOut + +class ProjectDB(Document, ProjectBase): + class Settings: + name = "projects" + +class ProjectIn(ProjectBase): + pass + +class ProjectOut(ProjectDB): + class Config: + fields = {"id": "id"} \ No newline at end of file From d4c3904a0d3a8142fdc9dc0b41c38e1bba1d93c8 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 29 Jul 2024 13:46:45 -0500 Subject: [PATCH 06/32] save and get methods --- backend/app/routers/projects.py | 83 +++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 backend/app/routers/projects.py diff --git a/backend/app/routers/projects.py b/backend/app/routers/projects.py new file mode 100644 index 000000000..f9c1fd086 --- /dev/null +++ b/backend/app/routers/projects.py @@ -0,0 +1,83 @@ +import datetime +import hashlib +import io +import os +import shutil +import tempfile +import zipfile +from collections.abc import Iterable, Mapping +from typing import List, Optional +from app.models.pages import Paged, _construct_page_metadata, _get_page_query + + +from app import dependencies +from app.config import settings + +from app.models.project import ( + ProjectBase, + ProjectDB, + ProjectIn, + ProjectOut, +) +from app.keycloak_auth import get_current_user, get_token, get_user +from beanie import PydanticObjectId +from beanie.operators import And, Or +from bson import ObjectId, json_util +from elasticsearch import Elasticsearch +from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile +from fastapi.responses import StreamingResponse +from fastapi.security import HTTPBearer +from minio import Minio +from pika.adapters.blocking_connection import BlockingChannel +from pymongo import DESCENDING +from rocrate.model.person import Person +from rocrate.rocrate import ROCrate + +router = APIRouter() +security = HTTPBearer() + +clowder_bucket = os.getenv("MINIO_BUCKET_NAME", "clowder") + +@router.post("", response_model=ProjectOut) +async def save_project( + project_in: ProjectIn, + license_id: str, + user=Depends(get_current_user), + es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), +): + + project = ProjectDB( + **project_in.dict(), + creator=user, + ) + await project.insert() + + # TODO Add new entry to elasticsearch + return project.dict() + +@router.get("", response_model=Paged) +async def get_projects( + user_id=Depends(get_user), + skip: int = 0, + limit: int = 10, + mine: bool = False, + enable_admin: bool = False, +): + + query = (ProjectDB.creator.email == user_id) + + projects_and_count = await ProjectDB.find(*query).to_list() + + page_metadata = _construct_page_metadata(projects_and_count, skip, limit) + # TODO have to change _id this way otherwise it won't work + # TODO need to research if there is other pydantic trick to make it work + + page = Paged( + metadata=page_metadata, + data=[ + ProjectOut(id=item.pop("_id"), **item) + for item in projects_and_count[0]["data"] + ], + ) + + return page.dict() \ No newline at end of file From 1c83600a997e6facf7f6a3ecafe9ec5b6351a656 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 29 Jul 2024 13:53:13 -0500 Subject: [PATCH 07/32] get method --- backend/app/routers/projects.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/backend/app/routers/projects.py b/backend/app/routers/projects.py index f9c1fd086..8ff50ca21 100644 --- a/backend/app/routers/projects.py +++ b/backend/app/routers/projects.py @@ -80,4 +80,13 @@ async def get_projects( ], ) - return page.dict() \ No newline at end of file + return page.dict() + +@router.get("/{project_id}", response_model=ProjectOut) +async def get_project( + project_id: str, +): + if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) + ) is not None: + return project.dict() + raise HTTPException(status_code=404, detail=f"Project {project_id} not found") \ No newline at end of file From a89ebbb1c162d6a262ccd3a762bd57a24e9465f4 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 29 Jul 2024 15:45:13 -0500 Subject: [PATCH 08/32] add users to project --- backend/app/models/project.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/app/models/project.py b/backend/app/models/project.py index 76bb27bb3..adaa9a7ff 100644 --- a/backend/app/models/project.py +++ b/backend/app/models/project.py @@ -10,6 +10,10 @@ from pydantic import BaseModel, Field +class Member(BaseModel): + user: UserOut + editor: bool = False + class ProjectBase(BaseModel): id: PydanticObjectId = Field(default_factory=PydanticObjectId, alias="_id") name: str @@ -20,6 +24,7 @@ class ProjectBase(BaseModel): folder_ids: Optional[List[PydanticObjectId]] = None file_ids: Optional[List[PydanticObjectId]] = None creator: UserOut + users: List[Member] = [] class ProjectDB(Document, ProjectBase): class Settings: From a15adad611334e2d3518b66e83e9f4b374833b44 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 29 Jul 2024 15:46:13 -0500 Subject: [PATCH 09/32] adding to do --- backend/app/routers/projects.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/app/routers/projects.py b/backend/app/routers/projects.py index 8ff50ca21..2d8c87927 100644 --- a/backend/app/routers/projects.py +++ b/backend/app/routers/projects.py @@ -64,6 +64,7 @@ async def get_projects( enable_admin: bool = False, ): + # TODO check if the current user is a member OR creator query = (ProjectDB.creator.email == user_id) projects_and_count = await ProjectDB.find(*query).to_list() From f5e69b3eb6991e013cee2995c3387f7203314136 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 29 Jul 2024 16:16:05 -0500 Subject: [PATCH 10/32] add a remove project member --- backend/app/routers/projects.py | 278 +++++++++++++++++++++++++++++++- 1 file changed, 277 insertions(+), 1 deletion(-) diff --git a/backend/app/routers/projects.py b/backend/app/routers/projects.py index 2d8c87927..878e882a5 100644 --- a/backend/app/routers/projects.py +++ b/backend/app/routers/projects.py @@ -90,4 +90,280 @@ async def get_project( if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) ) is not None: return project.dict() - raise HTTPException(status_code=404, detail=f"Project {project_id} not found") \ No newline at end of file + raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + +from datetime import datetime +from typing import Optional + +from app import dependencies +from app.deps.authorization_deps import AuthorizationDB, GroupAuthorization +from app.keycloak_auth import get_current_user, get_user +from app.models.authorization import RoleType +from app.models.datasets import DatasetDB, DatasetOut +from app.models.groups import GroupBase, GroupDB, GroupIn, GroupOut, Member +from app.models.pages import Paged, _construct_page_metadata, _get_page_query +from app.models.users import UserDB, UserOut +from app.routers.authentication import get_admin, get_admin_mode +from app.search.index import index_dataset, index_dataset_files +from beanie import PydanticObjectId +from beanie.operators import Or, Push, RegEx +from bson.objectid import ObjectId +from fastapi import APIRouter, Depends, HTTPException + +router = APIRouter() + + +@router.post("", response_model=GroupOut) +async def save_group( + group_in: GroupIn, + user=Depends(get_current_user), +): + group_db = GroupDB(**group_in.dict(), creator=user.email) + user_member = Member(user=user, editor=True) + if user_member not in group_db.users: + group_db.users.append(user_member) + await group_db.insert() + return group_db.dict() + + +@router.get("", response_model=Paged) +async def get_groups( + user_id=Depends(get_user), + skip: int = 0, + limit: int = 10, + enable_admin: bool = False, + admin_mode: bool = Depends(get_admin_mode), + admin=Depends(get_admin), +): + """Get a list of all Groups in the db the user is a member/owner of. + + Arguments: + skip -- number of initial recoto_list()rds to skip (i.e. for pagination) + limit -- restrict number of records to be returned (i.e. for pagination) + + + """ + criteria_list = [] + if not admin or not admin_mode: + criteria_list.append( + Or( + GroupDB.creator == user_id, + GroupDB.users.user.email == user_id, + ) + ) + + groups_and_count = ( + await GroupDB.find( + *criteria_list, + ) + .aggregate( + [_get_page_query(skip, limit, sort_field="created", ascending=False)], + ) + .to_list() + ) + page_metadata = _construct_page_metadata(groups_and_count, skip, limit) + page = Paged( + metadata=page_metadata, + data=[ + GroupOut(id=item.pop("_id"), **item) for item in groups_and_count[0]["data"] + ], + ) + return page.dict() + + +@router.get("/search/{search_term}", response_model=Paged) +async def search_group( + search_term: str, + user_id=Depends(get_user), + skip: int = 0, + limit: int = 10, + enable_admin: bool = False, + admin_mode: bool = Depends(get_admin_mode), + admin=Depends(get_admin), +): + """Search all groups in the db based on text. + + Arguments: + text -- any text matching name or description + skip -- number of initial records to skip (i.e. for pagination) + limit -- restrict number of records to be returned (i.e. for pagination) + """ + + criteria_list = [ + Or( + RegEx(field=GroupDB.name, pattern=search_term, options="i"), + RegEx(field=GroupDB.description, pattern=search_term, options="i"), + ), + ] + if not admin or not admin_mode: + criteria_list.append( + Or(GroupDB.creator == user_id, GroupDB.users.user.email == user_id) + ) + + # user has to be the creator or member first; then apply search + groups_and_count = ( + await GroupDB.find( + *criteria_list, + ) + .aggregate( + [_get_page_query(skip, limit, sort_field="created", ascending=False)], + ) + .to_list() + ) + page_metadata = _construct_page_metadata(groups_and_count, skip, limit) + page = Paged( + metadata=page_metadata, + data=[ + GroupOut(id=item.pop("_id"), **item) for item in groups_and_count[0]["data"] + ], + ) + return page.dict() + + +@router.get("/{group_id}", response_model=GroupOut) +async def get_group( + group_id: str, + allow: bool = Depends(GroupAuthorization("viewer")), +): + if (group := await GroupDB.get(PydanticObjectId(group_id))) is not None: + return group.dict() + raise HTTPException(status_code=404, detail=f"Group {group_id} not found") + + +@router.put("/{group_id}", response_model=GroupOut) +async def edit_group( + group_id: str, + group_info: GroupBase, + user_id=Depends(get_user), + allow: bool = Depends(GroupAuthorization("editor")), +): + if (group := await GroupDB.get(PydanticObjectId(group_id))) is not None: + group_dict = dict(group_info) if group_info is not None else {} + + if len(group_dict["name"]) == 0 or len(group_dict["users"]) == 0: + raise HTTPException( + status_code=400, + detail="Group name can't be null or user list can't be empty", + ) + return + + user = await UserDB.find_one(UserDB.email == user_id) + group_dict["creator"] = user.dict() + group_dict["modified"] = datetime.utcnow() + groups_users = group_dict["users"] + original_users = group.users + + # remove users that are no longer in this group + for original_user in original_users: + if original_user not in groups_users: + # remove them from auth + async for auth in AuthorizationDB.find( + {"group_ids": ObjectId(group_id)} + ): + auth.user_ids.remove(original_user.user.email) + await auth.replace() + # Update group itself + group.users.remove(original_user) + await group.replace() + # add new users to the group + for i in range(0, len(groups_users)): + user = groups_users[i] + if user in group.users: + for original_user in group.users: + if original_user.user.email == user.user.email: + original_editor = original_user.editor + new_editor = user.editor + # replace the user if editor has changed + if not new_editor == original_editor: + group.users.remove(original_user) + group.users.append(user) + await group.replace() + else: + # if user is not in the group add user + group.users.append(user) + await group.replace() + # Add user to all affected Authorization entries + await AuthorizationDB.find( + AuthorizationDB.group_ids == ObjectId(group_id), + ).update( + Push({AuthorizationDB.user_ids: user.email}), + ) + try: + group.name = group_dict["name"] + await group.replace() + if "description" in group_dict: + group.description = group_dict["description"] + await group.replace() + except Exception as e: + raise HTTPException(status_code=500, detail=e.args[0]) + return group.dict() + raise HTTPException(status_code=404, detail=f"Group {group_id} not found") + + +@router.delete("/{group_id}", response_model=GroupOut) +async def delete_group( + group_id: str, + allow: bool = Depends(GroupAuthorization("owner")), +): + if (group := await GroupDB.get(PydanticObjectId(group_id))) is not None: + await group.delete() + return group.dict() # TODO: Do we need to return what we just deleted? + else: + raise HTTPException(status_code=404, detail=f"Dataset {group_id} not found") + + +@router.post("/{project_id}/add/{username}", response_model=ProjectOut) +async def add_member( + project_id: str, + username: str, + role: Optional[str] = None, + es=Depends(dependencies.get_elasticsearchclient), + allow: bool = Depends(GroupAuthorization("editor")), +): + """Add a new user to a group.""" + if (user := await UserDB.find_one(UserDB.email == username)) is not None: + new_member = Member(user=UserOut(**user.dict())) + if (project := await ProjectDB.get(PydanticObjectId(project_id))) is not None: + found_already = False + for u in project.users: + if u.user.email == username: + found_already = True + break + if not found_already: + # If user is already in the group, skip directly to returning the group + # else add role and attach this member + + if role is not None and role == RoleType.EDITOR: + new_member.editor = True + else: + new_member.editor = False + project.users.append(new_member) + await project.replace() + return project.dict() + raise HTTPException(status_code=404, detail=f"Group {project_id} not found") + raise HTTPException(status_code=404, detail=f"User {username} not found") + +@router.post("/{project_id}/remove/{username}", response_model=ProjectOut) +async def remove_member( + project_id: str, + username: str, + es=Depends(dependencies.get_elasticsearchclient), + allow: bool = Depends(GroupAuthorization("editor")), +): + """Remove a user from a group.""" + + if (project := await ProjectDB.get(PydanticObjectId(project_id))) is not None: + # Is the user actually in the group already? + found_user = None + for u in project.users: + if u.user.email == username: + found_user = u + if not found_user: + # TODO: User wasn't in group, should this throw an error instead? Either way, the user is removed... + return project + # Update group itself + project.users.remove(found_user) + await project.replace() + return project.dict() + raise HTTPException(status_code=404, detail=f"Group {project_id} not found") + From 3b2cba9e1aa916406adebbc902e0b8194c854eab Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 29 Jul 2024 16:20:26 -0500 Subject: [PATCH 11/32] routes for deleting and adding, removing members --- backend/app/routers/projects.py | 235 ++------------------------------ 1 file changed, 9 insertions(+), 226 deletions(-) diff --git a/backend/app/routers/projects.py b/backend/app/routers/projects.py index 878e882a5..20dfa6749 100644 --- a/backend/app/routers/projects.py +++ b/backend/app/routers/projects.py @@ -12,12 +12,13 @@ from app import dependencies from app.config import settings - +from app.models.users import UserDB, UserOut, UserIn from app.models.project import ( ProjectBase, ProjectDB, ProjectIn, ProjectOut, + Member, ) from app.keycloak_auth import get_current_user, get_token, get_user from beanie import PydanticObjectId @@ -92,224 +93,15 @@ async def get_project( return project.dict() raise HTTPException(status_code=404, detail=f"Project {project_id} not found") -from datetime import datetime -from typing import Optional - -from app import dependencies -from app.deps.authorization_deps import AuthorizationDB, GroupAuthorization -from app.keycloak_auth import get_current_user, get_user -from app.models.authorization import RoleType -from app.models.datasets import DatasetDB, DatasetOut -from app.models.groups import GroupBase, GroupDB, GroupIn, GroupOut, Member -from app.models.pages import Paged, _construct_page_metadata, _get_page_query -from app.models.users import UserDB, UserOut -from app.routers.authentication import get_admin, get_admin_mode -from app.search.index import index_dataset, index_dataset_files -from beanie import PydanticObjectId -from beanie.operators import Or, Push, RegEx -from bson.objectid import ObjectId -from fastapi import APIRouter, Depends, HTTPException - -router = APIRouter() - - -@router.post("", response_model=GroupOut) -async def save_group( - group_in: GroupIn, - user=Depends(get_current_user), -): - group_db = GroupDB(**group_in.dict(), creator=user.email) - user_member = Member(user=user, editor=True) - if user_member not in group_db.users: - group_db.users.append(user_member) - await group_db.insert() - return group_db.dict() - - -@router.get("", response_model=Paged) -async def get_groups( - user_id=Depends(get_user), - skip: int = 0, - limit: int = 10, - enable_admin: bool = False, - admin_mode: bool = Depends(get_admin_mode), - admin=Depends(get_admin), -): - """Get a list of all Groups in the db the user is a member/owner of. - - Arguments: - skip -- number of initial recoto_list()rds to skip (i.e. for pagination) - limit -- restrict number of records to be returned (i.e. for pagination) - - - """ - criteria_list = [] - if not admin or not admin_mode: - criteria_list.append( - Or( - GroupDB.creator == user_id, - GroupDB.users.user.email == user_id, - ) - ) - - groups_and_count = ( - await GroupDB.find( - *criteria_list, - ) - .aggregate( - [_get_page_query(skip, limit, sort_field="created", ascending=False)], - ) - .to_list() - ) - page_metadata = _construct_page_metadata(groups_and_count, skip, limit) - page = Paged( - metadata=page_metadata, - data=[ - GroupOut(id=item.pop("_id"), **item) for item in groups_and_count[0]["data"] - ], - ) - return page.dict() - - -@router.get("/search/{search_term}", response_model=Paged) -async def search_group( - search_term: str, - user_id=Depends(get_user), - skip: int = 0, - limit: int = 10, - enable_admin: bool = False, - admin_mode: bool = Depends(get_admin_mode), - admin=Depends(get_admin), -): - """Search all groups in the db based on text. - - Arguments: - text -- any text matching name or description - skip -- number of initial records to skip (i.e. for pagination) - limit -- restrict number of records to be returned (i.e. for pagination) - """ - - criteria_list = [ - Or( - RegEx(field=GroupDB.name, pattern=search_term, options="i"), - RegEx(field=GroupDB.description, pattern=search_term, options="i"), - ), - ] - if not admin or not admin_mode: - criteria_list.append( - Or(GroupDB.creator == user_id, GroupDB.users.user.email == user_id) - ) - - # user has to be the creator or member first; then apply search - groups_and_count = ( - await GroupDB.find( - *criteria_list, - ) - .aggregate( - [_get_page_query(skip, limit, sort_field="created", ascending=False)], - ) - .to_list() - ) - page_metadata = _construct_page_metadata(groups_and_count, skip, limit) - page = Paged( - metadata=page_metadata, - data=[ - GroupOut(id=item.pop("_id"), **item) for item in groups_and_count[0]["data"] - ], - ) - return page.dict() - - -@router.get("/{group_id}", response_model=GroupOut) -async def get_group( - group_id: str, - allow: bool = Depends(GroupAuthorization("viewer")), -): - if (group := await GroupDB.get(PydanticObjectId(group_id))) is not None: - return group.dict() - raise HTTPException(status_code=404, detail=f"Group {group_id} not found") - - -@router.put("/{group_id}", response_model=GroupOut) -async def edit_group( - group_id: str, - group_info: GroupBase, - user_id=Depends(get_user), - allow: bool = Depends(GroupAuthorization("editor")), -): - if (group := await GroupDB.get(PydanticObjectId(group_id))) is not None: - group_dict = dict(group_info) if group_info is not None else {} - - if len(group_dict["name"]) == 0 or len(group_dict["users"]) == 0: - raise HTTPException( - status_code=400, - detail="Group name can't be null or user list can't be empty", - ) - return - - user = await UserDB.find_one(UserDB.email == user_id) - group_dict["creator"] = user.dict() - group_dict["modified"] = datetime.utcnow() - groups_users = group_dict["users"] - original_users = group.users - - # remove users that are no longer in this group - for original_user in original_users: - if original_user not in groups_users: - # remove them from auth - async for auth in AuthorizationDB.find( - {"group_ids": ObjectId(group_id)} - ): - auth.user_ids.remove(original_user.user.email) - await auth.replace() - # Update group itself - group.users.remove(original_user) - await group.replace() - # add new users to the group - for i in range(0, len(groups_users)): - user = groups_users[i] - if user in group.users: - for original_user in group.users: - if original_user.user.email == user.user.email: - original_editor = original_user.editor - new_editor = user.editor - # replace the user if editor has changed - if not new_editor == original_editor: - group.users.remove(original_user) - group.users.append(user) - await group.replace() - else: - # if user is not in the group add user - group.users.append(user) - await group.replace() - # Add user to all affected Authorization entries - await AuthorizationDB.find( - AuthorizationDB.group_ids == ObjectId(group_id), - ).update( - Push({AuthorizationDB.user_ids: user.email}), - ) - try: - group.name = group_dict["name"] - await group.replace() - if "description" in group_dict: - group.description = group_dict["description"] - await group.replace() - except Exception as e: - raise HTTPException(status_code=500, detail=e.args[0]) - return group.dict() - raise HTTPException(status_code=404, detail=f"Group {group_id} not found") - - -@router.delete("/{group_id}", response_model=GroupOut) -async def delete_group( - group_id: str, - allow: bool = Depends(GroupAuthorization("owner")), +@router.delete("/{project_id}", response_model=ProjectOut) +async def delete_project( + project_id: str, ): - if (group := await GroupDB.get(PydanticObjectId(group_id))) is not None: - await group.delete() - return group.dict() # TODO: Do we need to return what we just deleted? + if (project := await ProjectDB.get(PydanticObjectId(project_id))) is not None: + await project.delete() + return project.dict() # TODO: Do we need to return what we just deleted? else: - raise HTTPException(status_code=404, detail=f"Dataset {group_id} not found") + raise HTTPException(status_code=404, detail=f"Project {project_id} not found") @router.post("/{project_id}/add/{username}", response_model=ProjectOut) @@ -317,8 +109,6 @@ async def add_member( project_id: str, username: str, role: Optional[str] = None, - es=Depends(dependencies.get_elasticsearchclient), - allow: bool = Depends(GroupAuthorization("editor")), ): """Add a new user to a group.""" if (user := await UserDB.find_one(UserDB.email == username)) is not None: @@ -332,11 +122,6 @@ async def add_member( if not found_already: # If user is already in the group, skip directly to returning the group # else add role and attach this member - - if role is not None and role == RoleType.EDITOR: - new_member.editor = True - else: - new_member.editor = False project.users.append(new_member) await project.replace() return project.dict() @@ -347,8 +132,6 @@ async def add_member( async def remove_member( project_id: str, username: str, - es=Depends(dependencies.get_elasticsearchclient), - allow: bool = Depends(GroupAuthorization("editor")), ): """Remove a user from a group.""" From 250f22af0bfb43fc147c3e89bf2c28c501ed3beb Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 29 Jul 2024 16:20:59 -0500 Subject: [PATCH 12/32] "project" not "group" --- backend/app/routers/projects.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/app/routers/projects.py b/backend/app/routers/projects.py index 20dfa6749..d13336a5b 100644 --- a/backend/app/routers/projects.py +++ b/backend/app/routers/projects.py @@ -125,7 +125,7 @@ async def add_member( project.users.append(new_member) await project.replace() return project.dict() - raise HTTPException(status_code=404, detail=f"Group {project_id} not found") + raise HTTPException(status_code=404, detail=f"Project {project_id} not found") raise HTTPException(status_code=404, detail=f"User {username} not found") @router.post("/{project_id}/remove/{username}", response_model=ProjectOut) @@ -148,5 +148,5 @@ async def remove_member( project.users.remove(found_user) await project.replace() return project.dict() - raise HTTPException(status_code=404, detail=f"Group {project_id} not found") + raise HTTPException(status_code=404, detail=f"Project {project_id} not found") From 4dc5b9420cbfabc7bb23611af66a9c2b1ce0bb54 Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 30 Jul 2024 09:11:41 -0500 Subject: [PATCH 13/32] methods for adding and removing datasets from projects folders and files will be the same --- backend/app/routers/projects.py | 36 +++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/backend/app/routers/projects.py b/backend/app/routers/projects.py index d13336a5b..ac4894ee8 100644 --- a/backend/app/routers/projects.py +++ b/backend/app/routers/projects.py @@ -13,6 +13,7 @@ from app import dependencies from app.config import settings from app.models.users import UserDB, UserOut, UserIn +from app.models.datasets import DatasetDB from app.models.project import ( ProjectBase, ProjectDB, @@ -56,6 +57,41 @@ async def save_project( # TODO Add new entry to elasticsearch return project.dict() +@router.post("/{project_id}/add_dataset/{dataset_id}", response_model=ProjectOut) +async def add_dataset( + project_id: str, + dataset_id: str, +): + if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) + ) is not None: + if dataset := await DatasetDB.find_one(DatasetDB.id == PydanticObjectId(dataset_id) + ) is not None: + project.dataset_ids.append(dataset_id) + await project.replace() + return project.dict() + raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found") + raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + +@router.post("/{project_id}/remove_dataset/{dataset_id}", response_model=ProjectOut) +async def add_dataset( + project_id: str, + dataset_id: str, +): + if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) + ) is not None: + if dataset := await DatasetDB.find_one(DatasetDB.id == PydanticObjectId(dataset_id) + ) is not None: + if dataset_id in project.dataset_ids: + project.dataset_ids.remove(dataset_id) + await project.replace() + return project.dict() + else: + return project.dict() + raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found") + raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + + + @router.get("", response_model=Paged) async def get_projects( user_id=Depends(get_user), From 9252bef42c1e3f89f1c2a7eec1368fcae1e0e313 Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 30 Jul 2024 09:14:51 -0500 Subject: [PATCH 14/32] remove license --- backend/app/routers/projects.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/app/routers/projects.py b/backend/app/routers/projects.py index ac4894ee8..1dd5fab60 100644 --- a/backend/app/routers/projects.py +++ b/backend/app/routers/projects.py @@ -43,7 +43,6 @@ @router.post("", response_model=ProjectOut) async def save_project( project_in: ProjectIn, - license_id: str, user=Depends(get_current_user), es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), ): @@ -73,7 +72,7 @@ async def add_dataset( raise HTTPException(status_code=404, detail=f"Project {project_id} not found") @router.post("/{project_id}/remove_dataset/{dataset_id}", response_model=ProjectOut) -async def add_dataset( +async def remove_dataset( project_id: str, dataset_id: str, ): From 6291cc044b3ceeb2719613191cd36da209107291 Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 30 Jul 2024 09:22:02 -0500 Subject: [PATCH 15/32] adding and removing folders and files --- backend/app/routers/projects.py | 68 +++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/backend/app/routers/projects.py b/backend/app/routers/projects.py index 1dd5fab60..037c86682 100644 --- a/backend/app/routers/projects.py +++ b/backend/app/routers/projects.py @@ -14,6 +14,8 @@ from app.config import settings from app.models.users import UserDB, UserOut, UserIn from app.models.datasets import DatasetDB +from app.models.files import FileDB +from app.models.folders import FolderDB from app.models.project import ( ProjectBase, ProjectDB, @@ -90,6 +92,72 @@ async def remove_dataset( raise HTTPException(status_code=404, detail=f"Project {project_id} not found") +@router.post("/{project_id}/add_folder/{folder_id}", response_model=ProjectOut) +async def add_folder( + project_id: str, + folder_id: str, +): + if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) + ) is not None: + if folder := await FolderDB.find_one(FolderDB.id == PydanticObjectId(dataset_id) + ) is not None: + project.folder_ids.append(folder_id) + await project.replace() + return project.dict() + raise HTTPException(status_code=404, detail=f"Folder {folder_id} not found") + raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + +@router.post("/{project_id}/remove_folder/{folder_id}", response_model=ProjectOut) +async def remove_folder( + project_id: str, + folder_id: str, +): + if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) + ) is not None: + if folder := await FolderDB.find_one(FolderDB.id == PydanticObjectId(folder_id) + ) is not None: + if folder_id in project.folder_ids: + project.folder_ids.remove(folder_id) + await project.replace() + return project.dict() + else: + return project.dict() + raise HTTPException(status_code=404, detail=f"Folder {folder_id} not found") + raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + +@router.post("/{project_id}/add_file/{file_id}", response_model=ProjectOut) +async def add_file( + project_id: str, + file_id: str, +): + if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) + ) is not None: + if file := await FileDB.find_one(FileDB.id == PydanticObjectId(file_id) + ) is not None: + project.file_ids.append(file_id) + await project.replace() + return project.dict() + raise HTTPException(status_code=404, detail=f"File {file_id} not found") + raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + +@router.post("/{project_id}/remove_file/{file_id}", response_model=ProjectOut) +async def remove_file( + project_id: str, + file_id: str, +): + if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) + ) is not None: + if file := await FileDB.find_one(FileDB.id == PydanticObjectId(file_id) + ) is not None: + if file_id in project.file_ids: + project.file_ids.remove(file_id) + await project.replace() + return project.dict() + else: + return project.dict() + raise HTTPException(status_code=404, detail=f"File {file_id} not found") + raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + @router.get("", response_model=Paged) async def get_projects( From a679d7cc75d723710a9acbd30c6b73f4745ece60 Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 30 Jul 2024 09:26:55 -0500 Subject: [PATCH 16/32] only add if a file, folder, dataset id not in the project already --- backend/app/routers/projects.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/backend/app/routers/projects.py b/backend/app/routers/projects.py index 037c86682..d37982279 100644 --- a/backend/app/routers/projects.py +++ b/backend/app/routers/projects.py @@ -67,8 +67,9 @@ async def add_dataset( ) is not None: if dataset := await DatasetDB.find_one(DatasetDB.id == PydanticObjectId(dataset_id) ) is not None: - project.dataset_ids.append(dataset_id) - await project.replace() + if dataset_id not in project.dataset_ids: + project.dataset_ids.append(dataset_id) + await project.replace() return project.dict() raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found") raise HTTPException(status_code=404, detail=f"Project {project_id} not found") @@ -99,10 +100,11 @@ async def add_folder( ): if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) ) is not None: - if folder := await FolderDB.find_one(FolderDB.id == PydanticObjectId(dataset_id) + if folder := await FolderDB.find_one(FolderDB.id == PydanticObjectId(folder_id) ) is not None: - project.folder_ids.append(folder_id) - await project.replace() + if folder_id not in project.folder_ids: + project.folder_ids.append(folder_id) + await project.replace() return project.dict() raise HTTPException(status_code=404, detail=f"Folder {folder_id} not found") raise HTTPException(status_code=404, detail=f"Project {project_id} not found") @@ -134,8 +136,9 @@ async def add_file( ) is not None: if file := await FileDB.find_one(FileDB.id == PydanticObjectId(file_id) ) is not None: - project.file_ids.append(file_id) - await project.replace() + if file_id not in project.file_ids: + project.file_ids.append(file_id) + await project.replace() return project.dict() raise HTTPException(status_code=404, detail=f"File {file_id} not found") raise HTTPException(status_code=404, detail=f"Project {project_id} not found") From 9f2c8acfc91419a7b94269f0640963a421d1dbc5 Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 30 Jul 2024 09:44:44 -0500 Subject: [PATCH 17/32] beginning project tests formatting --- backend/app/models/project.py | 6 +- backend/app/routers/projects.py | 142 ++++++++++++++++++++--------- backend/app/tests/test_projects.py | 50 ++++++++++ backend/app/tests/utils.py | 16 ++++ 4 files changed, 169 insertions(+), 45 deletions(-) create mode 100644 backend/app/tests/test_projects.py diff --git a/backend/app/models/project.py b/backend/app/models/project.py index adaa9a7ff..52b964696 100644 --- a/backend/app/models/project.py +++ b/backend/app/models/project.py @@ -14,6 +14,7 @@ class Member(BaseModel): user: UserOut editor: bool = False + class ProjectBase(BaseModel): id: PydanticObjectId = Field(default_factory=PydanticObjectId, alias="_id") name: str @@ -26,13 +27,16 @@ class ProjectBase(BaseModel): creator: UserOut users: List[Member] = [] + class ProjectDB(Document, ProjectBase): class Settings: name = "projects" + class ProjectIn(ProjectBase): pass + class ProjectOut(ProjectDB): class Config: - fields = {"id": "id"} \ No newline at end of file + fields = {"id": "id"} diff --git a/backend/app/routers/projects.py b/backend/app/routers/projects.py index d37982279..2c0eeb84c 100644 --- a/backend/app/routers/projects.py +++ b/backend/app/routers/projects.py @@ -42,13 +42,13 @@ clowder_bucket = os.getenv("MINIO_BUCKET_NAME", "clowder") + @router.post("", response_model=ProjectOut) async def save_project( project_in: ProjectIn, user=Depends(get_current_user), es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), ): - project = ProjectDB( **project_in.dict(), creator=user, @@ -58,15 +58,24 @@ async def save_project( # TODO Add new entry to elasticsearch return project.dict() + @router.post("/{project_id}/add_dataset/{dataset_id}", response_model=ProjectOut) async def add_dataset( - project_id: str, - dataset_id: str, + project_id: str, + dataset_id: str, ): - if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) - ) is not None: - if dataset := await DatasetDB.find_one(DatasetDB.id == PydanticObjectId(dataset_id) - ) is not None: + if ( + project := await ProjectDB.find_one( + ProjectDB.id == PydanticObjectId(project_id) + ) + is not None + ): + if ( + dataset := await DatasetDB.find_one( + DatasetDB.id == PydanticObjectId(dataset_id) + ) + is not None + ): if dataset_id not in project.dataset_ids: project.dataset_ids.append(dataset_id) await project.replace() @@ -74,15 +83,24 @@ async def add_dataset( raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found") raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + @router.post("/{project_id}/remove_dataset/{dataset_id}", response_model=ProjectOut) async def remove_dataset( - project_id: str, - dataset_id: str, + project_id: str, + dataset_id: str, ): - if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) - ) is not None: - if dataset := await DatasetDB.find_one(DatasetDB.id == PydanticObjectId(dataset_id) - ) is not None: + if ( + project := await ProjectDB.find_one( + ProjectDB.id == PydanticObjectId(project_id) + ) + is not None + ): + if ( + dataset := await DatasetDB.find_one( + DatasetDB.id == PydanticObjectId(dataset_id) + ) + is not None + ): if dataset_id in project.dataset_ids: project.dataset_ids.remove(dataset_id) await project.replace() @@ -95,13 +113,21 @@ async def remove_dataset( @router.post("/{project_id}/add_folder/{folder_id}", response_model=ProjectOut) async def add_folder( - project_id: str, - folder_id: str, + project_id: str, + folder_id: str, ): - if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) - ) is not None: - if folder := await FolderDB.find_one(FolderDB.id == PydanticObjectId(folder_id) - ) is not None: + if ( + project := await ProjectDB.find_one( + ProjectDB.id == PydanticObjectId(project_id) + ) + is not None + ): + if ( + folder := await FolderDB.find_one( + FolderDB.id == PydanticObjectId(folder_id) + ) + is not None + ): if folder_id not in project.folder_ids: project.folder_ids.append(folder_id) await project.replace() @@ -109,15 +135,24 @@ async def add_folder( raise HTTPException(status_code=404, detail=f"Folder {folder_id} not found") raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + @router.post("/{project_id}/remove_folder/{folder_id}", response_model=ProjectOut) async def remove_folder( - project_id: str, - folder_id: str, + project_id: str, + folder_id: str, ): - if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) - ) is not None: - if folder := await FolderDB.find_one(FolderDB.id == PydanticObjectId(folder_id) - ) is not None: + if ( + project := await ProjectDB.find_one( + ProjectDB.id == PydanticObjectId(project_id) + ) + is not None + ): + if ( + folder := await FolderDB.find_one( + FolderDB.id == PydanticObjectId(folder_id) + ) + is not None + ): if folder_id in project.folder_ids: project.folder_ids.remove(folder_id) await project.replace() @@ -127,15 +162,22 @@ async def remove_folder( raise HTTPException(status_code=404, detail=f"Folder {folder_id} not found") raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + @router.post("/{project_id}/add_file/{file_id}", response_model=ProjectOut) async def add_file( - project_id: str, - file_id: str, + project_id: str, + file_id: str, ): - if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) - ) is not None: - if file := await FileDB.find_one(FileDB.id == PydanticObjectId(file_id) - ) is not None: + if ( + project := await ProjectDB.find_one( + ProjectDB.id == PydanticObjectId(project_id) + ) + is not None + ): + if ( + file := await FileDB.find_one(FileDB.id == PydanticObjectId(file_id)) + is not None + ): if file_id not in project.file_ids: project.file_ids.append(file_id) await project.replace() @@ -143,15 +185,22 @@ async def add_file( raise HTTPException(status_code=404, detail=f"File {file_id} not found") raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + @router.post("/{project_id}/remove_file/{file_id}", response_model=ProjectOut) async def remove_file( - project_id: str, - file_id: str, + project_id: str, + file_id: str, ): - if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) - ) is not None: - if file := await FileDB.find_one(FileDB.id == PydanticObjectId(file_id) - ) is not None: + if ( + project := await ProjectDB.find_one( + ProjectDB.id == PydanticObjectId(project_id) + ) + is not None + ): + if ( + file := await FileDB.find_one(FileDB.id == PydanticObjectId(file_id)) + is not None + ): if file_id in project.file_ids: project.file_ids.remove(file_id) await project.replace() @@ -170,9 +219,8 @@ async def get_projects( mine: bool = False, enable_admin: bool = False, ): - # TODO check if the current user is a member OR creator - query = (ProjectDB.creator.email == user_id) + query = ProjectDB.creator.email == user_id projects_and_count = await ProjectDB.find(*query).to_list() @@ -190,15 +238,21 @@ async def get_projects( return page.dict() + @router.get("/{project_id}", response_model=ProjectOut) async def get_project( project_id: str, ): - if project := await ProjectDB.find_one(ProjectDB.id == PydanticObjectId(project_id) - ) is not None: + if ( + project := await ProjectDB.find_one( + ProjectDB.id == PydanticObjectId(project_id) + ) + is not None + ): return project.dict() raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + @router.delete("/{project_id}", response_model=ProjectOut) async def delete_project( project_id: str, @@ -210,7 +264,7 @@ async def delete_project( raise HTTPException(status_code=404, detail=f"Project {project_id} not found") -@router.post("/{project_id}/add/{username}", response_model=ProjectOut) +@router.post("/{project_id}/add_member/{username}", response_model=ProjectOut) async def add_member( project_id: str, username: str, @@ -234,7 +288,8 @@ async def add_member( raise HTTPException(status_code=404, detail=f"Project {project_id} not found") raise HTTPException(status_code=404, detail=f"User {username} not found") -@router.post("/{project_id}/remove/{username}", response_model=ProjectOut) + +@router.post("/{project_id}/remove_member/{username}", response_model=ProjectOut) async def remove_member( project_id: str, username: str, @@ -255,4 +310,3 @@ async def remove_member( await project.replace() return project.dict() raise HTTPException(status_code=404, detail=f"Project {project_id} not found") - diff --git a/backend/app/tests/test_projects.py b/backend/app/tests/test_projects.py new file mode 100644 index 000000000..0196e0dec --- /dev/null +++ b/backend/app/tests/test_projects.py @@ -0,0 +1,50 @@ +from app.config import settings +from app.tests.utils import ( + create_dataset, + create_group, + create_user, + get_user_token, + user_alt, + create_project, +) +from fastapi.testclient import TestClient + +member_alt = {"user": user_alt, "editor": False} + + +def test_create_project(client: TestClient, headers: dict): + create_project(client, headers) + + +def test_get_project(client: TestClient, headers: dict): + project_id = create_project(client, headers).get("id") + response = client.get(f"{settings.API_V2_STR}/projects/{project_id}", headers=headers) + assert response.status_code == 200 + assert response.json().get("id") is not None + + +def test_delete_project(client: TestClient, headers: dict): + project_id = create_project(client, headers).get("id") + response = client.delete( + f"{settings.API_V2_STR}/projects/{project_id}", headers=headers + ) + assert response.status_code == 200 + + + +def test_add_member(client: TestClient, headers: dict): + new_project = create_project(client, headers) + project_id = new_project.get("id") + + create_user(client, headers) + new_project["users"].append(member_alt) + + response = client.post( + f"{settings.API_V2_STR}/projects/{project_id}/add_member/{member_alt['user']['email']}", + headers=headers, + ) + + assert response.status_code == 200 + assert response.json().get("id") is not None + for user in response.json().get("users"): + assert user.get("user").get("email") == member_alt["user"]["email"] diff --git a/backend/app/tests/utils.py b/backend/app/tests/utils.py index 395d2a676..341ffd516 100644 --- a/backend/app/tests/utils.py +++ b/backend/app/tests/utils.py @@ -61,6 +61,11 @@ }, } +project_example = { + "name": "test_project", + "description": "This project is a test" +} + extractor_info_v1_example = { "@context": "http://clowder.ncsa.illinois.edu/contexts/extractors.jsonld", "name": "test.extractor_info_v1_example", @@ -175,6 +180,17 @@ def create_dataset_with_custom_license(client: TestClient, headers: dict): assert response.json().get("id") is not None return response.json() +def create_project(client: TestClient, headers: dict): + """Creates a test dataset and returns the JSON.""" + response = client.post( + f"{settings.API_V2_STR}/projects", + headers=headers, + json=project_example, + ) + assert response.status_code == 200 + assert response.json().get("id") is not None + return response.json() + def upload_file( client: TestClient, From 7a8b3d2db7f2af21bceaa092b5935ceea880ea76 Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 30 Jul 2024 09:46:38 -0500 Subject: [PATCH 18/32] add test add dataset to project --- backend/app/tests/test_projects.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/backend/app/tests/test_projects.py b/backend/app/tests/test_projects.py index 0196e0dec..f734c4352 100644 --- a/backend/app/tests/test_projects.py +++ b/backend/app/tests/test_projects.py @@ -48,3 +48,19 @@ def test_add_member(client: TestClient, headers: dict): assert response.json().get("id") is not None for user in response.json().get("users"): assert user.get("user").get("email") == member_alt["user"]["email"] + +def test_add_dataset(client: TestClient, headers: dict): + new_project = create_project(client, headers) + project_id = new_project.get("id") + + dataset_id = create_dataset(client, headers).get("id") + + response = client.post( + f"{settings.API_V2_STR}/projects/{project_id}/add_dataset/{dataset_id}", + headers=headers, + ) + + assert response.status_code == 200 + assert response.json().get("id") is not None + assert dataset_id in response.json().get("dataset_ids") + From 0bd2f4bcec1e89005f2b3fc66cd4390707ffdf89 Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 30 Jul 2024 09:47:13 -0500 Subject: [PATCH 19/32] formatting --- backend/app/tests/test_projects.py | 7 ++++--- backend/app/tests/utils.py | 6 ++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/backend/app/tests/test_projects.py b/backend/app/tests/test_projects.py index f734c4352..e96420b26 100644 --- a/backend/app/tests/test_projects.py +++ b/backend/app/tests/test_projects.py @@ -18,7 +18,9 @@ def test_create_project(client: TestClient, headers: dict): def test_get_project(client: TestClient, headers: dict): project_id = create_project(client, headers).get("id") - response = client.get(f"{settings.API_V2_STR}/projects/{project_id}", headers=headers) + response = client.get( + f"{settings.API_V2_STR}/projects/{project_id}", headers=headers + ) assert response.status_code == 200 assert response.json().get("id") is not None @@ -31,7 +33,6 @@ def test_delete_project(client: TestClient, headers: dict): assert response.status_code == 200 - def test_add_member(client: TestClient, headers: dict): new_project = create_project(client, headers) project_id = new_project.get("id") @@ -49,6 +50,7 @@ def test_add_member(client: TestClient, headers: dict): for user in response.json().get("users"): assert user.get("user").get("email") == member_alt["user"]["email"] + def test_add_dataset(client: TestClient, headers: dict): new_project = create_project(client, headers) project_id = new_project.get("id") @@ -63,4 +65,3 @@ def test_add_dataset(client: TestClient, headers: dict): assert response.status_code == 200 assert response.json().get("id") is not None assert dataset_id in response.json().get("dataset_ids") - diff --git a/backend/app/tests/utils.py b/backend/app/tests/utils.py index 341ffd516..e6fedec52 100644 --- a/backend/app/tests/utils.py +++ b/backend/app/tests/utils.py @@ -61,10 +61,7 @@ }, } -project_example = { - "name": "test_project", - "description": "This project is a test" -} +project_example = {"name": "test_project", "description": "This project is a test"} extractor_info_v1_example = { "@context": "http://clowder.ncsa.illinois.edu/contexts/extractors.jsonld", @@ -180,6 +177,7 @@ def create_dataset_with_custom_license(client: TestClient, headers: dict): assert response.json().get("id") is not None return response.json() + def create_project(client: TestClient, headers: dict): """Creates a test dataset and returns the JSON.""" response = client.post( From 5c508e270276aa10cf1d039d252d972417209b71 Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 30 Jul 2024 10:15:44 -0500 Subject: [PATCH 20/32] adding routes to main fixing create and get routes --- backend/app/main.py | 8 ++++++++ backend/app/routers/projects.py | 9 ++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/backend/app/main.py b/backend/app/main.py index 84f54d38b..f40c4172a 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -39,6 +39,7 @@ VisualizationDataDBViewList, VisualizationDataFreezeDB, ) +from app.models.project import ProjectDB from app.routers import ( authentication, authorization, @@ -66,6 +67,7 @@ thumbnails, users, visualization, + projects, ) # setup loggers @@ -250,6 +252,11 @@ prefix="/public_thumbnails", tags=["public_thumbnails"], ) +api_router.include_router( + projects.router, + prefix="/projects", + tags=["projects"], +) api_router.include_router( licenses.router, prefix="/licenses", @@ -315,6 +322,7 @@ async def startup_beanie(): ThumbnailFreezeDB, ThumbnailDBViewList, LicenseDB, + ProjectDB, ], recreate_views=True, ) diff --git a/backend/app/routers/projects.py b/backend/app/routers/projects.py index 2c0eeb84c..3d57a9aab 100644 --- a/backend/app/routers/projects.py +++ b/backend/app/routers/projects.py @@ -50,8 +50,7 @@ async def save_project( es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), ): project = ProjectDB( - **project_in.dict(), - creator=user, + **project_in.dict() ) await project.insert() @@ -220,9 +219,9 @@ async def get_projects( enable_admin: bool = False, ): # TODO check if the current user is a member OR creator - query = ProjectDB.creator.email == user_id - - projects_and_count = await ProjectDB.find(*query).to_list() + projects_and_count = await ProjectDB.aggregate( + [_get_page_query(skip, limit, sort_field="email", ascending=True)], + ).to_list() page_metadata = _construct_page_metadata(projects_and_count, skip, limit) # TODO have to change _id this way otherwise it won't work From 91e14cb2f192fd208c29c02b4aa3a2789157d8d7 Mon Sep 17 00:00:00 2001 From: toddn Date: Tue, 30 Jul 2024 10:17:58 -0500 Subject: [PATCH 21/32] get project works --- .run/Python tests in tests.run.xml | 7 ++++--- backend/app/routers/projects.py | 11 ++++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.run/Python tests in tests.run.xml b/.run/Python tests in tests.run.xml index 2fd8345de..2f33fbef9 100644 --- a/.run/Python tests in tests.run.xml +++ b/.run/Python tests in tests.run.xml @@ -3,9 +3,10 @@