diff --git a/.env.development b/.env.development index 9b5b9682c..ddb981af6 100644 --- a/.env.development +++ b/.env.development @@ -84,7 +84,7 @@ SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection # NEXT_PUBLIC_SOURCEBOT_VERSION= # CONFIG_MAX_REPOS_NO_TOKEN= -# NODE_ENV= +NODE_ENV=development # SOURCEBOT_TENANCY_MODE=single # NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT= diff --git a/CHANGELOG.md b/CHANGELOG.md index 73092cb23..c63625beb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Added - [Experimental][Sourcebot EE] Added GitLab permission syncing. [#585](https://github.com/sourcebot-dev/sourcebot/pull/585) - [Sourcebot EE] Added external identity provider config and support for multiple accounts. [#595](https://github.com/sourcebot-dev/sourcebot/pull/595) +- Added ability to configure environment variables from the config. [#597](https://github.com/sourcebot-dev/sourcebot/pull/597) ### Fixed - [ask sb] Fixed issue where reasoning tokens would appear in `text` content for openai compatible models. [#582](https://github.com/sourcebot-dev/sourcebot/pull/582) diff --git a/Dockerfile b/Dockerfile index 43d410ac6..ae90c8c73 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,16 +42,10 @@ COPY package.json yarn.lock* .yarnrc.yml ./ COPY .yarn ./.yarn COPY ./packages/db ./packages/db COPY ./packages/schemas ./packages/schemas -COPY ./packages/crypto ./packages/crypto -COPY ./packages/error ./packages/error -COPY ./packages/logger ./packages/logger COPY ./packages/shared ./packages/shared RUN yarn workspace @sourcebot/db install RUN yarn workspace @sourcebot/schemas install -RUN yarn workspace @sourcebot/crypto install -RUN yarn workspace @sourcebot/error install -RUN yarn workspace @sourcebot/logger install RUN yarn workspace @sourcebot/shared install # ------------------------------------ @@ -97,9 +91,6 @@ COPY ./packages/web ./packages/web COPY --from=shared-libs-builder /app/node_modules ./node_modules COPY --from=shared-libs-builder /app/packages/db ./packages/db COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas -COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto -COPY --from=shared-libs-builder /app/packages/error ./packages/error -COPY --from=shared-libs-builder /app/packages/logger ./packages/logger COPY --from=shared-libs-builder /app/packages/shared ./packages/shared # Fixes arm64 timeouts @@ -138,9 +129,6 @@ COPY ./packages/backend ./packages/backend COPY --from=shared-libs-builder /app/node_modules ./node_modules COPY --from=shared-libs-builder /app/packages/db ./packages/db COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas -COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto -COPY --from=shared-libs-builder /app/packages/error ./packages/error -COPY --from=shared-libs-builder /app/packages/logger ./packages/logger COPY --from=shared-libs-builder /app/packages/shared ./packages/shared RUN yarn workspace @sourcebot/backend install RUN yarn workspace @sourcebot/backend build @@ -185,7 +173,6 @@ ENV DATA_DIR=/data ENV DATA_CACHE_DIR=$DATA_DIR/.sourcebot ENV DATABASE_DATA_DIR=$DATA_CACHE_DIR/db ENV REDIS_DATA_DIR=$DATA_CACHE_DIR/redis -ENV REDIS_URL="redis://localhost:6379" ENV SRC_TENANT_ENFORCEMENT_MODE=strict ENV SOURCEBOT_PUBLIC_KEY_PATH=/app/public.pem @@ -225,9 +212,6 @@ COPY --from=backend-builder /app/packages/backend ./packages/backend COPY --from=shared-libs-builder /app/node_modules ./node_modules COPY --from=shared-libs-builder /app/packages/db ./packages/db COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas -COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto -COPY --from=shared-libs-builder /app/packages/error ./packages/error -COPY --from=shared-libs-builder /app/packages/logger ./packages/logger COPY --from=shared-libs-builder /app/packages/shared ./packages/shared # Configure dependencies diff --git a/Makefile b/Makefile index 538a4e5da..13be47772 100644 --- a/Makefile +++ b/Makefile @@ -28,10 +28,6 @@ clean: packages/db/dist \ packages/schemas/node_modules \ packages/schemas/dist \ - packages/crypto/node_modules \ - packages/crypto/dist \ - packages/error/node_modules \ - packages/error/dist \ packages/mcp/node_modules \ packages/mcp/dist \ packages/shared/node_modules \ diff --git a/docs/docs/configuration/config-file.mdx b/docs/docs/configuration/config-file.mdx index 58d7a1b13..9985e4578 100644 --- a/docs/docs/configuration/config-file.mdx +++ b/docs/docs/configuration/config-file.mdx @@ -3,6 +3,9 @@ title: Config File sidebarTitle: Config file --- +import ConfigSchema from '/snippets/schemas/v3/index.schema.mdx' +import EnvironmentOverridesSchema from '/snippets/schemas/v3/environmentOverrides.schema.mdx' + When self-hosting Sourcebot, you **must** provide it a config file. This is done by defining a config file in a volume that's mounted to Sourcebot, and providing the path to this file in the `CONFIG_PATH` environment variable. For example: @@ -49,3 +52,103 @@ The following are settings that can be provided in your config file to modify So | `enablePublicAccess` **(deprecated)** | boolean | false | — | Use the `FORCE_ENABLE_ANONYMOUS_ACCESS` environment variable instead. | | `experiment_repoDrivenPermissionSyncIntervalMs` | number | 24 hours | 1 | Interval at which the repo permission syncer should run. | | `experiment_userDrivenPermissionSyncIntervalMs` | number | 24 hours | 1 | Interval at which the user permission syncer should run. | + +# Tokens + +Tokens are used to securely pass secrets to Sourcebot in a config file. They are used in various places, including connections, language model providers, auth providers, etc. Tokens can be passed as either environment variables or Google Cloud secrets: + + + + ```json + { + "token": { + "env": "TOKEN_NAME" + } + } + ``` + + + ```json + { + "token": { + "googleCloudSecret": "projects//secrets//versions/" + } + } + ``` + + + +# Overriding environment variables from the config + +You can override / set environment variables from the config file by using the `environmentOverrides` property. Overrides can be of type `string`, `number`, `boolean`, or a [token](/docs/configuration/config-file#tokens). Tokens are useful when you want to configure a environment variable using a Google Cloud Secret or other supported secret management service. + + + + ```jsonc + { + "environmentOverrides": { + "DATABASE_URL": { + "type": "token", + "value": { + "googleCloudSecret": "projects//secrets/postgres-connection-string/versions/latest" + } + }, + "REDIS_URL": { + "type": "token", + "value": { + "googleCloudSecret": "projects//secrets/redis-connection-string/versions/latest" + } + } + }, + } + ``` + + + + ```jsonc + { + "environmentOverrides": { + "EMAIL_FROM_ADDRESS": { + "type": "string", + "value": "hello@sourcebot.dev" + } + } + } + ``` + + + + ```jsonc + { + "environmentOverrides": { + "SOURCEBOT_CHAT_MODEL_TEMPERATURE": { + "type": "number", + "value": 0.5 + } + } + } + ``` + + + + ```jsonc + { + "environmentOverrides": { + "SOURCEBOT_TELEMETRY_DISABLED": { + "type": "boolean", + "value": false + } + } + } + ``` + + + + +**Note:** Overrides are **not** set as system environment variables, and instead are resolved at runtime on startup and stored in memory. + + +[schemas/v3/environmentOverrides.json](https://github.com/sourcebot-dev/sourcebot/blob/main/schemas/v3/environmentOverrides.json) + + + \ No newline at end of file diff --git a/docs/docs/configuration/environment-variables.mdx b/docs/docs/configuration/environment-variables.mdx index b6fba9eba..1fc78e2f5 100644 --- a/docs/docs/configuration/environment-variables.mdx +++ b/docs/docs/configuration/environment-variables.mdx @@ -1,7 +1,6 @@ --- title: Environment variables sidebarTitle: Environment variables -mode: "wide" --- This page provides a detailed reference of all environment variables supported by Sourcebot. If you're just looking to get up and running, we recommend starting with the [deployment guide](/docs/deployment-guide) instead. @@ -71,3 +70,6 @@ The following environment variables allow you to configure your Sourcebot deploy | `REVIEW_AGENT_LOGGING_ENABLED` | `true` |

Enables/disables logging for the review agent. Logs are saved in `DATA_CACHE_DIR/review-agent`

| | `REVIEW_AGENT_REVIEW_COMMAND` | `review` |

The command used to trigger a code review by the review agent.

| +### Overriding environment variables from the config + +You can override environment variables from the config file by using the `environmentOverrides` property. See [this doc](/docs/configuration/config-file#overriding-environment-variables-from-the-config) for more info. \ No newline at end of file diff --git a/docs/docs/configuration/idp.mdx b/docs/docs/configuration/idp.mdx index af4f8e053..21ae756d4 100644 --- a/docs/docs/configuration/idp.mdx +++ b/docs/docs/configuration/idp.mdx @@ -20,11 +20,6 @@ External identity providers can be used for [authentication](/docs/configuration "provider": "github", "purpose": "account_linking", "accountLinkingRequired": true, -/* -Secrets are provided through environment variables. Set the secret into -an env var and provide the name here to tell Sourcebot where to get -the value -*/ "clientId": { "env": "GITHUB_IDENTITY_PROVIDER_CLIENT_ID" }, @@ -45,6 +40,8 @@ the value } ``` +Secret values (such as `clientId` and `clientSecret`) can be provided as environment variables or Google Cloud secrets via [tokens](/docs/configuration/config-file#tokens). + # Supported External Identity Providers Sourcebot uses [Auth.js](https://authjs.dev/) to connect to external identity providers. If there's a provider supported by Auth.js that you don't see below, please submit a diff --git a/docs/docs/connections/ado-cloud.mdx b/docs/docs/connections/ado-cloud.mdx index 7071afb1d..92bde0481 100644 --- a/docs/docs/connections/ado-cloud.mdx +++ b/docs/docs/connections/ado-cloud.mdx @@ -86,7 +86,7 @@ If you're not familiar with Sourcebot [connections](/docs/connections/overview), Azure Devops Cloud requires you to provide a PAT in order to index your repositories. To learn how to create PAT, check out the [Azure Devops docs](https://learn.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=Windows). Sourcebot needs the `Read` access for the `Code` scope in order to find and clone your repos. -Next, provide the access token via an environment variable which is referenced in the `token` property: +Next, provide the access [token](/docs/configuration/config-file#tokens) via an environment variable which is referenced in the `token` property: diff --git a/docs/docs/connections/ado-server.mdx b/docs/docs/connections/ado-server.mdx index a62d94c2a..9687f827d 100644 --- a/docs/docs/connections/ado-server.mdx +++ b/docs/docs/connections/ado-server.mdx @@ -100,7 +100,7 @@ If you're not familiar with Sourcebot [connections](/docs/connections/overview), Azure Devops Server requires you to provide a PAT in order to index your repositories. To learn how to create PAT, check out the [Azure Devops docs](https://learn.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=Windows). Sourcebot needs the `Read` access for the `Code` scope in order to find and clone your repos. -Next, provide the access token via an environment variable which is referenced in the `token` property: +Next, provide the access [token](/docs/configuration/config-file#tokens) via an environment variable which is referenced in the `token` property: diff --git a/docs/docs/connections/bitbucket-cloud.mdx b/docs/docs/connections/bitbucket-cloud.mdx index bde7a6656..56b140f83 100644 --- a/docs/docs/connections/bitbucket-cloud.mdx +++ b/docs/docs/connections/bitbucket-cloud.mdx @@ -78,7 +78,7 @@ If you're not familiar with Sourcebot [connections](/docs/connections/overview), ## Authenticating with Bitbucket Cloud -In order to index private repositories, you'll need to provide authentication credentials. You can do this using an `App Password` or an `Access Token` +In order to index private repositories, you'll need to provide authentication credentials via a [token](/docs/configuration/config-file#tokens). You can do this using an `App Password` or an `Access Token` diff --git a/docs/docs/connections/bitbucket-data-center.mdx b/docs/docs/connections/bitbucket-data-center.mdx index 77479e461..bfda09342 100644 --- a/docs/docs/connections/bitbucket-data-center.mdx +++ b/docs/docs/connections/bitbucket-data-center.mdx @@ -70,7 +70,7 @@ If you're not familiar with Sourcebot [connections](/docs/connections/overview), ## Authenticating with Bitbucket Data Center -In order to index private repositories, you'll need to provide an access token to Sourcebot. +In order to index private repositories, you'll need to provide an access token to Sourcebot via a [token](/docs/configuration/config-file#tokens). Create an access token for the desired scope (repo, project, or workspace). Visit the official [Bitbucket Data Center docs](https://confluence.atlassian.com/bitbucketserver/http-access-tokens-939515499.html) for more info. diff --git a/docs/docs/connections/gitea.mdx b/docs/docs/connections/gitea.mdx index 0f8505cb3..0b4efe15b 100644 --- a/docs/docs/connections/gitea.mdx +++ b/docs/docs/connections/gitea.mdx @@ -81,7 +81,7 @@ In order to index private repositories, you'll need to generate a Gitea access t ![Gitea Access token creation](/images/gitea_pat_creation.png) -Next, provide the access token via an environment variable which is referenced in the `token` property: +Next, provide the access token via an environment variable [token](/docs/configuration/config-file#tokens) which is referenced in the `token` property: diff --git a/docs/docs/connections/github.mdx b/docs/docs/connections/github.mdx index e87a1f6d8..4d3384b44 100644 --- a/docs/docs/connections/github.mdx +++ b/docs/docs/connections/github.mdx @@ -128,7 +128,7 @@ In order to index private repositories, you'll need to generate a access token a -Next, provide the access token via an environment variable which is referenced in the `token` property: +Next, provide the access token via an environment variable [token](/docs/configuration/config-file#tokens) which is referenced in the `token` property: diff --git a/docs/docs/connections/gitlab.mdx b/docs/docs/connections/gitlab.mdx index 1b71b7ae3..256035376 100644 --- a/docs/docs/connections/gitlab.mdx +++ b/docs/docs/connections/gitlab.mdx @@ -116,7 +116,7 @@ In order to index private projects, you'll need to generate a GitLab Personal Ac ![GitLab PAT Scope](/images/gitlab_pat_scopes.png) -Next, provide the PAT via an environment variable which is referenced in the `token` property: +Next, provide the PAT via an environment variable [token](/docs/configuration/config-file#tokens) which is referenced in the `token` property: diff --git a/docs/snippets/schemas/v3/environmentOverrides.schema.mdx b/docs/snippets/schemas/v3/environmentOverrides.schema.mdx new file mode 100644 index 000000000..bca6ec083 --- /dev/null +++ b/docs/snippets/schemas/v3/environmentOverrides.schema.mdx @@ -0,0 +1,115 @@ +{/* THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! */} +```json +{ + "type": "object", + "description": "Environment variable overrides.", + "title": "EnvironmentOverrides", + "not": { + "$comment": "List of environment variables that are not allowed to be overridden.", + "anyOf": [ + { + "required": [ + "CONFIG_PATH" + ] + } + ] + }, + "patternProperties": { + "^[a-zA-Z0-9_-]+$": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "const": "token" + }, + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token." + } + }, + "required": [ + "env" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "googleCloudSecret": { + "type": "string", + "description": "The resource name of a Google Cloud secret. Must be in the format `projects//secrets//versions/`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets" + } + }, + "required": [ + "googleCloudSecret" + ], + "additionalProperties": false + } + ] + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "number" + }, + "value": { + "type": "number" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "boolean" + }, + "value": { + "type": "boolean" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + } + ] + } + } +} +``` diff --git a/docs/snippets/schemas/v3/index.schema.mdx b/docs/snippets/schemas/v3/index.schema.mdx index 615c058a8..413e51bd6 100644 --- a/docs/snippets/schemas/v3/index.schema.mdx +++ b/docs/snippets/schemas/v3/index.schema.mdx @@ -279,6 +279,118 @@ }, "additionalProperties": false }, + "environmentOverrides": { + "type": "object", + "description": "Environment variable overrides.", + "title": "EnvironmentOverrides", + "not": { + "$comment": "List of environment variables that are not allowed to be overridden.", + "anyOf": [ + { + "required": [ + "CONFIG_PATH" + ] + } + ] + }, + "patternProperties": { + "^[a-zA-Z0-9_-]+$": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "const": "token" + }, + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token." + } + }, + "required": [ + "env" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "googleCloudSecret": { + "type": "string", + "description": "The resource name of a Google Cloud secret. Must be in the format `projects//secrets//versions/`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets" + } + }, + "required": [ + "googleCloudSecret" + ], + "additionalProperties": false + } + ] + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "number" + }, + "value": { + "type": "number" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "boolean" + }, + "value": { + "type": "boolean" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + } + ] + } + } + }, "connections": { "type": "object", "description": "Defines a collection of connections from varying code hosts that Sourcebot should sync with. This is only available in single-tenancy mode.", @@ -297,7 +409,6 @@ "description": "GitHub Configuration" }, "token": { - "description": "A Personal Access Token (PAT).", "anyOf": [ { "type": "object", @@ -325,7 +436,8 @@ ], "additionalProperties": false } - ] + ], + "description": "A Personal Access Token (PAT)." }, "url": { "type": "string", @@ -505,7 +617,6 @@ "description": "GitLab Configuration" }, "token": { - "description": "An authentication token.", "anyOf": [ { "type": "object", @@ -533,7 +644,8 @@ ], "additionalProperties": false } - ] + ], + "description": "An authentication token." }, "url": { "type": "string", @@ -707,7 +819,6 @@ "description": "Gitea Configuration" }, "token": { - "description": "A Personal Access Token (PAT).", "anyOf": [ { "type": "object", @@ -735,7 +846,8 @@ ], "additionalProperties": false } - ] + ], + "description": "A Personal Access Token (PAT)." }, "url": { "type": "string", @@ -974,7 +1086,6 @@ "description": "The username to use for authentication. Only needed if token is an app password." }, "token": { - "description": "An authentication token.", "anyOf": [ { "type": "object", @@ -1002,7 +1113,8 @@ ], "additionalProperties": false } - ] + ], + "description": "An authentication token." }, "url": { "type": "string", @@ -1142,7 +1254,6 @@ "description": "Azure DevOps Configuration" }, "token": { - "description": "A Personal Access Token (PAT).", "anyOf": [ { "type": "object", @@ -1170,7 +1281,8 @@ ], "additionalProperties": false } - ] + ], + "description": "A Personal Access Token (PAT)." }, "url": { "type": "string", @@ -1426,7 +1538,6 @@ "description": "Optional display name." }, "accessKeyId": { - "description": "Optional access key ID to use with the model. Defaults to the `AWS_ACCESS_KEY_ID` environment variable.", "anyOf": [ { "type": "object", @@ -1454,10 +1565,10 @@ ], "additionalProperties": false } - ] + ], + "description": "Optional access key ID to use with the model. Defaults to the `AWS_ACCESS_KEY_ID` environment variable." }, "accessKeySecret": { - "description": "Optional secret access key to use with the model. Defaults to the `AWS_SECRET_ACCESS_KEY` environment variable.", "anyOf": [ { "type": "object", @@ -1485,10 +1596,10 @@ ], "additionalProperties": false } - ] + ], + "description": "Optional secret access key to use with the model. Defaults to the `AWS_SECRET_ACCESS_KEY` environment variable." }, "sessionToken": { - "description": "Optional session token to use with the model. Defaults to the `AWS_SESSION_TOKEN` environment variable.", "anyOf": [ { "type": "object", @@ -1516,7 +1627,8 @@ ], "additionalProperties": false } - ] + ], + "description": "Optional session token to use with the model. Defaults to the `AWS_SESSION_TOKEN` environment variable." }, "region": { "type": "string", @@ -2855,7 +2967,6 @@ "description": "Optional display name." }, "accessKeyId": { - "description": "Optional access key ID to use with the model. Defaults to the `AWS_ACCESS_KEY_ID` environment variable.", "anyOf": [ { "type": "object", @@ -2883,10 +2994,10 @@ ], "additionalProperties": false } - ] + ], + "description": "Optional access key ID to use with the model. Defaults to the `AWS_ACCESS_KEY_ID` environment variable." }, "accessKeySecret": { - "description": "Optional secret access key to use with the model. Defaults to the `AWS_SECRET_ACCESS_KEY` environment variable.", "anyOf": [ { "type": "object", @@ -2914,10 +3025,10 @@ ], "additionalProperties": false } - ] + ], + "description": "Optional secret access key to use with the model. Defaults to the `AWS_SECRET_ACCESS_KEY` environment variable." }, "sessionToken": { - "description": "Optional session token to use with the model. Defaults to the `AWS_SESSION_TOKEN` environment variable.", "anyOf": [ { "type": "object", @@ -2945,7 +3056,8 @@ ], "additionalProperties": false } - ] + ], + "description": "Optional session token to use with the model. Defaults to the `AWS_SESSION_TOKEN` environment variable." }, "region": { "type": "string", diff --git a/entrypoint.sh b/entrypoint.sh index b031b326c..cf90a377b 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,28 +1,57 @@ #!/bin/sh -set -e -# Check if DATABASE_URL is not set -if [ -z "$DATABASE_URL" ]; then - # Check if the individual database variables are set and construct the URL - if [ -n "$DATABASE_HOST" ] && [ -n "$DATABASE_USERNAME" ] && [ -n "$DATABASE_PASSWORD" ] && [ -n "$DATABASE_NAME" ]; then - DATABASE_URL="postgresql://${DATABASE_USERNAME}:${DATABASE_PASSWORD}@${DATABASE_HOST}/${DATABASE_NAME}" +# Exit immediately if a command fails +set -e +# Disable auto-exporting of variables +set +a + +# If a CONFIG_PATH is set, resolve the environment overrides from the config file. +# The overrides will be written into variables scopped to the current shell. This is +# required in case one of the variables used in this entrypoint is overriden (e.g., +# DATABASE_URL, REDIS_URL, etc.) +if [ -n "$CONFIG_PATH" ]; then + echo -e "\e[34m[Info] Resolving environment overrides from $CONFIG_PATH...\e[0m" + + set +e # Disable exist on error so we can capture EXIT_CODE + OVERRIDES_OUTPUT=$(SKIP_ENV_VALIDATION=1 yarn tool:resolve-env-overrides 2>&1) + EXIT_CODE=$? + set -e # Re-enable exit on error + + if [ $EXIT_CODE -eq 0 ]; then + eval "$OVERRIDES_OUTPUT" + else + echo -e "\e[31m[Error] Failed to resolve environment overrides.\e[0m" + echo "$OVERRIDES_OUTPUT" + exit 1 + fi +fi - if [ -n "$DATABASE_ARGS" ]; then - DATABASE_URL="${DATABASE_URL}?$DATABASE_ARGS" - fi +# Descontruct the database URL from the individual variables if DATABASE_URL is not set +if [ -z "$DATABASE_URL" ] && [ -n "$DATABASE_HOST" ] && [ -n "$DATABASE_USERNAME" ] && [ -n "$DATABASE_PASSWORD" ] && [ -n "$DATABASE_NAME" ]; then + DATABASE_URL="postgresql://${DATABASE_USERNAME}:${DATABASE_PASSWORD}@${DATABASE_HOST}/${DATABASE_NAME}" - export DATABASE_URL - else - # Otherwise, fallback to a default value - DATABASE_URL="postgresql://postgres@localhost:5432/sourcebot" - export DATABASE_URL + if [ -n "$DATABASE_ARGS" ]; then + DATABASE_URL="${DATABASE_URL}?$DATABASE_ARGS" fi fi -if [ "$DATABASE_URL" = "postgresql://postgres@localhost:5432/sourcebot" ]; then - DATABASE_EMBEDDED="true" +if [ -z "$DATABASE_URL" ]; then + echo -e "\e[34m[Info] DATABASE_URL is not set. Using embeded database.\e[0m" + export DATABASE_EMBEDDED="true" + export DATABASE_URL="postgresql://postgres@localhost:5432/sourcebot" +else + export DATABASE_EMBEDDED="false" +fi + +if [ -z "$REDIS_URL" ]; then + echo -e "\e[34m[Info] REDIS_URL is not set. Using embeded redis.\e[0m" + export REDIS_EMBEDDED="true" + export REDIS_URL="redis://localhost:6379" +else + export REDIS_EMBEDDED="false" fi + echo -e "\e[34m[Info] Sourcebot version: $NEXT_PUBLIC_SOURCEBOT_VERSION\e[0m" # If we don't have a PostHog key, then we need to disable telemetry. @@ -59,7 +88,7 @@ if [ "$DATABASE_EMBEDDED" = "true" ] && [ ! -d "$DATABASE_DATA_DIR" ]; then fi # Create the redis data directory if it doesn't exist -if [ ! -d "$REDIS_DATA_DIR" ]; then +if [ "$REDIS_EMBEDDED" = "true" ] && [ ! -d "$REDIS_DATA_DIR" ]; then mkdir -p $REDIS_DATA_DIR fi @@ -149,7 +178,6 @@ fi echo "{\"version\": \"$NEXT_PUBLIC_SOURCEBOT_VERSION\", \"install_id\": \"$SOURCEBOT_INSTALL_ID\"}" > "$FIRST_RUN_FILE" - # Start the database and wait for it to be ready before starting any other service if [ "$DATABASE_EMBEDDED" = "true" ]; then su postgres -c "postgres -D $DATABASE_DATA_DIR" & @@ -171,7 +199,7 @@ fi # Run a Database migration echo -e "\e[34m[Info] Running database migration...\e[0m" -yarn workspace @sourcebot/db prisma:migrate:prod +DATABASE_URL="$DATABASE_URL" yarn workspace @sourcebot/db prisma:migrate:prod # Create the log directory mkdir -p /var/log/sourcebot diff --git a/package.json b/package.json index 7c726c7eb..a70bab991 100644 --- a/package.json +++ b/package.json @@ -4,8 +4,8 @@ "packages/*" ], "scripts": { - "build": "cross-env SKIP_ENV_VALIDATION=1 yarn workspaces foreach -A run build", - "test": "yarn workspaces foreach -A run test", + "build": "cross-env SKIP_ENV_VALIDATION=1 yarn workspaces foreach --all --topological run build", + "test": "yarn workspaces foreach --all --topological run test", "dev": "concurrently --kill-others --names \"zoekt,worker,web,mcp,schemas\" 'yarn dev:zoekt' 'yarn dev:backend' 'yarn dev:web' 'yarn watch:mcp' 'yarn watch:schemas'", "with-env": "cross-env PATH=\"$PWD/bin:$PATH\" dotenv -e .env.development -c --", "dev:zoekt": "yarn with-env zoekt-webserver -index .sourcebot/index -rpc", @@ -18,7 +18,7 @@ "dev:prisma:studio": "yarn with-env yarn workspace @sourcebot/db prisma:studio", "dev:prisma:migrate:reset": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:reset", "dev:prisma:db:push": "yarn with-env yarn workspace @sourcebot/db prisma:db:push", - "build:deps": "yarn workspaces foreach -R --from '{@sourcebot/schemas,@sourcebot/error,@sourcebot/crypto,@sourcebot/db,@sourcebot/shared}' run build" + "build:deps": "yarn workspaces foreach --recursive --topological --from '{@sourcebot/schemas,@sourcebot/db,@sourcebot/shared}' run build" }, "devDependencies": { "concurrently": "^9.2.1", diff --git a/packages/backend/package.json b/packages/backend/package.json index 8b0659d21..5369bde1e 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -29,13 +29,9 @@ "@sentry/cli": "^2.42.2", "@sentry/node": "^9.3.0", "@sentry/profiling-node": "^9.3.0", - "@sourcebot/crypto": "workspace:*", "@sourcebot/db": "workspace:*", - "@sourcebot/error": "workspace:*", - "@sourcebot/logger": "workspace:*", "@sourcebot/schemas": "workspace:*", "@sourcebot/shared": "workspace:*", - "@t3-oss/env-core": "^0.12.0", "@types/express": "^5.0.0", "argparse": "^2.0.1", "azure-devops-node-api": "^15.1.1", @@ -55,6 +51,6 @@ "posthog-node": "^4.2.1", "prom-client": "^15.1.3", "simple-git": "^3.27.0", - "zod": "^3.24.3" + "zod": "^3.25.74" } } diff --git a/packages/backend/src/azuredevops.ts b/packages/backend/src/azuredevops.ts index aa0fbf550..0714276e8 100644 --- a/packages/backend/src/azuredevops.ts +++ b/packages/backend/src/azuredevops.ts @@ -1,13 +1,12 @@ import { AzureDevOpsConnectionConfig } from "@sourcebot/schemas/v3/azuredevops.type"; -import { createLogger } from "@sourcebot/logger"; +import { createLogger } from "@sourcebot/shared"; import { measure, fetchWithRetry } from "./utils.js"; import micromatch from "micromatch"; -import { BackendException, BackendError } from "@sourcebot/error"; import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js"; import * as Sentry from "@sentry/node"; import * as azdev from "azure-devops-node-api"; import { GitRepository } from "azure-devops-node-api/interfaces/GitInterfaces.js"; -import { getTokenFromConfig } from "@sourcebot/crypto"; +import { getTokenFromConfig } from "@sourcebot/shared"; const logger = createLogger('azuredevops'); const AZUREDEVOPS_CLOUD_HOSTNAME = "dev.azure.com"; @@ -36,9 +35,7 @@ export const getAzureDevOpsReposFromConfig = async ( undefined; if (!token) { - const e = new BackendException(BackendError.CONNECTION_SYNC_INVALID_TOKEN, { - message: 'Azure DevOps requires a Personal Access Token', - }); + const e = new Error('Azure DevOps requires a Personal Access Token'); Sentry.captureException(e); throw e; } diff --git a/packages/backend/src/bitbucket.ts b/packages/backend/src/bitbucket.ts index c6fa87ffe..3f0e18ffa 100644 --- a/packages/backend/src/bitbucket.ts +++ b/packages/backend/src/bitbucket.ts @@ -2,7 +2,7 @@ import { createBitbucketCloudClient } from "@coderabbitai/bitbucket/cloud"; import { createBitbucketServerClient } from "@coderabbitai/bitbucket/server"; import { BitbucketConnectionConfig } from "@sourcebot/schemas/v3/bitbucket.type"; import type { ClientOptions, ClientPathsWithMethod } from "openapi-fetch"; -import { createLogger } from "@sourcebot/logger"; +import { createLogger } from "@sourcebot/shared"; import { measure, fetchWithRetry } from "./utils.js"; import * as Sentry from "@sentry/node"; import { @@ -11,7 +11,7 @@ import { import { SchemaRestRepository as ServerRepository } from "@coderabbitai/bitbucket/server/openapi"; import { processPromiseResults } from "./connectionUtils.js"; import { throwIfAnyFailed } from "./connectionUtils.js"; -import { getTokenFromConfig } from "@sourcebot/crypto"; +import { getTokenFromConfig } from "@sourcebot/shared"; const logger = createLogger('bitbucket'); const BITBUCKET_CLOUD_GIT = 'https://bitbucket.org'; diff --git a/packages/backend/src/configManager.ts b/packages/backend/src/configManager.ts index 14c6b0deb..55dbd6edc 100644 --- a/packages/backend/src/configManager.ts +++ b/packages/backend/src/configManager.ts @@ -1,5 +1,5 @@ import { Prisma, PrismaClient } from "@sourcebot/db"; -import { createLogger } from "@sourcebot/logger"; +import { createLogger } from "@sourcebot/shared"; import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; import { loadConfig } from "@sourcebot/shared"; import chokidar, { FSWatcher } from 'chokidar'; diff --git a/packages/backend/src/connectionManager.ts b/packages/backend/src/connectionManager.ts index 006a3bf2a..8b54fa4d0 100644 --- a/packages/backend/src/connectionManager.ts +++ b/packages/backend/src/connectionManager.ts @@ -1,11 +1,10 @@ import * as Sentry from "@sentry/node"; import { Connection, ConnectionSyncJobStatus, PrismaClient } from "@sourcebot/db"; -import { createLogger } from "@sourcebot/logger"; +import { createLogger } from "@sourcebot/shared"; import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; -import { loadConfig } from "@sourcebot/shared"; +import { loadConfig, env } from "@sourcebot/shared"; import { Job, Queue, ReservedJob, Worker } from "groupmq"; import { Redis } from 'ioredis'; -import { env } from "./env.js"; import { compileAzureDevOpsConfig, compileBitbucketConfig, compileGenericGitHostConfig, compileGerritConfig, compileGiteaConfig, compileGithubConfig, compileGitlabConfig } from "./repoCompileUtils.js"; import { Settings } from "./types.js"; import { groupmqLifecycleExceptionWrapper } from "./utils.js"; diff --git a/packages/backend/src/constants.ts b/packages/backend/src/constants.ts index f073bac5f..a52d822ee 100644 --- a/packages/backend/src/constants.ts +++ b/packages/backend/src/constants.ts @@ -1,5 +1,5 @@ import { CodeHostType } from "@sourcebot/db"; -import { env } from "./env.js"; +import { env } from "@sourcebot/shared"; import path from "path"; export const SINGLE_TENANT_ORG_ID = 1; diff --git a/packages/backend/src/ee/accountPermissionSyncer.ts b/packages/backend/src/ee/accountPermissionSyncer.ts index 70ff0e132..eb32ba3ba 100644 --- a/packages/backend/src/ee/accountPermissionSyncer.ts +++ b/packages/backend/src/ee/accountPermissionSyncer.ts @@ -1,13 +1,11 @@ import * as Sentry from "@sentry/node"; -import { PrismaClient, AccountPermissionSyncJobStatus, Account } from "@sourcebot/db"; -import { createLogger } from "@sourcebot/logger"; +import { PrismaClient, AccountPermissionSyncJobStatus, Account} from "@sourcebot/db"; +import { env, hasEntitlement, createLogger } from "@sourcebot/shared"; import { Job, Queue, Worker } from "bullmq"; import { Redis } from "ioredis"; import { PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES } from "../constants.js"; -import { env } from "../env.js"; import { createOctokitFromToken, getReposForAuthenticatedUser } from "../github.js"; import { createGitLabFromOAuthToken, getProjectsForAuthenticatedUser } from "../gitlab.js"; -import { hasEntitlement } from "@sourcebot/shared"; import { Settings } from "../types.js"; const LOG_TAG = 'user-permission-syncer'; diff --git a/packages/backend/src/ee/githubAppManager.ts b/packages/backend/src/ee/githubAppManager.ts index 892e637bd..0ee50acf6 100644 --- a/packages/backend/src/ee/githubAppManager.ts +++ b/packages/backend/src/ee/githubAppManager.ts @@ -1,10 +1,9 @@ -import { loadConfig } from "@sourcebot/shared"; -import { env } from "../env.js"; -import { createLogger } from "@sourcebot/logger"; -import { getTokenFromConfig } from "@sourcebot/crypto"; -import { PrismaClient } from "@sourcebot/db"; import { App } from "@octokit/app"; +import { getTokenFromConfig } from "@sourcebot/shared"; +import { PrismaClient } from "@sourcebot/db"; +import { createLogger } from "@sourcebot/shared"; import { GitHubAppConfig } from "@sourcebot/schemas/v3/index.type"; +import { env, loadConfig } from "@sourcebot/shared"; const logger = createLogger('githubAppManager'); const GITHUB_DEFAULT_DEPLOYMENT_HOSTNAME = 'github.com'; @@ -45,7 +44,7 @@ export class GithubAppManager { public async init(db: PrismaClient) { this.db = db; - const config = await loadConfig(env.CONFIG_PATH!); + const config = await loadConfig(env.CONFIG_PATH); if (!config.apps) { return; } diff --git a/packages/backend/src/ee/repoPermissionSyncer.ts b/packages/backend/src/ee/repoPermissionSyncer.ts index 2e9be5f49..078055ff0 100644 --- a/packages/backend/src/ee/repoPermissionSyncer.ts +++ b/packages/backend/src/ee/repoPermissionSyncer.ts @@ -1,11 +1,10 @@ import * as Sentry from "@sentry/node"; import { PrismaClient, Repo, RepoPermissionSyncJobStatus } from "@sourcebot/db"; -import { createLogger } from "@sourcebot/logger"; -import { hasEntitlement } from "@sourcebot/shared"; +import { createLogger } from "@sourcebot/shared"; +import { env, hasEntitlement } from "@sourcebot/shared"; import { Job, Queue, Worker } from 'bullmq'; import { Redis } from 'ioredis'; import { PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES } from "../constants.js"; -import { env } from "../env.js"; import { createOctokitFromToken, getRepoCollaborators, GITHUB_CLOUD_HOSTNAME } from "../github.js"; import { createGitLabFromPersonalAccessToken, getProjectMembers } from "../gitlab.js"; import { Settings } from "../types.js"; diff --git a/packages/backend/src/ee/syncSearchContexts.ts b/packages/backend/src/ee/syncSearchContexts.ts index e53679aee..8f9265a38 100644 --- a/packages/backend/src/ee/syncSearchContexts.ts +++ b/packages/backend/src/ee/syncSearchContexts.ts @@ -1,5 +1,5 @@ import micromatch from "micromatch"; -import { createLogger } from "@sourcebot/logger"; +import { createLogger } from "@sourcebot/shared"; import { PrismaClient } from "@sourcebot/db"; import { getPlan, hasEntitlement, SOURCEBOT_SUPPORT_EMAIL } from "@sourcebot/shared"; import { SearchContext } from "@sourcebot/schemas/v3/index.type"; diff --git a/packages/backend/src/env.ts b/packages/backend/src/env.ts deleted file mode 100644 index c3ea36797..000000000 --- a/packages/backend/src/env.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { createEnv } from "@t3-oss/env-core"; -import { z } from "zod"; -import dotenv from 'dotenv'; - -// Booleans are specified as 'true' or 'false' strings. -const booleanSchema = z.enum(["true", "false"]); - -// Numbers are treated as strings in .env files. -// coerce helps us convert them to numbers. -// @see: https://zod.dev/?id=coercion-for-primitives -const numberSchema = z.coerce.number(); - -dotenv.config({ - path: './.env', -}); - -dotenv.config({ - path: './.env.local', - override: true -}); - -export const env = createEnv({ - server: { - SOURCEBOT_ENCRYPTION_KEY: z.string(), - SOURCEBOT_TELEMETRY_DISABLED: booleanSchema.default("false"), - SOURCEBOT_INSTALL_ID: z.string().default("unknown"), - NEXT_PUBLIC_SOURCEBOT_VERSION: z.string().default("unknown"), - - DATA_CACHE_DIR: z.string(), - - NEXT_PUBLIC_POSTHOG_PAPIK: z.string().optional(), - - FALLBACK_GITHUB_CLOUD_TOKEN: z.string().optional(), - FALLBACK_GITLAB_CLOUD_TOKEN: z.string().optional(), - FALLBACK_GITEA_CLOUD_TOKEN: z.string().optional(), - - REDIS_URL: z.string().url().default("redis://localhost:6379"), - REDIS_REMOVE_ON_COMPLETE: numberSchema.default(0), - REDIS_REMOVE_ON_FAIL: numberSchema.default(100), - - NEXT_PUBLIC_SENTRY_BACKEND_DSN: z.string().optional(), - NEXT_PUBLIC_SENTRY_ENVIRONMENT: z.string().optional(), - - LOGTAIL_TOKEN: z.string().optional(), - LOGTAIL_HOST: z.string().url().optional(), - SOURCEBOT_LOG_LEVEL: z.enum(["info", "debug", "warn", "error"]).default("info"), - DEBUG_ENABLE_GROUPMQ_LOGGING: booleanSchema.default('false'), - - DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres"), - CONFIG_PATH: z.string(), - - CONNECTION_MANAGER_UPSERT_TIMEOUT_MS: numberSchema.default(300000), - REPO_SYNC_RETRY_BASE_SLEEP_SECONDS: numberSchema.default(60), - - GITLAB_CLIENT_QUERY_TIMEOUT_SECONDS: numberSchema.default(60 * 10), - - EXPERIMENT_EE_PERMISSION_SYNC_ENABLED: booleanSchema.default('false'), - AUTH_EE_GITHUB_BASE_URL: z.string().optional(), - AUTH_EE_GITLAB_BASE_URL: z.string().default("https://gitlab.com"), - }, - runtimeEnv: process.env, - emptyStringAsUndefined: true, - skipValidation: process.env.SKIP_ENV_VALIDATION === "1", -}); \ No newline at end of file diff --git a/packages/backend/src/gerrit.ts b/packages/backend/src/gerrit.ts index ef149648b..34feb2081 100644 --- a/packages/backend/src/gerrit.ts +++ b/packages/backend/src/gerrit.ts @@ -1,11 +1,8 @@ +import { GerritConnectionConfig } from "@sourcebot/schemas/v3/index.type"; +import { createLogger } from '@sourcebot/shared'; import fetch from 'cross-fetch'; -import { GerritConnectionConfig } from "@sourcebot/schemas/v3/index.type" -import { createLogger } from '@sourcebot/logger'; import micromatch from "micromatch"; -import { measure, fetchWithRetry } from './utils.js'; -import { BackendError } from '@sourcebot/error'; -import { BackendException } from '@sourcebot/error'; -import * as Sentry from "@sentry/node"; +import { fetchWithRetry, measure } from './utils.js'; // https://gerrit-review.googlesource.com/Documentation/rest-api.html interface GerritProjects { @@ -39,26 +36,10 @@ export const getGerritReposFromConfig = async (config: GerritConnectionConfig): const url = config.url.endsWith('/') ? config.url : `${config.url}/`; let { durationMs, data: projects } = await measure(async () => { - try { - const fetchFn = () => fetchAllProjects(url); - return fetchWithRetry(fetchFn, `projects from ${url}`, logger); - } catch (err) { - Sentry.captureException(err); - if (err instanceof BackendException) { - throw err; - } - - logger.error(`Failed to fetch projects from ${url}`, err); - return null; - } + const fetchFn = () => fetchAllProjects(url); + return fetchWithRetry(fetchFn, `projects from ${url}`, logger); }); - if (!projects) { - const e = new Error(`Failed to fetch projects from ${url}`); - Sentry.captureException(e); - throw e; - } - // include repos by glob if specified in config if (config.projects) { projects = projects.filter((project) => { @@ -91,27 +72,9 @@ const fetchAllProjects = async (url: string): Promise => { logger.debug(`Fetching projects from Gerrit at ${endpointWithParams}`); let response: Response; - try { - response = await fetch(endpointWithParams); - if (!response.ok) { - logger.error(`Failed to fetch projects from Gerrit at ${endpointWithParams} with status ${response.status}`); - const e = new BackendException(BackendError.CONNECTION_SYNC_FAILED_TO_FETCH_GERRIT_PROJECTS, { - status: response.status, - }); - Sentry.captureException(e); - throw e; - } - } catch (err) { - Sentry.captureException(err); - if (err instanceof BackendException) { - throw err; - } - - const status = (err as any).code; - logger.error(`Failed to fetch projects from Gerrit at ${endpointWithParams} with status ${status}`); - throw new BackendException(BackendError.CONNECTION_SYNC_FAILED_TO_FETCH_GERRIT_PROJECTS, { - status: status, - }); + response = await fetch(endpointWithParams); + if (!response.ok) { + throw new Error(`Failed to fetch projects from Gerrit at ${endpointWithParams} with status ${response.status}`); } const text = await response.text(); @@ -151,11 +114,11 @@ const shouldExcludeProject = ({ const shouldExclude = (() => { if ([ - 'All-Projects', - 'All-Users', - 'All-Avatars', - 'All-Archived-Projects' - ].includes(project.name)) { + 'All-Projects', + 'All-Users', + 'All-Avatars', + 'All-Archived-Projects' + ].includes(project.name)) { reason = `Project is a special project.`; return true; } diff --git a/packages/backend/src/git.ts b/packages/backend/src/git.ts index dbb602e1c..948b1f3bc 100644 --- a/packages/backend/src/git.ts +++ b/packages/backend/src/git.ts @@ -1,8 +1,8 @@ -import { CheckRepoActions, GitConfigScope, simpleGit, SimpleGitProgressEvent } from 'simple-git'; +import { env } from "@sourcebot/shared"; +import { existsSync } from 'node:fs'; import { mkdir } from 'node:fs/promises'; -import { env } from './env.js'; import { dirname, resolve } from 'node:path'; -import { existsSync } from 'node:fs'; +import { CheckRepoActions, GitConfigScope, simpleGit, SimpleGitProgressEvent } from 'simple-git'; type onProgressFn = (event: SimpleGitProgressEvent) => void; diff --git a/packages/backend/src/gitea.ts b/packages/backend/src/gitea.ts index 91493b0ff..23f5c520f 100644 --- a/packages/backend/src/gitea.ts +++ b/packages/backend/src/gitea.ts @@ -1,13 +1,13 @@ -import { Api, giteaApi, HttpResponse, Repository as GiteaRepository } from 'gitea-js'; +import * as Sentry from "@sentry/node"; +import { getTokenFromConfig } from "@sourcebot/shared"; +import { createLogger } from '@sourcebot/shared'; import { GiteaConnectionConfig } from '@sourcebot/schemas/v3/gitea.type'; -import { measure } from './utils.js'; +import { env } from "@sourcebot/shared"; import fetch from 'cross-fetch'; -import { createLogger } from '@sourcebot/logger'; +import { Api, giteaApi, Repository as GiteaRepository, HttpResponse } from 'gitea-js'; import micromatch from 'micromatch'; import { processPromiseResults, throwIfAnyFailed } from './connectionUtils.js'; -import * as Sentry from "@sentry/node"; -import { env } from './env.js'; -import { getTokenFromConfig } from "@sourcebot/crypto"; +import { measure } from './utils.js'; const logger = createLogger('gitea'); const GITEA_CLOUD_HOSTNAME = "gitea.com"; diff --git a/packages/backend/src/github.ts b/packages/backend/src/github.ts index 5b8aaa5a7..49ce1d37e 100644 --- a/packages/backend/src/github.ts +++ b/packages/backend/src/github.ts @@ -1,15 +1,14 @@ import { Octokit } from "@octokit/rest"; import * as Sentry from "@sentry/node"; -import { createLogger } from "@sourcebot/logger"; +import { getTokenFromConfig } from "@sourcebot/shared"; +import { createLogger } from "@sourcebot/shared"; import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type"; -import { hasEntitlement } from "@sourcebot/shared"; +import { env, hasEntitlement } from "@sourcebot/shared"; import micromatch from "micromatch"; import pLimit from "p-limit"; import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js"; import { GithubAppManager } from "./ee/githubAppManager.js"; -import { env } from "./env.js"; import { fetchWithRetry, measure } from "./utils.js"; -import { getTokenFromConfig } from "@sourcebot/crypto"; export const GITHUB_CLOUD_HOSTNAME = "github.com"; diff --git a/packages/backend/src/gitlab.ts b/packages/backend/src/gitlab.ts index 6063f7bda..b461d59ba 100644 --- a/packages/backend/src/gitlab.ts +++ b/packages/backend/src/gitlab.ts @@ -1,12 +1,12 @@ import { Gitlab, ProjectSchema } from "@gitbeaker/rest"; +import * as Sentry from "@sentry/node"; +import { getTokenFromConfig } from "@sourcebot/shared"; +import { createLogger } from "@sourcebot/shared"; +import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type"; +import { env } from "@sourcebot/shared"; import micromatch from "micromatch"; -import { createLogger } from "@sourcebot/logger"; -import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type" -import { measure, fetchWithRetry } from "./utils.js"; import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js"; -import * as Sentry from "@sentry/node"; -import { env } from "./env.js"; -import { getTokenFromConfig } from "@sourcebot/crypto"; +import { fetchWithRetry, measure } from "./utils.js"; const logger = createLogger('gitlab'); export const GITLAB_CLOUD_HOSTNAME = "gitlab.com"; diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index ed66a390f..6c1cbeba8 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -1,8 +1,8 @@ import "./instrument.js"; import { PrismaClient } from "@sourcebot/db"; -import { createLogger } from "@sourcebot/logger"; -import { getConfigSettings, hasEntitlement } from '@sourcebot/shared'; +import { createLogger } from "@sourcebot/shared"; +import { env, getConfigSettings, hasEntitlement, getDBConnectionString } from '@sourcebot/shared'; import { existsSync } from 'fs'; import { mkdir } from 'fs/promises'; import { Redis } from 'ioredis'; @@ -12,7 +12,6 @@ import { INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js'; import { GithubAppManager } from "./ee/githubAppManager.js"; import { RepoPermissionSyncer } from './ee/repoPermissionSyncer.js'; import { AccountPermissionSyncer } from "./ee/accountPermissionSyncer.js"; -import { env } from "./env.js"; import { PromClient } from './promClient.js'; import { RepoIndexManager } from "./repoIndexManager.js"; @@ -29,7 +28,13 @@ if (!existsSync(indexPath)) { await mkdir(indexPath, { recursive: true }); } -const prisma = new PrismaClient(); +const prisma = new PrismaClient({ + datasources: { + db: { + url: getDBConnectionString(), + }, + }, +}); const redis = new Redis(env.REDIS_URL, { maxRetriesPerRequest: null diff --git a/packages/backend/src/instrument.ts b/packages/backend/src/instrument.ts index 926bf2aec..5e0faa28e 100644 --- a/packages/backend/src/instrument.ts +++ b/packages/backend/src/instrument.ts @@ -1,6 +1,6 @@ import * as Sentry from "@sentry/node"; -import { env } from "./env.js"; -import { createLogger } from "@sourcebot/logger"; +import { createLogger } from "@sourcebot/shared"; +import { env } from "@sourcebot/shared/client"; const logger = createLogger('instrument'); diff --git a/packages/backend/src/posthog.ts b/packages/backend/src/posthog.ts index 9984643bc..d61ee46e0 100644 --- a/packages/backend/src/posthog.ts +++ b/packages/backend/src/posthog.ts @@ -1,12 +1,13 @@ +import { env as clientEnv } from "@sourcebot/shared/client"; +import { env } from "@sourcebot/shared"; import { PostHog } from 'posthog-node'; import { PosthogEvent, PosthogEventMap } from './posthogEvents.js'; -import { env } from './env.js'; let posthog: PostHog | undefined = undefined; -if (env.NEXT_PUBLIC_POSTHOG_PAPIK) { +if (clientEnv.NEXT_PUBLIC_POSTHOG_PAPIK) { posthog = new PostHog( - env.NEXT_PUBLIC_POSTHOG_PAPIK, + clientEnv.NEXT_PUBLIC_POSTHOG_PAPIK, { host: "https://us.i.posthog.com", } @@ -23,7 +24,7 @@ export function captureEvent(event: E, properties: Posth event: event, properties: { ...properties, - sourcebot_version: env.NEXT_PUBLIC_SOURCEBOT_VERSION, + sourcebot_version: clientEnv.NEXT_PUBLIC_SOURCEBOT_VERSION, }, }); } diff --git a/packages/backend/src/promClient.ts b/packages/backend/src/promClient.ts index 67ed6f3fb..2fa7718f0 100644 --- a/packages/backend/src/promClient.ts +++ b/packages/backend/src/promClient.ts @@ -1,7 +1,7 @@ import express, { Request, Response } from 'express'; import { Server } from 'http'; import client, { Registry, Counter, Gauge } from 'prom-client'; -import { createLogger } from "@sourcebot/logger"; +import { createLogger } from "@sourcebot/shared"; const logger = createLogger('prometheus-client'); diff --git a/packages/backend/src/repoCompileUtils.ts b/packages/backend/src/repoCompileUtils.ts index 04a8b3b5d..10c748a80 100644 --- a/packages/backend/src/repoCompileUtils.ts +++ b/packages/backend/src/repoCompileUtils.ts @@ -10,7 +10,7 @@ import { SchemaRepository as BitbucketCloudRepository } from "@coderabbitai/bitb import { CodeHostType, Prisma } from '@sourcebot/db'; import { WithRequired } from "./types.js" import { marshalBool } from "./utils.js"; -import { createLogger } from '@sourcebot/logger'; +import { createLogger } from '@sourcebot/shared'; import { BitbucketConnectionConfig, GerritConnectionConfig, GiteaConnectionConfig, GitlabConnectionConfig, GenericGitHostConnectionConfig, AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/connection.type'; import { ProjectVisibility } from "azure-devops-node-api/interfaces/CoreInterfaces.js"; import path from 'path'; diff --git a/packages/backend/src/repoIndexManager.ts b/packages/backend/src/repoIndexManager.ts index 3b98e0e08..cccc1b90a 100644 --- a/packages/backend/src/repoIndexManager.ts +++ b/packages/backend/src/repoIndexManager.ts @@ -1,14 +1,13 @@ import * as Sentry from '@sentry/node'; import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; -import { createLogger, Logger } from "@sourcebot/logger"; -import { repoMetadataSchema, RepoIndexingJobMetadata, repoIndexingJobMetadataSchema, RepoMetadata } from '@sourcebot/shared'; +import { createLogger, Logger } from "@sourcebot/shared"; +import { env, RepoIndexingJobMetadata, repoIndexingJobMetadataSchema, RepoMetadata, repoMetadataSchema } from '@sourcebot/shared'; import { existsSync } from 'fs'; import { readdir, rm } from 'fs/promises'; import { Job, Queue, ReservedJob, Worker } from "groupmq"; import { Redis } from 'ioredis'; import micromatch from 'micromatch'; import { INDEX_CACHE_DIR } from './constants.js'; -import { env } from './env.js'; import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName, getTags, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js'; import { captureEvent } from './posthog.js'; import { PromClient } from './promClient.js'; diff --git a/packages/backend/src/utils.ts b/packages/backend/src/utils.ts index dd1e7dfa0..e85ce7665 100644 --- a/packages/backend/src/utils.ts +++ b/packages/backend/src/utils.ts @@ -2,7 +2,7 @@ import { Logger } from "winston"; import { RepoAuthCredentials, RepoWithConnections } from "./types.js"; import path from 'path'; import { Repo } from "@sourcebot/db"; -import { getTokenFromConfig } from "@sourcebot/crypto"; +import { getTokenFromConfig } from "@sourcebot/shared"; import * as Sentry from "@sentry/node"; import { GithubConnectionConfig, GitlabConnectionConfig, GiteaConnectionConfig, BitbucketConnectionConfig, AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/connection.type'; import { GithubAppManager } from "./ee/githubAppManager.js"; diff --git a/packages/backend/src/zoekt.ts b/packages/backend/src/zoekt.ts index 54ae615eb..9f65f4734 100644 --- a/packages/backend/src/zoekt.ts +++ b/packages/backend/src/zoekt.ts @@ -1,5 +1,5 @@ import { Repo } from "@sourcebot/db"; -import { createLogger } from "@sourcebot/logger"; +import { createLogger } from "@sourcebot/shared"; import { exec } from "child_process"; import { INDEX_CACHE_DIR } from "./constants.js"; import { Settings } from "./types.js"; diff --git a/packages/crypto/.gitignore b/packages/crypto/.gitignore deleted file mode 100644 index 3a8fe5ede..000000000 --- a/packages/crypto/.gitignore +++ /dev/null @@ -1 +0,0 @@ -.env.local \ No newline at end of file diff --git a/packages/crypto/package.json b/packages/crypto/package.json deleted file mode 100644 index d25e412fa..000000000 --- a/packages/crypto/package.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "name": "@sourcebot/crypto", - "version": "0.1.0", - "main": "dist/index.js", - "private": true, - "scripts": { - "build": "tsc", - "postinstall": "yarn build" - }, - "dependencies": { - "@google-cloud/secret-manager": "^6.1.1", - "@sourcebot/db": "*", - "@sourcebot/schemas": "*", - "dotenv": "^16.4.5" - }, - "devDependencies": { - "@types/node": "^22.7.5", - "typescript": "^5.7.3" - } -} diff --git a/packages/crypto/src/environment.ts b/packages/crypto/src/environment.ts deleted file mode 100644 index 8efe296d0..000000000 --- a/packages/crypto/src/environment.ts +++ /dev/null @@ -1,13 +0,0 @@ -import dotenv from 'dotenv'; - -export const getEnv = (env: string | undefined, defaultValue?: string) => { - return env ?? defaultValue; -} - -dotenv.config({ - path: './.env.local', - override: true -}); - -// @note: You can use https://generate-random.org/encryption-key-generator to create a new 32 byte key -export const SOURCEBOT_ENCRYPTION_KEY = getEnv(process.env.SOURCEBOT_ENCRYPTION_KEY); \ No newline at end of file diff --git a/packages/crypto/src/tokenUtils.ts b/packages/crypto/src/tokenUtils.ts deleted file mode 100644 index abefa7ef9..000000000 --- a/packages/crypto/src/tokenUtils.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { SecretManagerServiceClient } from "@google-cloud/secret-manager"; -import { Token } from "@sourcebot/schemas/v3/shared.type"; - -export const getTokenFromConfig = async (token: Token): Promise => { - if ('env' in token) { - const envToken = process.env[token.env]; - if (!envToken) { - throw new Error(`Environment variable ${token.env} not found.`); - } - - return envToken; - } else if ('googleCloudSecret' in token) { - try { - const client = new SecretManagerServiceClient(); - const [response] = await client.accessSecretVersion({ - name: token.googleCloudSecret, - }); - - if (!response.payload?.data) { - throw new Error(`Secret ${token.googleCloudSecret} not found.`); - } - - return response.payload.data.toString(); - } catch (error) { - throw new Error(`Failed to access Google Cloud secret ${token.googleCloudSecret}: ${error instanceof Error ? error.message : String(error)}`); - } - } else { - throw new Error('Invalid token configuration'); - } -}; \ No newline at end of file diff --git a/packages/crypto/tsconfig.json b/packages/crypto/tsconfig.json deleted file mode 100644 index b364feca2..000000000 --- a/packages/crypto/tsconfig.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2022", - "module": "Node16", - "lib": ["ES2023"], - "outDir": "dist", - "rootDir": "src", - "declaration": true, - "declarationMap": true, - "sourceMap": true, - "strict": true, - "noImplicitAny": true, - "strictNullChecks": true, - "moduleResolution": "Node16", - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "skipLibCheck": true, - "isolatedModules": true, - "resolveJsonModule": true - }, - "include": ["src/**/*"], - "exclude": ["node_modules", "dist"] - } - \ No newline at end of file diff --git a/packages/db/package.json b/packages/db/package.json index 8dc07d69d..7af80bb4c 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -25,7 +25,6 @@ }, "dependencies": { "@prisma/client": "6.2.1", - "@sourcebot/logger": "workspace:*", "@types/readline-sync": "^1.4.8", "readline-sync": "^1.4.10" } diff --git a/packages/db/tools/scriptRunner.ts b/packages/db/tools/scriptRunner.ts index 499a37701..6bb619494 100644 --- a/packages/db/tools/scriptRunner.ts +++ b/packages/db/tools/scriptRunner.ts @@ -3,7 +3,6 @@ import { ArgumentParser } from "argparse"; import { migrateDuplicateConnections } from "./scripts/migrate-duplicate-connections"; import { injectAuditData } from "./scripts/inject-audit-data"; import { confirmAction } from "./utils"; -import { createLogger } from "@sourcebot/logger"; import { injectRepoData } from "./scripts/inject-repo-data"; import { testRepoQueryPerf } from "./scripts/test-repo-query-perf"; @@ -23,19 +22,17 @@ parser.add_argument("--url", { required: true, help: "Database URL" }); parser.add_argument("--script", { required: true, help: "Script to run" }); const args = parser.parse_args(); -const logger = createLogger('db-script-runner'); - (async () => { if (!(args.script in scripts)) { - logger.error("Invalid script"); + console.error("Invalid script"); process.exit(1); } const selectedScript = scripts[args.script]; - logger.info("\nTo confirm:"); - logger.info(`- Database URL: ${args.url}`); - logger.info(`- Script: ${args.script}`); + console.log("\nTo confirm:"); + console.log(`- Database URL: ${args.url}`); + console.log(`- Script: ${args.script}`); confirmAction(); @@ -45,7 +42,7 @@ const logger = createLogger('db-script-runner'); await selectedScript.run(prisma); - logger.info("\nDone."); + console.log("\nDone."); process.exit(0); })(); diff --git a/packages/db/tools/scripts/inject-audit-data.ts b/packages/db/tools/scripts/inject-audit-data.ts index 11f3e8cc0..c0c202d9f 100644 --- a/packages/db/tools/scripts/inject-audit-data.ts +++ b/packages/db/tools/scripts/inject-audit-data.ts @@ -1,9 +1,6 @@ import { Script } from "../scriptRunner"; import { PrismaClient } from "../../dist"; import { confirmAction } from "../utils"; -import { createLogger } from "@sourcebot/logger"; - -const logger = createLogger('inject-audit-data'); // Generate realistic audit data for analytics testing // Simulates 50 engineers with varying activity patterns @@ -17,11 +14,11 @@ export const injectAuditData: Script = { }); if (!org) { - logger.error(`Organization with id ${orgId} not found. Please create it first.`); + console.error(`Organization with id ${orgId} not found. Please create it first.`); return; } - logger.info(`Injecting audit data for organization: ${org.name} (${org.domain})`); + console.log(`Injecting audit data for organization: ${org.name} (${org.domain})`); // Generate 50 fake user IDs const userIds = Array.from({ length: 50 }, (_, i) => `user_${String(i + 1).padStart(3, '0')}`); @@ -38,7 +35,7 @@ export const injectAuditData: Script = { const startDate = new Date(); startDate.setDate(startDate.getDate() - 90); - logger.info(`Generating data from ${startDate.toISOString().split('T')[0]} to ${endDate.toISOString().split('T')[0]}`); + console.log(`Generating data from ${startDate.toISOString().split('T')[0]} to ${endDate.toISOString().split('T')[0]}`); confirmAction(); @@ -125,9 +122,9 @@ export const injectAuditData: Script = { } } - logger.info(`\nAudit data injection complete!`); - logger.info(`Users: ${userIds.length}`); - logger.info(`Date range: ${startDate.toISOString().split('T')[0]} to ${endDate.toISOString().split('T')[0]}`); + console.log(`\nAudit data injection complete!`); + console.log(`Users: ${userIds.length}`); + console.log(`Date range: ${startDate.toISOString().split('T')[0]} to ${endDate.toISOString().split('T')[0]}`); // Show some statistics const stats = await prisma.audit.groupBy({ @@ -136,9 +133,9 @@ export const injectAuditData: Script = { _count: { action: true } }); - logger.info('\nAction breakdown:'); + console.log('\nAction breakdown:'); stats.forEach(stat => { - logger.info(` ${stat.action}: ${stat._count.action}`); + console.log(` ${stat.action}: ${stat._count.action}`); }); }, }; \ No newline at end of file diff --git a/packages/db/tools/scripts/inject-repo-data.ts b/packages/db/tools/scripts/inject-repo-data.ts index 209880f39..bb427cbe3 100644 --- a/packages/db/tools/scripts/inject-repo-data.ts +++ b/packages/db/tools/scripts/inject-repo-data.ts @@ -1,8 +1,5 @@ import { Script } from "../scriptRunner"; import { PrismaClient } from "../../dist"; -import { createLogger } from "@sourcebot/logger"; - -const logger = createLogger('inject-repo-data'); const NUM_REPOS = 100000; @@ -35,7 +32,7 @@ export const injectRepoData: Script = { }); - logger.info(`Creating ${NUM_REPOS} repos...`); + console.log(`Creating ${NUM_REPOS} repos...`); for (let i = 0; i < NUM_REPOS; i++) { await prisma.repo.create({ @@ -59,6 +56,6 @@ export const injectRepoData: Script = { }); } - logger.info(`Created ${NUM_REPOS} repos.`); + console.log(`Created ${NUM_REPOS} repos.`); } }; \ No newline at end of file diff --git a/packages/db/tools/scripts/migrate-duplicate-connections.ts b/packages/db/tools/scripts/migrate-duplicate-connections.ts index fe3fa949c..7093e429e 100644 --- a/packages/db/tools/scripts/migrate-duplicate-connections.ts +++ b/packages/db/tools/scripts/migrate-duplicate-connections.ts @@ -1,9 +1,6 @@ import { Script } from "../scriptRunner"; import { PrismaClient } from "../../dist"; import { confirmAction } from "../utils"; -import { createLogger } from "@sourcebot/logger"; - -const logger = createLogger('migrate-duplicate-connections'); // Handles duplicate connections by renaming them to be unique. // @see: 20250320215449_unique_connection_name_constraint_within_org @@ -18,7 +15,7 @@ export const migrateDuplicateConnections: Script = { }, })).filter(({ _count }) => _count._all > 1); - logger.info(`Found ${duplicates.reduce((acc, { _count }) => acc + _count._all, 0)} duplicate connections.`); + console.log(`Found ${duplicates.reduce((acc, { _count }) => acc + _count._all, 0)} duplicate connections.`); confirmAction(); @@ -40,7 +37,7 @@ export const migrateDuplicateConnections: Script = { const connection = connections[i]; const newName = `${name}-${i + 1}`; - logger.info(`Migrating connection with id ${connection.id} from name=${name} to name=${newName}`); + console.log(`Migrating connection with id ${connection.id} from name=${name} to name=${newName}`); await prisma.connection.update({ where: { id: connection.id }, @@ -50,6 +47,6 @@ export const migrateDuplicateConnections: Script = { } } - logger.info(`Migrated ${migrated} connections.`); + console.log(`Migrated ${migrated} connections.`); }, }; diff --git a/packages/db/tools/scripts/test-repo-query-perf.ts b/packages/db/tools/scripts/test-repo-query-perf.ts index ee07d14f1..c41beaca0 100644 --- a/packages/db/tools/scripts/test-repo-query-perf.ts +++ b/packages/db/tools/scripts/test-repo-query-perf.ts @@ -1,8 +1,5 @@ import { Script } from "../scriptRunner"; import { PrismaClient } from "../../dist"; -import { createLogger } from "@sourcebot/logger"; - -const logger = createLogger('test-repo-query-perf'); export const testRepoQueryPerf: Script = { run: async (prisma: PrismaClient) => { @@ -23,6 +20,6 @@ export const testRepoQueryPerf: Script = { }); const durationMs = Date.now() - start; - logger.info(`Found ${allRepos.length} repos in ${durationMs}ms`); + console.log(`Found ${allRepos.length} repos in ${durationMs}ms`); } }; \ No newline at end of file diff --git a/packages/db/tools/utils.ts b/packages/db/tools/utils.ts index a096ac9fb..ed1886b4f 100644 --- a/packages/db/tools/utils.ts +++ b/packages/db/tools/utils.ts @@ -1,17 +1,14 @@ import readline from 'readline-sync'; -import { createLogger } from "@sourcebot/logger"; - -const logger = createLogger('db-utils'); export const confirmAction = (message: string = "Are you sure you want to proceed? [N/y]") => { const response = readline.question(message).toLowerCase(); if (response !== 'y') { - logger.info("Aborted."); + console.log("Aborted."); process.exit(0); } } export const abort = () => { - logger.info("Aborted."); + console.log("Aborted."); process.exit(0); }; diff --git a/packages/error/package.json b/packages/error/package.json deleted file mode 100644 index 8b97d5f16..000000000 --- a/packages/error/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "@sourcebot/error", - "main": "dist/index.js", - "version": "0.1.0", - "private": true, - "scripts": { - "build": "tsc", - "postinstall": "yarn build" - }, - "devDependencies": { - "@types/node": "^22.7.5", - "typescript": "^5.7.3" - } -} diff --git a/packages/error/src/index.ts b/packages/error/src/index.ts deleted file mode 100644 index f18f40057..000000000 --- a/packages/error/src/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -export enum BackendError { - CONNECTION_SYNC_SECRET_DNE = 'CONNECTION_SYNC_SECRET_DNE', - CONNECTION_SYNC_INVALID_TOKEN = 'CONNECTION_SYNC_INVALID_TOKEN', - CONNECTION_SYNC_SYSTEM_ERROR = 'CONNECTION_SYNC_SYSTEM_ERROR', - CONNECTION_SYNC_FAILED_TO_FETCH_GERRIT_PROJECTS = 'CONNECTION_SYNC_FAILED_TO_FETCH_GERRIT_PROJECTS', - CONNECTION_SYNC_CONNECTION_NOT_FOUND = 'CONNECTION_SYNC_CONNECTION_NOT_FOUND', -} - -export class BackendException extends Error { - constructor( - public readonly code: BackendError, - public readonly metadata: Record = {} - ) { - super(code); - this.name = 'BackendException'; - } -} \ No newline at end of file diff --git a/packages/error/tsconfig.json b/packages/error/tsconfig.json deleted file mode 100644 index a27277b93..000000000 --- a/packages/error/tsconfig.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "compilerOptions": { - "target": "ES6", - "module": "CommonJS", - "lib": ["ES6"], - "outDir": "dist", - "rootDir": "src", - "declaration": true, - "declarationMap": true, - "sourceMap": true, - "strict": true, - "noImplicitAny": true, - "strictNullChecks": true, - "moduleResolution": "node", - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "skipLibCheck": true, - "isolatedModules": true - }, - "include": ["src/**/*"], - "exclude": ["node_modules", "dist"] -} diff --git a/packages/logger/.gitignore b/packages/logger/.gitignore deleted file mode 100644 index 96351007d..000000000 --- a/packages/logger/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -dist/ -*.tsbuildinfo \ No newline at end of file diff --git a/packages/logger/package.json b/packages/logger/package.json deleted file mode 100644 index 2e2279a35..000000000 --- a/packages/logger/package.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name": "@sourcebot/logger", - "version": "0.1.0", - "main": "dist/index.js", - "type": "module", - "private": true, - "scripts": { - "build": "tsc", - "postinstall": "yarn build" - }, - "dependencies": { - "@logtail/node": "^0.5.2", - "@logtail/winston": "^0.5.2", - "@t3-oss/env-core": "^0.12.0", - "dotenv": "^16.4.5", - "triple-beam": "^1.4.1", - "winston": "^3.15.0", - "zod": "^3.24.3" - }, - "devDependencies": { - "@types/node": "^22.7.5", - "typescript": "^5.7.3" - } -} diff --git a/packages/logger/src/env.ts b/packages/logger/src/env.ts deleted file mode 100644 index 5f582e0d2..000000000 --- a/packages/logger/src/env.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { createEnv } from "@t3-oss/env-core"; -import { z } from "zod"; -import dotenv from 'dotenv'; - -// Booleans are specified as 'true' or 'false' strings. -const booleanSchema = z.enum(["true", "false"]); - -dotenv.config({ - path: './.env', -}); - -dotenv.config({ - path: './.env.local', - override: true -}); - -export const env = createEnv({ - server: { - SOURCEBOT_LOG_LEVEL: z.enum(["info", "debug", "warn", "error"]).default("info"), - SOURCEBOT_STRUCTURED_LOGGING_ENABLED: booleanSchema.default("false"), - SOURCEBOT_STRUCTURED_LOGGING_FILE: z.string().optional(), - LOGTAIL_TOKEN: z.string().optional(), - LOGTAIL_HOST: z.string().url().optional(), - }, - runtimeEnv: process.env, - emptyStringAsUndefined: true, - skipValidation: process.env.SKIP_ENV_VALIDATION === "1", -}); \ No newline at end of file diff --git a/packages/logger/tsconfig.json b/packages/logger/tsconfig.json deleted file mode 100644 index 88ae91dd7..000000000 --- a/packages/logger/tsconfig.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2022", - "module": "Node16", - "moduleResolution": "Node16", - "lib": ["ES2023"], - "outDir": "dist", - "rootDir": "src", - "declaration": true, - "declarationMap": true, - "sourceMap": true, - "strict": true, - "noImplicitAny": true, - "strictNullChecks": true, - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "skipLibCheck": true, - "isolatedModules": true, - "resolveJsonModule": true - }, - "include": ["src/**/*"], - "exclude": ["node_modules", "dist"] -} \ No newline at end of file diff --git a/packages/schemas/src/v3/environmentOverrides.schema.ts b/packages/schemas/src/v3/environmentOverrides.schema.ts new file mode 100644 index 000000000..49742827c --- /dev/null +++ b/packages/schemas/src/v3/environmentOverrides.schema.ts @@ -0,0 +1,114 @@ +// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! +const schema = { + "type": "object", + "description": "Environment variable overrides.", + "title": "EnvironmentOverrides", + "not": { + "$comment": "List of environment variables that are not allowed to be overridden.", + "anyOf": [ + { + "required": [ + "CONFIG_PATH" + ] + } + ] + }, + "patternProperties": { + "^[a-zA-Z0-9_-]+$": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "const": "token" + }, + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token." + } + }, + "required": [ + "env" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "googleCloudSecret": { + "type": "string", + "description": "The resource name of a Google Cloud secret. Must be in the format `projects//secrets//versions/`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets" + } + }, + "required": [ + "googleCloudSecret" + ], + "additionalProperties": false + } + ] + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "number" + }, + "value": { + "type": "number" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "boolean" + }, + "value": { + "type": "boolean" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + } + ] + } + } +} as const; +export { schema as environmentOverridesSchema }; \ No newline at end of file diff --git a/packages/schemas/src/v3/environmentOverrides.type.ts b/packages/schemas/src/v3/environmentOverrides.type.ts new file mode 100644 index 000000000..e9adfaec8 --- /dev/null +++ b/packages/schemas/src/v3/environmentOverrides.type.ts @@ -0,0 +1,40 @@ +// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! + +/** + * Environment variable overrides. + */ +export interface EnvironmentOverrides { + /** + * This interface was referenced by `EnvironmentOverrides`'s JSON-Schema definition + * via the `patternProperty` "^[a-zA-Z0-9_-]+$". + */ + [k: string]: + | { + type: "token"; + value: + | { + /** + * The name of the environment variable that contains the token. + */ + env: string; + } + | { + /** + * The resource name of a Google Cloud secret. Must be in the format `projects//secrets//versions/`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets + */ + googleCloudSecret: string; + }; + } + | { + type: "string"; + value: string; + } + | { + type: "number"; + value: number; + } + | { + type: "boolean"; + value: boolean; + }; +} diff --git a/packages/schemas/src/v3/index.schema.ts b/packages/schemas/src/v3/index.schema.ts index ee0452435..c7bf374d4 100644 --- a/packages/schemas/src/v3/index.schema.ts +++ b/packages/schemas/src/v3/index.schema.ts @@ -278,6 +278,118 @@ const schema = { }, "additionalProperties": false }, + "environmentOverrides": { + "type": "object", + "description": "Environment variable overrides.", + "title": "EnvironmentOverrides", + "not": { + "$comment": "List of environment variables that are not allowed to be overridden.", + "anyOf": [ + { + "required": [ + "CONFIG_PATH" + ] + } + ] + }, + "patternProperties": { + "^[a-zA-Z0-9_-]+$": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "const": "token" + }, + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token." + } + }, + "required": [ + "env" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "googleCloudSecret": { + "type": "string", + "description": "The resource name of a Google Cloud secret. Must be in the format `projects//secrets//versions/`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets" + } + }, + "required": [ + "googleCloudSecret" + ], + "additionalProperties": false + } + ] + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "number" + }, + "value": { + "type": "number" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "boolean" + }, + "value": { + "type": "boolean" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + } + ] + } + } + }, "connections": { "type": "object", "description": "Defines a collection of connections from varying code hosts that Sourcebot should sync with. This is only available in single-tenancy mode.", @@ -296,7 +408,6 @@ const schema = { "description": "GitHub Configuration" }, "token": { - "description": "A Personal Access Token (PAT).", "anyOf": [ { "type": "object", @@ -324,7 +435,8 @@ const schema = { ], "additionalProperties": false } - ] + ], + "description": "A Personal Access Token (PAT)." }, "url": { "type": "string", @@ -504,7 +616,6 @@ const schema = { "description": "GitLab Configuration" }, "token": { - "description": "An authentication token.", "anyOf": [ { "type": "object", @@ -532,7 +643,8 @@ const schema = { ], "additionalProperties": false } - ] + ], + "description": "An authentication token." }, "url": { "type": "string", @@ -706,7 +818,6 @@ const schema = { "description": "Gitea Configuration" }, "token": { - "description": "A Personal Access Token (PAT).", "anyOf": [ { "type": "object", @@ -734,7 +845,8 @@ const schema = { ], "additionalProperties": false } - ] + ], + "description": "A Personal Access Token (PAT)." }, "url": { "type": "string", @@ -973,7 +1085,6 @@ const schema = { "description": "The username to use for authentication. Only needed if token is an app password." }, "token": { - "description": "An authentication token.", "anyOf": [ { "type": "object", @@ -1001,7 +1112,8 @@ const schema = { ], "additionalProperties": false } - ] + ], + "description": "An authentication token." }, "url": { "type": "string", @@ -1141,7 +1253,6 @@ const schema = { "description": "Azure DevOps Configuration" }, "token": { - "description": "A Personal Access Token (PAT).", "anyOf": [ { "type": "object", @@ -1169,7 +1280,8 @@ const schema = { ], "additionalProperties": false } - ] + ], + "description": "A Personal Access Token (PAT)." }, "url": { "type": "string", @@ -1425,7 +1537,6 @@ const schema = { "description": "Optional display name." }, "accessKeyId": { - "description": "Optional access key ID to use with the model. Defaults to the `AWS_ACCESS_KEY_ID` environment variable.", "anyOf": [ { "type": "object", @@ -1453,10 +1564,10 @@ const schema = { ], "additionalProperties": false } - ] + ], + "description": "Optional access key ID to use with the model. Defaults to the `AWS_ACCESS_KEY_ID` environment variable." }, "accessKeySecret": { - "description": "Optional secret access key to use with the model. Defaults to the `AWS_SECRET_ACCESS_KEY` environment variable.", "anyOf": [ { "type": "object", @@ -1484,10 +1595,10 @@ const schema = { ], "additionalProperties": false } - ] + ], + "description": "Optional secret access key to use with the model. Defaults to the `AWS_SECRET_ACCESS_KEY` environment variable." }, "sessionToken": { - "description": "Optional session token to use with the model. Defaults to the `AWS_SESSION_TOKEN` environment variable.", "anyOf": [ { "type": "object", @@ -1515,7 +1626,8 @@ const schema = { ], "additionalProperties": false } - ] + ], + "description": "Optional session token to use with the model. Defaults to the `AWS_SESSION_TOKEN` environment variable." }, "region": { "type": "string", @@ -2854,7 +2966,6 @@ const schema = { "description": "Optional display name." }, "accessKeyId": { - "description": "Optional access key ID to use with the model. Defaults to the `AWS_ACCESS_KEY_ID` environment variable.", "anyOf": [ { "type": "object", @@ -2882,10 +2993,10 @@ const schema = { ], "additionalProperties": false } - ] + ], + "description": "Optional access key ID to use with the model. Defaults to the `AWS_ACCESS_KEY_ID` environment variable." }, "accessKeySecret": { - "description": "Optional secret access key to use with the model. Defaults to the `AWS_SECRET_ACCESS_KEY` environment variable.", "anyOf": [ { "type": "object", @@ -2913,10 +3024,10 @@ const schema = { ], "additionalProperties": false } - ] + ], + "description": "Optional secret access key to use with the model. Defaults to the `AWS_SECRET_ACCESS_KEY` environment variable." }, "sessionToken": { - "description": "Optional session token to use with the model. Defaults to the `AWS_SESSION_TOKEN` environment variable.", "anyOf": [ { "type": "object", @@ -2944,7 +3055,8 @@ const schema = { ], "additionalProperties": false } - ] + ], + "description": "Optional session token to use with the model. Defaults to the `AWS_SESSION_TOKEN` environment variable." }, "region": { "type": "string", diff --git a/packages/schemas/src/v3/index.type.ts b/packages/schemas/src/v3/index.type.ts index 0e88d8ab7..669a9511b 100644 --- a/packages/schemas/src/v3/index.type.ts +++ b/packages/schemas/src/v3/index.type.ts @@ -44,6 +44,7 @@ export interface SourcebotConfig { contexts?: { [k: string]: SearchContext; }; + environmentOverrides?: EnvironmentOverrides; /** * Defines a collection of connections from varying code hosts that Sourcebot should sync with. This is only available in single-tenancy mode. */ @@ -159,6 +160,44 @@ export interface SearchContext { */ description?: string; } +/** + * Environment variable overrides. + */ +export interface EnvironmentOverrides { + /** + * This interface was referenced by `EnvironmentOverrides`'s JSON-Schema definition + * via the `patternProperty` "^[a-zA-Z0-9_-]+$". + */ + [k: string]: + | { + type: "token"; + value: + | { + /** + * The name of the environment variable that contains the token. + */ + env: string; + } + | { + /** + * The resource name of a Google Cloud secret. Must be in the format `projects//secrets//versions/`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets + */ + googleCloudSecret: string; + }; + } + | { + type: "string"; + value: string; + } + | { + type: "number"; + value: number; + } + | { + type: "boolean"; + value: boolean; + }; +} export interface GithubConnectionConfig { /** * GitHub Configuration diff --git a/packages/shared/package.json b/packages/shared/package.json index 902a34853..23d4463b2 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -6,23 +6,28 @@ "scripts": { "build": "tsc", "build:watch": "tsc-watch --preserveWatchOutput", - "postinstall": "yarn build" + "postinstall": "yarn build", + "tool:resolve-env-overrides": "tsx tools/resolveEnvOverrides.ts" }, "dependencies": { - "@sourcebot/crypto": "workspace:*", + "@google-cloud/secret-manager": "^6.1.1", + "@logtail/node": "^0.5.2", + "@logtail/winston": "^0.5.2", "@sourcebot/db": "workspace:*", - "@sourcebot/logger": "workspace:*", "@sourcebot/schemas": "workspace:*", "@t3-oss/env-core": "^0.12.0", "ajv": "^8.17.1", "micromatch": "^4.0.8", "strip-json-comments": "^5.0.1", - "zod": "^3.24.3" + "triple-beam": "^1.4.1", + "winston": "^3.15.0", + "zod": "^3.25.74" }, "devDependencies": { "@types/micromatch": "^4.0.9", "@types/node": "^22.7.5", "tsc-watch": "6.2.1", + "tsx": "^4.19.1", "typescript": "^5.7.3" }, "exports": { diff --git a/packages/crypto/src/index.ts b/packages/shared/src/crypto.ts similarity index 62% rename from packages/crypto/src/index.ts rename to packages/shared/src/crypto.ts index 8f6ca211d..7a88e5ab2 100644 --- a/packages/crypto/src/index.ts +++ b/packages/shared/src/crypto.ts @@ -1,6 +1,8 @@ import crypto from 'crypto'; import fs from 'fs'; -import { SOURCEBOT_ENCRYPTION_KEY } from './environment'; +import { env } from './env.server.js'; +import { Token } from '@sourcebot/schemas/v3/shared.type'; +import { SecretManagerServiceClient } from "@google-cloud/secret-manager"; const algorithm = 'aes-256-cbc'; const ivLength = 16; // 16 bytes for CBC @@ -12,11 +14,7 @@ const generateIV = (): Buffer => { }; export function encrypt(text: string): { iv: string; encryptedData: string } { - if (!SOURCEBOT_ENCRYPTION_KEY) { - throw new Error('Encryption key is not set'); - } - - const encryptionKey = Buffer.from(SOURCEBOT_ENCRYPTION_KEY, 'ascii'); + const encryptionKey = Buffer.from(env.SOURCEBOT_ENCRYPTION_KEY, 'ascii'); const iv = generateIV(); const cipher = crypto.createCipheriv(algorithm, encryptionKey, iv); @@ -28,18 +26,10 @@ export function encrypt(text: string): { iv: string; encryptedData: string } { } export function hashSecret(text: string): string { - if (!SOURCEBOT_ENCRYPTION_KEY) { - throw new Error('Encryption key is not set'); - } - - return crypto.createHmac('sha256', SOURCEBOT_ENCRYPTION_KEY).update(text).digest('hex'); + return crypto.createHmac('sha256', env.SOURCEBOT_ENCRYPTION_KEY).update(text).digest('hex'); } export function generateApiKey(): { key: string; hash: string } { - if (!SOURCEBOT_ENCRYPTION_KEY) { - throw new Error('Encryption key is not set'); - } - const secret = crypto.randomBytes(32).toString('hex'); const hash = hashSecret(secret); @@ -50,11 +40,7 @@ export function generateApiKey(): { key: string; hash: string } { } export function decrypt(iv: string, encryptedText: string): string { - if (!SOURCEBOT_ENCRYPTION_KEY) { - throw new Error('Encryption key is not set'); - } - - const encryptionKey = Buffer.from(SOURCEBOT_ENCRYPTION_KEY, 'ascii'); + const encryptionKey = Buffer.from(env.SOURCEBOT_ENCRYPTION_KEY, 'ascii'); const ivBuffer = Buffer.from(iv, 'hex'); const encryptedBuffer = Buffer.from(encryptedText, 'hex'); @@ -92,4 +78,30 @@ export function verifySignature(data: string, signature: string, publicKeyPath: } } -export { getTokenFromConfig } from './tokenUtils.js'; \ No newline at end of file +export const getTokenFromConfig = async (token: Token): Promise => { + if ('env' in token) { + const envToken = process.env[token.env]; + if (!envToken) { + throw new Error(`Environment variable ${token.env} not found.`); + } + + return envToken; + } else if ('googleCloudSecret' in token) { + try { + const client = new SecretManagerServiceClient(); + const [response] = await client.accessSecretVersion({ + name: token.googleCloudSecret, + }); + + if (!response.payload?.data) { + throw new Error(`Secret ${token.googleCloudSecret} not found.`); + } + + return response.payload.data.toString(); + } catch (error) { + throw new Error(`Failed to access Google Cloud secret ${token.googleCloudSecret}: ${error instanceof Error ? error.message : String(error)}`); + } + } else { + throw new Error('Invalid token configuration'); + } +}; \ No newline at end of file diff --git a/packages/shared/src/db.ts b/packages/shared/src/db.ts new file mode 100644 index 000000000..b3588829b --- /dev/null +++ b/packages/shared/src/db.ts @@ -0,0 +1,15 @@ +import { env } from "./env.server.js"; + +export const getDBConnectionString = (): string | undefined => { + if (env.DATABASE_URL) { + return env.DATABASE_URL; + } + else if (env.DATABASE_HOST && env.DATABASE_USERNAME && env.DATABASE_PASSWORD && env.DATABASE_NAME) { + let databaseUrl = `postgresql://${env.DATABASE_USERNAME}:${env.DATABASE_PASSWORD}@${env.DATABASE_HOST}/${env.DATABASE_NAME}`; + if (env.DATABASE_ARGS) { + databaseUrl += `?${env.DATABASE_ARGS}`; + } + + return databaseUrl; + } +} \ No newline at end of file diff --git a/packages/shared/src/entitlements.ts b/packages/shared/src/entitlements.ts index 1c0c688ff..e1bba6027 100644 --- a/packages/shared/src/entitlements.ts +++ b/packages/shared/src/entitlements.ts @@ -1,9 +1,10 @@ import { base64Decode } from "./utils.js"; import { z } from "zod"; -import { createLogger } from "@sourcebot/logger"; -import { verifySignature } from "@sourcebot/crypto"; -import { env } from "./env.js"; +import { createLogger } from "./logger.js"; +import { env } from "./env.server.js"; +import { env as clientEnv } from "./env.client.js"; import { SOURCEBOT_SUPPORT_EMAIL, SOURCEBOT_UNLIMITED_SEATS } from "./constants.js"; +import { verifySignature } from "./crypto.js"; const logger = createLogger('entitlements'); @@ -89,8 +90,8 @@ export const getLicenseKey = (): LicenseKeyPayload | null => { } export const getPlan = (): Plan => { - if (env.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT) { - if (env.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT === "demo") { + if (clientEnv.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT) { + if (clientEnv.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT === "demo") { return "cloud:demo"; } diff --git a/packages/shared/src/env.client.ts b/packages/shared/src/env.client.ts new file mode 100644 index 000000000..727feb59b --- /dev/null +++ b/packages/shared/src/env.client.ts @@ -0,0 +1,27 @@ +import { createEnv } from "@t3-oss/env-core"; +import { z } from "zod"; +import { SOURCEBOT_CLOUD_ENVIRONMENT } from "./constants.js"; + +export const env = createEnv({ + clientPrefix: "NEXT_PUBLIC_", + client: { + NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT: z.enum(SOURCEBOT_CLOUD_ENVIRONMENT).optional(), + NEXT_PUBLIC_SOURCEBOT_VERSION: z.string().default("unknown"), + NEXT_PUBLIC_POSTHOG_PAPIK: z.string().optional(), + NEXT_PUBLIC_SENTRY_BACKEND_DSN: z.string().optional(), + NEXT_PUBLIC_SENTRY_ENVIRONMENT: z.string().optional(), + NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY: z.string().optional(), + NEXT_PUBLIC_LANGFUSE_BASE_URL: z.string().optional() + }, + runtimeEnvStrict: { + NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT: process.env.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT, + NEXT_PUBLIC_SOURCEBOT_VERSION: process.env.NEXT_PUBLIC_SOURCEBOT_VERSION, + NEXT_PUBLIC_POSTHOG_PAPIK: process.env.NEXT_PUBLIC_POSTHOG_PAPIK, + NEXT_PUBLIC_SENTRY_BACKEND_DSN: process.env.NEXT_PUBLIC_SENTRY_BACKEND_DSN, + NEXT_PUBLIC_SENTRY_ENVIRONMENT: process.env.NEXT_PUBLIC_SENTRY_ENVIRONMENT, + NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY: process.env.NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY, + NEXT_PUBLIC_LANGFUSE_BASE_URL: process.env.NEXT_PUBLIC_LANGFUSE_BASE_URL, + }, + emptyStringAsUndefined: true, + skipValidation: process.env.SKIP_ENV_VALIDATION === "1", +}); \ No newline at end of file diff --git a/packages/web/src/env.mjs b/packages/shared/src/env.server.ts similarity index 63% rename from packages/web/src/env.mjs rename to packages/shared/src/env.server.ts index 00b6e736d..87e487588 100644 --- a/packages/web/src/env.mjs +++ b/packages/shared/src/env.server.ts @@ -1,16 +1,65 @@ -import { createEnv } from "@t3-oss/env-nextjs"; +import { createEnv } from "@t3-oss/env-core"; import { z } from "zod"; -import { SOURCEBOT_CLOUD_ENVIRONMENT } from "@sourcebot/shared/client"; +import { loadConfig } from "./utils.js"; +import { tenancyModeSchema } from "./types.js"; +import { SourcebotConfig } from "@sourcebot/schemas/v3/index.type"; +import { getTokenFromConfig } from "./crypto.js"; // Booleans are specified as 'true' or 'false' strings. const booleanSchema = z.enum(["true", "false"]); -export const tenancyModeSchema = z.enum(["multi", "single"]); // Numbers are treated as strings in .env files. // coerce helps us convert them to numbers. // @see: https://zod.dev/?id=coercion-for-primitives const numberSchema = z.coerce.number(); +export const resolveEnvironmentVariableOverridesFromConfig = async (config: SourcebotConfig): Promise> => { + if (!config.environmentOverrides) { + return {}; + } + + const resolved: Record = {}; + + const start = performance.now(); + + for (const [key, override] of Object.entries(config.environmentOverrides)) { + switch (override.type) { + case 'token': + resolved[key] = await getTokenFromConfig(override.value); + break; + case 'boolean': + resolved[key] = override.value ? 'true' : 'false'; + break; + case 'number': + resolved[key] = override.value.toString(); + break; + case 'string': + resolved[key] = override.value; + break; + } + } + + const end = performance.now(); + console.debug(`resolved environment variable overrides in ${end - start}ms`); + + return resolved; +} + +// Merge process.env with environment variables resolved from config.json +const runtimeEnv = await (async () => { + const configPath = process.env.CONFIG_PATH; + if (!configPath) { + return process.env; + } + + const config = await loadConfig(configPath); + const overrides = await resolveEnvironmentVariableOverridesFromConfig(config); + return { + ...process.env, + ...overrides, + } +})(); + export const env = createEnv({ server: { // Zoekt @@ -18,7 +67,6 @@ export const env = createEnv({ // Auth FORCE_ENABLE_ANONYMOUS_ACCESS: booleanSchema.default('false'), - AUTH_SECRET: z.string(), AUTH_URL: z.string().url(), AUTH_CREDENTIALS_LOGIN_ENABLED: booleanSchema.default('true'), @@ -72,10 +120,19 @@ export const env = createEnv({ CONFIG_MAX_REPOS_NO_TOKEN: numberSchema.default(Number.MAX_SAFE_INTEGER), NODE_ENV: z.enum(["development", "test", "production"]), SOURCEBOT_TELEMETRY_DISABLED: booleanSchema.default('false'), - DATABASE_URL: z.string().url(), + + // Database variables + // Either DATABASE_URL or DATABASE_HOST, DATABASE_USERNAME, DATABASE_PASSWORD, and DATABASE_NAME must be set. + // @see: shared/src/db.ts for more details. + DATABASE_URL: z.string().url().optional(), + DATABASE_HOST: z.string().optional(), + DATABASE_USERNAME: z.string().optional(), + DATABASE_PASSWORD: z.string().optional(), + DATABASE_NAME: z.string().optional(), + DATABASE_ARGS: z.string().optional(), SOURCEBOT_TENANCY_MODE: tenancyModeSchema.default("single"), - CONFIG_PATH: z.string().optional(), + CONFIG_PATH: z.string(), // Misc UI flags SECURITY_CARD_ENABLED: booleanSchema.default('false'), @@ -137,31 +194,30 @@ export const env = createEnv({ // @NOTE: Take care to update actions.ts when changing the name of this. EXPERIMENT_SELF_SERVE_REPO_INDEXING_GITHUB_TOKEN: z.string().optional(), EXPERIMENT_EE_PERMISSION_SYNC_ENABLED: booleanSchema.default('false'), - }, - // @NOTE: Please make sure of the following: - // - Make sure you destructure all client variables in - // the `experimental__runtimeEnv` block below. - // - Update the Dockerfile to pass these variables as build-args. - client: { - // PostHog - NEXT_PUBLIC_POSTHOG_PAPIK: z.string().optional(), - // Misc - NEXT_PUBLIC_SOURCEBOT_VERSION: z.string().default('unknown'), + SOURCEBOT_ENCRYPTION_KEY: z.string(), + SOURCEBOT_INSTALL_ID: z.string().default("unknown"), - NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT: z.enum(SOURCEBOT_CLOUD_ENVIRONMENT).optional(), + FALLBACK_GITHUB_CLOUD_TOKEN: z.string().optional(), + FALLBACK_GITLAB_CLOUD_TOKEN: z.string().optional(), + FALLBACK_GITEA_CLOUD_TOKEN: z.string().optional(), - NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY: z.string().optional(), - NEXT_PUBLIC_LANGFUSE_BASE_URL: z.string().optional() - }, - // For Next.js >= 13.4.4, you only need to destructure client variables: - experimental__runtimeEnv: { - NEXT_PUBLIC_POSTHOG_PAPIK: process.env.NEXT_PUBLIC_POSTHOG_PAPIK, - NEXT_PUBLIC_SOURCEBOT_VERSION: process.env.NEXT_PUBLIC_SOURCEBOT_VERSION, - NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT: process.env.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT, - NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY: process.env.NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY, - NEXT_PUBLIC_LANGFUSE_BASE_URL: process.env.NEXT_PUBLIC_LANGFUSE_BASE_URL, + REDIS_URL: z.string().url().default("redis://localhost:6379"), + REDIS_REMOVE_ON_COMPLETE: numberSchema.default(0), + REDIS_REMOVE_ON_FAIL: numberSchema.default(100), + + DEBUG_ENABLE_GROUPMQ_LOGGING: booleanSchema.default('false'), + + CONNECTION_MANAGER_UPSERT_TIMEOUT_MS: numberSchema.default(300000), + REPO_SYNC_RETRY_BASE_SLEEP_SECONDS: numberSchema.default(60), + + GITLAB_CLIENT_QUERY_TIMEOUT_SECONDS: numberSchema.default(60 * 10), + + SOURCEBOT_LOG_LEVEL: z.enum(["info", "debug", "warn", "error"]).default("info"), + SOURCEBOT_STRUCTURED_LOGGING_ENABLED: booleanSchema.default("false"), + SOURCEBOT_STRUCTURED_LOGGING_FILE: z.string().optional(), }, - skipValidation: process.env.SKIP_ENV_VALIDATION === "1", + runtimeEnv, emptyStringAsUndefined: true, + skipValidation: process.env.SKIP_ENV_VALIDATION === "1", }); \ No newline at end of file diff --git a/packages/shared/src/env.ts b/packages/shared/src/env.ts deleted file mode 100644 index c11629236..000000000 --- a/packages/shared/src/env.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { createEnv } from "@t3-oss/env-core"; -import { z } from "zod"; -import { SOURCEBOT_CLOUD_ENVIRONMENT } from "./constants.js"; - -export const env = createEnv({ - server: { - SOURCEBOT_EE_LICENSE_KEY: z.string().optional(), - SOURCEBOT_PUBLIC_KEY_PATH: z.string(), - }, - client: { - NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT: z.enum(SOURCEBOT_CLOUD_ENVIRONMENT).optional(), - }, - clientPrefix: "NEXT_PUBLIC_", - runtimeEnvStrict: { - SOURCEBOT_EE_LICENSE_KEY: process.env.SOURCEBOT_EE_LICENSE_KEY, - SOURCEBOT_PUBLIC_KEY_PATH: process.env.SOURCEBOT_PUBLIC_KEY_PATH, - NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT: process.env.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT, - }, - emptyStringAsUndefined: true, - skipValidation: process.env.SKIP_ENV_VALIDATION === "1", -}); \ No newline at end of file diff --git a/packages/shared/src/index.client.ts b/packages/shared/src/index.client.ts index ca2bfad2d..8d185ba67 100644 --- a/packages/shared/src/index.client.ts +++ b/packages/shared/src/index.client.ts @@ -1,2 +1,4 @@ - -export * from "./constants.js"; \ No newline at end of file +export * from "./constants.js"; +export { + env +} from "./env.client.js"; \ No newline at end of file diff --git a/packages/shared/src/index.server.ts b/packages/shared/src/index.server.ts index f3303c149..fabe608e7 100644 --- a/packages/shared/src/index.server.ts +++ b/packages/shared/src/index.server.ts @@ -16,6 +16,7 @@ export type { export { repoMetadataSchema, repoIndexingJobMetadataSchema, + tenancyModeSchema, } from "./types.js"; export { base64Decode, @@ -24,4 +25,25 @@ export { isRemotePath, getConfigSettings, } from "./utils.js"; -export * from "./constants.js"; \ No newline at end of file +export * from "./constants.js"; +export { + env, + resolveEnvironmentVariableOverridesFromConfig, +} from "./env.server.js"; +export { + createLogger, +} from "./logger.js"; +export type { + Logger, +} from "./logger.js"; +export { + getTokenFromConfig, + encrypt, + decrypt, + hashSecret, + generateApiKey, + verifySignature, +} from "./crypto.js"; +export { + getDBConnectionString, +} from "./db.js"; \ No newline at end of file diff --git a/packages/logger/src/index.ts b/packages/shared/src/logger.ts similarity index 95% rename from packages/logger/src/index.ts rename to packages/shared/src/logger.ts index 635c8b3ce..a3f89e2cc 100644 --- a/packages/logger/src/index.ts +++ b/packages/shared/src/logger.ts @@ -2,7 +2,7 @@ import winston, { format, Logger } from 'winston'; import { Logtail } from '@logtail/node'; import { LogtailTransport } from '@logtail/winston'; import { MESSAGE } from 'triple-beam'; -import { env } from './env.js'; +import { env } from './env.server.js'; /** * Logger configuration with support for structured JSON logging. @@ -16,7 +16,7 @@ import { env } from './env.js'; * - Logs will be formatted as: "timestamp level: [label] message" */ -const { combine, colorize, timestamp, prettyPrint, errors, printf, label: labelFn, json } = format; +const { combine, colorize, timestamp, errors, printf, label: labelFn, json } = format; const datadogFormat = format((info) => { info.status = info.level.toLowerCase(); diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index a03b2e9d2..0d5fb5209 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -43,3 +43,5 @@ export const repoIndexingJobMetadataSchema = z.object({ }); export type RepoIndexingJobMetadata = z.infer; + +export const tenancyModeSchema = z.enum(["multi", "single"]); \ No newline at end of file diff --git a/packages/shared/tools/resolveEnvOverrides.ts b/packages/shared/tools/resolveEnvOverrides.ts new file mode 100644 index 000000000..def001c38 --- /dev/null +++ b/packages/shared/tools/resolveEnvOverrides.ts @@ -0,0 +1,30 @@ +// The following script loads the config.json file and resolves any environment variable overrides. +// It then writes then to stdout in the format of `KEY="VALUE"`. +// This is used by entrypoint.sh to set them as variables. +(async () => { + if (!process.env.CONFIG_PATH) { + console.error('CONFIG_PATH is not set'); + process.exit(1); + } + + // Silence all console logs so we don't pollute stdout. + const originalConsoleLog = console.log; + console.log = () => {}; + console.debug = () => {}; + console.info = () => {}; + console.warn = () => {}; + // console.error = () => {}; // Keep errors + + const { loadConfig } = await import("../src/utils.js"); + const { resolveEnvironmentVariableOverridesFromConfig } = await import("../src/env.server.js"); + + const config = await loadConfig(process.env.CONFIG_PATH); + const overrides = await resolveEnvironmentVariableOverridesFromConfig(config); + + for (const [key, value] of Object.entries(overrides)) { + const escapedValue = value.replace(/"/g, '\\"'); + originalConsoleLog(`${key}="${escapedValue}"`); + } + + process.exit(0); +})(); \ No newline at end of file diff --git a/packages/web/next.config.mjs b/packages/web/next.config.mjs index 2ab0b6421..d344d1214 100644 --- a/packages/web/next.config.mjs +++ b/packages/web/next.config.mjs @@ -1,4 +1,3 @@ -await import("./src/env.mjs"); import { withSentryConfig } from "@sentry/nextjs"; @@ -8,7 +7,7 @@ const nextConfig = { // This is required when using standalone builds. // @see: https://env.t3.gg/docs/nextjs#create-your-schema - transpilePackages: ["@t3-oss/env-nextjs", "@t3-oss/env-core"], + transpilePackages: ["@t3-oss/env-core"], // @see : https://posthog.com/docs/advanced/proxy/nextjs async rewrites() { diff --git a/packages/web/package.json b/packages/web/package.json index a7a798abb..ba3c3dbc4 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -90,16 +90,12 @@ "@sentry/nextjs": "^9", "@shopify/lang-jsonc": "^1.0.0", "@sourcebot/codemirror-lang-tcl": "^1.0.12", - "@sourcebot/crypto": "workspace:*", "@sourcebot/db": "workspace:*", - "@sourcebot/error": "workspace:*", - "@sourcebot/logger": "workspace:*", "@sourcebot/schemas": "workspace:*", "@sourcebot/shared": "workspace:*", "@ssddanbrown/codemirror-lang-twig": "^1.0.0", "@stripe/react-stripe-js": "^3.1.1", "@stripe/stripe-js": "^5.6.0", - "@t3-oss/env-nextjs": "^0.12.0", "@tailwindcss/typography": "^0.5.16", "@tanstack/react-query": "^5.53.3", "@tanstack/react-table": "^8.20.5", diff --git a/packages/web/sentry.server.config.ts b/packages/web/sentry.server.config.ts index 548160c8c..c9416bc64 100644 --- a/packages/web/sentry.server.config.ts +++ b/packages/web/sentry.server.config.ts @@ -3,7 +3,7 @@ // https://docs.sentry.io/platforms/javascript/guides/nextjs/ import * as Sentry from "@sentry/nextjs"; -import { createLogger } from "@sourcebot/logger"; +import { createLogger } from "@sourcebot/shared"; const logger = createLogger('sentry-server-config'); diff --git a/packages/web/src/actions.ts b/packages/web/src/actions.ts index 4ebfd552a..e194f808a 100644 --- a/packages/web/src/actions.ts +++ b/packages/web/src/actions.ts @@ -1,7 +1,7 @@ 'use server'; import { getAuditService } from "@/ee/features/audit/factory"; -import { env } from "@/env.mjs"; +import { env } from "@sourcebot/shared"; import { addUserToOrganization, orgHasAvailability } from "@/lib/authUtils"; import { ErrorCode } from "@/lib/errorCodes"; import { notAuthenticated, notFound, orgNotFound, ServiceError, ServiceErrorException, unexpectedError } from "@/lib/serviceError"; @@ -9,9 +9,9 @@ import { getOrgMetadata, isHttpError, isServiceError } from "@/lib/utils"; import { prisma } from "@/prisma"; import { render } from "@react-email/components"; import * as Sentry from '@sentry/nextjs'; -import { generateApiKey, getTokenFromConfig, hashSecret } from "@sourcebot/crypto"; +import { generateApiKey, getTokenFromConfig, hashSecret } from "@sourcebot/shared"; import { ApiKey, ConnectionSyncJobStatus, Org, OrgRole, Prisma, RepoIndexingJobStatus, RepoIndexingJobType, StripeSubscriptionStatus } from "@sourcebot/db"; -import { createLogger } from "@sourcebot/logger"; +import { createLogger } from "@sourcebot/shared"; import { GiteaConnectionConfig } from "@sourcebot/schemas/v3/gitea.type"; import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type"; import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type"; diff --git a/packages/web/src/app/[domain]/agents/page.tsx b/packages/web/src/app/[domain]/agents/page.tsx index 1da98ff20..fd564268a 100644 --- a/packages/web/src/app/[domain]/agents/page.tsx +++ b/packages/web/src/app/[domain]/agents/page.tsx @@ -1,7 +1,7 @@ import Link from "next/link"; import { NavigationMenu } from "../components/navigationMenu"; import { FaCogs } from "react-icons/fa"; -import { env } from "@/env.mjs"; +import { env } from "@sourcebot/shared"; const agents = [ { diff --git a/packages/web/src/app/[domain]/chat/page.tsx b/packages/web/src/app/[domain]/chat/page.tsx index 5b7afef2d..5317328d9 100644 --- a/packages/web/src/app/[domain]/chat/page.tsx +++ b/packages/web/src/app/[domain]/chat/page.tsx @@ -9,7 +9,7 @@ import { RepositoryCarousel } from "../components/repositoryCarousel"; import { NavigationMenu } from "../components/navigationMenu"; import { Separator } from "@/components/ui/separator"; import { DemoCards } from "./components/demoCards"; -import { env } from "@/env.mjs"; +import { env } from "@sourcebot/shared"; import { loadJsonFile } from "@sourcebot/shared"; import { DemoExamples, demoExamplesSchema } from "@/types"; diff --git a/packages/web/src/app/[domain]/components/navigationMenu/index.tsx b/packages/web/src/app/[domain]/components/navigationMenu/index.tsx index f350fe92a..66c7c36ff 100644 --- a/packages/web/src/app/[domain]/components/navigationMenu/index.tsx +++ b/packages/web/src/app/[domain]/components/navigationMenu/index.tsx @@ -6,7 +6,7 @@ import { NavigationMenu as NavigationMenuBase } from "@/components/ui/navigation import { Separator } from "@/components/ui/separator"; import { getSubscriptionInfo } from "@/ee/features/billing/actions"; import { IS_BILLING_ENABLED } from "@/ee/features/billing/stripe"; -import { env } from "@/env.mjs"; +import { env } from "@sourcebot/shared"; import { ServiceErrorException } from "@/lib/serviceError"; import { isServiceError } from "@/lib/utils"; import { DiscordLogoIcon, GitHubLogoIcon } from "@radix-ui/react-icons"; diff --git a/packages/web/src/app/[domain]/components/settingsDropdown.tsx b/packages/web/src/app/[domain]/components/settingsDropdown.tsx index 92a5705aa..ec8ad0f0b 100644 --- a/packages/web/src/app/[domain]/components/settingsDropdown.tsx +++ b/packages/web/src/app/[domain]/components/settingsDropdown.tsx @@ -32,7 +32,7 @@ import { useKeymapType } from "@/hooks/useKeymapType" import { useSession } from "next-auth/react"; import { Avatar, AvatarFallback, AvatarImage } from "@/components/ui/avatar"; import { signOut } from "next-auth/react" -import { env } from "@/env.mjs"; +import { env } from "@sourcebot/shared/client"; import posthog from "posthog-js"; import { useDomain } from "@/hooks/useDomain"; diff --git a/packages/web/src/app/[domain]/layout.tsx b/packages/web/src/app/[domain]/layout.tsx index a3079d7f0..de506b400 100644 --- a/packages/web/src/app/[domain]/layout.tsx +++ b/packages/web/src/app/[domain]/layout.tsx @@ -16,7 +16,7 @@ import { getSubscriptionInfo } from "@/ee/features/billing/actions"; import { PendingApprovalCard } from "./components/pendingApproval"; import { SubmitJoinRequest } from "./components/submitJoinRequest"; import { hasEntitlement } from "@sourcebot/shared"; -import { env } from "@/env.mjs"; +import { env } from "@sourcebot/shared"; import { GcpIapAuth } from "./components/gcpIapAuth"; import { getAnonymousAccessStatus, getMemberApprovalRequired } from "@/actions"; import { JoinOrganizationCard } from "@/app/components/joinOrganizationCard"; diff --git a/packages/web/src/app/[domain]/repos/[id]/page.tsx b/packages/web/src/app/[domain]/repos/[id]/page.tsx index a3255c040..8986f7f60 100644 --- a/packages/web/src/app/[domain]/repos/[id]/page.tsx +++ b/packages/web/src/app/[domain]/repos/[id]/page.tsx @@ -4,7 +4,7 @@ import { Button } from "@/components/ui/button" import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card" import { Skeleton } from "@/components/ui/skeleton" import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip" -import { env } from "@/env.mjs" +import { env } from "@sourcebot/shared" import { SINGLE_TENANT_ORG_DOMAIN } from "@/lib/constants" import { ServiceErrorException } from "@/lib/serviceError" import { cn, getCodeHostInfoForRepo, isServiceError } from "@/lib/utils" diff --git a/packages/web/src/app/[domain]/settings/connections/[id]/page.tsx b/packages/web/src/app/[domain]/settings/connections/[id]/page.tsx index c36a33424..4cebd7db0 100644 --- a/packages/web/src/app/[domain]/settings/connections/[id]/page.tsx +++ b/packages/web/src/app/[domain]/settings/connections/[id]/page.tsx @@ -4,7 +4,7 @@ import { DisplayDate } from "@/app/[domain]/components/DisplayDate"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { Skeleton } from "@/components/ui/skeleton"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; -import { env } from "@/env.mjs"; +import { env } from "@sourcebot/shared"; import { SINGLE_TENANT_ORG_DOMAIN } from "@/lib/constants"; import { notFound, ServiceErrorException } from "@/lib/serviceError"; import { isServiceError } from "@/lib/utils"; diff --git a/packages/web/src/app/[domain]/settings/layout.tsx b/packages/web/src/app/[domain]/settings/layout.tsx index b0aa8f5b6..0b508903a 100644 --- a/packages/web/src/app/[domain]/settings/layout.tsx +++ b/packages/web/src/app/[domain]/settings/layout.tsx @@ -10,8 +10,8 @@ import { getConnectionStats, getMe, getOrgAccountRequests } from "@/actions"; import { ServiceErrorException } from "@/lib/serviceError"; import { getOrgFromDomain } from "@/data/org"; import { OrgRole } from "@prisma/client"; -import { env } from "@/env.mjs"; import { hasEntitlement } from "@sourcebot/shared"; +import { env } from "@sourcebot/shared/client"; interface LayoutProps { children: React.ReactNode; diff --git a/packages/web/src/app/[domain]/settings/license/page.tsx b/packages/web/src/app/[domain]/settings/license/page.tsx index 7c6be35ef..3e9df37dc 100644 --- a/packages/web/src/app/[domain]/settings/license/page.tsx +++ b/packages/web/src/app/[domain]/settings/license/page.tsx @@ -4,7 +4,7 @@ import { Info, Mail } from "lucide-react"; import { getOrgMembers } from "@/actions"; import { isServiceError } from "@/lib/utils"; import { notFound, ServiceErrorException } from "@/lib/serviceError"; -import { env } from "@/env.mjs"; +import { env } from "@sourcebot/shared/client"; interface LicensePageProps { params: Promise<{ diff --git a/packages/web/src/app/[domain]/upgrade/page.tsx b/packages/web/src/app/[domain]/upgrade/page.tsx index ed931e8dd..3d4e0e306 100644 --- a/packages/web/src/app/[domain]/upgrade/page.tsx +++ b/packages/web/src/app/[domain]/upgrade/page.tsx @@ -8,7 +8,7 @@ import { isServiceError } from "@/lib/utils"; import Link from "next/link"; import { ArrowLeftIcon } from "@radix-ui/react-icons"; import { LogoutEscapeHatch } from "@/app/components/logoutEscapeHatch"; -import { env } from "@/env.mjs"; +import { env } from "@sourcebot/shared"; import { IS_BILLING_ENABLED } from "@/ee/features/billing/stripe"; import { getSubscriptionInfo } from "@/ee/features/billing/actions"; diff --git a/packages/web/src/app/api/(server)/chat/route.ts b/packages/web/src/app/api/(server)/chat/route.ts index 8f045305e..1abc1a1c7 100644 --- a/packages/web/src/app/api/(server)/chat/route.ts +++ b/packages/web/src/app/api/(server)/chat/route.ts @@ -10,7 +10,7 @@ import { prisma } from "@/prisma"; import { LanguageModelV2 as AISDKLanguageModelV2 } from "@ai-sdk/provider"; import * as Sentry from "@sentry/nextjs"; import { OrgRole } from "@sourcebot/db"; -import { createLogger } from "@sourcebot/logger"; +import { createLogger } from "@sourcebot/shared"; import { createUIMessageStream, createUIMessageStreamResponse, diff --git a/packages/web/src/app/api/(server)/ee/audit/route.ts b/packages/web/src/app/api/(server)/ee/audit/route.ts index 80a05e2ef..c1b8c86c8 100644 --- a/packages/web/src/app/api/(server)/ee/audit/route.ts +++ b/packages/web/src/app/api/(server)/ee/audit/route.ts @@ -6,7 +6,7 @@ import { isServiceError } from "@/lib/utils"; import { serviceErrorResponse } from "@/lib/serviceError"; import { StatusCodes } from "http-status-codes"; import { ErrorCode } from "@/lib/errorCodes"; -import { env } from "@/env.mjs"; +import { env } from "@sourcebot/shared"; import { getEntitlements } from "@sourcebot/shared"; export const GET = async (request: NextRequest) => { diff --git a/packages/web/src/app/api/(server)/ee/user/route.ts b/packages/web/src/app/api/(server)/ee/user/route.ts index e308924b6..539f4e7c2 100644 --- a/packages/web/src/app/api/(server)/ee/user/route.ts +++ b/packages/web/src/app/api/(server)/ee/user/route.ts @@ -4,7 +4,7 @@ import { withAuthV2, withMinimumOrgRole } from "@/withAuthV2"; import { OrgRole } from "@sourcebot/db"; import { isServiceError } from "@/lib/utils"; import { serviceErrorResponse, missingQueryParam, notFound } from "@/lib/serviceError"; -import { createLogger } from "@sourcebot/logger"; +import { createLogger } from "@sourcebot/shared"; import { NextRequest } from "next/server"; import { StatusCodes } from "http-status-codes"; import { ErrorCode } from "@/lib/errorCodes"; diff --git a/packages/web/src/app/api/(server)/ee/users/route.ts b/packages/web/src/app/api/(server)/ee/users/route.ts index cc37c4199..e77edb18b 100644 --- a/packages/web/src/app/api/(server)/ee/users/route.ts +++ b/packages/web/src/app/api/(server)/ee/users/route.ts @@ -4,7 +4,7 @@ import { withAuthV2, withMinimumOrgRole } from "@/withAuthV2"; import { OrgRole } from "@sourcebot/db"; import { isServiceError } from "@/lib/utils"; import { serviceErrorResponse } from "@/lib/serviceError"; -import { createLogger } from "@sourcebot/logger"; +import { createLogger } from "@sourcebot/shared"; import { getAuditService } from "@/ee/features/audit/factory"; const logger = createLogger('ee-users-api'); diff --git a/packages/web/src/app/api/(server)/health/route.ts b/packages/web/src/app/api/(server)/health/route.ts index ac1a2ba1d..24b698764 100644 --- a/packages/web/src/app/api/(server)/health/route.ts +++ b/packages/web/src/app/api/(server)/health/route.ts @@ -1,6 +1,6 @@ 'use server'; -import { createLogger } from "@sourcebot/logger"; +import { createLogger } from "@sourcebot/shared"; const logger = createLogger('health-check'); diff --git a/packages/web/src/app/api/(server)/stripe/route.ts b/packages/web/src/app/api/(server)/stripe/route.ts index 84b671141..212860437 100644 --- a/packages/web/src/app/api/(server)/stripe/route.ts +++ b/packages/web/src/app/api/(server)/stripe/route.ts @@ -4,8 +4,8 @@ import Stripe from 'stripe'; import { prisma } from '@/prisma'; import { StripeSubscriptionStatus } from '@sourcebot/db'; import { stripeClient } from '@/ee/features/billing/stripe'; -import { env } from '@/env.mjs'; -import { createLogger } from "@sourcebot/logger"; +import { env } from '@sourcebot/shared'; +import { createLogger } from "@sourcebot/shared"; const logger = createLogger('stripe-webhook'); diff --git a/packages/web/src/app/api/(server)/version/route.ts b/packages/web/src/app/api/(server)/version/route.ts index 750955de5..08008e761 100644 --- a/packages/web/src/app/api/(server)/version/route.ts +++ b/packages/web/src/app/api/(server)/version/route.ts @@ -1,4 +1,4 @@ -import { env } from "@/env.mjs"; +import { env } from "@sourcebot/shared/client"; import { GetVersionResponse } from "@/lib/types"; // Note: In Next.JS 14, GET methods with no params are cached by default at build time. diff --git a/packages/web/src/app/api/(server)/webhook/route.ts b/packages/web/src/app/api/(server)/webhook/route.ts index ade6e54d0..cfd01a250 100644 --- a/packages/web/src/app/api/(server)/webhook/route.ts +++ b/packages/web/src/app/api/(server)/webhook/route.ts @@ -4,12 +4,12 @@ import { NextRequest } from "next/server"; import { App, Octokit } from "octokit"; import { WebhookEventDefinition} from "@octokit/webhooks/types"; import { EndpointDefaults } from "@octokit/types"; -import { env } from "@/env.mjs"; +import { env } from "@sourcebot/shared"; import { processGitHubPullRequest } from "@/features/agents/review-agent/app"; import { throttling } from "@octokit/plugin-throttling"; import fs from "fs"; import { GitHubPullRequest } from "@/features/agents/review-agent/types"; -import { createLogger } from "@sourcebot/logger"; +import { createLogger } from "@sourcebot/shared"; const logger = createLogger('github-webhook'); diff --git a/packages/web/src/app/components/authSecurityNotice.tsx b/packages/web/src/app/components/authSecurityNotice.tsx index e903383f3..2bdda7a8d 100644 --- a/packages/web/src/app/components/authSecurityNotice.tsx +++ b/packages/web/src/app/components/authSecurityNotice.tsx @@ -1,7 +1,7 @@ 'use client'; import React, { useState, useEffect } from "react"; -import { env } from "@/env.mjs"; +import { env } from "@sourcebot/shared/client"; interface AuthSecurityNoticeProps { closable?: boolean; diff --git a/packages/web/src/app/components/organizationAccessSettings.tsx b/packages/web/src/app/components/organizationAccessSettings.tsx index e240a374c..705281640 100644 --- a/packages/web/src/app/components/organizationAccessSettings.tsx +++ b/packages/web/src/app/components/organizationAccessSettings.tsx @@ -6,7 +6,7 @@ import { getOrgMetadata } from "@/lib/utils" import { headers } from "next/headers" import { SINGLE_TENANT_ORG_DOMAIN } from "@/lib/constants" import { hasEntitlement } from "@sourcebot/shared" -import { env } from "@/env.mjs" +import { env } from "@sourcebot/shared" export async function OrganizationAccessSettings() { const org = await getOrgFromDomain(SINGLE_TENANT_ORG_DOMAIN); diff --git a/packages/web/src/app/layout.tsx b/packages/web/src/app/layout.tsx index 441e5e58e..4249c00fc 100644 --- a/packages/web/src/app/layout.tsx +++ b/packages/web/src/app/layout.tsx @@ -6,7 +6,7 @@ import { PostHogProvider } from "./posthogProvider"; import { Toaster } from "@/components/ui/toaster"; import { TooltipProvider } from "@/components/ui/tooltip"; import { SessionProvider } from "next-auth/react"; -import { env } from "@/env.mjs"; +import { env } from "@sourcebot/shared"; import { PlanProvider } from "@/features/entitlements/planProvider"; import { getEntitlements } from "@sourcebot/shared"; diff --git a/packages/web/src/app/onboard/page.tsx b/packages/web/src/app/onboard/page.tsx index 33077a9be..5f22cddd6 100644 --- a/packages/web/src/app/onboard/page.tsx +++ b/packages/web/src/app/onboard/page.tsx @@ -17,7 +17,7 @@ import { LogoutEscapeHatch } from "@/app/components/logoutEscapeHatch"; import { redirect } from "next/navigation"; import { BetweenHorizontalStart, Brain, GitBranchIcon, LockIcon } from "lucide-react"; import { hasEntitlement } from "@sourcebot/shared"; -import { env } from "@/env.mjs"; +import { env } from "@sourcebot/shared"; import { GcpIapAuth } from "@/app/[domain]/components/gcpIapAuth"; interface OnboardingProps { diff --git a/packages/web/src/app/posthogProvider.tsx b/packages/web/src/app/posthogProvider.tsx index 44da24eae..8f3bccc7b 100644 --- a/packages/web/src/app/posthogProvider.tsx +++ b/packages/web/src/app/posthogProvider.tsx @@ -4,7 +4,7 @@ import { usePostHog } from 'posthog-js/react' import { PostHogProvider as PHProvider } from 'posthog-js/react' import { usePathname, useSearchParams } from "next/navigation" import { Suspense, useEffect } from "react" -import { env } from '@/env.mjs' +import { env } from '@sourcebot/shared/client' import { useSession } from 'next-auth/react' import { captureEvent } from '@/hooks/useCaptureEvent' diff --git a/packages/web/src/app/signup/page.tsx b/packages/web/src/app/signup/page.tsx index fd00c87df..fe84e3d9e 100644 --- a/packages/web/src/app/signup/page.tsx +++ b/packages/web/src/app/signup/page.tsx @@ -2,8 +2,8 @@ import { auth } from "@/auth"; import { LoginForm } from "../login/components/loginForm"; import { redirect } from "next/navigation"; import { Footer } from "@/app/components/footer"; -import { createLogger } from "@sourcebot/logger"; import { getIdentityProviderMetadata } from "@/lib/identityProviders"; +import { createLogger } from "@sourcebot/shared"; import { getOrgFromDomain } from "@/data/org"; import { SINGLE_TENANT_ORG_DOMAIN } from "@/lib/constants"; diff --git a/packages/web/src/auth.ts b/packages/web/src/auth.ts index 64dab588f..f61219d7b 100644 --- a/packages/web/src/auth.ts +++ b/packages/web/src/auth.ts @@ -4,7 +4,7 @@ import Credentials from "next-auth/providers/credentials" import EmailProvider from "next-auth/providers/nodemailer"; import { PrismaAdapter } from "@auth/prisma-adapter" import { prisma } from "@/prisma"; -import { env } from "@/env.mjs"; +import { env } from "@sourcebot/shared"; import { User } from '@sourcebot/db'; import 'next-auth/jwt'; import type { Provider } from "next-auth/providers"; diff --git a/packages/web/src/ee/features/audit/actions.ts b/packages/web/src/ee/features/audit/actions.ts index 57455cb0a..60814946e 100644 --- a/packages/web/src/ee/features/audit/actions.ts +++ b/packages/web/src/ee/features/audit/actions.ts @@ -5,7 +5,7 @@ import { ErrorCode } from "@/lib/errorCodes"; import { StatusCodes } from "http-status-codes"; import { sew, withAuth, withOrgMembership } from "@/actions"; import { OrgRole } from "@sourcebot/db"; -import { createLogger } from "@sourcebot/logger"; +import { createLogger } from "@sourcebot/shared"; import { ServiceError } from "@/lib/serviceError"; import { getAuditService } from "@/ee/features/audit/factory"; import { AuditEvent } from "./types"; diff --git a/packages/web/src/ee/features/audit/auditService.ts b/packages/web/src/ee/features/audit/auditService.ts index 09cc647b0..a0bee39a3 100644 --- a/packages/web/src/ee/features/audit/auditService.ts +++ b/packages/web/src/ee/features/audit/auditService.ts @@ -1,7 +1,7 @@ import { IAuditService, AuditEvent } from '@/ee/features/audit/types'; import { prisma } from '@/prisma'; import { Audit } from '@prisma/client'; -import { createLogger } from '@sourcebot/logger'; +import { createLogger } from '@sourcebot/shared'; const logger = createLogger('audit-service'); diff --git a/packages/web/src/ee/features/audit/factory.ts b/packages/web/src/ee/features/audit/factory.ts index 5fe7d0f48..9a2a43006 100644 --- a/packages/web/src/ee/features/audit/factory.ts +++ b/packages/web/src/ee/features/audit/factory.ts @@ -2,7 +2,7 @@ import { IAuditService } from '@/ee/features/audit/types'; import { MockAuditService } from '@/ee/features/audit/mockAuditService'; import { AuditService } from '@/ee/features/audit/auditService'; import { hasEntitlement } from '@sourcebot/shared'; -import { env } from '@/env.mjs'; +import { env } from '@sourcebot/shared'; let enterpriseService: IAuditService | undefined; diff --git a/packages/web/src/ee/features/billing/actions.ts b/packages/web/src/ee/features/billing/actions.ts index 48b115810..d66e94eec 100644 --- a/packages/web/src/ee/features/billing/actions.ts +++ b/packages/web/src/ee/features/billing/actions.ts @@ -7,12 +7,12 @@ import { prisma } from "@/prisma"; import { OrgRole } from "@sourcebot/db"; import { stripeClient } from "./stripe"; import { isServiceError } from "@/lib/utils"; -import { env } from "@/env.mjs"; +import { env } from "@sourcebot/shared"; import { StatusCodes } from "http-status-codes"; import { ErrorCode } from "@/lib/errorCodes"; import { headers } from "next/headers"; import { getSubscriptionForOrg } from "./serverUtils"; -import { createLogger } from "@sourcebot/logger"; +import { createLogger } from "@sourcebot/shared"; const logger = createLogger('billing-actions'); diff --git a/packages/web/src/ee/features/billing/stripe.ts b/packages/web/src/ee/features/billing/stripe.ts index efc69668b..c8ca0af7a 100644 --- a/packages/web/src/ee/features/billing/stripe.ts +++ b/packages/web/src/ee/features/billing/stripe.ts @@ -1,5 +1,5 @@ import 'server-only'; -import { env } from '@/env.mjs' +import { env } from '@sourcebot/shared' import Stripe from "stripe"; import { hasEntitlement } from '@sourcebot/shared'; diff --git a/packages/web/src/ee/features/permissionSyncing/actions.ts b/packages/web/src/ee/features/permissionSyncing/actions.ts index 956670271..a52b0c3e5 100644 --- a/packages/web/src/ee/features/permissionSyncing/actions.ts +++ b/packages/web/src/ee/features/permissionSyncing/actions.ts @@ -1,10 +1,9 @@ 'use server'; import { sew } from "@/actions"; -import { createLogger } from "@sourcebot/logger"; +import { createLogger, env } from "@sourcebot/shared"; import { withAuthV2, withMinimumOrgRole } from "@/withAuthV2"; import { loadConfig } from "@sourcebot/shared"; -import { env } from "@/env.mjs"; import { OrgRole } from "@sourcebot/db"; import { cookies } from "next/headers"; import { OPTIONAL_PROVIDERS_LINK_SKIPPED_COOKIE_NAME } from "@/lib/constants"; diff --git a/packages/web/src/ee/features/permissionSyncing/tokenRefresh.ts b/packages/web/src/ee/features/permissionSyncing/tokenRefresh.ts index f6ebef067..7d2c69eb3 100644 --- a/packages/web/src/ee/features/permissionSyncing/tokenRefresh.ts +++ b/packages/web/src/ee/features/permissionSyncing/tokenRefresh.ts @@ -1,7 +1,5 @@ import { loadConfig } from "@sourcebot/shared"; -import { env } from "@/env.mjs"; -import { createLogger } from "@sourcebot/logger"; -import { getTokenFromConfig } from '@sourcebot/crypto'; +import { getTokenFromConfig, createLogger, env } from "@sourcebot/shared"; import { GitHubIdentityProviderConfig, GitLabIdentityProviderConfig } from "@sourcebot/schemas/v3/index.type"; import { LinkedAccountTokensMap } from "@/auth" const { prisma } = await import('@/prisma'); diff --git a/packages/web/src/ee/features/sso/sso.ts b/packages/web/src/ee/features/sso/sso.ts index 9a5153ff7..a0fe415c9 100644 --- a/packages/web/src/ee/features/sso/sso.ts +++ b/packages/web/src/ee/features/sso/sso.ts @@ -1,21 +1,18 @@ -import { env } from "@/env.mjs"; -import GitHub from "next-auth/providers/github"; -import Google from "next-auth/providers/google"; -import Okta from "next-auth/providers/okta"; -import Keycloak from "next-auth/providers/keycloak"; -import Gitlab from "next-auth/providers/gitlab"; -import MicrosoftEntraID from "next-auth/providers/microsoft-entra-id"; +import type { IdentityProvider } from "@/auth"; +import { onCreateUser } from "@/lib/authUtils"; import { prisma } from "@/prisma"; +import { GCPIAPIdentityProviderConfig, GitHubIdentityProviderConfig, GitLabIdentityProviderConfig, GoogleIdentityProviderConfig, KeycloakIdentityProviderConfig, MicrosoftEntraIDIdentityProviderConfig, OktaIdentityProviderConfig } from "@sourcebot/schemas/v3/index.type"; +import { createLogger, env, getTokenFromConfig, hasEntitlement, loadConfig } from "@sourcebot/shared"; import { OAuth2Client } from "google-auth-library"; -import Credentials from "next-auth/providers/credentials"; import type { User as AuthJsUser } from "next-auth"; import type { Provider } from "next-auth/providers"; -import { onCreateUser } from "@/lib/authUtils"; -import { createLogger } from "@sourcebot/logger"; -import { hasEntitlement, loadConfig } from "@sourcebot/shared"; -import { getTokenFromConfig } from "@sourcebot/crypto"; -import type { IdentityProvider } from "@/auth"; -import { GCPIAPIdentityProviderConfig, GitHubIdentityProviderConfig, GitLabIdentityProviderConfig, GoogleIdentityProviderConfig, KeycloakIdentityProviderConfig, MicrosoftEntraIDIdentityProviderConfig, OktaIdentityProviderConfig } from "@sourcebot/schemas/v3/index.type"; +import Credentials from "next-auth/providers/credentials"; +import GitHub from "next-auth/providers/github"; +import Gitlab from "next-auth/providers/gitlab"; +import Google from "next-auth/providers/google"; +import Keycloak from "next-auth/providers/keycloak"; +import MicrosoftEntraID from "next-auth/providers/microsoft-entra-id"; +import Okta from "next-auth/providers/okta"; const logger = createLogger('web-sso'); diff --git a/packages/web/src/features/agents/review-agent/app.ts b/packages/web/src/features/agents/review-agent/app.ts index 0ea2af8f2..80d5a2f38 100644 --- a/packages/web/src/features/agents/review-agent/app.ts +++ b/packages/web/src/features/agents/review-agent/app.ts @@ -2,11 +2,11 @@ import { Octokit } from "octokit"; import { generatePrReviews } from "@/features/agents/review-agent/nodes/generatePrReview"; import { githubPushPrReviews } from "@/features/agents/review-agent/nodes/githubPushPrReviews"; import { githubPrParser } from "@/features/agents/review-agent/nodes/githubPrParser"; -import { env } from "@/env.mjs"; +import { env } from "@sourcebot/shared"; import { GitHubPullRequest } from "@/features/agents/review-agent/types"; import path from "path"; import fs from "fs"; -import { createLogger } from "@sourcebot/logger"; +import { createLogger } from "@sourcebot/shared"; const rules = [ "Do NOT provide general feedback, summaries, explanations of changes, or praises for making good additions.", diff --git a/packages/web/src/features/agents/review-agent/nodes/fetchFileContent.ts b/packages/web/src/features/agents/review-agent/nodes/fetchFileContent.ts index 7617c959a..d9903b728 100644 --- a/packages/web/src/features/agents/review-agent/nodes/fetchFileContent.ts +++ b/packages/web/src/features/agents/review-agent/nodes/fetchFileContent.ts @@ -2,7 +2,7 @@ import { sourcebot_context, sourcebot_pr_payload } from "@/features/agents/revie import { getFileSource } from "@/features/search/fileSourceApi"; import { fileSourceResponseSchema } from "@/features/search/schemas"; import { isServiceError } from "@/lib/utils"; -import { createLogger } from "@sourcebot/logger"; +import { createLogger } from "@sourcebot/shared"; const logger = createLogger('fetch-file-content'); diff --git a/packages/web/src/features/agents/review-agent/nodes/generateDiffReviewPrompt.ts b/packages/web/src/features/agents/review-agent/nodes/generateDiffReviewPrompt.ts index 59060408e..f5eda1ca0 100644 --- a/packages/web/src/features/agents/review-agent/nodes/generateDiffReviewPrompt.ts +++ b/packages/web/src/features/agents/review-agent/nodes/generateDiffReviewPrompt.ts @@ -1,6 +1,6 @@ import { sourcebot_diff, sourcebot_context, sourcebot_file_diff_review_schema } from "@/features/agents/review-agent/types"; import { zodToJsonSchema } from "zod-to-json-schema"; -import { createLogger } from "@sourcebot/logger"; +import { createLogger } from "@sourcebot/shared"; const logger = createLogger('generate-diff-review-prompt'); diff --git a/packages/web/src/features/agents/review-agent/nodes/generatePrReview.ts b/packages/web/src/features/agents/review-agent/nodes/generatePrReview.ts index 48f6bda03..3e1fb0edb 100644 --- a/packages/web/src/features/agents/review-agent/nodes/generatePrReview.ts +++ b/packages/web/src/features/agents/review-agent/nodes/generatePrReview.ts @@ -2,7 +2,7 @@ import { sourcebot_pr_payload, sourcebot_diff_review, sourcebot_file_diff_review import { generateDiffReviewPrompt } from "@/features/agents/review-agent/nodes/generateDiffReviewPrompt"; import { invokeDiffReviewLlm } from "@/features/agents/review-agent/nodes/invokeDiffReviewLlm"; import { fetchFileContent } from "@/features/agents/review-agent/nodes/fetchFileContent"; -import { createLogger } from "@sourcebot/logger"; +import { createLogger } from "@sourcebot/shared"; const logger = createLogger('generate-pr-review'); diff --git a/packages/web/src/features/agents/review-agent/nodes/githubPrParser.ts b/packages/web/src/features/agents/review-agent/nodes/githubPrParser.ts index b1cee1987..b633450ea 100644 --- a/packages/web/src/features/agents/review-agent/nodes/githubPrParser.ts +++ b/packages/web/src/features/agents/review-agent/nodes/githubPrParser.ts @@ -2,7 +2,7 @@ import { sourcebot_pr_payload, sourcebot_file_diff, sourcebot_diff } from "@/fea import parse from "parse-diff"; import { Octokit } from "octokit"; import { GitHubPullRequest } from "@/features/agents/review-agent/types"; -import { createLogger } from "@sourcebot/logger"; +import { createLogger } from "@sourcebot/shared"; const logger = createLogger('github-pr-parser'); diff --git a/packages/web/src/features/agents/review-agent/nodes/githubPushPrReviews.ts b/packages/web/src/features/agents/review-agent/nodes/githubPushPrReviews.ts index e0a9e5973..a6ef88d2b 100644 --- a/packages/web/src/features/agents/review-agent/nodes/githubPushPrReviews.ts +++ b/packages/web/src/features/agents/review-agent/nodes/githubPushPrReviews.ts @@ -1,6 +1,6 @@ import { Octokit } from "octokit"; import { sourcebot_pr_payload, sourcebot_file_diff_review } from "@/features/agents/review-agent/types"; -import { createLogger } from "@sourcebot/logger"; +import { createLogger } from "@sourcebot/shared"; const logger = createLogger('github-push-pr-reviews'); diff --git a/packages/web/src/features/agents/review-agent/nodes/invokeDiffReviewLlm.ts b/packages/web/src/features/agents/review-agent/nodes/invokeDiffReviewLlm.ts index c726ba01a..f3f41be80 100644 --- a/packages/web/src/features/agents/review-agent/nodes/invokeDiffReviewLlm.ts +++ b/packages/web/src/features/agents/review-agent/nodes/invokeDiffReviewLlm.ts @@ -1,8 +1,8 @@ import OpenAI from "openai"; import { sourcebot_file_diff_review, sourcebot_file_diff_review_schema } from "@/features/agents/review-agent/types"; -import { env } from "@/env.mjs"; +import { env } from "@sourcebot/shared"; import fs from "fs"; -import { createLogger } from "@sourcebot/logger"; +import { createLogger } from "@sourcebot/shared"; const logger = createLogger('invoke-diff-review-llm'); diff --git a/packages/web/src/features/chat/actions.ts b/packages/web/src/features/chat/actions.ts index 19ec9abd5..86f863012 100644 --- a/packages/web/src/features/chat/actions.ts +++ b/packages/web/src/features/chat/actions.ts @@ -1,7 +1,6 @@ 'use server'; import { sew, withAuth, withOrgMembership } from "@/actions"; -import { env } from "@/env.mjs"; import { SOURCEBOT_GUEST_USER_ID } from "@/lib/constants"; import { ErrorCode } from "@/lib/errorCodes"; import { chatIsReadonly, notFound, ServiceError, serviceErrorResponse } from "@/lib/serviceError"; @@ -20,9 +19,8 @@ import { LanguageModelV2 as AISDKLanguageModelV2 } from "@ai-sdk/provider"; import { createXai } from '@ai-sdk/xai'; import { fromNodeProviderChain } from '@aws-sdk/credential-providers'; import { createOpenRouter } from '@openrouter/ai-sdk-provider'; -import { getTokenFromConfig } from "@sourcebot/crypto"; +import { getTokenFromConfig, createLogger, env } from "@sourcebot/shared"; import { ChatVisibility, OrgRole, Prisma } from "@sourcebot/db"; -import { createLogger } from "@sourcebot/logger"; import { LanguageModel } from "@sourcebot/schemas/v3/languageModel.type"; import { Token } from "@sourcebot/schemas/v3/shared.type"; import { generateText, JSONValue, extractReasoningMiddleware, wrapLanguageModel } from "ai"; diff --git a/packages/web/src/features/chat/agent.ts b/packages/web/src/features/chat/agent.ts index 7df5a6d48..0c722c27c 100644 --- a/packages/web/src/features/chat/agent.ts +++ b/packages/web/src/features/chat/agent.ts @@ -1,8 +1,9 @@ -import { env } from "@/env.mjs"; +import { env } from "@sourcebot/shared"; +import { env as clientEnv } from "@sourcebot/shared/client"; import { getFileSource } from "@/features/search/fileSourceApi"; import { isServiceError } from "@/lib/utils"; import { ProviderOptions } from "@ai-sdk/provider-utils"; -import { createLogger } from "@sourcebot/logger"; +import { createLogger } from "@sourcebot/shared"; import { LanguageModel, ModelMessage, StopCondition, streamText } from "ai"; import { ANSWER_TAG, FILE_REFERENCE_PREFIX, toolNames } from "./constants"; import { createCodeSearchTool, findSymbolDefinitionsTool, findSymbolReferencesTool, readFilesTool, searchReposTool, listAllReposTool } from "./tools"; @@ -140,7 +141,7 @@ export const createAgentStream = async ({ }, // Only enable langfuse traces in cloud environments. experimental_telemetry: { - isEnabled: env.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT !== undefined, + isEnabled: clientEnv.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT !== undefined, metadata: { langfuseTraceId: traceId, }, diff --git a/packages/web/src/features/chat/components/chatThread/answerCard.tsx b/packages/web/src/features/chat/components/chatThread/answerCard.tsx index d37ee67ee..e3f89f627 100644 --- a/packages/web/src/features/chat/components/chatThread/answerCard.tsx +++ b/packages/web/src/features/chat/components/chatThread/answerCard.tsx @@ -17,7 +17,7 @@ import { isServiceError } from "@/lib/utils"; import { useDomain } from "@/hooks/useDomain"; import useCaptureEvent from "@/hooks/useCaptureEvent"; import { LangfuseWeb } from "langfuse"; -import { env } from "@/env.mjs"; +import { env } from "@sourcebot/shared/client"; interface AnswerCardProps { answerText: string; diff --git a/packages/web/src/features/chat/utils.test.ts b/packages/web/src/features/chat/utils.test.ts index 3ebd4e31f..698fbe421 100644 --- a/packages/web/src/features/chat/utils.test.ts +++ b/packages/web/src/features/chat/utils.test.ts @@ -4,7 +4,7 @@ import { FILE_REFERENCE_REGEX, ANSWER_TAG } from './constants'; import { SBChatMessage, SBChatMessagePart } from './types'; // Mock the env module -vi.mock('@/env.mjs', () => ({ +vi.mock('@sourcebot/shared', () => ({ env: { SOURCEBOT_CHAT_FILE_MAX_CHARACTERS: 4000, } diff --git a/packages/web/src/features/fileTree/actions.ts b/packages/web/src/features/fileTree/actions.ts index 6111ea701..a861670d7 100644 --- a/packages/web/src/features/fileTree/actions.ts +++ b/packages/web/src/features/fileTree/actions.ts @@ -1,11 +1,11 @@ 'use server'; import { sew } from '@/actions'; -import { env } from '@/env.mjs'; +import { env } from '@sourcebot/shared'; import { notFound, unexpectedError } from '@/lib/serviceError'; import { withOptionalAuthV2 } from '@/withAuthV2'; import { Repo } from '@sourcebot/db'; -import { createLogger } from '@sourcebot/logger'; +import { createLogger } from '@sourcebot/shared'; import path from 'path'; import { simpleGit } from 'simple-git'; diff --git a/packages/web/src/features/search/zoektClient.ts b/packages/web/src/features/search/zoektClient.ts index 3379d2cea..bd30fcd14 100644 --- a/packages/web/src/features/search/zoektClient.ts +++ b/packages/web/src/features/search/zoektClient.ts @@ -1,4 +1,4 @@ -import { env } from "@/env.mjs"; +import { env } from "@sourcebot/shared"; interface ZoektRequest { path: string, diff --git a/packages/web/src/hooks/useCaptureEvent.ts b/packages/web/src/hooks/useCaptureEvent.ts index 8dd03ffe2..597f73ca3 100644 --- a/packages/web/src/hooks/useCaptureEvent.ts +++ b/packages/web/src/hooks/useCaptureEvent.ts @@ -3,7 +3,7 @@ import { CaptureOptions } from "posthog-js"; import posthog from "posthog-js"; import { PosthogEvent, PosthogEventMap } from "../lib/posthogEvents"; -import { env } from "@/env.mjs"; +import { env } from "@sourcebot/shared/client"; export function captureEvent(event: E, properties: PosthogEventMap[E], options?: CaptureOptions) { if(!options) { diff --git a/packages/web/src/initialize.ts b/packages/web/src/initialize.ts index b08685b2b..fa27febd2 100644 --- a/packages/web/src/initialize.ts +++ b/packages/web/src/initialize.ts @@ -2,10 +2,8 @@ import { createGuestUser } from '@/lib/authUtils'; import { SOURCEBOT_SUPPORT_EMAIL } from "@/lib/constants"; import { prisma } from "@/prisma"; import { OrgRole } from '@sourcebot/db'; -import { createLogger } from "@sourcebot/logger"; -import { hasEntitlement, loadConfig } from '@sourcebot/shared'; +import { createLogger, env, hasEntitlement, loadConfig } from "@sourcebot/shared"; import { getOrgFromDomain } from './data/org'; -import { env } from './env.mjs'; import { SINGLE_TENANT_ORG_DOMAIN, SINGLE_TENANT_ORG_ID, SOURCEBOT_GUEST_USER_ID } from './lib/constants'; import { ServiceErrorException } from './lib/serviceError'; import { getOrgMetadata, isServiceError } from './lib/utils'; diff --git a/packages/web/src/lib/authUtils.ts b/packages/web/src/lib/authUtils.ts index bf7e5ea95..e11942a49 100644 --- a/packages/web/src/lib/authUtils.ts +++ b/packages/web/src/lib/authUtils.ts @@ -5,7 +5,7 @@ import { SINGLE_TENANT_ORG_ID, SOURCEBOT_GUEST_USER_EMAIL, SOURCEBOT_GUEST_USER_ import { getPlan, getSeats, hasEntitlement, SOURCEBOT_UNLIMITED_SEATS } from "@sourcebot/shared"; import { isServiceError } from "@/lib/utils"; import { orgNotFound, ServiceError, userNotFound } from "@/lib/serviceError"; -import { createLogger } from "@sourcebot/logger"; +import { createLogger } from "@sourcebot/shared"; import { getAuditService } from "@/ee/features/audit/factory"; import { StatusCodes } from "http-status-codes"; import { ErrorCode } from "./errorCodes"; diff --git a/packages/web/src/lib/types.ts b/packages/web/src/lib/types.ts index 545dbbf47..e27e7057f 100644 --- a/packages/web/src/lib/types.ts +++ b/packages/web/src/lib/types.ts @@ -1,6 +1,6 @@ import { z } from "zod"; import { getReposResponseSchema, getVersionResponseSchema, repositoryQuerySchema, searchContextQuerySchema } from "./schemas"; -import { tenancyModeSchema } from "@/env.mjs"; +import { tenancyModeSchema } from "@sourcebot/shared"; export type KeymapType = "default" | "vim"; diff --git a/packages/web/src/middleware.ts b/packages/web/src/middleware.ts index 1b127f41f..b59e207de 100644 --- a/packages/web/src/middleware.ts +++ b/packages/web/src/middleware.ts @@ -1,15 +1,10 @@ import { NextResponse } from 'next/server' import type { NextRequest } from 'next/server' -import { env } from './env.mjs' import { SINGLE_TENANT_ORG_DOMAIN } from '@/lib/constants' export async function middleware(request: NextRequest) { const url = request.nextUrl.clone(); - if (env.SOURCEBOT_TENANCY_MODE !== 'single') { - return NextResponse.next(); - } - if ( url.pathname.startsWith('/login') || url.pathname.startsWith('/redeem') || diff --git a/packages/web/src/prisma.ts b/packages/web/src/prisma.ts index d9e488ceb..0d520de78 100644 --- a/packages/web/src/prisma.ts +++ b/packages/web/src/prisma.ts @@ -1,11 +1,13 @@ import 'server-only'; -import { env } from "@/env.mjs"; +import { env, getDBConnectionString } from "@sourcebot/shared"; import { Prisma, PrismaClient } from "@sourcebot/db"; import { hasEntitlement } from "@sourcebot/shared"; // @see: https://authjs.dev/getting-started/adapters/prisma const globalForPrisma = globalThis as unknown as { prisma: PrismaClient } +const dbConnectionString = getDBConnectionString(); + // @NOTE: In almost all cases, the userScopedPrismaClientExtension should be used // (since actions & queries are scoped to a particular user). There are some exceptions // (e.g., in initialize.ts). @@ -13,7 +15,17 @@ const globalForPrisma = globalThis as unknown as { prisma: PrismaClient } // @todo: we can mark this as `__unsafePrisma` in the future once we've migrated // all of the actions & queries to use the userScopedPrismaClientExtension to avoid // accidental misuse. -export const prisma = globalForPrisma.prisma || new PrismaClient() +export const prisma = globalForPrisma.prisma || new PrismaClient({ + // @note: this code is evaluated at build time, and will throw exceptions if these env vars are not set. + // Here we explicitly check if the DATABASE_URL or the individual database variables are set, and only + ...(dbConnectionString !== undefined ? { + datasources: { + db: { + url: dbConnectionString, + }, + } + }: {}), +}) if (env.NODE_ENV !== "production") globalForPrisma.prisma = prisma /** diff --git a/packages/web/src/withAuthV2.test.ts b/packages/web/src/withAuthV2.test.ts index 5a1dd343d..1b9360057 100644 --- a/packages/web/src/withAuthV2.test.ts +++ b/packages/web/src/withAuthV2.test.ts @@ -18,18 +18,10 @@ vi.mock('./auth', () => ({ auth: mocks.auth, })); -vi.mock('@/env.mjs', () => ({ - env: {} -})); - vi.mock('next/headers', () => ({ headers: mocks.headers, })); -vi.mock('@/env.mjs', () => ({ - env: {} -})); - vi.mock('@/prisma', async () => { // @see: https://github.com/prisma/prisma/discussions/20244#discussioncomment-7976447 const actual = await vi.importActual('@/__mocks__/prisma'); @@ -38,16 +30,14 @@ vi.mock('@/prisma', async () => { }; }); -vi.mock('@sourcebot/crypto', () => ({ - hashSecret: vi.fn((secret: string) => secret), -})); - vi.mock('server-only', () => ({ default: vi.fn(), })); vi.mock('@sourcebot/shared', () => ({ hasEntitlement: mocks.hasEntitlement, + hashSecret: vi.fn((secret: string) => secret), + env: {} })); // Test utility to set the mock session diff --git a/packages/web/src/withAuthV2.ts b/packages/web/src/withAuthV2.ts index c4bf80956..65ebb0547 100644 --- a/packages/web/src/withAuthV2.ts +++ b/packages/web/src/withAuthV2.ts @@ -1,5 +1,5 @@ import { prisma as __unsafePrisma, userScopedPrismaClientExtension } from "@/prisma"; -import { hashSecret } from "@sourcebot/crypto"; +import { hashSecret } from "@sourcebot/shared"; import { ApiKey, Org, OrgRole, PrismaClient, User } from "@sourcebot/db"; import { headers } from "next/headers"; import { auth } from "./auth"; diff --git a/packages/web/tsconfig.json b/packages/web/tsconfig.json index a41d0da0f..1abfee26f 100644 --- a/packages/web/tsconfig.json +++ b/packages/web/tsconfig.json @@ -35,8 +35,7 @@ "next-env.d.ts", "**/*.ts", "**/*.tsx", - ".next/types/**/*.ts", - "src/env.mjs" + ".next/types/**/*.ts" ], "exclude": [ "node_modules" diff --git a/schemas/v3/environmentOverrides.json b/schemas/v3/environmentOverrides.json new file mode 100644 index 000000000..8def7d207 --- /dev/null +++ b/schemas/v3/environmentOverrides.json @@ -0,0 +1,85 @@ +{ + "type": "object", + "description": "Environment variable overrides.", + "title": "EnvironmentOverrides", + "not": { + "$comment": "List of environment variables that are not allowed to be overridden.", + "anyOf": [ + { + "required": [ + "CONFIG_PATH" + ] + } + ] + }, + "patternProperties": { + "^[a-zA-Z0-9_-]+$": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "const": "token" + }, + "value": { + "$ref": "./shared.json#/definitions/Token" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "number" + }, + "value": { + "type": "number" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "boolean" + }, + "value": { + "type": "boolean" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + } + ] + } + } +} \ No newline at end of file diff --git a/schemas/v3/index.json b/schemas/v3/index.json index 392a2eb98..10048a347 100644 --- a/schemas/v3/index.json +++ b/schemas/v3/index.json @@ -102,6 +102,9 @@ }, "additionalProperties": false }, + "environmentOverrides": { + "$ref": "./environmentOverrides.json" + }, "connections": { "type": "object", "description": "Defines a collection of connections from varying code hosts that Sourcebot should sync with. This is only available in single-tenancy mode.", diff --git a/supervisord.conf b/supervisord.conf index eda6d4301..19d308502 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -36,7 +36,7 @@ redirect_stderr=true [program:redis] command=redis-server --dir %(ENV_REDIS_DATA_DIR)s priority=10 -autostart=true +autostart=%(ENV_REDIS_EMBEDDED)s autorestart=true startretries=3 stdout_logfile=/dev/fd/1 diff --git a/yarn.lock b/yarn.lock index f6f9af628..2f19bfa9a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7893,13 +7893,9 @@ __metadata: "@sentry/cli": "npm:^2.42.2" "@sentry/node": "npm:^9.3.0" "@sentry/profiling-node": "npm:^9.3.0" - "@sourcebot/crypto": "workspace:*" "@sourcebot/db": "workspace:*" - "@sourcebot/error": "workspace:*" - "@sourcebot/logger": "workspace:*" "@sourcebot/schemas": "workspace:*" "@sourcebot/shared": "workspace:*" - "@t3-oss/env-core": "npm:^0.12.0" "@types/argparse": "npm:^2.0.16" "@types/express": "npm:^5.0.0" "@types/micromatch": "npm:^4.0.9" @@ -7928,7 +7924,7 @@ __metadata: tsx: "npm:^4.19.1" typescript: "npm:^5.6.2" vitest: "npm:^2.1.9" - zod: "npm:^3.24.3" + zod: "npm:^3.25.74" languageName: unknown linkType: soft @@ -7946,25 +7942,11 @@ __metadata: languageName: node linkType: hard -"@sourcebot/crypto@workspace:*, @sourcebot/crypto@workspace:packages/crypto": - version: 0.0.0-use.local - resolution: "@sourcebot/crypto@workspace:packages/crypto" - dependencies: - "@google-cloud/secret-manager": "npm:^6.1.1" - "@sourcebot/db": "npm:*" - "@sourcebot/schemas": "npm:*" - "@types/node": "npm:^22.7.5" - dotenv: "npm:^16.4.5" - typescript: "npm:^5.7.3" - languageName: unknown - linkType: soft - -"@sourcebot/db@npm:*, @sourcebot/db@workspace:*, @sourcebot/db@workspace:packages/db": +"@sourcebot/db@workspace:*, @sourcebot/db@workspace:packages/db": version: 0.0.0-use.local resolution: "@sourcebot/db@workspace:packages/db" dependencies: "@prisma/client": "npm:6.2.1" - "@sourcebot/logger": "workspace:*" "@types/argparse": "npm:^2.0.16" "@types/readline-sync": "npm:^1.4.8" argparse: "npm:^2.0.1" @@ -7975,31 +7957,6 @@ __metadata: languageName: unknown linkType: soft -"@sourcebot/error@workspace:*, @sourcebot/error@workspace:packages/error": - version: 0.0.0-use.local - resolution: "@sourcebot/error@workspace:packages/error" - dependencies: - "@types/node": "npm:^22.7.5" - typescript: "npm:^5.7.3" - languageName: unknown - linkType: soft - -"@sourcebot/logger@workspace:*, @sourcebot/logger@workspace:packages/logger": - version: 0.0.0-use.local - resolution: "@sourcebot/logger@workspace:packages/logger" - dependencies: - "@logtail/node": "npm:^0.5.2" - "@logtail/winston": "npm:^0.5.2" - "@t3-oss/env-core": "npm:^0.12.0" - "@types/node": "npm:^22.7.5" - dotenv: "npm:^16.4.5" - triple-beam: "npm:^1.4.1" - typescript: "npm:^5.7.3" - winston: "npm:^3.15.0" - zod: "npm:^3.24.3" - languageName: unknown - linkType: soft - "@sourcebot/mcp@workspace:packages/mcp": version: 0.0.0-use.local resolution: "@sourcebot/mcp@workspace:packages/mcp" @@ -8019,7 +7976,7 @@ __metadata: languageName: unknown linkType: soft -"@sourcebot/schemas@npm:*, @sourcebot/schemas@workspace:*, @sourcebot/schemas@workspace:packages/schemas": +"@sourcebot/schemas@workspace:*, @sourcebot/schemas@workspace:packages/schemas": version: 0.0.0-use.local resolution: "@sourcebot/schemas@workspace:packages/schemas" dependencies: @@ -8036,9 +7993,10 @@ __metadata: version: 0.0.0-use.local resolution: "@sourcebot/shared@workspace:packages/shared" dependencies: - "@sourcebot/crypto": "workspace:*" + "@google-cloud/secret-manager": "npm:^6.1.1" + "@logtail/node": "npm:^0.5.2" + "@logtail/winston": "npm:^0.5.2" "@sourcebot/db": "workspace:*" - "@sourcebot/logger": "workspace:*" "@sourcebot/schemas": "workspace:*" "@t3-oss/env-core": "npm:^0.12.0" "@types/micromatch": "npm:^4.0.9" @@ -8046,9 +8004,12 @@ __metadata: ajv: "npm:^8.17.1" micromatch: "npm:^4.0.8" strip-json-comments: "npm:^5.0.1" + triple-beam: "npm:^1.4.1" tsc-watch: "npm:6.2.1" + tsx: "npm:^4.19.1" typescript: "npm:^5.7.3" - zod: "npm:^3.24.3" + winston: "npm:^3.15.0" + zod: "npm:^3.25.74" languageName: unknown linkType: soft @@ -8135,16 +8096,12 @@ __metadata: "@sentry/nextjs": "npm:^9" "@shopify/lang-jsonc": "npm:^1.0.0" "@sourcebot/codemirror-lang-tcl": "npm:^1.0.12" - "@sourcebot/crypto": "workspace:*" "@sourcebot/db": "workspace:*" - "@sourcebot/error": "workspace:*" - "@sourcebot/logger": "workspace:*" "@sourcebot/schemas": "workspace:*" "@sourcebot/shared": "workspace:*" "@ssddanbrown/codemirror-lang-twig": "npm:^1.0.0" "@stripe/react-stripe-js": "npm:^3.1.1" "@stripe/stripe-js": "npm:^5.6.0" - "@t3-oss/env-nextjs": "npm:^0.12.0" "@tailwindcss/typography": "npm:^0.5.16" "@tanstack/eslint-plugin-query": "npm:^5.74.7" "@tanstack/react-query": "npm:^5.53.3" @@ -8323,7 +8280,7 @@ __metadata: languageName: node linkType: hard -"@t3-oss/env-core@npm:0.12.0, @t3-oss/env-core@npm:^0.12.0": +"@t3-oss/env-core@npm:^0.12.0": version: 0.12.0 resolution: "@t3-oss/env-core@npm:0.12.0" peerDependencies: @@ -8360,26 +8317,6 @@ __metadata: languageName: node linkType: hard -"@t3-oss/env-nextjs@npm:^0.12.0": - version: 0.12.0 - resolution: "@t3-oss/env-nextjs@npm:0.12.0" - dependencies: - "@t3-oss/env-core": "npm:0.12.0" - peerDependencies: - typescript: ">=5.0.0" - valibot: ^1.0.0-beta.7 || ^1.0.0 - zod: ^3.24.0 - peerDependenciesMeta: - typescript: - optional: true - valibot: - optional: true - zod: - optional: true - checksum: 10c0/f39cae67353c09818b13836cf1f0ab8186058f6fd2f5e8983f43c8601f1cd6948d3a159a2f166420049080ca56e73ba69e483229f21593aba0d6d8f4cf79ef32 - languageName: node - linkType: hard - "@tailwindcss/typography@npm:^0.5.16": version: 0.5.16 resolution: "@tailwindcss/typography@npm:0.5.16"