feat: migrate to the deepnote/database-integrations package (#158)

* refactor: rename IntegrationConfig to LegacyIntegrationConfig

* replace notebook scanning for integrations with full project integration list

* move duckdb integration inclusion from env var provider to integration storage

* fix: scope integrations for env vars by the open project

* consolidate duckdb and other integration processing

* add `@deepnote/database-integrations` package

* fix import restriction violations in integration env var logic

* rename IntegrationType to LegacyIntegrationType

* add legacy integration config upgrading logic

* prefix integration config types with "legacy" to differentiate from new

* migrate integration storage to format of deepnote/database-integrations

* replace integration env var logic with deepnote/database-integrations

* prevent saving and loading of duckdb integrations via integration storage

* migrate all components to the new database integration config format

* localize "unsupported integration type" message

* test: add tests for updated sql env var provider

* test: add tests for updated integrationStorage

* test: add tests for legacy->new config conversions

* fix tests

* fix postgres ssl flag migration from legacy config

* use node assert in SQL env var tests

* update INTEGRATIONS_CREDENTIALS document

* upgrade deepnote/database-integrations (fix snowflake URL construction)

Author: jankuca

INTEGRATIONS_CREDENTIALS.md

@@ -2455,6 +2455,7 @@
         "@c4312/evt": "^0.1.1",
         "@deepnote/blocks": "^1.3.5",
         "@deepnote/convert": "^1.2.0",
+        "@deepnote/database-integrations": "^1.1.1",
         "@enonic/fnv-plus": "^1.3.0",
         "@jupyter-widgets/base": "^6.0.8",
         "@jupyter-widgets/controls": "^5.0.9",

package-lock.json

@@ -235,6 +235,7 @@ export type LocalizedMessages = {
     integrationsRequiredField: string;
     integrationsOptionalField: string;
     integrationsUnnamedIntegration: string;
+    integrationsUnsupportedIntegrationType: string;
     // Select input settings strings
     selectInputSettingsTitle: string;
     allowMultipleValues: string;

package.json

@@ -2,14 +2,9 @@ import { inject, injectable } from 'inversify';
 
 import { logger } from '../../../platform/logging';
 import { IDeepnoteNotebookManager } from '../../types';
-import {
-    DATAFRAME_SQL_INTEGRATION_ID,
-    DEEPNOTE_TO_INTEGRATION_TYPE,
-    IntegrationStatus,
-    IntegrationWithStatus,
-    RawIntegrationType
-} from '../../../platform/notebooks/deepnote/integrationTypes';
+import { IntegrationStatus, IntegrationWithStatus } from '../../../platform/notebooks/deepnote/integrationTypes';
 import { IIntegrationDetector, IIntegrationStorage } from './types';
+import { DatabaseIntegrationType, databaseIntegrationTypes } from '@deepnote/database-integrations';
 
 /**
  * Service for detecting integrations used in Deepnote notebooks
@@ -40,39 +35,25 @@ export class IntegrationDetector implements IIntegrationDetector {
         const integrations = new Map<string, IntegrationWithStatus>();
 
         // Use the project's integrations field as the source of truth
-        const projectIntegrations = project.project.integrations || [];
+        const projectIntegrations = project.project.integrations?.slice() ?? [];
         logger.debug(`IntegrationDetector: Found ${projectIntegrations.length} integrations in project.integrations`);
 
         for (const projectIntegration of projectIntegrations) {
             const integrationId = projectIntegration.id;
-
-            // Skip the internal DuckDB integration
-            if (integrationId === DATAFRAME_SQL_INTEGRATION_ID) {
-                continue;
-            }
-
-            logger.debug(`IntegrationDetector: Found integration: ${integrationId} (${projectIntegration.type})`);
-
-            // Map the Deepnote integration type to our IntegrationType
-            const integrationType = DEEPNOTE_TO_INTEGRATION_TYPE[projectIntegration.type as RawIntegrationType];
-
-            // Skip unknown integration types
-            if (!integrationType) {
-                logger.warn(
-                    `IntegrationDetector: Unknown integration type '${projectIntegration.type}' for integration ID '${integrationId}'. Skipping.`
-                );
+            const integrationType = projectIntegration.type;
+            if (!(databaseIntegrationTypes as readonly string[]).includes(integrationType)) {
+                logger.debug(`IntegrationDetector: Skipping unsupported integration type: ${integrationType}`);
                 continue;
             }
 
             // Check if the integration is configured
             const config = await this.integrationStorage.getIntegrationConfig(integrationId);
-
             const status: IntegrationWithStatus = {
-                config: config || null,
+                config: config ?? null,
                 status: config ? IntegrationStatus.Connected : IntegrationStatus.Disconnected,
                 // Include integration metadata from project for prefilling when config is null
                 integrationName: projectIntegration.name,
-                integrationType: integrationType
+                integrationType: integrationType as DatabaseIntegrationType
             };
 
             integrations.set(integrationId, status);

src/messageTypes.ts

@@ -1,19 +1,13 @@
 import { inject, injectable } from 'inversify';
-import { commands, l10n, NotebookDocument, window, workspace } from 'vscode';
+import { commands, l10n, window, workspace } from 'vscode';
 
 import { IExtensionContext } from '../../../platform/common/types';
 import { Commands } from '../../../platform/common/constants';
 import { logger } from '../../../platform/logging';
 import { IIntegrationDetector, IIntegrationManager, IIntegrationStorage, IIntegrationWebviewProvider } from './types';
-import {
-    DEEPNOTE_TO_INTEGRATION_TYPE,
-    IntegrationStatus,
-    IntegrationType,
-    IntegrationWithStatus,
-    RawIntegrationType
-} from '../../../platform/notebooks/deepnote/integrationTypes';
-import { BlockWithIntegration, scanBlocksForIntegrations } from './integrationUtils';
+import { IntegrationStatus } from '../../../platform/notebooks/deepnote/integrationTypes';
 import { IDeepnoteNotebookManager } from '../../types';
+import { DatabaseIntegrationType, databaseIntegrationTypes } from '@deepnote/database-integrations';
 
 /**
  * Manages integration UI and commands for Deepnote notebooks
@@ -143,14 +137,6 @@ export class IntegrationManager implements IIntegrationManager {
 
         // First try to detect integrations from the stored project
         let integrations = await this.integrationDetector.detectIntegrations(projectId);
-
-        // If no integrations found in stored project, scan cells directly
-        // This handles the case where the notebook was already open when the extension loaded
-        if (integrations.size === 0) {
-            logger.debug(`IntegrationManager: No integrations found in stored project, scanning cells directly`);
-            integrations = await this.detectIntegrationsFromCells(activeNotebook);
-        }
-
         logger.debug(`IntegrationManager: Found ${integrations.size} integrations`);
 
         // If a specific integration was requested (e.g., from status bar click),
@@ -164,20 +150,15 @@ export class IntegrationManager implements IIntegrationManager {
             const projectIntegration = project?.project.integrations?.find((i) => i.id === selectedIntegrationId);
 
             let integrationName: string | undefined;
-            let integrationType: IntegrationType | undefined;
+            let integrationType: DatabaseIntegrationType | undefined;
 
-            if (projectIntegration) {
+            // Validate that projectIntegration.type against supported types
+            if (
+                projectIntegration &&
+                (databaseIntegrationTypes as readonly string[]).includes(projectIntegration.type)
+            ) {
                 integrationName = projectIntegration.name;
-
-                // Validate that projectIntegration.type exists in the mapping before lookup
-                if (projectIntegration.type in DEEPNOTE_TO_INTEGRATION_TYPE) {
-                    // Map the Deepnote integration type to our IntegrationType
-                    integrationType = DEEPNOTE_TO_INTEGRATION_TYPE[projectIntegration.type as RawIntegrationType];
-                } else {
-                    logger.warn(
-                        `IntegrationManager: Unknown integration type '${projectIntegration.type}' for integration ID '${selectedIntegrationId}' in project '${projectId}'. Integration type will be undefined.`
-                    );
-                }
+                integrationType = projectIntegration.type as DatabaseIntegrationType;
             }
 
             integrations.set(selectedIntegrationId, {
@@ -196,35 +177,4 @@ export class IntegrationManager implements IIntegrationManager {
         // Show the webview with optional selected integration
         await this.webviewProvider.show(projectId, integrations, selectedIntegrationId);
     }
-
-    /**
-     * Detect integrations by scanning cells directly (fallback method)
-     * This is used when the project isn't stored in the notebook manager
-     */
-    private async detectIntegrationsFromCells(notebook: NotebookDocument): Promise<Map<string, IntegrationWithStatus>> {
-        // Collect all cells with SQL integration metadata
-        const blocksWithIntegrations: BlockWithIntegration[] = [];
-
-        for (const cell of notebook.getCells()) {
-            const metadata = cell.metadata;
-            logger.trace(`IntegrationManager: Cell ${cell.index} metadata:`, metadata);
-
-            // Check cell metadata for sql_integration_id
-            if (metadata && typeof metadata === 'object') {
-                const integrationId = (metadata as Record<string, unknown>).sql_integration_id;
-                if (typeof integrationId === 'string') {
-                    logger.debug(`IntegrationManager: Found integration ${integrationId} in cell ${cell.index}`);
-                    blocksWithIntegrations.push({
-                        id: `cell-${cell.index}`,
-                        sql_integration_id: integrationId
-                    });
-                }
-            }
-        }
-
-        logger.debug(`IntegrationManager: Found ${blocksWithIntegrations.length} cells with integrations`);
-
-        // Use the shared utility to scan blocks and build the status map
-        return scanBlocksForIntegrations(blocksWithIntegrations, this.integrationStorage, 'IntegrationManager');
-    }
 }

src/notebooks/deepnote/integrations/integrationDetector.ts

@@ -7,13 +7,8 @@ import { logger } from '../../../platform/logging';
 import { LocalizedMessages, SharedMessages } from '../../../messageTypes';
 import { IDeepnoteNotebookManager, ProjectIntegration } from '../../types';
 import { IIntegrationStorage, IIntegrationWebviewProvider } from './types';
-import {
-    INTEGRATION_TYPE_TO_DEEPNOTE,
-    IntegrationConfig,
-    IntegrationStatus,
-    IntegrationWithStatus,
-    RawIntegrationType
-} from '../../../platform/notebooks/deepnote/integrationTypes';
+import { IntegrationStatus, IntegrationWithStatus } from '../../../platform/notebooks/deepnote/integrationTypes';
+import { DatabaseIntegrationConfig } from '@deepnote/database-integrations';
 
 /**
  * Manages the webview panel for integration configuration
@@ -182,7 +177,8 @@ export class IntegrationWebviewProvider implements IIntegrationWebviewProvider {
             integrationsSnowflakeRolePlaceholder: localize.Integrations.snowflakeRolePlaceholder,
             integrationsSnowflakeWarehouseLabel: localize.Integrations.snowflakeWarehouseLabel,
             integrationsSnowflakeWarehousePlaceholder: localize.Integrations.snowflakeWarehousePlaceholder,
-            integrationsUnnamedIntegration: localize.Integrations.unnamedIntegration('{0}')
+            integrationsUnnamedIntegration: localize.Integrations.unnamedIntegration('{0}'),
+            integrationsUnsupportedIntegrationType: localize.Integrations.unsupportedIntegrationType('{0}')
         };
 
         await this.currentPanel.webview.postMessage({
@@ -221,7 +217,7 @@ export class IntegrationWebviewProvider implements IIntegrationWebviewProvider {
     private async handleMessage(message: {
         type: string;
         integrationId?: string;
-        config?: IntegrationConfig;
+        config?: DatabaseIntegrationConfig;
     }): Promise<void> {
         switch (message.type) {
             case 'configure':
@@ -263,7 +259,7 @@ export class IntegrationWebviewProvider implements IIntegrationWebviewProvider {
     /**
      * Save the configuration for an integration
      */
-    private async saveConfiguration(integrationId: string, config: IntegrationConfig): Promise<void> {
+    private async saveConfiguration(integrationId: string, config: DatabaseIntegrationConfig): Promise<void> {
         try {
             await this.integrationStorage.save(config);
 
@@ -349,17 +345,16 @@ export class IntegrationWebviewProvider implements IIntegrationWebviewProvider {
                     return null;
                 }
 
-                // Map to Deepnote integration type
-                const deepnoteType: RawIntegrationType | undefined = INTEGRATION_TYPE_TO_DEEPNOTE[type];
-                if (!deepnoteType) {
-                    logger.warn(`IntegrationWebviewProvider: Cannot map type ${type} for integration ${id}, skipping`);
+                // Skip DuckDB integration (internal, not a real Deepnote integration)
+                if (type === 'pandas-dataframe') {
+                    logger.trace(`IntegrationWebviewProvider: Skipping internal DuckDB integration ${id}`);
                     return null;
                 }
 
                 return {
                     id,
                     name: integration.config?.name || integration.integrationName || id,
-                    type: deepnoteType
+                    type
                 };
             })
             .filter((integration): integration is ProjectIntegration => integration !== null);