diff --git a/build/esbuild/build.ts b/build/esbuild/build.ts index 80f4393b80..eeafc701d0 100644 --- a/build/esbuild/build.ts +++ b/build/esbuild/build.ts @@ -343,6 +343,11 @@ async function buildAll() { path.join(extensionFolder, 'src', 'webviews', 'webview-side', 'data-explorer', 'index.tsx'), path.join(extensionFolder, 'dist', 'webviews', 'webview-side', 'viewers', 'dataExplorer.js'), { target: 'web', watch: watchAll } + ), + build( + path.join(extensionFolder, 'src', 'webviews', 'webview-side', 'integrations', 'index.tsx'), + path.join(extensionFolder, 'dist', 'webviews', 'webview-side', 'integrations', 'index.js'), + { target: 'web', watch: watchAll } ) ); diff --git a/package.json b/package.json index 911beb70c1..47e13becb8 100644 --- a/package.json +++ b/package.json @@ -85,6 +85,12 @@ "category": "Deepnote", "icon": "$(reveal)" }, + { + "command": "deepnote.manageIntegrations", + "title": "%deepnote.commands.manageIntegrations.title%", + "category": "Deepnote", + "icon": "$(plug)" + }, { "command": "dataScience.ClearCache", "title": "%jupyter.command.dataScience.clearCache.title%", @@ -707,6 +713,11 @@ } ], "notebook/toolbar": [ + { + "command": "deepnote.manageIntegrations", + "group": "navigation@0", + "when": "notebookType == 'deepnote'" + }, { "command": "jupyter.restartkernel", "group": "navigation/execute@5", diff --git a/package.nls.json b/package.nls.json index b4c31235b8..fbc258484b 100644 --- a/package.nls.json +++ b/package.nls.json @@ -249,6 +249,7 @@ "deepnote.commands.openNotebook.title": "Open Notebook", "deepnote.commands.openFile.title": "Open File", "deepnote.commands.revealInExplorer.title": "Reveal in Explorer", + "deepnote.commands.manageIntegrations.title": "Manage Integrations", "deepnote.views.explorer.name": "Explorer", "deepnote.command.selectNotebook.title": "Select Notebook" } diff --git a/src/notebooks/deepnote/deepnoteActivationService.ts b/src/notebooks/deepnote/deepnoteActivationService.ts index f99686ba78..546a26f349 100644 --- a/src/notebooks/deepnote/deepnoteActivationService.ts +++ b/src/notebooks/deepnote/deepnoteActivationService.ts @@ -5,6 +5,7 @@ import { IExtensionContext } from '../../platform/common/types'; import { IDeepnoteNotebookManager } from '../types'; import { DeepnoteNotebookSerializer } from './deepnoteSerializer'; import { DeepnoteExplorerView } from './deepnoteExplorerView'; +import { IIntegrationManager } from './integrations/types'; /** * Service responsible for activating and configuring Deepnote notebook support in VS Code. @@ -13,12 +14,18 @@ import { DeepnoteExplorerView } from './deepnoteExplorerView'; @injectable() export class DeepnoteActivationService implements IExtensionSyncActivationService { private explorerView: DeepnoteExplorerView; + + private integrationManager: IIntegrationManager; + private serializer: DeepnoteNotebookSerializer; constructor( @inject(IExtensionContext) private extensionContext: IExtensionContext, - @inject(IDeepnoteNotebookManager) private readonly notebookManager: IDeepnoteNotebookManager - ) {} + @inject(IDeepnoteNotebookManager) private readonly notebookManager: IDeepnoteNotebookManager, + @inject(IIntegrationManager) integrationManager: IIntegrationManager + ) { + this.integrationManager = integrationManager; + } /** * Activates Deepnote support by registering serializers and commands. @@ -31,5 +38,6 @@ export class DeepnoteActivationService implements IExtensionSyncActivationServic this.extensionContext.subscriptions.push(workspace.registerNotebookSerializer('deepnote', this.serializer)); this.explorerView.activate(); + this.integrationManager.activate(); } } diff --git a/src/notebooks/deepnote/deepnoteActivationService.unit.test.ts b/src/notebooks/deepnote/deepnoteActivationService.unit.test.ts index 57fc5dd463..c7ee287329 100644 --- a/src/notebooks/deepnote/deepnoteActivationService.unit.test.ts +++ b/src/notebooks/deepnote/deepnoteActivationService.unit.test.ts @@ -3,11 +3,13 @@ import { assert } from 'chai'; import { DeepnoteActivationService } from './deepnoteActivationService'; import { DeepnoteNotebookManager } from './deepnoteNotebookManager'; import { IExtensionContext } from '../../platform/common/types'; +import { IIntegrationManager } from './integrations/types'; suite('DeepnoteActivationService', () => { let activationService: DeepnoteActivationService; let mockExtensionContext: IExtensionContext; let manager: DeepnoteNotebookManager; + let mockIntegrationManager: IIntegrationManager; setup(() => { mockExtensionContext = { @@ -15,7 +17,12 @@ suite('DeepnoteActivationService', () => { } as any; manager = new DeepnoteNotebookManager(); - activationService = new DeepnoteActivationService(mockExtensionContext, manager); + mockIntegrationManager = { + activate: () => { + return; + } + }; + activationService = new DeepnoteActivationService(mockExtensionContext, manager, mockIntegrationManager); }); suite('constructor', () => { @@ -75,8 +82,18 @@ suite('DeepnoteActivationService', () => { const manager1 = new DeepnoteNotebookManager(); const manager2 = new DeepnoteNotebookManager(); - const service1 = new DeepnoteActivationService(context1, manager1); - const service2 = new DeepnoteActivationService(context2, manager2); + const mockIntegrationManager1: IIntegrationManager = { + activate: () => { + return; + } + }; + const mockIntegrationManager2: IIntegrationManager = { + activate: () => { + return; + } + }; + const service1 = new DeepnoteActivationService(context1, manager1, mockIntegrationManager1); + const service2 = new DeepnoteActivationService(context2, manager2, mockIntegrationManager2); // Verify each service has its own context assert.strictEqual((service1 as any).extensionContext, context1); @@ -101,8 +118,18 @@ suite('DeepnoteActivationService', () => { const manager1 = new DeepnoteNotebookManager(); const manager2 = new DeepnoteNotebookManager(); - new DeepnoteActivationService(context1, manager1); - new DeepnoteActivationService(context2, manager2); + const mockIntegrationManager1: IIntegrationManager = { + activate: () => { + return; + } + }; + const mockIntegrationManager2: IIntegrationManager = { + activate: () => { + return; + } + }; + new DeepnoteActivationService(context1, manager1, mockIntegrationManager1); + new DeepnoteActivationService(context2, manager2, mockIntegrationManager2); assert.strictEqual(context1.subscriptions.length, 0); assert.strictEqual(context2.subscriptions.length, 1); diff --git a/src/notebooks/deepnote/deepnoteSerializer.ts b/src/notebooks/deepnote/deepnoteSerializer.ts index eafbbe5e91..6ff7d8fc14 100644 --- a/src/notebooks/deepnote/deepnoteSerializer.ts +++ b/src/notebooks/deepnote/deepnoteSerializer.ts @@ -2,6 +2,7 @@ import { injectable, inject } from 'inversify'; import { l10n, type CancellationToken, type NotebookData, type NotebookSerializer, workspace } from 'vscode'; import * as yaml from 'js-yaml'; +import { logger } from '../../platform/logging'; import { IDeepnoteNotebookManager } from '../types'; import type { DeepnoteProject } from './deepnoteTypes'; import { DeepnoteDataConverter } from './deepnoteDataConverter'; @@ -35,7 +36,7 @@ export class DeepnoteNotebookSerializer implements NotebookSerializer { * @returns Promise resolving to notebook data */ async deserializeNotebook(content: Uint8Array, token: CancellationToken): Promise { - console.log('Deserializing Deepnote notebook'); + logger.debug('DeepnoteSerializer: Deserializing Deepnote notebook'); if (token?.isCancellationRequested) { throw new Error('Serialization cancelled'); @@ -52,7 +53,7 @@ export class DeepnoteNotebookSerializer implements NotebookSerializer { const projectId = deepnoteProject.project.id; const notebookId = this.findCurrentNotebookId(projectId); - console.log(`Selected notebook ID: ${notebookId}.`); + logger.debug(`DeepnoteSerializer: Project ID: ${projectId}, Selected notebook ID: ${notebookId}`); const selectedNotebook = notebookId ? deepnoteProject.project.notebooks.find((nb) => nb.id === notebookId) @@ -64,9 +65,10 @@ export class DeepnoteNotebookSerializer implements NotebookSerializer { const cells = this.converter.convertBlocksToCells(selectedNotebook.blocks); - console.log(`Converted ${cells.length} cells from notebook blocks.`); + logger.debug(`DeepnoteSerializer: Converted ${cells.length} cells from notebook blocks`); this.notebookManager.storeOriginalProject(deepnoteProject.project.id, deepnoteProject, selectedNotebook.id); + logger.debug(`DeepnoteSerializer: Stored project ${projectId} in notebook manager`); return { cells, @@ -81,7 +83,7 @@ export class DeepnoteNotebookSerializer implements NotebookSerializer { } }; } catch (error) { - console.error('Error deserializing Deepnote notebook:', error); + logger.error('DeepnoteSerializer: Error deserializing Deepnote notebook', error); throw new Error( `Failed to parse Deepnote file: ${error instanceof Error ? error.message : 'Unknown error'}` @@ -148,7 +150,7 @@ export class DeepnoteNotebookSerializer implements NotebookSerializer { return new TextEncoder().encode(yamlString); } catch (error) { - console.error('Error serializing Deepnote notebook:', error); + logger.error('DeepnoteSerializer: Error serializing Deepnote notebook', error); throw new Error( `Failed to save Deepnote file: ${error instanceof Error ? error.message : 'Unknown error'}` ); diff --git a/src/notebooks/deepnote/integrations/integrationDetector.ts b/src/notebooks/deepnote/integrations/integrationDetector.ts new file mode 100644 index 0000000000..974b4a24fd --- /dev/null +++ b/src/notebooks/deepnote/integrations/integrationDetector.ts @@ -0,0 +1,75 @@ +import { inject, injectable } from 'inversify'; + +import { logger } from '../../../platform/logging'; +import { IDeepnoteNotebookManager } from '../../types'; +import { IntegrationStatus, IntegrationWithStatus } from './integrationTypes'; +import { IIntegrationDetector, IIntegrationStorage } from './types'; +import { BlockWithIntegration, scanBlocksForIntegrations } from './integrationUtils'; + +/** + * Service for detecting integrations used in Deepnote notebooks + */ +@injectable() +export class IntegrationDetector implements IIntegrationDetector { + constructor( + @inject(IIntegrationStorage) private readonly integrationStorage: IIntegrationStorage, + @inject(IDeepnoteNotebookManager) private readonly notebookManager: IDeepnoteNotebookManager + ) {} + + /** + * Detect all integrations used in the given project + */ + async detectIntegrations(projectId: string): Promise> { + // Get the project + const project = this.notebookManager.getOriginalProject(projectId); + if (!project) { + logger.warn( + `IntegrationDetector: No project found for ID: ${projectId}. The project may not have been loaded yet.` + ); + return new Map(); + } + + logger.debug( + `IntegrationDetector: Scanning project ${projectId} with ${project.project.notebooks.length} notebooks` + ); + + // Collect all blocks with SQL integration metadata from all notebooks + const blocksWithIntegrations: BlockWithIntegration[] = []; + for (const notebook of project.project.notebooks) { + logger.trace(`IntegrationDetector: Scanning notebook ${notebook.id} with ${notebook.blocks.length} blocks`); + + for (const block of notebook.blocks) { + // Check if this is a code block with SQL integration metadata + if (block.type === 'code' && block.metadata?.sql_integration_id) { + blocksWithIntegrations.push({ + id: block.id, + sql_integration_id: block.metadata.sql_integration_id + }); + } else if (block.type === 'code') { + logger.trace( + `IntegrationDetector: Block ${block.id} has no sql_integration_id. Metadata:`, + block.metadata + ); + } + } + } + + // Use the shared utility to scan blocks and build the status map + return scanBlocksForIntegrations(blocksWithIntegrations, this.integrationStorage, 'IntegrationDetector'); + } + + /** + * Check if a project has any unconfigured integrations + */ + async hasUnconfiguredIntegrations(projectId: string): Promise { + const integrations = await this.detectIntegrations(projectId); + + for (const integration of integrations.values()) { + if (integration.status === IntegrationStatus.Disconnected) { + return true; + } + } + + return false; + } +} diff --git a/src/notebooks/deepnote/integrations/integrationManager.ts b/src/notebooks/deepnote/integrations/integrationManager.ts new file mode 100644 index 0000000000..1335ecb1fe --- /dev/null +++ b/src/notebooks/deepnote/integrations/integrationManager.ts @@ -0,0 +1,160 @@ +import { inject, injectable } from 'inversify'; +import { commands, NotebookDocument, window, workspace } from 'vscode'; + +import { IExtensionContext } from '../../../platform/common/types'; +import { Commands } from '../../../platform/common/constants'; +import { logger } from '../../../platform/logging'; +import { IIntegrationDetector, IIntegrationManager, IIntegrationStorage, IIntegrationWebviewProvider } from './types'; +import { IntegrationStatus, IntegrationWithStatus } from './integrationTypes'; +import { BlockWithIntegration, scanBlocksForIntegrations } from './integrationUtils'; + +/** + * Manages integration UI and commands for Deepnote notebooks + */ +@injectable() +export class IntegrationManager implements IIntegrationManager { + private hasIntegrationsContext = 'deepnote.hasIntegrations'; + + private hasUnconfiguredIntegrationsContext = 'deepnote.hasUnconfiguredIntegrations'; + + constructor( + @inject(IExtensionContext) private readonly extensionContext: IExtensionContext, + @inject(IIntegrationDetector) private readonly integrationDetector: IIntegrationDetector, + @inject(IIntegrationStorage) private readonly integrationStorage: IIntegrationStorage, + @inject(IIntegrationWebviewProvider) private readonly webviewProvider: IIntegrationWebviewProvider + ) {} + + public activate(): void { + // Register the manage integrations command + this.extensionContext.subscriptions.push( + commands.registerCommand(Commands.ManageIntegrations, () => this.showIntegrationsUI()) + ); + + // Listen for active notebook changes to update context + this.extensionContext.subscriptions.push( + window.onDidChangeActiveNotebookEditor(() => + this.updateContext().catch((err) => + logger.error('IntegrationManager: Failed to update context on notebook editor change', err) + ) + ) + ); + + // Listen for notebook document changes + this.extensionContext.subscriptions.push( + workspace.onDidOpenNotebookDocument(() => + this.updateContext().catch((err) => + logger.error('IntegrationManager: Failed to update context on notebook open', err) + ) + ) + ); + + this.extensionContext.subscriptions.push( + workspace.onDidCloseNotebookDocument(() => + this.updateContext().catch((err) => + logger.error('IntegrationManager: Failed to update context on notebook close', err) + ) + ) + ); + + // Initial context update + this.updateContext().catch((err) => + logger.error('IntegrationManager: Failed to update context on activation', err) + ); + } + + /** + * Update the context keys based on the active notebook + */ + private async updateContext(): Promise { + const activeNotebook = window.activeNotebookEditor?.notebook; + + if (!activeNotebook || activeNotebook.notebookType !== 'deepnote') { + await commands.executeCommand('setContext', this.hasIntegrationsContext, false); + await commands.executeCommand('setContext', this.hasUnconfiguredIntegrationsContext, false); + return; + } + + // Get the project ID from the notebook metadata + const projectId = activeNotebook.metadata?.deepnoteProjectId; + if (!projectId) { + await commands.executeCommand('setContext', this.hasIntegrationsContext, false); + await commands.executeCommand('setContext', this.hasUnconfiguredIntegrationsContext, false); + return; + } + + // Detect integrations in the project + const integrations = await this.integrationDetector.detectIntegrations(projectId); + const hasIntegrations = integrations.size > 0; + const hasUnconfigured = Array.from(integrations.values()).some( + (integration) => integration.status === IntegrationStatus.Disconnected + ); + + await commands.executeCommand('setContext', this.hasIntegrationsContext, hasIntegrations); + await commands.executeCommand('setContext', this.hasUnconfiguredIntegrationsContext, hasUnconfigured); + } + + /** + * Show the integrations management UI + */ + private async showIntegrationsUI(): Promise { + const activeNotebook = window.activeNotebookEditor?.notebook; + + if (!activeNotebook || activeNotebook.notebookType !== 'deepnote') { + void window.showErrorMessage('No active Deepnote notebook'); + return; + } + + const projectId = activeNotebook.metadata?.deepnoteProjectId; + if (!projectId) { + void window.showErrorMessage('Cannot determine project ID'); + return; + } + + logger.debug(`IntegrationManager: Project ID: ${projectId}`); + logger.trace(`IntegrationManager: Notebook metadata:`, activeNotebook.metadata); + + // First try to detect integrations from the stored project + let integrations = await this.integrationDetector.detectIntegrations(projectId); + + // If no integrations found in stored project, scan cells directly + // This handles the case where the notebook was already open when the extension loaded + if (integrations.size === 0) { + logger.debug(`IntegrationManager: No integrations found in stored project, scanning cells directly`); + integrations = await this.detectIntegrationsFromCells(activeNotebook); + } + + logger.debug(`IntegrationManager: Found ${integrations.size} integrations`); + + if (integrations.size === 0) { + void window.showInformationMessage(`No integrations found in this project.`); + return; + } + + // Show the webview + await this.webviewProvider.show(integrations); + } + + /** + * Detect integrations by scanning cells directly (fallback method) + * This is used when the project isn't stored in the notebook manager + */ + private async detectIntegrationsFromCells(notebook: NotebookDocument): Promise> { + // Collect all cells with SQL integration metadata + const blocksWithIntegrations: BlockWithIntegration[] = []; + + for (const cell of notebook.getCells()) { + const deepnoteMetadata = cell.metadata?.deepnoteMetadata; + logger.trace(`IntegrationManager: Cell ${cell.index} metadata:`, deepnoteMetadata); + + if (deepnoteMetadata?.sql_integration_id) { + blocksWithIntegrations.push({ + id: `cell-${cell.index}`, + sql_integration_id: deepnoteMetadata.sql_integration_id + }); + } + } + + // Use the shared utility to scan blocks and build the status map + return scanBlocksForIntegrations(blocksWithIntegrations, this.integrationStorage, 'IntegrationManager'); + } +} diff --git a/src/notebooks/deepnote/integrations/integrationStorage.ts b/src/notebooks/deepnote/integrations/integrationStorage.ts new file mode 100644 index 0000000000..3f97dce782 --- /dev/null +++ b/src/notebooks/deepnote/integrations/integrationStorage.ts @@ -0,0 +1,151 @@ +import { inject, injectable } from 'inversify'; + +import { IEncryptedStorage } from '../../../platform/common/application/types'; +import { logger } from '../../../platform/logging'; +import { IntegrationConfig, IntegrationType } from './integrationTypes'; + +const INTEGRATION_SERVICE_NAME = 'deepnote-integrations'; + +/** + * Storage service for integration configurations. + * Uses VSCode's SecretStorage API to securely store credentials. + * Storage is scoped to the user's machine and shared across all deepnote projects. + */ +@injectable() +export class IntegrationStorage { + private readonly cache: Map = new Map(); + + private cacheLoaded = false; + + constructor(@inject(IEncryptedStorage) private readonly encryptedStorage: IEncryptedStorage) {} + + /** + * Get all stored integration configurations + */ + async getAll(): Promise { + await this.ensureCacheLoaded(); + return Array.from(this.cache.values()); + } + + /** + * Get a specific integration configuration by ID + */ + async get(integrationId: string): Promise { + await this.ensureCacheLoaded(); + return this.cache.get(integrationId); + } + + /** + * Get all integrations of a specific type + */ + async getByType(type: IntegrationType): Promise { + await this.ensureCacheLoaded(); + return Array.from(this.cache.values()).filter((config) => config.type === type); + } + + /** + * Save or update an integration configuration + */ + async save(config: IntegrationConfig): Promise { + await this.ensureCacheLoaded(); + + // Store the configuration as JSON in encrypted storage + const configJson = JSON.stringify(config); + await this.encryptedStorage.store(INTEGRATION_SERVICE_NAME, config.id, configJson); + + // Update cache + this.cache.set(config.id, config); + + // Update the index + await this.updateIndex(); + } + + /** + * Delete an integration configuration + */ + async delete(integrationId: string): Promise { + await this.ensureCacheLoaded(); + + // Remove from encrypted storage + await this.encryptedStorage.store(INTEGRATION_SERVICE_NAME, integrationId, undefined); + + // Remove from cache + this.cache.delete(integrationId); + + // Update the index + await this.updateIndex(); + } + + /** + * Check if an integration exists + */ + async exists(integrationId: string): Promise { + await this.ensureCacheLoaded(); + return this.cache.has(integrationId); + } + + /** + * Clear all integration configurations + */ + async clear(): Promise { + await this.ensureCacheLoaded(); + + // Delete all integrations from encrypted storage + const integrationIds = Array.from(this.cache.keys()); + for (const id of integrationIds) { + await this.encryptedStorage.store(INTEGRATION_SERVICE_NAME, id, undefined); + } + + // Clear the index + await this.encryptedStorage.store(INTEGRATION_SERVICE_NAME, 'index', undefined); + + // Clear cache + this.cache.clear(); + } + + /** + * Ensure the cache is loaded from storage + */ + private async ensureCacheLoaded(): Promise { + if (this.cacheLoaded) { + return; + } + + // Load the index of integration IDs + const indexJson = await this.encryptedStorage.retrieve(INTEGRATION_SERVICE_NAME, 'index'); + if (!indexJson) { + this.cacheLoaded = true; + return; + } + + try { + const integrationIds: string[] = JSON.parse(indexJson); + + // Load each integration configuration + for (const id of integrationIds) { + const configJson = await this.encryptedStorage.retrieve(INTEGRATION_SERVICE_NAME, id); + if (configJson) { + try { + const config: IntegrationConfig = JSON.parse(configJson); + this.cache.set(id, config); + } catch (error) { + logger.error(`Failed to parse integration config for ${id}:`, error); + } + } + } + } catch (error) { + logger.error('Failed to parse integration index:', error); + } + + this.cacheLoaded = true; + } + + /** + * Update the index of integration IDs in storage + */ + private async updateIndex(): Promise { + const integrationIds = Array.from(this.cache.keys()); + const indexJson = JSON.stringify(integrationIds); + await this.encryptedStorage.store(INTEGRATION_SERVICE_NAME, 'index', indexJson); + } +} diff --git a/src/notebooks/deepnote/integrations/integrationTypes.ts b/src/notebooks/deepnote/integrations/integrationTypes.ts new file mode 100644 index 0000000000..048c0ff13a --- /dev/null +++ b/src/notebooks/deepnote/integrations/integrationTypes.ts @@ -0,0 +1,67 @@ +/** + * Special integration ID that should be excluded from management. + * This is the internal DuckDB integration that doesn't require configuration. + */ +export const DATAFRAME_SQL_INTEGRATION_ID = 'deepnote-dataframe-sql'; + +/** + * Supported integration types + */ +export enum IntegrationType { + Postgres = 'postgres', + BigQuery = 'bigquery' +} + +/** + * Base interface for all integration configurations + */ +export interface BaseIntegrationConfig { + id: string; + name: string; + type: IntegrationType; +} + +/** + * PostgreSQL integration configuration + */ +export interface PostgresIntegrationConfig extends BaseIntegrationConfig { + type: IntegrationType.Postgres; + host: string; + port: number; + database: string; + username: string; + password: string; + ssl?: boolean; +} + +/** + * BigQuery integration configuration + */ +export interface BigQueryIntegrationConfig extends BaseIntegrationConfig { + type: IntegrationType.BigQuery; + projectId: string; + credentials: string; // JSON string of service account credentials +} + +/** + * Union type of all integration configurations + */ +export type IntegrationConfig = PostgresIntegrationConfig | BigQueryIntegrationConfig; + +/** + * Integration connection status + */ +export enum IntegrationStatus { + Connected = 'connected', + Disconnected = 'disconnected', + Error = 'error' +} + +/** + * Integration with its current status + */ +export interface IntegrationWithStatus { + config: IntegrationConfig | null; + status: IntegrationStatus; + error?: string; +} diff --git a/src/notebooks/deepnote/integrations/integrationUtils.ts b/src/notebooks/deepnote/integrations/integrationUtils.ts new file mode 100644 index 0000000000..01b62b081e --- /dev/null +++ b/src/notebooks/deepnote/integrations/integrationUtils.ts @@ -0,0 +1,64 @@ +import { logger } from '../../../platform/logging'; +import { IIntegrationStorage } from './types'; +import { DATAFRAME_SQL_INTEGRATION_ID, IntegrationStatus, IntegrationWithStatus } from './integrationTypes'; + +/** + * Represents a block with SQL integration metadata + */ +export interface BlockWithIntegration { + id: string; + sql_integration_id: string; +} + +/** + * Scans blocks for SQL integrations and builds a status map. + * This is the core logic shared between IntegrationDetector and IntegrationManager. + * + * @param blocks - Iterator of blocks to scan (can be from Deepnote project or VSCode notebook cells) + * @param integrationStorage - Storage service to check configuration status + * @param logContext - Context string for logging (e.g., "IntegrationDetector", "IntegrationManager") + * @returns Map of integration IDs to their status + */ +export async function scanBlocksForIntegrations( + blocks: Iterable, + integrationStorage: IIntegrationStorage, + logContext: string +): Promise> { + const integrations = new Map(); + + for (const block of blocks) { + const integrationId = block.sql_integration_id; + + // Skip blocks without integration IDs + if (!integrationId) { + continue; + } + + // Skip excluded integrations (e.g., internal DuckDB integration) + if (integrationId === DATAFRAME_SQL_INTEGRATION_ID) { + logger.trace(`${logContext}: Skipping excluded integration: ${integrationId} in block ${block.id}`); + continue; + } + + // Skip if we've already detected this integration + if (integrations.has(integrationId)) { + continue; + } + + logger.debug(`${logContext}: Found integration: ${integrationId} in block ${block.id}`); + + // Check if the integration is configured + const config = await integrationStorage.get(integrationId); + + const status: IntegrationWithStatus = { + config: config || null, + status: config ? IntegrationStatus.Connected : IntegrationStatus.Disconnected + }; + + integrations.set(integrationId, status); + } + + logger.debug(`${logContext}: Found ${integrations.size} integrations`); + + return integrations; +} diff --git a/src/notebooks/deepnote/integrations/integrationWebview.ts b/src/notebooks/deepnote/integrations/integrationWebview.ts new file mode 100644 index 0000000000..50f888336a --- /dev/null +++ b/src/notebooks/deepnote/integrations/integrationWebview.ts @@ -0,0 +1,277 @@ +import { inject, injectable } from 'inversify'; +import { Disposable, l10n, Uri, ViewColumn, WebviewPanel, window } from 'vscode'; + +import { IExtensionContext } from '../../../platform/common/types'; +import { logger } from '../../../platform/logging'; +import { IIntegrationStorage, IIntegrationWebviewProvider } from './types'; +import { IntegrationConfig, IntegrationStatus, IntegrationWithStatus } from './integrationTypes'; + +/** + * Manages the webview panel for integration configuration + */ +@injectable() +export class IntegrationWebviewProvider implements IIntegrationWebviewProvider { + private currentPanel: WebviewPanel | undefined; + + private readonly disposables: Disposable[] = []; + + private integrations: Map = new Map(); + + constructor( + @inject(IExtensionContext) private readonly extensionContext: IExtensionContext, + @inject(IIntegrationStorage) private readonly integrationStorage: IIntegrationStorage + ) {} + + /** + * Show the integration management webview + */ + public async show(integrations: Map): Promise { + // Update the stored integrations with the latest data + this.integrations = integrations; + + const column = window.activeTextEditor ? window.activeTextEditor.viewColumn : ViewColumn.One; + + // If we already have a panel, show it + if (this.currentPanel) { + this.currentPanel.reveal(column); + await this.updateWebview(); + return; + } + + // Create a new panel + this.currentPanel = window.createWebviewPanel( + 'deepnoteIntegrations', + 'Deepnote Integrations', + column || ViewColumn.One, + { + enableScripts: true, + retainContextWhenHidden: true, + localResourceRoots: [this.extensionContext.extensionUri], + enableForms: true + } + ); + + // Set the webview's initial html content + this.currentPanel.webview.html = this.getWebviewContent(); + + // Handle messages from the webview + this.currentPanel.webview.onDidReceiveMessage( + async (message) => { + await this.handleMessage(message); + }, + null, + this.disposables + ); + + // Reset when the current panel is closed + this.currentPanel.onDidDispose( + () => { + this.currentPanel = undefined; + this.integrations = new Map(); + this.disposables.forEach((d) => d.dispose()); + this.disposables.length = 0; + }, + null, + this.disposables + ); + + await this.updateWebview(); + } + + /** + * Update the webview with current integration data + */ + private async updateWebview(): Promise { + if (!this.currentPanel) { + return; + } + + const integrationsData = Array.from(this.integrations.entries()).map(([id, integration]) => ({ + config: integration.config, + id, + status: integration.status + })); + + await this.currentPanel.webview.postMessage({ + integrations: integrationsData, + type: 'update' + }); + } + + /** + * Handle messages from the webview + */ + private async handleMessage(message: { + type: string; + integrationId?: string; + config?: IntegrationConfig; + }): Promise { + switch (message.type) { + case 'configure': + if (message.integrationId) { + await this.showConfigurationForm(message.integrationId); + } + break; + case 'save': + if (message.integrationId && message.config) { + await this.saveConfiguration(message.integrationId, message.config); + } + break; + case 'delete': + if (message.integrationId) { + await this.deleteConfiguration(message.integrationId); + } + break; + } + } + + /** + * Show the configuration form for an integration + */ + private async showConfigurationForm(integrationId: string): Promise { + const integration = this.integrations.get(integrationId); + if (!integration) { + return; + } + + await this.currentPanel?.webview.postMessage({ + config: integration.config, + integrationId, + type: 'showForm' + }); + } + + /** + * Save the configuration for an integration + */ + private async saveConfiguration(integrationId: string, config: IntegrationConfig): Promise { + try { + await this.integrationStorage.save(config); + + // Update local state + const integration = this.integrations.get(integrationId); + if (integration) { + integration.config = config; + integration.status = IntegrationStatus.Connected; + this.integrations.set(integrationId, integration); + } + + await this.updateWebview(); + await this.currentPanel?.webview.postMessage({ + message: l10n.t('Configuration saved successfully'), + type: 'success' + }); + } catch (error) { + logger.error('Failed to save integration configuration', error); + await this.currentPanel?.webview.postMessage({ + message: l10n.t( + 'Failed to save configuration: {0}', + error instanceof Error ? error.message : 'Unknown error' + ), + type: 'error' + }); + } + } + + /** + * Delete the configuration for an integration + */ + private async deleteConfiguration(integrationId: string): Promise { + try { + await this.integrationStorage.delete(integrationId); + + // Update local state + const integration = this.integrations.get(integrationId); + if (integration) { + integration.config = null; + integration.status = IntegrationStatus.Disconnected; + this.integrations.set(integrationId, integration); + } + + await this.updateWebview(); + await this.currentPanel?.webview.postMessage({ + message: l10n.t('Configuration deleted successfully'), + type: 'success' + }); + } catch (error) { + logger.error('Failed to delete integration configuration', error); + await this.currentPanel?.webview.postMessage({ + message: l10n.t( + 'Failed to delete configuration: {0}', + error instanceof Error ? error.message : 'Unknown error' + ), + type: 'error' + }); + } + } + + /** + * Get the HTML content for the webview (React-based) + */ + private getWebviewContent(): string { + if (!this.currentPanel) { + return ''; + } + + const webview = this.currentPanel.webview; + const nonce = this.getNonce(); + + // Get URIs for the React app + const scriptUri = webview.asWebviewUri( + Uri.joinPath( + this.extensionContext.extensionUri, + 'dist', + 'webviews', + 'webview-side', + 'integrations', + 'index.js' + ) + ); + const styleUri = webview.asWebviewUri( + Uri.joinPath( + this.extensionContext.extensionUri, + 'dist', + 'webviews', + 'webview-side', + 'integrations', + 'integrations.css' + ) + ); + const codiconUri = webview.asWebviewUri( + Uri.joinPath( + this.extensionContext.extensionUri, + 'dist', + 'webviews', + 'webview-side', + 'react-common', + 'codicon', + 'codicon.css' + ) + ); + + return ` + + + + + + + + Deepnote Integrations + + +
+ + +`; + } + + private getNonce(): string { + let text = ''; + const possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; + for (let i = 0; i < 32; i++) { + text += possible.charAt(Math.floor(Math.random() * possible.length)); + } + return text; + } +} diff --git a/src/notebooks/deepnote/integrations/types.ts b/src/notebooks/deepnote/integrations/types.ts new file mode 100644 index 0000000000..2daa217ae1 --- /dev/null +++ b/src/notebooks/deepnote/integrations/types.ts @@ -0,0 +1,40 @@ +import { IntegrationConfig, IntegrationWithStatus } from './integrationTypes'; + +export const IIntegrationStorage = Symbol('IIntegrationStorage'); +export interface IIntegrationStorage { + getAll(): Promise; + get(integrationId: string): Promise; + save(config: IntegrationConfig): Promise; + delete(integrationId: string): Promise; + exists(integrationId: string): Promise; + clear(): Promise; +} + +export const IIntegrationDetector = Symbol('IIntegrationDetector'); +export interface IIntegrationDetector { + /** + * Detect all integrations used in the given project + */ + detectIntegrations(projectId: string): Promise>; + + /** + * Check if a project has any unconfigured integrations + */ + hasUnconfiguredIntegrations(projectId: string): Promise; +} + +export const IIntegrationWebviewProvider = Symbol('IIntegrationWebviewProvider'); +export interface IIntegrationWebviewProvider { + /** + * Show the integration management webview + */ + show(integrations: Map): Promise; +} + +export const IIntegrationManager = Symbol('IIntegrationManager'); +export interface IIntegrationManager { + /** + * Activate the integration manager by registering commands and event listeners + */ + activate(): void; +} diff --git a/src/notebooks/serviceRegistry.node.ts b/src/notebooks/serviceRegistry.node.ts index 6cb19391a5..9b7ed3a7c3 100644 --- a/src/notebooks/serviceRegistry.node.ts +++ b/src/notebooks/serviceRegistry.node.ts @@ -43,6 +43,16 @@ import { INotebookEditorProvider, INotebookPythonEnvironmentService } from './ty import { DeepnoteActivationService } from './deepnote/deepnoteActivationService'; import { DeepnoteNotebookManager } from './deepnote/deepnoteNotebookManager'; import { IDeepnoteNotebookManager } from './types'; +import { IntegrationStorage } from './deepnote/integrations/integrationStorage'; +import { IntegrationDetector } from './deepnote/integrations/integrationDetector'; +import { IntegrationManager } from './deepnote/integrations/integrationManager'; +import { IntegrationWebviewProvider } from './deepnote/integrations/integrationWebview'; +import { + IIntegrationDetector, + IIntegrationManager, + IIntegrationStorage, + IIntegrationWebviewProvider +} from './deepnote/integrations/types'; import { IDeepnoteToolkitInstaller, IDeepnoteServerStarter, @@ -128,6 +138,10 @@ export function registerTypes(serviceManager: IServiceManager, isDevMode: boolea DeepnoteActivationService ); serviceManager.addSingleton(IDeepnoteNotebookManager, DeepnoteNotebookManager); + serviceManager.addSingleton(IIntegrationStorage, IntegrationStorage); + serviceManager.addSingleton(IIntegrationDetector, IntegrationDetector); + serviceManager.addSingleton(IIntegrationWebviewProvider, IntegrationWebviewProvider); + serviceManager.addSingleton(IIntegrationManager, IntegrationManager); // Deepnote kernel services serviceManager.addSingleton(IDeepnoteToolkitInstaller, DeepnoteToolkitInstaller); diff --git a/src/notebooks/serviceRegistry.web.ts b/src/notebooks/serviceRegistry.web.ts index 9bfb894ced..8c1728f231 100644 --- a/src/notebooks/serviceRegistry.web.ts +++ b/src/notebooks/serviceRegistry.web.ts @@ -38,6 +38,16 @@ import { INotebookEditorProvider, INotebookPythonEnvironmentService } from './ty import { DeepnoteActivationService } from './deepnote/deepnoteActivationService'; import { DeepnoteNotebookManager } from './deepnote/deepnoteNotebookManager'; import { IDeepnoteNotebookManager } from './types'; +import { IntegrationStorage } from './deepnote/integrations/integrationStorage'; +import { IntegrationDetector } from './deepnote/integrations/integrationDetector'; +import { IntegrationManager } from './deepnote/integrations/integrationManager'; +import { IntegrationWebviewProvider } from './deepnote/integrations/integrationWebview'; +import { + IIntegrationDetector, + IIntegrationManager, + IIntegrationStorage, + IIntegrationWebviewProvider +} from './deepnote/integrations/types'; export function registerTypes(serviceManager: IServiceManager, isDevMode: boolean) { registerControllerTypes(serviceManager, isDevMode); @@ -92,6 +102,10 @@ export function registerTypes(serviceManager: IServiceManager, isDevMode: boolea DeepnoteActivationService ); serviceManager.addSingleton(IDeepnoteNotebookManager, DeepnoteNotebookManager); + serviceManager.addSingleton(IIntegrationStorage, IntegrationStorage); + serviceManager.addSingleton(IIntegrationDetector, IntegrationDetector); + serviceManager.addSingleton(IIntegrationWebviewProvider, IntegrationWebviewProvider); + serviceManager.addSingleton(IIntegrationManager, IntegrationManager); serviceManager.addSingleton(IExportBase, ExportBase); serviceManager.addSingleton(IFileConverter, FileConverter); diff --git a/src/notebooks/types.ts b/src/notebooks/types.ts index 377c27988a..21f5c92a34 100644 --- a/src/notebooks/types.ts +++ b/src/notebooks/types.ts @@ -4,6 +4,7 @@ import { NotebookDocument, NotebookEditor, Uri, type Event } from 'vscode'; import { Resource } from '../platform/common/types'; import type { EnvironmentPath } from '@vscode/python-extension'; +import { DeepnoteProject } from './deepnote/deepnoteTypes'; export interface IEmbedNotebookEditorProvider { findNotebookEditor(resource: Resource): NotebookEditor | undefined; @@ -27,10 +28,10 @@ export interface INotebookPythonEnvironmentService { export const IDeepnoteNotebookManager = Symbol('IDeepnoteNotebookManager'); export interface IDeepnoteNotebookManager { getCurrentNotebookId(projectId: string): string | undefined; - getOriginalProject(projectId: string): unknown | undefined; + getOriginalProject(projectId: string): DeepnoteProject | undefined; getTheSelectedNotebookForAProject(projectId: string): string | undefined; selectNotebookForProject(projectId: string, notebookId: string): void; - storeOriginalProject(projectId: string, project: unknown, notebookId: string): void; + storeOriginalProject(projectId: string, project: DeepnoteProject, notebookId: string): void; updateCurrentNotebookId(projectId: string, notebookId: string): void; hasInitNotebookBeenRun(projectId: string): boolean; markInitNotebookAsRun(projectId: string): void; diff --git a/src/platform/common/constants.ts b/src/platform/common/constants.ts index 711265499b..3217b95cc4 100644 --- a/src/platform/common/constants.ts +++ b/src/platform/common/constants.ts @@ -222,6 +222,7 @@ export namespace Commands { export const OpenDeepnoteNotebook = 'deepnote.openNotebook'; export const OpenDeepnoteFile = 'deepnote.openFile'; export const RevealInDeepnoteExplorer = 'deepnote.revealInExplorer'; + export const ManageIntegrations = 'deepnote.manageIntegrations'; export const ExportAsPythonScript = 'jupyter.exportAsPythonScript'; export const ExportToHTML = 'jupyter.exportToHTML'; export const ExportToPDF = 'jupyter.exportToPDF'; diff --git a/src/webviews/webview-side/integrations/BigQueryForm.tsx b/src/webviews/webview-side/integrations/BigQueryForm.tsx new file mode 100644 index 0000000000..627aa04c30 --- /dev/null +++ b/src/webviews/webview-side/integrations/BigQueryForm.tsx @@ -0,0 +1,135 @@ +import * as React from 'react'; +import { l10n } from 'vscode'; + +import { BigQueryIntegrationConfig } from './types'; + +export interface IBigQueryFormProps { + integrationId: string; + existingConfig: BigQueryIntegrationConfig | null; + onSave: (config: BigQueryIntegrationConfig) => void; + onCancel: () => void; +} + +export const BigQueryForm: React.FC = ({ integrationId, existingConfig, onSave, onCancel }) => { + const [name, setName] = React.useState(existingConfig?.name || ''); + const [projectId, setProjectId] = React.useState(existingConfig?.projectId || ''); + const [credentials, setCredentials] = React.useState(existingConfig?.credentials || ''); + const [credentialsError, setCredentialsError] = React.useState(null); + + // Update form fields when existingConfig changes + React.useEffect(() => { + if (existingConfig) { + setName(existingConfig.name || ''); + setProjectId(existingConfig.projectId || ''); + setCredentials(existingConfig.credentials || ''); + setCredentialsError(null); + } + }, [existingConfig]); + + const validateCredentials = (value: string): boolean => { + if (!value.trim()) { + setCredentialsError(l10n.t('Credentials are required')); + return false; + } + + try { + JSON.parse(value); + setCredentialsError(null); + return true; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Invalid JSON format'; + setCredentialsError(l10n.t('Invalid JSON: {0}', errorMessage)); + return false; + } + }; + + const handleCredentialsChange = (e: React.ChangeEvent) => { + const value = e.target.value; + setCredentials(value); + validateCredentials(value); + }; + + const handleSubmit = (e: React.FormEvent) => { + e.preventDefault(); + + // Validate credentials before submitting + if (!validateCredentials(credentials)) { + return; + } + + const config: BigQueryIntegrationConfig = { + id: integrationId, + name: name || l10n.t('Unnamed BigQuery Integration ({0})', integrationId), + type: 'bigquery', + projectId, + credentials + }; + + onSave(config); + }; + + return ( +
+
+ + setName(e.target.value)} + placeholder="My BigQuery Project" + autoComplete="off" + /> +
+ +
+ + setProjectId(e.target.value)} + placeholder="my-project-id" + autoComplete="off" + required + /> +
+ +
+ +