[Drizzle Kit]: Extend api (#4999)

* feat: Add external kit api to start Drizzle Studio

* feat: Separate API build process and fix ESM dynamic require handling

* fix: Update import paths for Relations in api.ts, pgImports.ts, and studio.ts

* dprint

* fix: Remove '-pooler' suffix from NEON_CONNECTION_STRING in Vercel client tests

* dprint

* chore: update version to 0.31.6 and add changelog for bug fixes

Author: RomanNabukhotnyi

changelogs/drizzle-kit/0.31.6.md

@@ -0,0 +1,3 @@
+### Bug fixes
+
+- [[BUG]: Importing drizzle-kit/api fails in ESM modules](https://github.com/drizzle-team/drizzle-orm/issues/2853)

drizzle-kit/build.ts

@@ -82,7 +82,7 @@ esbuild.buildSync({
 
 const main = async () => {
 	await tsup.build({
-		entryPoints: ['./src/index.ts', './src/api.ts'],
+		entryPoints: ['./src/index.ts'],
 		outDir: './dist',
 		external: ['bun:sqlite'],
 		splitting: false,
@@ -102,6 +102,39 @@ const main = async () => {
 		},
 	});
 
+	await tsup.build({
+		entryPoints: ['./src/api.ts'],
+		outDir: './dist',
+		external: ['bun:sqlite'],
+		splitting: false,
+		dts: true,
+		format: ['cjs', 'esm'],
+		banner: (ctx) => {
+			/**
+			 * fix dynamic require in ESM ("glob" -> "fs.realpath" requires 'fs' module)
+			 * @link https://github.com/drizzle-team/drizzle-orm/issues/2853
+			 */
+			if (ctx.format === 'esm') {
+				return {
+					js: "import { createRequire } from 'module'; const require = createRequire(import.meta.url);",
+				};
+			}
+			return undefined;
+		},
+		outExtension: (ctx) => {
+			if (ctx.format === 'cjs') {
+				return {
+					dts: '.d.ts',
+					js: '.js',
+				};
+			}
+			return {
+				dts: '.d.mts',
+				js: '.mjs',
+			};
+		},
+	});
+
 	const apiCjs = readFileSync('./dist/api.js', 'utf8').replace(/await import\(/g, 'require(');
 	writeFileSync('./dist/api.js', apiCjs);
 };

drizzle-kit/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "drizzle-kit",
-	"version": "0.31.5",
+	"version": "0.31.6",
 	"homepage": "https://orm.drizzle.team",
 	"keywords": [
 		"drizzle",

drizzle-kit/src/api.ts

@@ -1,8 +1,18 @@
+import type { PGlite } from '@electric-sql/pglite';
 import { randomUUID } from 'crypto';
+import { is } from 'drizzle-orm';
 import { LibSQLDatabase } from 'drizzle-orm/libsql';
+import { AnyMySqlTable, getTableConfig as mysqlTableConfig, MySqlTable } from 'drizzle-orm/mysql-core';
 import type { MySql2Database } from 'drizzle-orm/mysql2';
-import { PgDatabase } from 'drizzle-orm/pg-core';
+import { AnyPgTable, getTableConfig as pgTableConfig, PgDatabase, PgTable } from 'drizzle-orm/pg-core';
+import { Relations } from 'drizzle-orm/relations';
 import { SingleStoreDriverDatabase } from 'drizzle-orm/singlestore';
+import {
+	AnySingleStoreTable,
+	getTableConfig as singlestoreTableConfig,
+	SingleStoreTable,
+} from 'drizzle-orm/singlestore-core';
+import { AnySQLiteTable, SQLiteTable } from 'drizzle-orm/sqlite-core';
 import {
 	columnsResolver,
 	enumsResolver,
@@ -22,10 +32,13 @@ import { updateUpToV6 as upPgV6, updateUpToV7 as upPgV7 } from './cli/commands/p
 import { sqlitePushIntrospect } from './cli/commands/sqliteIntrospect';
 import { logSuggestionsAndReturn } from './cli/commands/sqlitePushUtils';
 import type { CasingType } from './cli/validations/common';
+import type { MysqlCredentials } from './cli/validations/mysql';
+import type { PostgresCredentials } from './cli/validations/postgres';
+import type { SingleStoreCredentials } from './cli/validations/singlestore';
+import type { SqliteCredentials } from './cli/validations/sqlite';
 import { getTablesFilterByExtensions } from './extensions/getTablesFilterByExtensions';
 import { originUUID } from './global';
 import type { Config } from './index';
-import { fillPgSnapshot } from './migrationPreparator';
 import { MySqlSchema as MySQLSchemaKit, mysqlSchema, squashMysqlScheme } from './serializer/mysqlSchema';
 import { generateMySqlSnapshot } from './serializer/mysqlSerializer';
 import { prepareFromExports } from './serializer/pgImports';
@@ -39,7 +52,9 @@ import {
 import { generateSingleStoreSnapshot } from './serializer/singlestoreSerializer';
 import { SQLiteSchema as SQLiteSchemaKit, sqliteSchema, squashSqliteScheme } from './serializer/sqliteSchema';
 import { generateSqliteSnapshot } from './serializer/sqliteSerializer';
+import type { Setup } from './serializer/studio';
 import type { DB, SQLiteDB } from './utils';
+import { certs } from './utils/certs';
 export type DrizzleSnapshotJSON = PgSchemaKit;
 export type DrizzleSQLiteSnapshotJSON = SQLiteSchemaKit;
 export type DrizzleMySQLSnapshotJSON = MySQLSchemaKit;
@@ -68,11 +83,11 @@ export const generateDrizzleJson = (
 		schemaFilters,
 	);
 
-	return fillPgSnapshot({
-		serialized: snapshot,
+	return {
+		...snapshot,
 		id,
-		idPrev: prevId ?? originUUID,
-	});
+		prevId: prevId ?? originUUID,
+	};
 };
 
 export const generateMigration = async (
@@ -171,6 +186,39 @@ export const pushSchema = async (
 	};
 };
 
+export const startStudioPostgresServer = async (
+	imports: Record<string, unknown>,
+	credentials: PostgresCredentials | {
+		driver: 'pglite';
+		client: PGlite;
+	},
+	options?: {
+		host?: string;
+		port?: number;
+		casing?: CasingType;
+	},
+) => {
+	const { drizzleForPostgres } = await import('./serializer/studio');
+
+	const pgSchema: Record<string, Record<string, AnyPgTable>> = {};
+	const relations: Record<string, Relations> = {};
+
+	Object.entries(imports).forEach(([k, t]) => {
+		if (is(t, PgTable)) {
+			const schema = pgTableConfig(t).schema || 'public';
+			pgSchema[schema] = pgSchema[schema] || {};
+			pgSchema[schema][k] = t;
+		}
+
+		if (is(t, Relations)) {
+			relations[k] = t;
+		}
+	});
+
+	const setup = await drizzleForPostgres(credentials, pgSchema, relations, [], options?.casing);
+	await startServerFromSetup(setup, options);
+};
+
 // SQLite
 
 export const generateSQLiteDrizzleJson = async (
@@ -277,6 +325,36 @@ export const pushSQLiteSchema = async (
 	};
 };
 
+export const startStudioSQLiteServer = async (
+	imports: Record<string, unknown>,
+	credentials: SqliteCredentials,
+	options?: {
+		host?: string;
+		port?: number;
+		casing?: CasingType;
+	},
+) => {
+	const { drizzleForSQLite } = await import('./serializer/studio');
+
+	const sqliteSchema: Record<string, Record<string, AnySQLiteTable>> = {};
+	const relations: Record<string, Relations> = {};
+
+	Object.entries(imports).forEach(([k, t]) => {
+		if (is(t, SQLiteTable)) {
+			const schema = 'public'; // sqlite does not have schemas
+			sqliteSchema[schema] = sqliteSchema[schema] || {};
+			sqliteSchema[schema][k] = t;
+		}
+
+		if (is(t, Relations)) {
+			relations[k] = t;
+		}
+	});
+
+	const setup = await drizzleForSQLite(credentials, sqliteSchema, relations, [], options?.casing);
+	await startServerFromSetup(setup, options);
+};
+
 // MySQL
 
 export const generateMySQLDrizzleJson = async (
@@ -382,6 +460,36 @@ export const pushMySQLSchema = async (
 	};
 };
 
+export const startStudioMySQLServer = async (
+	imports: Record<string, unknown>,
+	credentials: MysqlCredentials,
+	options?: {
+		host?: string;
+		port?: number;
+		casing?: CasingType;
+	},
+) => {
+	const { drizzleForMySQL } = await import('./serializer/studio');
+
+	const mysqlSchema: Record<string, Record<string, AnyMySqlTable>> = {};
+	const relations: Record<string, Relations> = {};
+
+	Object.entries(imports).forEach(([k, t]) => {
+		if (is(t, MySqlTable)) {
+			const schema = mysqlTableConfig(t).schema || 'public';
+			mysqlSchema[schema] = mysqlSchema[schema] || {};
+			mysqlSchema[schema][k] = t;
+		}
+
+		if (is(t, Relations)) {
+			relations[k] = t;
+		}
+	});
+
+	const setup = await drizzleForMySQL(credentials, mysqlSchema, relations, [], options?.casing);
+	await startServerFromSetup(setup, options);
+};
+
 // SingleStore
 
 export const generateSingleStoreDrizzleJson = async (
@@ -489,6 +597,62 @@ export const pushSingleStoreSchema = async (
 	};
 };
 
+export const startStudioSingleStoreServer = async (
+	imports: Record<string, unknown>,
+	credentials: SingleStoreCredentials,
+	options?: {
+		host?: string;
+		port?: number;
+		casing?: CasingType;
+	},
+) => {
+	const { drizzleForSingleStore } = await import('./serializer/studio');
+
+	const singleStoreSchema: Record<string, Record<string, AnySingleStoreTable>> = {};
+	const relations: Record<string, Relations> = {};
+
+	Object.entries(imports).forEach(([k, t]) => {
+		if (is(t, SingleStoreTable)) {
+			const schema = singlestoreTableConfig(t).schema || 'public';
+			singleStoreSchema[schema] = singleStoreSchema[schema] || {};
+			singleStoreSchema[schema][k] = t;
+		}
+
+		if (is(t, Relations)) {
+			relations[k] = t;
+		}
+	});
+
+	const setup = await drizzleForSingleStore(credentials, singleStoreSchema, relations, [], options?.casing);
+	await startServerFromSetup(setup, options);
+};
+
+const startServerFromSetup = async (setup: Setup, options?: {
+	host?: string;
+	port?: number;
+}) => {
+	const { prepareServer } = await import('./serializer/studio');
+
+	const server = await prepareServer(setup);
+
+	const host = options?.host || '127.0.0.1';
+	const port = options?.port || 4983;
+	const { key, cert } = (await certs()) || {};
+	server.start({
+		host,
+		port,
+		key,
+		cert,
+		cb: (err) => {
+			if (err) {
+				console.error(err);
+			} else {
+				console.log(`Studio is running at ${key ? 'https' : 'http'}://${host}:${port}`);
+			}
+		},
+	});
+};
+
 export const upPgSnapshot = (snapshot: Record<string, unknown>) => {
 	if (snapshot.version === '5') {
 		return upPgV7(upPgV6(snapshot));

drizzle-kit/src/cli/connections.ts

@@ -1,3 +1,4 @@
+import type { PGlite } from '@electric-sql/pglite';
 import type { AwsDataApiPgQueryResult, AwsDataApiSessionOptions } from 'drizzle-orm/aws-data-api/pg';
 import type { MigrationConfig } from 'drizzle-orm/migrator';
 import type { PreparedQueryConfig } from 'drizzle-orm/pg-core';
@@ -24,7 +25,10 @@ import { SingleStoreCredentials } from './validations/singlestore';
 import type { SqliteCredentials } from './validations/sqlite';
 
 export const preparePostgresDB = async (
-	credentials: PostgresCredentials,
+	credentials: PostgresCredentials | {
+		driver: 'pglite';
+		client: PGlite;
+	},
 ): Promise<
 	DB & {
 		packageName:
@@ -123,7 +127,7 @@ export const preparePostgresDB = async (
 			const { drizzle } = await import('drizzle-orm/pglite');
 			const { migrate } = await import('drizzle-orm/pglite/migrator');
 
-			const pglite = new PGlite(normalisePGliteUrl(credentials.url));
+			const pglite = 'client' in credentials ? credentials.client : new PGlite(normalisePGliteUrl(credentials.url));
 			await pglite.waitReady;
 			const drzl = drizzle(pglite);
 			const migrateFn = async (config: MigrationConfig) => {

drizzle-kit/src/migrationPreparator.ts

@@ -3,7 +3,7 @@ import fs from 'fs';
 import { CasingType } from './cli/validations/common';
 import { serializeMySql, serializePg, serializeSingleStore, serializeSQLite } from './serializer';
 import { dryMySql, MySqlSchema, mysqlSchema } from './serializer/mysqlSchema';
-import { dryPg, PgSchema, pgSchema, PgSchemaInternal } from './serializer/pgSchema';
+import { dryPg, PgSchema, pgSchema } from './serializer/pgSchema';
 import { drySingleStore, SingleStoreSchema, singlestoreSchema } from './serializer/singlestoreSchema';
 import { drySQLite, SQLiteSchema, sqliteSchema } from './serializer/sqliteSchema';
 
@@ -168,19 +168,6 @@ export const prepareSqliteMigrationSnapshot = async (
 	return { prev: prevSnapshot, cur: result, custom };
 };
 
-export const fillPgSnapshot = ({
-	serialized,
-	id,
-	idPrev,
-}: {
-	serialized: PgSchemaInternal;
-	id: string;
-	idPrev: string;
-}): PgSchema => {
-	// const id = randomUUID();
-	return { id, prevId: idPrev, ...serialized };
-};
-
 export const preparePgMigrationSnapshot = async (
 	snapshots: string[],
 	schemaPath: string | string[],
@@ -199,7 +186,11 @@ export const preparePgMigrationSnapshot = async (
 	const { id: _ignoredId, prevId: _ignoredPrevId, ...prevRest } = prevSnapshot;
 
 	// that's for custom migrations, when we need new IDs, but old snapshot
-	const custom: PgSchema = fillPgSnapshot({ serialized: prevRest, id, idPrev });
+	const custom: PgSchema = {
+		id,
+		prevId: idPrev,
+		...prevRest,
+	};
 
 	return { prev: prevSnapshot, cur: result, custom };
 };