diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index ae5952f726af9..af7f124664ab2 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -244,6 +244,9 @@ const variables: Record any> = { nestedFoldersDelimiter: () => get('CUBEJS_NESTED_FOLDERS_DELIMITER') .default('') .asString(), + preciseDecimalInCubestore: () => get('CUBEJS_DB_PRECISE_DECIMAL_IN_CUBESTORE') + .default('false') + .asBoolStrict(), /** **************************************************************** * Common db options * diff --git a/packages/cubejs-base-driver/src/BaseDriver.ts b/packages/cubejs-base-driver/src/BaseDriver.ts index c89b1637eb13f..d566111d76a1f 100644 --- a/packages/cubejs-base-driver/src/BaseDriver.ts +++ b/packages/cubejs-base-driver/src/BaseDriver.ts @@ -640,14 +640,16 @@ export abstract class BaseDriver implements DriverInterface { `SELECT columns.column_name as ${this.quoteIdentifier('column_name')}, columns.table_name as ${this.quoteIdentifier('table_name')}, columns.table_schema as ${this.quoteIdentifier('table_schema')}, - columns.data_type as ${this.quoteIdentifier('data_type')} + columns.data_type as ${this.quoteIdentifier('data_type')}, + columns.numeric_precision as ${this.quoteIdentifier('numeric_precision')}, + columns.numeric_scale as ${this.quoteIdentifier('numeric_scale')} FROM information_schema.columns WHERE table_name = ${this.param(0)} AND table_schema = ${this.param(1)} ${getEnv('fetchColumnsByOrdinalPosition') ? 'ORDER BY columns.ordinal_position' : ''}`, [name, schema] ); - return columns.map(c => ({ name: c.column_name, type: this.toGenericType(c.data_type) })); + return columns.map(c => ({ name: c.column_name, type: this.toGenericType(c.data_type, c.numeric_precision, c.numeric_scale) })); } // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -673,8 +675,14 @@ export abstract class BaseDriver implements DriverInterface { return `CREATE TABLE ${quotedTableName} (${columnNames.join(', ')})`; } - protected toGenericType(columnType: string): string { - return DbTypeToGenericType[columnType.toLowerCase()] || columnType; + protected toGenericType(columnType: string, precision?: number | null, scale?: number | null): string { + const genericType = DbTypeToGenericType[columnType.toLowerCase()] || columnType; + + if (genericType === 'decimal' && precision && scale && getEnv('CUBEJS_DB_PRECISE_DECIMAL_IN_CUBESTORE')) { + return `decimal(${precision}, ${scale})`; + } + + return genericType; } protected fromGenericType(columnType: string): string { diff --git a/packages/cubejs-base-driver/src/driver.interface.ts b/packages/cubejs-base-driver/src/driver.interface.ts index 213774ebe000a..5fa35e06b3e7d 100644 --- a/packages/cubejs-base-driver/src/driver.interface.ts +++ b/packages/cubejs-base-driver/src/driver.interface.ts @@ -19,6 +19,10 @@ export interface TableColumnQueryResult { column_name: string; // eslint-disable-next-line camelcase data_type: GenericDataBaseType; + // eslint-disable-next-line camelcase + numeric_precision?: number | null; + // eslint-disable-next-line camelcase + numeric_scale?: number | null; attributes?: string[] // eslint-disable-next-line camelcase foreign_keys?: ForeignKey[] diff --git a/packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts b/packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts index 395da8d5f98cb..df6b9b6154616 100644 --- a/packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts +++ b/packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts @@ -401,7 +401,7 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface { return emptyKeys.filter(key => key !== 'keyId' && key !== 'secretKey'); } } - + return emptyKeys; } @@ -421,7 +421,7 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface { const emptyKeys = Object.keys(exportBucket) .filter((key: string) => exportBucket[key] === undefined); const keysToValidate = this.getRequiredExportBucketKeys(exportBucket, emptyKeys); - + if (keysToValidate.length) { throw new Error( `Unsupported configuration exportBucket, some configuration keys are empty: ${keysToValidate.join(',')}` @@ -695,11 +695,13 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface { /** * Returns an array of table fields meta info. */ - public async tableColumnTypes(table: string) { + public override async tableColumnTypes(table: string) { const [schema, name] = table.split('.'); const columns = await this.query<{ COLUMN_NAME: string, - DATA_TYPE: string + DATA_TYPE: string, + NUMERIC_PRECISION: number | null, + NUMERIC_SCALE: number | null }[]>( `SELECT COLUMNS.COLUMN_NAME, CASE @@ -708,7 +710,9 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface { COLUMNS.DATA_TYPE = 'NUMBER' THEN 'int' ELSE COLUMNS.DATA_TYPE - END as DATA_TYPE + END as DATA_TYPE, + COLUMNS.NUMERIC_PRECISION, + COLUMNS.NUMERIC_SCALE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = ${this.param(0)} AND @@ -718,7 +722,7 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface { ); return columns.map(c => ({ name: c.COLUMN_NAME, - type: this.toGenericType(c.DATA_TYPE), + type: this.toGenericType(c.DATA_TYPE, c.NUMERIC_PRECISION, c.NUMERIC_SCALE), })); } @@ -941,13 +945,15 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface { type: '', }; if (column.isNumber()) { - // @ts-ignore - if (column.getScale() === 0) { + const precision = column.getPrecision(); + const scale = column.getScale(); + + if (scale === 0) { type.type = 'int'; - } else if (column.getScale() && column.getScale() <= 10) { + } else if (precision && scale && scale <= 10) { type.type = 'decimal'; } else { - type.type = this.toGenericType(column.getType()); + type.type = this.toGenericType(column.getType(), precision, scale); } } else { type.type = this.toGenericType(column.getType()); @@ -1026,8 +1032,14 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface { } } - public toGenericType(columnType: string) { - return SnowflakeToGenericType[columnType.toLowerCase()] || super.toGenericType(columnType); + public override toGenericType(columnType: string, precision?: number | null, scale?: number | null) { + const genericType = SnowflakeToGenericType[columnType.toLowerCase()] || super.toGenericType(columnType); + + if (genericType === 'decimal' && precision && scale && getEnv('CUBEJS_DB_PRECISE_DECIMAL_IN_CUBESTORE')) { + return `decimal(${precision}, ${scale})`; + } + + return genericType; } public async getTablesQuery(schemaName: string) {