perf(schema-compiler): Add Server Core shared caches for model compilation (#10144)

* chore(server-core): move driver deps to ts

* move logger to ts

* move CompilerApi to ts

# Conflicts:
#	packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts

* add top compile cache in the server core

* fix tsc errors

Author: KSDaemon

packages/cubejs-api-gateway/src/interfaces.ts

@@ -21,6 +21,7 @@ import {
   Query,
   NormalizedQueryFilter,
   NormalizedQuery,
+  MemberExpression,
 } from './types/query';
 
 import {
@@ -69,6 +70,7 @@ export {
   Query,
   NormalizedQueryFilter,
   NormalizedQuery,
+  MemberExpression,
   JWTOptions,
   CheckAuthFn,
   CheckSQLAuthSuccessResponse,

packages/cubejs-api-gateway/src/types/query.ts

@@ -146,11 +146,10 @@ interface Query {
   cache?: CacheMode; // Used in public interface
   ungrouped?: boolean;
   responseFormat?: ResultType;
-
   // TODO incoming query, query with parsed exprs and query with evaluated exprs are all different types
-  subqueryJoins?: Array<SubqueryJoins>,
-
-  joinHints?: Array<JoinHint>
+  subqueryJoins?: Array<SubqueryJoins>;
+  joinHints?: Array<JoinHint>;
+  requestId?: string;
 }
 
 /**

packages/cubejs-schema-compiler/src/adapter/index.ts

@@ -18,6 +18,8 @@ export * from './PostgresQuery';
 export * from './MssqlQuery';
 export * from './PrestodbQuery';
 
+export { PreAggregationReferences } from '../compiler/CubeEvaluator';
+
 // Candidates to move from this package to drivers packages
 // export * from './RedshiftQuery';
 // export * from './SnowflakeQuery';

packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts

@@ -9,7 +9,8 @@ import {
   JoinDefinition,
   PreAggregationDefinition,
   PreAggregationDefinitionRollup,
-  type ToString
+  type ToString,
+  ViewIncludedMember
 } from './CubeSymbols';
 import { UserError } from './UserError';
 import { BaseQuery, PreAggregationDefinitionExtended } from '../adapter';
@@ -100,7 +101,7 @@ export type PreAggregationReferences = {
 export type PreAggregationInfo = {
   id: string,
   preAggregationName: string,
-  preAggregation: unknown,
+  preAggregation: any,
   cube: string,
   references: PreAggregationReferences,
   refreshKey: unknown,
@@ -124,6 +125,7 @@ export type EvaluatedFolder = {
 };
 
 export type EvaluatedCube = {
+  name: string;
   measures: Record<string, MeasureDefinition>;
   dimensions: Record<string, DimensionDefinition>;
   segments: Record<string, SegmentDefinition>;
@@ -136,6 +138,8 @@ export type EvaluatedCube = {
   sql?: (...args: any[]) => string;
   sqlTable?: (...args: any[]) => string;
   accessPolicy?: AccessPolicyDefinition[];
+  isView?: boolean;
+  includedMembers?: ViewIncludedMember[];
 };
 
 export class CubeEvaluator extends CubeSymbols {

packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts

@@ -133,6 +133,9 @@ export type AccessPolicyDefinition = {
     includesMembers?: string[];
     excludesMembers?: string[];
   };
+  conditions?: {
+    if: Function;
+  }[]
 };
 
 export type ViewIncludedMember = {
@@ -233,7 +236,7 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface
 
   private builtCubes: Record<string, CubeDefinitionExtended>;
 
-  private cubeDefinitions: Record<string, CubeDefinition>;
+  public cubeDefinitions: Record<string, CubeDefinition>;
 
   private funcArgumentsValues: Record<string, string[]>;
 
@@ -925,7 +928,7 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface
    * resolveSymbolsCall are sync. Async support may be added later with deeper
    * refactoring.
    */
-  protected evaluateContextFunction(cube: any, contextFn: any, context: any = {}) {
+  public evaluateContextFunction(cube: any, contextFn: any, context: any = {}) {
     return this.resolveSymbolsCall(contextFn, (name: string) => {
       const resolvedSymbol = this.resolveSymbol(cube, name);
       if (resolvedSymbol) {

packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts

@@ -135,7 +135,7 @@ export class CubeToMetaTransformer implements CompilerInterface {
 
   private readonly cubeSymbols: CubeEvaluator;
 
-  private readonly cubeEvaluator: CubeEvaluator;
+  public readonly cubeEvaluator: CubeEvaluator;
 
   private readonly contextEvaluator: ContextEvaluator;
 

packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts

@@ -45,7 +45,18 @@ export interface CompilerInterface {
   compile: (cubes: any[], errorReporter: ErrorReporter) => void;
 }
 
-export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareCompilerOptions = {}) => {
+export type Compiler = {
+    compiler: DataSchemaCompiler;
+    metaTransformer: CubeToMetaTransformer;
+    cubeEvaluator: CubeEvaluator;
+    contextEvaluator: ContextEvaluator;
+    joinGraph: JoinGraph;
+    compilerCache: CompilerCache;
+    headCommitId?: string;
+    compilerId: string;
+};
+
+export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareCompilerOptions = {}): Compiler => {
   const nativeInstance = options.nativeInstance || new NativeInstance();
   const cubeDictionary = new CubeDictionary();
   const cubeSymbols = new CubeSymbols();
@@ -118,9 +129,8 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp
   };
 };
 
-export const compile = (repo: SchemaFileRepository, options?: PrepareCompilerOptions) => {
+export const compile = async (repo: SchemaFileRepository, options?: PrepareCompilerOptions): Promise<Compiler> => {
   const compilers = prepareCompiler(repo, options);
-  return compilers.compiler.compile().then(
-    () => compilers
-  );
+  await compilers.compiler.compile();
+  return compilers;
 };

packages/cubejs-schema-compiler/src/compiler/index.ts

@@ -1,3 +1,13 @@
 export * from './PrepareCompiler';
 export * from './UserError';
 export * from './converters';
+export {
+  CubeDefinition,
+  AccessPolicyDefinition,
+  ViewIncludedMember,
+} from './CubeSymbols';
+export {
+  PreAggregationFilters,
+  PreAggregationInfo,
+  EvaluatedCube,
+} from './CubeEvaluator';

packages/cubejs-server-core/index.js

@@ -3,7 +3,7 @@ const { CubejsServerCore } = require('./dist/src/core/server');
 
 /**
  * After 5 years working with TypeScript, now I know
- * that commonjs and nodejs require is not compatibility with using export default
+ * that commonjs and Node.js require is not compatible with using export default
  */
 const toExport = CubejsServerCore;
 

packages/cubejs-server-core/package.json

@@ -41,6 +41,7 @@
     "codesandbox-import-utils": "^2.1.12",
     "cross-spawn": "^7.0.1",
     "fs-extra": "^8.1.0",
+    "graphql": "^15.8.0",
     "http-proxy-agent": "^7.0.2",
     "https-proxy-agent": "^7.0.6",
     "is-docker": "^2.1.1",