refactor: rename warehouse to warehouse long table and extract some logic

moonrailgun · moonrailgun · commit 476553b67c5e · 2025-08-08T21:01:43.000+08:00
diff --git a/src/server/model/insights/__snapshots__/warehouse.spec.ts.snap b/src/server/model/insights/__snapshots__/warehouse.spec.ts.snap
diff --git a/src/server/model/insights/index.ts b/src/server/model/insights/index.ts
@@ -8,7 +8,7 @@ import { insightsSurvey } from './survey.js';
 import { insightsAIGateway } from './aiGateway.js';
 import { compact, omit } from 'lodash-es';
 import { prisma } from '../_client.js';
-import { insightsWarehouse } from './warehouse.js';
+import { insightsWarehouse } from './warehouse/longTable.js';
 import { INIT_WORKSPACE_ID } from '../../utils/const.js';
 
 export function queryInsight(
diff --git a/src/server/model/insights/warehouse/__snapshots__/longTable.spec.ts.snap b/src/server/model/insights/warehouse/__snapshots__/longTable.spec.ts.snap
@@ -0,0 +1,21 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`WarehouseInsightsSqlBuilder > default > sql 1`] = `
+"select
+        DATE_FORMAT(CONVERT_TZ(FROM_UNIXTIME("events"."event_timestamp" / 1000), '+00:00', 'UTC'), '%Y-%m-%d') date , count(1) as "$all_event"
+      from "events" 
+      where "events"."event_timestamp" BETWEEN '1753977600000' AND '1754064000000' AND (1 = 1)
+      group by 1
+      order by 1 desc
+      limit 1000"
+`;
+
+exports[`WarehouseInsightsSqlBuilder > with filter > sql 1`] = `
+"select
+        DATE_FORMAT(CONVERT_TZ(FROM_UNIXTIME("events"."event_timestamp" / 1000), '+00:00', 'UTC'), '%Y-%m-%d') date , count(1) as "$all_event"
+      from "events" INNER JOIN "event_parameters" ON "events"."event_name" = "event_parameters"."event_name" AND "event_parameters"."event_param_value" = 'value'
+      where "events"."event_timestamp" BETWEEN '1753977600000' AND '1754064000000' AND (1 = 1)
+      group by 1
+      order by 1 desc
+      limit 1000"
+`;
diff --git a/src/server/model/insights/warehouse/longTable.spec.ts b/src/server/model/insights/warehouse/longTable.spec.ts
@@ -1,8 +1,8 @@
 import { beforeAll, describe, expect, test } from 'vitest';
-import { WarehouseInsightsSqlBuilder } from './warehouse.js';
-import { unwrapSQL } from '../../utils/prisma.js';
+import { WarehouseLongTableInsightsSqlBuilder } from './longTable.js';
+import { unwrapSQL } from '../../../utils/prisma.js';
 import dayjs from 'dayjs';
-import { env } from '../../utils/env.js';
+import { env } from '../../../utils/env.js';
 
 describe('WarehouseInsightsSqlBuilder', () => {
   const insightId = 'test'; // application name
@@ -29,7 +29,7 @@ describe('WarehouseInsightsSqlBuilder', () => {
   });
 
   test('default', () => {
-    const builder = new WarehouseInsightsSqlBuilder(
+    const builder = new WarehouseLongTableInsightsSqlBuilder(
       {
         insightId,
         insightType,
@@ -58,7 +58,7 @@ describe('WarehouseInsightsSqlBuilder', () => {
   });
 
   test('with filter', () => {
-    const builder = new WarehouseInsightsSqlBuilder(
+    const builder = new WarehouseLongTableInsightsSqlBuilder(
       {
         insightId,
         insightType,
diff --git a/src/server/model/insights/warehouse/longTable.ts b/src/server/model/insights/warehouse/longTable.ts
@@ -3,85 +3,22 @@
  */
 
 import { Prisma } from '@prisma/client';
-import { InsightsSqlBuilder } from './shared.js';
-import { createPool, Pool } from 'mysql2/promise';
-import { insightsQuerySchema } from '../../utils/schema.js';
+import { InsightsSqlBuilder } from '../shared.js';
+import { insightsQuerySchema } from '../../../utils/schema.js';
 import { z } from 'zod';
 import dayjs from 'dayjs';
-import { processGroupedTimeSeriesData } from './utils.js';
-import { env } from '../../utils/env.js';
+import { processGroupedTimeSeriesData } from '../utils.js';
 import { FilterInfoType, FilterInfoValue } from '@tianji/shared';
 import { compact, get } from 'lodash-es';
-
-// MySQL date formats for different time units
-const MYSQL_DATE_FORMATS = {
-  minute: '%Y-%m-%d %H:%i:00',
-  hour: '%Y-%m-%d %H:00:00',
-  day: '%Y-%m-%d',
-  month: '%Y-%m-01',
-  year: '%Y-01-01',
-} as const;
-
-const dateTypeSchema = z.enum([
-  'timestamp', // for example: 1739203200
-  'timestampMs', // for example: 1739203200000
-  'date', // for example: 2025-08-01
-  'datetime', // for example: 2025-08-01 00:00:00
-]);
-
-export const warehouseInsightsApplicationSchema = z.object({
-  name: z.string(),
-  type: z.enum(['longTable', 'wideTable']).default('longTable'), // long table or wide table, TODO: implement wide table support
-  eventTable: z.object({
-    name: z.string(),
-    eventNameField: z.string(),
-    createdAtField: z.string(),
-    createdAtFieldType: dateTypeSchema.default('timestampMs'),
-    dateBasedCreatedAtField: z.string().optional(), // for improve performance, treat as date type
-  }),
-  eventParametersTable: z.object({
-    name: z.string(),
-    eventNameField: z.string(),
-    paramsNameField: z.string(),
-    paramsValueField: z.string(),
-    paramsValueNumberField: z.string().optional(),
-    paramsValueStringField: z.string().optional(),
-    paramsValueDateField: z.string().optional(),
-    createdAtField: z.string(),
-    createdAtFieldType: dateTypeSchema.default('timestampMs'),
-    dateBasedCreatedAtField: z.string().optional(), // for improve performance, treat as date type
-  }),
-});
-
-export type WarehouseInsightsApplication = z.infer<
-  typeof warehouseInsightsApplicationSchema
->;
-
-let applications: WarehouseInsightsApplication[] | null = null;
-export function getWarehouseApplications() {
-  if (!applications) {
-    applications = warehouseInsightsApplicationSchema
-      .array()
-      .parse(JSON.parse(env.insights.warehouse.applicationsJson || '[]'));
-  }
-
-  return applications;
-}
-
-let connection: Pool | null = null;
-export function getWarehouseConnection() {
-  if (!env.insights.warehouse.enable || !env.insights.warehouse.url) {
-    throw new Error('Warehouse is not enabled');
-  }
-
-  if (!connection) {
-    connection = createPool(env.insights.warehouse.url);
-  }
-
-  return connection;
-}
-
-export class WarehouseInsightsSqlBuilder extends InsightsSqlBuilder {
+import {
+  dateTypeSchema,
+  getWarehouseApplications,
+  getWarehouseConnection,
+  MYSQL_DATE_FORMATS,
+  WarehouseInsightsApplication,
+} from './utils.js';
+
+export class WarehouseLongTableInsightsSqlBuilder extends InsightsSqlBuilder {
   getApplication(): WarehouseInsightsApplication {
     const name = this.query.insightId;
 
@@ -430,7 +367,7 @@ export async function insightsWarehouse(
   query: z.infer<typeof insightsQuerySchema>,
   context: { timezone: string }
 ) {
-  const builder = new WarehouseInsightsSqlBuilder(query, context);
+  const builder = new WarehouseLongTableInsightsSqlBuilder(query, context);
   const sql = builder.build();
 
   const data = await builder.executeQuery(sql);
diff --git a/src/server/model/insights/warehouse/utils.ts b/src/server/model/insights/warehouse/utils.ts
@@ -0,0 +1,71 @@
+import { z } from 'zod';
+import { createPool, Pool } from 'mysql2/promise';
+import { env } from '../../../utils/env.js';
+
+// MySQL date formats for different time units
+export const MYSQL_DATE_FORMATS = {
+  minute: '%Y-%m-%d %H:%i:00',
+  hour: '%Y-%m-%d %H:00:00',
+  day: '%Y-%m-%d',
+  month: '%Y-%m-01',
+  year: '%Y-01-01',
+} as const;
+
+export const dateTypeSchema = z.enum([
+  'timestamp', // for example: 1739203200
+  'timestampMs', // for example: 1739203200000
+  'date', // for example: 2025-08-01
+  'datetime', // for example: 2025-08-01 00:00:00
+]);
+
+export const warehouseInsightsApplicationSchema = z.object({
+  name: z.string(),
+  type: z.enum(['longTable', 'wideTable']).default('longTable'), // long table or wide table, TODO: implement wide table support
+  eventTable: z.object({
+    name: z.string(),
+    eventNameField: z.string(),
+    createdAtField: z.string(),
+    createdAtFieldType: dateTypeSchema.default('timestampMs'),
+    dateBasedCreatedAtField: z.string().optional(), // for improve performance, treat as date type
+  }),
+  eventParametersTable: z.object({
+    name: z.string(),
+    eventNameField: z.string(),
+    paramsNameField: z.string(),
+    paramsValueField: z.string(),
+    paramsValueNumberField: z.string().optional(),
+    paramsValueStringField: z.string().optional(),
+    paramsValueDateField: z.string().optional(),
+    createdAtField: z.string(),
+    createdAtFieldType: dateTypeSchema.default('timestampMs'),
+    dateBasedCreatedAtField: z.string().optional(), // for improve performance, treat as date type
+  }),
+});
+
+export type WarehouseInsightsApplication = z.infer<
+  typeof warehouseInsightsApplicationSchema
+>;
+
+let applications: WarehouseInsightsApplication[] | null = null;
+export function getWarehouseApplications() {
+  if (!applications) {
+    applications = warehouseInsightsApplicationSchema
+      .array()
+      .parse(JSON.parse(env.insights.warehouse.applicationsJson || '[]'));
+  }
+
+  return applications;
+}
+
+let connection: Pool | null = null;
+export function getWarehouseConnection() {
+  if (!env.insights.warehouse.enable || !env.insights.warehouse.url) {
+    throw new Error('Warehouse is not enabled');
+  }
+
+  if (!connection) {
+    connection = createPool(env.insights.warehouse.url);
+  }
+
+  return connection;
+}
diff --git a/src/server/trpc/routers/insights.ts b/src/server/trpc/routers/insights.ts
@@ -12,10 +12,10 @@ import { queryEvents, queryInsight } from '../../model/insights/index.js';
 import { insightsSurveyBuiltinFields } from '../../model/insights/utils.js';
 import { uniq } from 'lodash-es';
 import {
-  getWarehouseApplications,
   insightsWarehouseEvents,
   insightsWarehouseFilterParams,
-} from '../../model/insights/warehouse.js';
+} from '../../model/insights/warehouse/longTable.js';
+import { getWarehouseApplications } from '../../model/insights/warehouse/utils.js';
 
 export const insightsRouter = router({
   query: workspaceProcedure
