Skip to content

Commit 13e3fb8

Browse files
moonrailgunmoonrailgun
committed
feat: add insights event handling with warehouse wide table application
1 parent 37b0640 commit 13e3fb8

File tree

9 files changed

+200
-57
lines changed

9 files changed

+200
-57
lines changed

‎src/client/routes/insights/events.tsx‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { createFileRoute } from '@tanstack/react-router';
22
import { routeAuthBeforeLoad } from '@/utils/route';
33
import { useCurrentWorkspaceId } from '@/store/user';
4-
import { useInsightsStore } from '@/store/insights';
4+
import { InsightType, useInsightsStore } from '@/store/insights';
55
import { trpc } from '@/api/trpc';
66
import dayjs, { Dayjs } from 'dayjs';
77
import { useMemo, useState } from 'react';
@@ -29,6 +29,7 @@ import Identicon from 'react-identicons';
2929
import { getUserTimezone } from '@/api/model/user';
3030
import { LuChevronDown } from 'react-icons/lu';
3131
import { cn } from '@/utils/style';
32+
import { get } from 'lodash-es';
3233

3334
export const Route = createFileRoute('/insights/events')({
3435
beforeLoad: routeAuthBeforeLoad,
@@ -44,6 +45,8 @@ function PageComponent() {
4445
state.currentFilters.filter((f): f is NonNullable<typeof f> => !!f)
4546
);
4647
const { data: websites = [] } = trpc.website.all.useQuery({ workspaceId });
48+
const { data: warehouseApplicationIds = [] } =
49+
trpc.insights.warehouseApplications.useQuery({ workspaceId });
4750

4851
// Date range state
4952
const [dateRange, setDateRange] = useState<[Dayjs, Dayjs]>([
@@ -55,7 +58,11 @@ function PageComponent() {
5558
const [jsonMode, setJsonMode] = useState(false);
5659

5760
const handleValueChange = useEvent((value: string) => {
58-
const type = 'website';
61+
let type: InsightType = 'website';
62+
63+
if (warehouseApplicationIds.includes(value)) {
64+
type = 'warehouse';
65+
}
5966

6067
useInsightsStore.setState({
6168
insightId: value,
@@ -143,6 +150,16 @@ function PageComponent() {
143150
</SelectItem>
144151
))}
145152
</SelectGroup>
153+
{warehouseApplicationIds.length > 0 && (
154+
<SelectGroup>
155+
<SelectLabel>{t('Warehouse')}</SelectLabel>
156+
{warehouseApplicationIds.map((item) => (
157+
<SelectItem key={item} value={item}>
158+
{item}
159+
</SelectItem>
160+
))}
161+
</SelectGroup>
162+
)}
146163
</SelectContent>
147164
</Select>
148165
</div>
@@ -199,7 +216,7 @@ function PageComponent() {
199216
/>
200217
)}
201218
>
202-
{data.map((event) => {
219+
{data.map((event, index) => {
203220
// Calculate date difference
204221
const eventDate = dayjs(event.createdAt);
205222
const diffNow = eventDate.isValid()
@@ -208,14 +225,19 @@ function PageComponent() {
208225

209226
// Get user ID
210227
const userId =
228+
event.properties.distinctId ||
211229
event.properties.userId ||
212230
event.properties.user_id ||
213231
event.properties.sessionId ||
214232
'';
215233

216234
// Clean properties and sessions objects to remove falsy values
217-
const cleanedProperties = cleanObject(event.properties);
218-
const cleanedSessions = cleanObject(event.sessions);
235+
const cleanedProperties = cleanObject(
236+
get(event, 'properties', {})
237+
);
238+
const cleanedSessions = cleanObject(
239+
get(event, 'sessions', {})
240+
);
219241

220242
return (
221243
<Collapse.Panel
@@ -249,7 +271,7 @@ function PageComponent() {
249271
</div>
250272
</div>
251273
}
252-
key={event.id}
274+
key={`${event.id}-${index}`}
253275
>
254276
{jsonMode ? (
255277
<pre className="overflow-x-auto rounded p-2 text-xs">

‎src/client/store/insights.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,11 @@ import { DateUnit } from '@tianji/shared';
77
import { TimeEventChartType } from '../components/chart/TimeEventChart';
88
import dayjs from 'dayjs';
99

10+
export type InsightType = 'website' | 'survey' | 'aigateway' | 'warehouse';
11+
1012
interface InsightsState {
1113
insightId: string;
12-
insightType: 'website' | 'survey' | 'aigateway' | 'warehouse';
14+
insightType: InsightType;
1315
currentMetrics: (MetricsInfo | null)[];
1416
currentFilters: (FilterInfo | null)[];
1517
currentGroups: (GroupInfo | null)[];

‎src/server/model/insights/index.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import { insightsLongTableWarehouse } from './warehouse/longTable.js';
1212
import { INIT_WORKSPACE_ID } from '../../utils/const.js';
1313
import { findWarehouseApplication } from './warehouse/utils.js';
1414
import { insightsWideTableWarehouse } from './warehouse/wideTable.js';
15+
import { queryWarehouseEvents } from './warehouse/index.js';
1516

1617
export function queryInsight(
1718
query: z.infer<typeof insightsQuerySchema>,
@@ -92,5 +93,9 @@ export async function queryEvents(
9293
});
9394
}
9495

96+
if (insightType === 'warehouse' && query.workspaceId === INIT_WORKSPACE_ID) {
97+
return queryWarehouseEvents(query, context);
98+
}
99+
95100
throw new Error('Unknown Insight Type');
96101
}

‎src/server/model/insights/shared.ts‎

Lines changed: 62 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ import { clickhouseHealthManager } from '../../clickhouse/health.js';
1919
import { logger } from '../../utils/logger.js';
2020
import { prisma } from '../_client.js';
2121

22+
const { sql, raw } = Prisma;
23+
2224
// ClickHouse date formats for different time units
2325
export const CLICKHOUSE_DATE_FORMATS = {
2426
minute: '%Y-%m-%d %H:%M:00',
@@ -44,6 +46,12 @@ export interface InsightsQueryContext {
4446

4547
export abstract class InsightsSqlBuilder {
4648
protected abstract getTableName(): string;
49+
protected getDistinctFieldName(): string {
50+
return 'userId';
51+
}
52+
protected getCreateAtFieldName(): string {
53+
return 'createdAt';
54+
}
4755

4856
protected resultLimit = 100; // set default result limit for insights query
4957
protected maxResultLimit = 1000; // set max result limit for insights query to avoid performance issues
@@ -80,7 +88,7 @@ export abstract class InsightsSqlBuilder {
8088
throw new Error(`Invalid date unit: ${unit}`);
8189
}
8290

83-
return Prisma.sql`to_char(date_trunc(${unit}, ${Prisma.raw(field)} at time zone ${timezone}), '${Prisma.raw(POSTGRESQL_DATE_FORMATS[unit as keyof typeof POSTGRESQL_DATE_FORMATS])}')`;
91+
return sql`to_char(date_trunc(${unit}, ${raw(field)} at time zone ${timezone}), '${raw(POSTGRESQL_DATE_FORMATS[unit as keyof typeof POSTGRESQL_DATE_FORMATS])}')`;
8492
}
8593

8694
protected getClickHouseDateQuery(
@@ -97,20 +105,20 @@ export abstract class InsightsSqlBuilder {
97105
CLICKHOUSE_DATE_FORMATS[unit as keyof typeof CLICKHOUSE_DATE_FORMATS];
98106

99107
// ClickHouse uses different functions for date manipulation
100-
// Use Prisma.raw for string literals to avoid parameter binding issues
108+
// Use raw for string literals to avoid parameter binding issues
101109
if (unit === 'minute') {
102-
return Prisma.sql`formatDateTime(toStartOfMinute(${Prisma.raw(field)}, '${Prisma.raw(timezone)}'), '${Prisma.raw(format)}')`;
110+
return sql`formatDateTime(toStartOfMinute(${raw(field)}, '${raw(timezone)}'), '${raw(format)}')`;
103111
} else if (unit === 'hour') {
104-
return Prisma.sql`formatDateTime(toStartOfHour(${Prisma.raw(field)}, '${Prisma.raw(timezone)}'), '${Prisma.raw(format)}')`;
112+
return sql`formatDateTime(toStartOfHour(${raw(field)}, '${raw(timezone)}'), '${raw(format)}')`;
105113
} else if (unit === 'day') {
106-
return Prisma.sql`formatDateTime(toStartOfDay(${Prisma.raw(field)}, '${Prisma.raw(timezone)}'), '${Prisma.raw(format)}')`;
114+
return sql`formatDateTime(toStartOfDay(${raw(field)}, '${raw(timezone)}'), '${raw(format)}')`;
107115
} else if (unit === 'month') {
108-
return Prisma.sql`formatDateTime(toStartOfMonth(${Prisma.raw(field)}, '${Prisma.raw(timezone)}'), '${Prisma.raw(format)}')`;
116+
return sql`formatDateTime(toStartOfMonth(${raw(field)}, '${raw(timezone)}'), '${raw(format)}')`;
109117
} else if (unit === 'year') {
110-
return Prisma.sql`formatDateTime(toStartOfYear(${Prisma.raw(field)}, '${Prisma.raw(timezone)}'), '${Prisma.raw(format)}')`;
118+
return sql`formatDateTime(toStartOfYear(${raw(field)}, '${raw(timezone)}'), '${raw(format)}')`;
111119
}
112120

113-
return Prisma.sql`formatDateTime(toStartOfDay(${Prisma.raw(field)}, '${Prisma.raw(timezone)}'), '${Prisma.raw(format)}')`;
121+
return sql`formatDateTime(toStartOfDay(${raw(field)}, '${raw(timezone)}'), '${raw(format)}')`;
114122
}
115123

116124
protected buildGroupByText(length: number): string {
@@ -128,10 +136,10 @@ export abstract class InsightsSqlBuilder {
128136
// NOTICE: ClickHouse needs proper DateTime conversion functions
129137
const startTime = dayjs(startAt).utc().format('YYYY-MM-DD HH:mm:ss');
130138
const endTime = dayjs(endAt).utc().format('YYYY-MM-DD HH:mm:ss');
131-
return Prisma.sql`${Prisma.raw(field)} BETWEEN toDateTime(${startTime}, 'UTC') AND toDateTime(${endTime}, 'UTC')`;
139+
return sql`${raw(field)} BETWEEN toDateTime(${startTime}, 'UTC') AND toDateTime(${endTime}, 'UTC')`;
132140
}
133141

134-
return Prisma.sql`${Prisma.raw(field)} between ${dayjs(startAt).toISOString()}::timestamptz and ${dayjs(endAt).toISOString()}::timestamptz`;
142+
return sql`${raw(field)} between ${dayjs(startAt).toISOString()}::timestamptz and ${dayjs(endAt).toISOString()}::timestamptz`;
135143
}
136144

137145
public buildCommonFilterQueryOperator(
@@ -166,7 +174,7 @@ export abstract class InsightsSqlBuilder {
166174
);
167175
}
168176

169-
return Prisma.sql`1 = 1`;
177+
return sql`1 = 1`;
170178
}
171179

172180
private buildNumberFilterOperator(
@@ -175,34 +183,34 @@ export abstract class InsightsSqlBuilder {
175183
valueField: Prisma.Sql
176184
): Prisma.Sql {
177185
if (operator === 'equals') {
178-
return Prisma.sql`${valueField} = ${castToNumber(value)}`;
186+
return sql`${valueField} = ${castToNumber(value)}`;
179187
}
180188
if (operator === 'not equals') {
181-
return Prisma.sql`${valueField} != ${castToNumber(value)}`;
189+
return sql`${valueField} != ${castToNumber(value)}`;
182190
}
183191
if (operator === 'in list' && Array.isArray(value)) {
184-
return Prisma.sql`${valueField} IN (${value.join(',')})`;
192+
return sql`${valueField} IN (${value.join(',')})`;
185193
}
186194
if (operator === 'not in list' && Array.isArray(value)) {
187-
return Prisma.sql`${valueField} NOT IN (${value.join(',')})`;
195+
return sql`${valueField} NOT IN (${value.join(',')})`;
188196
}
189197
if (operator === 'greater than') {
190-
return Prisma.sql`${valueField} > ${castToNumber(value)}`;
198+
return sql`${valueField} > ${castToNumber(value)}`;
191199
}
192200
if (operator === 'greater than or equal') {
193-
return Prisma.sql`${valueField} >= ${castToNumber(value)}`;
201+
return sql`${valueField} >= ${castToNumber(value)}`;
194202
}
195203
if (operator === 'less than') {
196-
return Prisma.sql`${valueField} < ${castToNumber(value)}`;
204+
return sql`${valueField} < ${castToNumber(value)}`;
197205
}
198206
if (operator === 'less than or equal') {
199-
return Prisma.sql`${valueField} <= ${castToNumber(value)}`;
207+
return sql`${valueField} <= ${castToNumber(value)}`;
200208
}
201209
if (operator === 'between') {
202-
return Prisma.sql`${valueField} BETWEEN ${castToNumber(get(value, '0'))} AND ${castToNumber(get(value, '1'))}`;
210+
return sql`${valueField} BETWEEN ${castToNumber(get(value, '0'))} AND ${castToNumber(get(value, '1'))}`;
203211
}
204212

205-
return Prisma.sql`1 = 1`;
213+
return sql`1 = 1`;
206214
}
207215

208216
private buildStringFilterOperator(
@@ -211,25 +219,25 @@ export abstract class InsightsSqlBuilder {
211219
valueField: Prisma.Sql
212220
): Prisma.Sql {
213221
if (operator === 'equals') {
214-
return Prisma.sql`${valueField} = ${castToString(value)}`;
222+
return sql`${valueField} = ${castToString(value)}`;
215223
}
216224
if (operator === 'not equals') {
217-
return Prisma.sql`${valueField} != ${castToString(value)}`;
225+
return sql`${valueField} != ${castToString(value)}`;
218226
}
219227
if (operator === 'contains') {
220-
return Prisma.sql`${valueField} LIKE ${`%${castToString(value)}%`}`;
228+
return sql`${valueField} LIKE ${`%${castToString(value)}%`}`;
221229
}
222230
if (operator === 'not contains') {
223-
return Prisma.sql`${valueField} NOT LIKE ${`%${castToString(value)}%`}`;
231+
return sql`${valueField} NOT LIKE ${`%${castToString(value)}%`}`;
224232
}
225233
if (operator === 'in list' && Array.isArray(value)) {
226-
return Prisma.sql`${valueField} IN (${Prisma.join(value, ' , ')})`;
234+
return sql`${valueField} IN (${Prisma.join(value, ' , ')})`;
227235
}
228236
if (operator === 'not in list' && Array.isArray(value)) {
229-
return Prisma.sql`${valueField} NOT IN (${value.join(',')})`;
237+
return sql`${valueField} NOT IN (${value.join(',')})`;
230238
}
231239

232-
return Prisma.sql`1 = 1`;
240+
return sql`1 = 1`;
233241
}
234242

235243
private buildBooleanFilterOperator(
@@ -238,13 +246,13 @@ export abstract class InsightsSqlBuilder {
238246
valueField: Prisma.Sql
239247
): Prisma.Sql {
240248
if (operator === 'equals') {
241-
return Prisma.sql`${valueField} = ${castToNumber(value)}`;
249+
return sql`${valueField} = ${castToNumber(value)}`;
242250
}
243251
if (operator === 'not equals') {
244-
return Prisma.sql`${valueField} != ${castToNumber(value)}`;
252+
return sql`${valueField} != ${castToNumber(value)}`;
245253
}
246254

247-
return Prisma.sql`1 = 1`;
255+
return sql`1 = 1`;
248256
}
249257

250258
private buildDateFilterOperator(
@@ -253,13 +261,13 @@ export abstract class InsightsSqlBuilder {
253261
valueField: Prisma.Sql
254262
): Prisma.Sql {
255263
if (operator === 'between') {
256-
return Prisma.sql`${valueField} BETWEEN ${castToDate(get(value, '0')).toISOString()} AND ${castToDate(get(value, '1')).toISOString()}`;
264+
return sql`${valueField} BETWEEN ${castToDate(get(value, '0')).toISOString()} AND ${castToDate(get(value, '1')).toISOString()}`;
257265
}
258266
if (operator === 'in day') {
259-
return Prisma.sql`${valueField} = DATE(${castToDate(value).toISOString()})`;
267+
return sql`${valueField} = DATE(${castToDate(value).toISOString()})`;
260268
}
261269

262-
return Prisma.sql`1 = 1`;
270+
return sql`1 = 1`;
263271
}
264272

265273
protected buildSelectQueryArr(): (Prisma.Sql | null)[] {
@@ -283,7 +291,7 @@ export abstract class InsightsSqlBuilder {
283291
const { unit, timezone = 'UTC' } = time;
284292
const tableName = this.getTableName();
285293

286-
return Prisma.sql`${this.getDateQuery(`"${tableName}"."createdAt"`, unit, timezone)} date`;
294+
return sql`${this.getDateQuery(`"${tableName}"."${this.getCreateAtFieldName()}"`, unit, timezone)} date`;
287295
}
288296

289297
public build(): Prisma.Sql {
@@ -295,9 +303,9 @@ export abstract class InsightsSqlBuilder {
295303

296304
const groupByText = this.buildGroupByText(groupSelectQueryArr.length + 1);
297305

298-
let sql: Prisma.Sql;
306+
let _sql: Prisma.Sql;
299307

300-
sql = Prisma.sql`select
308+
_sql = sql`select
301309
${Prisma.join(
302310
compact([
303311
this.buildDateQuerySql(),
@@ -306,32 +314,37 @@ export abstract class InsightsSqlBuilder {
306314
]),
307315
' , '
308316
)}
309-
from "${Prisma.raw(tableName)}" ${innerJoinQuery}
317+
from "${raw(tableName)}" ${innerJoinQuery}
310318
where ${Prisma.join(whereQueryArr, ' AND ')}
311-
group by ${Prisma.raw(groupByText)}
319+
group by ${raw(groupByText)}
312320
order by 1 desc
313-
limit ${Prisma.raw(String(this.maxResultLimit))}`;
321+
limit ${raw(String(this.maxResultLimit))}`;
314322

315323
if (env.debugInsights) {
316-
printSQL(sql);
324+
printSQL(_sql);
317325
}
318326

319-
return sql;
327+
return _sql;
320328
}
321329

322330
public buildFetchEventsQuery(cursor: string | undefined): Prisma.Sql {
323331
const tableName = this.getTableName();
324332
const whereQueryArr = this.buildWhereQueryArr();
325333
const innerJoinQuery = this.buildInnerJoinQuery();
326-
327-
return Prisma.sql`
328-
select *
329-
from "${Prisma.raw(tableName)}"
334+
const distinctFieldName = this.getDistinctFieldName();
335+
const createAtFieldName = this.getCreateAtFieldName();
336+
337+
return sql`
338+
select
339+
*,
340+
"${raw(distinctFieldName)}" as "distinctId",
341+
"${raw(createAtFieldName)}" as "createdAt"
342+
from "${raw(tableName)}"
330343
${innerJoinQuery}
331344
where ${Prisma.join(whereQueryArr, ' AND ')}
332-
${cursor ? Prisma.sql`AND "${Prisma.raw(tableName)}"."id" < ${cursor}` : Prisma.empty}
333-
order by "${Prisma.raw(tableName)}"."createdAt" desc
334-
limit ${this.resultLimit}
345+
${cursor ? sql`AND "${raw(tableName)}"."id" < ${cursor}` : Prisma.empty}
346+
order by "${raw(tableName)}"."${raw(createAtFieldName)}" desc
347+
limit ${raw(String(this.resultLimit))}
335348
`;
336349
}
337350

0 commit comments

Comments
 (0)