feat(backend): fix

Author: kidkidkid

backend/modules/observability/infra/repo/ck/annotation.go

@@ -87,6 +87,9 @@ func (a *AnnotationCkDaoImpl) Get(ctx context.Context, params *GetAnnotationPara
 	if err != nil {
 		return nil, err
 	}
+	logs.CtxInfo(ctx, "Get Annotation SQL: %s", db.ToSQL(func(tx *gorm.DB) *gorm.DB {
+		return tx.Find(nil)
+	}))
 	var annotations []*model.ObservabilityAnnotation
 	if err := db.Find(&annotations).Error; err != nil {
 		return nil, err
@@ -114,6 +117,9 @@ func (a *AnnotationCkDaoImpl) List(ctx context.Context, params *ListAnnotationsP
 	if err != nil {
 		return nil, err
 	}
+	logs.CtxInfo(ctx, "List Annotations SQL: %s", db.ToSQL(func(tx *gorm.DB) *gorm.DB {
+		return tx.Find(nil)
+	}))
 	var annotations []*model.ObservabilityAnnotation
 	if err := db.Find(&annotations).Error; err != nil {
 		return nil, err
@@ -144,7 +150,11 @@ func (a *AnnotationCkDaoImpl) buildSql(ctx context.Context, param *annoSqlParam)
 	if len(tableQueries) == 0 {
 		return nil, fmt.Errorf("no table configured")
 	} else if len(tableQueries) == 1 {
-		return tableQueries[0], nil
+		query := tableQueries[0].ToSQL(func(tx *gorm.DB) *gorm.DB {
+			return tx.Find(nil)
+		})
+		query += " SETTINGS final = 1"
+		return db.Raw(query), nil
 	} else {
 		queries := make([]string, 0)
 		for i := 0; i < len(tableQueries); i++ {
@@ -159,7 +169,7 @@ func (a *AnnotationCkDaoImpl) buildSql(ctx context.Context, param *annoSqlParam)
 		} else {
 			sql += " ORDER BY created_at ASC"
 		}
-		sql += fmt.Sprintf(" LIMIT %d", param.Limit)
+		sql += fmt.Sprintf(" LIMIT %d SETTINGS final = 1", param.Limit)
 		return db.Raw(sql), nil
 	}
 }

backend/modules/observability/infra/repo/ck/spans.go

@@ -99,6 +99,9 @@ func (s *SpansCkDaoImpl) Get(ctx context.Context, param *QueryParam) ([]*model.O
 	if err := sql.Find(&spans).Error; err != nil {
 		return nil, errorx.WrapByCode(err, obErrorx.CommercialCommonRPCErrorCodeCode)
 	}
+	for _, span := range spans {
+		span.SystemTagsString[loop_span.SpanFieldTenant] = "cozeloop" // tenant
+	}
 	return spans, nil
 }
 
@@ -167,28 +170,6 @@ func (s *SpansCkDaoImpl) buildSingleSql(ctx context.Context, param *buildSqlPara
 	return sqlQuery, nil
 }
 
-// 根据start_date查询start_time的子查询
-func (s *SpansCkDaoImpl) buildSubQuerySql(ctx context.Context, param *buildSqlParam) (*gorm.DB, error) {
-	sqlQuery, err := s.buildSqlForFilterFields(ctx, param, param.queryParam.Filters)
-	if err != nil {
-		return nil, err
-	}
-	sqlQuery = param.db.
-		Table(param.spanTable).
-		Select("start_time").
-		Where(sqlQuery).
-		Where("start_time >= ?", param.queryParam.StartTime).
-		Where("start_time <= ?", param.queryParam.EndTime)
-	if param.queryParam.OrderByStartTime {
-		sqlQuery = sqlQuery.Order(clause.OrderBy{Columns: []clause.OrderByColumn{
-			{Column: clause.Column{Name: "start_time"}, Desc: true},
-			{Column: clause.Column{Name: "span_id"}, Desc: true},
-		}})
-	}
-	sqlQuery = sqlQuery.Limit(int(param.queryParam.Limit))
-	return sqlQuery, nil
-}
-
 // chain
 func (s *SpansCkDaoImpl) buildSqlForFilterFields(ctx context.Context, param *buildSqlParam, filter *loop_span.FilterFields) (*gorm.DB, error) {
 	if filter == nil {
@@ -396,7 +377,10 @@ func (s *SpansCkDaoImpl) buildAnnotationSql(ctx context.Context, param *buildSql
 		Where("deleted_at = 0").
 		Where("start_time >= ?", param.queryParam.StartTime).
 		Where("start_time <= ?", param.queryParam.EndTime)
-	return param.db.Where("span_id in (?)", subsql), nil
+	query := subsql.ToSQL(func(tx *gorm.DB) *gorm.DB {
+		return tx.Find(nil)
+	})
+	return param.db.Where("span_id in (?)", param.db.Raw(query+" SETTINGS final = 1")), nil
 }
 
 func (s *SpansCkDaoImpl) getSuperFieldsMap(ctx context.Context) map[string]bool {

backend/modules/observability/infra/repo/ck/spans_test.go

@@ -197,10 +197,24 @@ func TestBuildSql(t *testing.T) {
 			},
 			expectedSql: "SELECT * FROM `observability_spans` WHERE `input` like '%123%' AND start_time >= 1 AND start_time <= 2 LIMIT 100",
 		},
+		{
+			filter: &loop_span.FilterFields{
+				FilterFields: []*loop_span.FilterField{
+					{
+						FieldName: "manual_feedback_abc",
+						FieldType: loop_span.FieldTypeString,
+						Values:    []string{"123"},
+						QueryType: ptr.Of(loop_span.QueryTypeEnumIn),
+					},
+				},
+			},
+			expectedSql: "SELECT * FROM `observability_spans` WHERE span_id in (SELECT span_id FROM `observability_annotations` WHERE (annotation_type = 'manual_feedback' AND key = 'abc' AND value_string IN ('123')) AND deleted_at = 0 AND start_time >= 1 AND start_time <= 2 FINAL) AND start_time >= 1 AND start_time <= 2 LIMIT 100",
+		},
 	}
 	for _, tc := range testCases {
 		qDb, err := new(SpansCkDaoImpl).buildSingleSql(context.Background(), &buildSqlParam{
 			spanTable: "observability_spans",
+			annoTable: "observability_annotations",
 			queryParam: &QueryParam{
 				StartTime: 1,
 				EndTime:   2,

release/deployment/docker-compose/bootstrap/clickhouse-init/init-sql/observability_annotations.sql

@@ -21,10 +21,12 @@ CREATE TABLE IF NOT EXISTS `observability_annotations` (
     `updated_by` String,
     `updated_at` UInt64,
     `deleted_at` UInt64,
+    `start_date` Date,
+    INDEX idx_id id TYPE bloom_filter() GRANULARITY 1,
     INDEX idx_span_id span_id TYPE bloom_filter() GRANULARITY 1,
     INDEX idx_trace_id trace_id TYPE bloom_filter() GRANULARITY 1,
     INDEX idx_space_id space_id TYPE bloom_filter() GRANULARITY 1,
     INDEX idx_annotation_type annotation_type TYPE bloom_filter() GRANULARITY 1
 ) ENGINE = ReplacingMergeTree(updated_at) PARTITION BY toDate(start_time / 1000000)
-PRIMARY KEY (id)
-ORDER BY (id, start_time);
+PRIMARY KEY (start_time)
+ORDER BY (start_time, id);

release/deployment/docker-compose/conf/observability.yaml

@@ -60,9 +60,10 @@ trace_tenant_cfg:
     cozeloop:
       365d:
         span_table: "observability_spans"
+        anno_table: "observability_annotations"
   default_ingest_tenant: "cozeloop"
   tenants_support_annotation:
-    cozeloop: false
+    cozeloop: true
 
 trace_field_meta_info:
   available_fields:
@@ -176,6 +177,8 @@ trace_field_meta_info:
         - "exist"
         - "not_exist"
       support_custom: true
+    feedback_manual:
+      support_custom: true
   field_metas:
     cozeloop:
       root_span:
@@ -189,6 +192,7 @@ trace_field_meta_info:
         - "span_type"
         - "message_id"
         - "user_id"
+        - "feedback_manual"
       all_span:
         - "input"
         - "output"
@@ -204,6 +208,7 @@ trace_field_meta_info:
         - "output_tokens"
         - "prompt_key"
         - "tokens"
+        - "feedback_manual"
       llm_span:
         - "input"
         - "output"
@@ -217,6 +222,7 @@ trace_field_meta_info:
         - "input_tokens"
         - "output_tokens"
         - "tokens"
+        - "feedback_manual"
     prompt:
       root_span:
         - "input"
@@ -229,6 +235,7 @@ trace_field_meta_info:
         - "span_type"
         - "prompt_key"
         - "user_id"
+        - "feedback_manual"
       all_span:
         - "input"
         - "output"
@@ -243,6 +250,7 @@ trace_field_meta_info:
         - "output_tokens"
         - "prompt_key"
         - "tokens"
+        - "feedback_manual"
       llm_span:
         - "input"
         - "output"
@@ -256,6 +264,7 @@ trace_field_meta_info:
         - "output_tokens"
         - "prompt_key"
         - "tokens"
+        - "feedback_manual"
 
 trace_collector_cfg:
   receivers: