support baidu vdb as doc engine and field weight assignments

Author: zhangping24

pyproject.toml

@@ -126,7 +126,7 @@ dependencies = [
     "debugpy>=1.8.13",
     "mcp>=1.9.4",
     "opensearch-py==2.7.1",
-    "pymochow==2.2.4"
+    "pymochow==2.2.9"
     "pluginlib==0.9.4",
     "click>=8.1.8",
     "python-calamine>=0.4.0",

rag/utils/baidu_vdb_conn.py

@@ -75,6 +75,14 @@ def __init__(self):
             raise Exception(f"Mapping file not found at {fp_mapping}")
         self.mapping = json.load(open(fp_mapping))
         healthy = self.health()
+        self.query_fields_boosts = {
+            "title_tks": 10,
+            "title_sm_tks": 5,
+            "important_tks": 20,
+            "question_tks": 20,
+            "content_ltks": 2,
+            "content_sm_ltks": 1,
+        }
         if healthy["err"] == "":
             logger.info(f"BaiduVDB {settings.BAIDUVDB['endpoint']} is healthy.")
         else:
@@ -333,7 +341,13 @@ def _select(self, db: Database, indexNames: list[str], projections: list[str], f
     def _matchTextExpr2Bm25SearchReq(self, m: MatchTextExpr, filter: str) -> BM25SearchRequest:
         search_text_cond = list()
         for field in m.fields:
-            search_text_cond.append(f"{field}:{m.matching_text}")
+            boost = 1
+            if field in self.query_fields_boosts:
+                boost = self.query_fields_boosts[field]
+            if boost != 1:
+                search_text_cond.append(f"{field}:{m.matching_text}^{boost}")
+            else:
+                search_text_cond.append(f"{field}:{m.matching_text}")
         search_text = " OR ".join(search_text_cond)
         return BM25SearchRequest(
             index_name=self.tks_inverted_idx,