knowledge_base.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369
  1. """向量知识库管理接口 - 直接操作 ChromaDB"""
  2. from fastapi import APIRouter, HTTPException
  3. from pydantic import BaseModel
  4. from typing import Optional, List
  5. import logging
  6. import asyncio
  7. import math
  8. import os
  9. # 模块级导入:verify 端点复用同一 Chroma/embedding 实例构造(测试可 monkeypatch)
  10. from langchain_chroma import Chroma
  11. from app.config import settings
  12. from app.rag.embeddings import get_embeddings
  13. logger = logging.getLogger(__name__)
  14. router = APIRouter(prefix="/api/v1/knowledge-base", tags=["knowledge-base"])
  15. # 后台同步任务引用
  16. _sync_task = None
  17. class DocumentInfo(BaseModel):
  18. id: str
  19. source: str
  20. title: str
  21. content: str = ""
  22. content_preview: str = ""
  23. metadata: dict = {}
  24. class ListRequest(BaseModel):
  25. source: Optional[str] = None
  26. keyword: Optional[str] = None
  27. page: int = 1
  28. size: int = 20
  29. class UpdateDocumentRequest(BaseModel):
  30. content: str
  31. title: Optional[str] = None
  32. metadata_override: Optional[dict] = None
  33. @router.post("/stats")
  34. async def get_stats():
  35. """获取向量库统计信息"""
  36. try:
  37. from app.rag.retriever import RagRetriever
  38. from app.rag.embeddings import get_embeddings
  39. from app.config import settings
  40. from langchain_chroma import Chroma
  41. retriever = RagRetriever()
  42. collection = retriever.vectorstore._collection
  43. count = collection.count()
  44. # 按 source 统计(兼容新旧两种 metadata 字段:source / type)
  45. all_data = collection.get(include=["metadatas"])
  46. metas = all_data.get("metadatas") or []
  47. source_counts = {}
  48. type_counts = {}
  49. for meta in metas:
  50. if not meta:
  51. continue
  52. src = meta.get("source") or meta.get("type") or "unknown"
  53. source_counts[src] = source_counts.get(src, 0) + 1
  54. doc_type = meta.get("type") or meta.get("source") or "unknown"
  55. type_counts[doc_type] = type_counts.get(doc_type, 0) + 1
  56. return {
  57. "total": count,
  58. "by_source": source_counts,
  59. "by_type": type_counts,
  60. "last_sync": _load_sync_state(),
  61. }
  62. except Exception as e:
  63. logger.error("获取向量库统计失败: %s", e)
  64. raise HTTPException(status_code=500, detail=str(e))
  65. @router.post("/documents")
  66. async def list_documents(req: ListRequest):
  67. """分页列出向量库文档"""
  68. try:
  69. from app.rag.retriever import RagRetriever
  70. from app.rag.embeddings import get_embeddings
  71. from app.config import settings
  72. from langchain_chroma import Chroma
  73. retriever = RagRetriever()
  74. collection = retriever.vectorstore._collection
  75. # 构建过滤条件(兼容新旧两种 metadata 字段:source / type)
  76. where_filter = None
  77. if req.source:
  78. where_filter = {"$or": [{"source": req.source}, {"type": req.source}]}
  79. # 使用 get() 分页获取,避免加载全部 embedding
  80. limit = req.size
  81. offset = (req.page - 1) * req.size
  82. result = collection.get(
  83. where=where_filter,
  84. include=["documents", "metadatas"],
  85. limit=limit,
  86. offset=offset,
  87. )
  88. ids = result.get("ids") or []
  89. docs = result.get("documents") or []
  90. metas = result.get("metadatas") or []
  91. documents = []
  92. for i, doc_id in enumerate(ids):
  93. meta = metas[i] if i < len(metas) else {}
  94. content = docs[i] if i < len(docs) else ""
  95. # 截取内容预览
  96. preview = content[:200] + "..." if len(content) > 200 else content
  97. documents.append(DocumentInfo(
  98. id=doc_id,
  99. source=meta.get("source") or meta.get("type") or "unknown",
  100. title=meta.get("title", ""),
  101. content=content,
  102. content_preview=preview,
  103. metadata=meta,
  104. ))
  105. # 获取总数
  106. count_result = collection.get(
  107. where=where_filter,
  108. include=[],
  109. )
  110. total = len(count_result.get("ids", []))
  111. return {
  112. "total": total,
  113. "page": req.page,
  114. "size": req.size,
  115. "documents": documents,
  116. }
  117. except Exception as e:
  118. logger.error("列出文档失败: %s", e)
  119. raise HTTPException(status_code=500, detail=str(e))
  120. @router.delete("/source/{source_prefix}")
  121. async def delete_by_source(source_prefix: str):
  122. """删除指定 source 前缀的所有文档"""
  123. try:
  124. from app.rag.retriever import RagRetriever
  125. from app.rag.embeddings import get_embeddings
  126. from app.config import settings
  127. from langchain_chroma import Chroma
  128. retriever = RagRetriever()
  129. collection = retriever.vectorstore._collection
  130. # 兼容新旧两种 metadata 字段:source / type
  131. existing = collection.get(
  132. where={"$or": [{"source": source_prefix}, {"type": source_prefix}]},
  133. include=[],
  134. )
  135. ids = existing.get("ids", []) or []
  136. if not ids:
  137. return {"deleted": 0, "message": f"无 {source_prefix} 文档"}
  138. collection.delete(ids=ids)
  139. logger.info("已删除 source=%s 的 %d 个文档", source_prefix, len(ids))
  140. return {"deleted": len(ids), "source": source_prefix}
  141. except Exception as e:
  142. logger.error("删除文档失败: %s", e)
  143. raise HTTPException(status_code=500, detail=str(e))
  144. @router.post("/sync")
  145. async def trigger_sync():
  146. """触发一次完整知识库同步"""
  147. global _sync_task
  148. try:
  149. # 如果已有同步任务在运行,返回提示
  150. if _sync_task and not _sync_task.done():
  151. return {"status": "running", "message": "同步任务已在运行中"}
  152. _sync_task = asyncio.create_task(_run_sync())
  153. return {"status": "started", "message": "已启动知识库同步任务"}
  154. except Exception as e:
  155. logger.error("触发同步失败: %s", e)
  156. raise HTTPException(status_code=500, detail=str(e))
  157. async def _run_sync():
  158. """执行完整同步"""
  159. try:
  160. from app.tasks.knowledge_sync import sync_knowledge_base
  161. await sync_knowledge_base()
  162. logger.info("知识库同步完成")
  163. except Exception as e:
  164. logger.error("知识库同步失败: %s", e)
  165. @router.put("/document/{doc_id}")
  166. async def update_document(doc_id: str, req: UpdateDocumentRequest):
  167. """更新单个文档内容(先获取原 metadata,再删除重建,保持 ID 不变)"""
  168. try:
  169. from app.rag.retriever import RagRetriever
  170. from app.rag.embeddings import get_embeddings
  171. from app.config import settings
  172. from langchain_chroma import Chroma
  173. retriever = RagRetriever()
  174. collection = retriever.vectorstore._collection
  175. # 先获取原文档 metadata,避免删除后丢失
  176. old_result = collection.get(ids=[doc_id], include=["metadatas", "documents"])
  177. old_metas = old_result.get("metadatas") or []
  178. old_docs = old_result.get("documents") or []
  179. if not old_metas:
  180. raise HTTPException(status_code=404, detail="文档不存在")
  181. original_meta = old_metas[0]
  182. original_content = old_docs[0] if old_docs else ""
  183. # 删除旧文档
  184. collection.delete(ids=[doc_id])
  185. logger.info("已删除旧文档: %s", doc_id)
  186. # 构建新文档:保留原有 metadata,仅覆盖 content/title
  187. meta = req.metadata_override or {}
  188. # 保留原有 source/type/title 等字段,除非显式覆盖
  189. for k, v in original_meta.items():
  190. if k not in meta:
  191. meta[k] = v
  192. if req.title:
  193. meta["title"] = req.title
  194. # 使用原 ID 重新写入(显式传 ids,避免生成新 ID)
  195. from langchain_core.documents import Document
  196. doc = Document(page_content=req.content, metadata=meta)
  197. await retriever.vectorstore.aadd_documents([doc], ids=[doc_id])
  198. logger.info("已更新文档: %s", doc_id)
  199. return {"success": True, "message": "文档已更新", "doc_id": doc_id}
  200. except HTTPException:
  201. raise
  202. except Exception as e:
  203. logger.error("更新文档失败: %s", e)
  204. raise HTTPException(status_code=500, detail=str(e))
  205. def _load_sync_state() -> str:
  206. """加载上次同步时间"""
  207. try:
  208. import json
  209. from app.config import settings
  210. import os
  211. sync_file = os.path.join(settings.chroma_db_path, ".sync_state")
  212. if os.path.exists(sync_file):
  213. with open(sync_file) as f:
  214. state = json.load(f)
  215. return state.get("last_sync", "未知")
  216. except Exception:
  217. pass
  218. return "未知"
  219. class VerifySentence(BaseModel):
  220. id: str
  221. text: str
  222. class VerifyRequest(BaseModel):
  223. sentences: List[VerifySentence]
  224. source: str = "microbiome"
  225. k: int = 3
  226. threshold: float = 0.7
  227. @router.post("/verify")
  228. async def verify(req: VerifyRequest):
  229. """知识库出处自检:对每个待检整句做 RAG 检索,返回最大 cosine 相似度。
  230. 度量说明:本库 ChromaDB collection 未配置 hnsw:space(默认 L2 距离),
  231. similarity_search_with_score 的 score 是 L2 距离(越小越相似),不是相似度,
  232. 因而返回前必须取候选向量与 query 向量独立计算 cosine 相似度,
  233. 不得把 L2 距离当相似度、也不得用 1 - d^2/2 近似(仅对归一化向量成立)。
  234. """
  235. try:
  236. store = Chroma(
  237. collection_name="cfc_knowledge",
  238. embedding_function=get_embeddings(),
  239. persist_directory=settings.chroma_db_path,
  240. )
  241. embedder = get_embeddings()
  242. results = []
  243. for s in req.sentences:
  244. text = (s.text or "").strip()
  245. if not text:
  246. results.append({"id": s.id, "max_similarity": 0.0, "passed": False,
  247. "matched_title": None, "matched_chunk": None})
  248. continue
  249. try:
  250. candidates = store.similarity_search_with_score(
  251. text, k=req.k, filter={"source": req.source})
  252. except Exception as e:
  253. logger.warning("verify 检索失败 sentence=%s: %s", s.id, e)
  254. results.append({"id": s.id, "max_similarity": 0.0, "passed": False,
  255. "matched_title": None, "matched_chunk": None})
  256. continue
  257. query_vec = await asyncio.to_thread(embedder.embed_query, text)
  258. best = {"max_similarity": 0.0, "matched_title": None, "matched_chunk": None}
  259. for doc, _dist in candidates:
  260. meta = doc.metadata or {}
  261. doc_id = meta.get("id")
  262. if not doc_id:
  263. continue
  264. try:
  265. got = store._collection.get(ids=[doc_id], include=["embeddings"])
  266. vecs = got.get("embeddings")
  267. # 真实 chromadb 返回 np.ndarray shape (1,dim);直接按长度判断,
  268. # 避免 对数组求真值抛 ValueError
  269. if vecs is None or len(vecs) == 0:
  270. continue
  271. cand_vec = vecs[0]
  272. except Exception as e:
  273. logger.warning("verify 取 embedding 失败 doc=%s: %s", doc_id, e)
  274. continue
  275. sim = _cosine(query_vec, cand_vec)
  276. if sim > best["max_similarity"]:
  277. best = {
  278. "max_similarity": sim,
  279. "matched_title": meta.get("title"),
  280. "matched_chunk": (doc.page_content[:120] if doc.page_content else None),
  281. }
  282. best["id"] = s.id
  283. best["passed"] = best["max_similarity"] >= req.threshold
  284. results.append(best)
  285. all_passed = all(r["passed"] for r in results)
  286. return {"code": 200, "data": {
  287. "source": req.source, "threshold": req.threshold,
  288. "results": results, "all_passed": all_passed,
  289. }}
  290. except Exception as e:
  291. logger.error("verify 端点异常: %s", e)
  292. raise HTTPException(status_code=500, detail=str(e))
  293. def _cosine(a, b) -> float:
  294. """余弦相似度(对任意范数向量成立,不依赖归一化)。
  295. 兼容 numpy 数组:真实 chromadb collection.get(include=["embeddings"])
  296. 返回 numpy.ndarray(shape (1,dim)),先 list() 序列化、再按长度判断,
  297. 避免对数组求真值(if not a)抛 ValueError。返回值为 pure float。
  298. """
  299. a = list(a)
  300. b = list(b)
  301. if not a or not b or len(a) != len(b):
  302. return 0.0
  303. dot = sum(x * y for x, y in zip(a, b))
  304. na = math.sqrt(sum(float(x) * float(x) for x in a))
  305. nb = math.sqrt(sum(float(y) * float(y) for y in b))
  306. if na == 0.0 or nb == 0.0:
  307. return 0.0
  308. return float(dot) / (float(na) * float(nb))