"""向量知识库管理接口 - 直接操作 ChromaDB""" from fastapi import APIRouter, HTTPException from pydantic import BaseModel from typing import Optional, List import logging import asyncio import math import os # 模块级导入:verify 端点复用同一 Chroma/embedding 实例构造(测试可 monkeypatch) from langchain_chroma import Chroma from app.config import settings from app.rag.embeddings import get_embeddings logger = logging.getLogger(__name__) router = APIRouter(prefix="/api/v1/knowledge-base", tags=["knowledge-base"]) # 后台同步任务引用 _sync_task = None class DocumentInfo(BaseModel): id: str source: str title: str content: str = "" content_preview: str = "" metadata: dict = {} class ListRequest(BaseModel): source: Optional[str] = None keyword: Optional[str] = None page: int = 1 size: int = 20 class UpdateDocumentRequest(BaseModel): content: str title: Optional[str] = None metadata_override: Optional[dict] = None @router.post("/stats") async def get_stats(): """获取向量库统计信息""" try: from app.rag.retriever import RagRetriever from app.rag.embeddings import get_embeddings from app.config import settings from langchain_chroma import Chroma retriever = RagRetriever() collection = retriever.vectorstore._collection count = collection.count() # 按 source 统计(兼容新旧两种 metadata 字段:source / type) all_data = collection.get(include=["metadatas"]) metas = all_data.get("metadatas") or [] source_counts = {} type_counts = {} for meta in metas: if not meta: continue src = meta.get("source") or meta.get("type") or "unknown" source_counts[src] = source_counts.get(src, 0) + 1 doc_type = meta.get("type") or meta.get("source") or "unknown" type_counts[doc_type] = type_counts.get(doc_type, 0) + 1 return { "total": count, "by_source": source_counts, "by_type": type_counts, "last_sync": _load_sync_state(), } except Exception as e: logger.error("获取向量库统计失败: %s", e) raise HTTPException(status_code=500, detail=str(e)) @router.post("/documents") async def list_documents(req: ListRequest): """分页列出向量库文档""" try: from app.rag.retriever import RagRetriever from app.rag.embeddings import get_embeddings from app.config import settings from langchain_chroma import Chroma retriever = RagRetriever() collection = retriever.vectorstore._collection # 构建过滤条件(兼容新旧两种 metadata 字段:source / type) where_filter = None if req.source: where_filter = {"$or": [{"source": req.source}, {"type": req.source}]} # 使用 get() 分页获取,避免加载全部 embedding limit = req.size offset = (req.page - 1) * req.size result = collection.get( where=where_filter, include=["documents", "metadatas"], limit=limit, offset=offset, ) ids = result.get("ids") or [] docs = result.get("documents") or [] metas = result.get("metadatas") or [] documents = [] for i, doc_id in enumerate(ids): meta = metas[i] if i < len(metas) else {} content = docs[i] if i < len(docs) else "" # 截取内容预览 preview = content[:200] + "..." if len(content) > 200 else content documents.append(DocumentInfo( id=doc_id, source=meta.get("source") or meta.get("type") or "unknown", title=meta.get("title", ""), content=content, content_preview=preview, metadata=meta, )) # 获取总数 count_result = collection.get( where=where_filter, include=[], ) total = len(count_result.get("ids", [])) return { "total": total, "page": req.page, "size": req.size, "documents": documents, } except Exception as e: logger.error("列出文档失败: %s", e) raise HTTPException(status_code=500, detail=str(e)) @router.delete("/source/{source_prefix}") async def delete_by_source(source_prefix: str): """删除指定 source 前缀的所有文档""" try: from app.rag.retriever import RagRetriever from app.rag.embeddings import get_embeddings from app.config import settings from langchain_chroma import Chroma retriever = RagRetriever() collection = retriever.vectorstore._collection # 兼容新旧两种 metadata 字段:source / type existing = collection.get( where={"$or": [{"source": source_prefix}, {"type": source_prefix}]}, include=[], ) ids = existing.get("ids", []) or [] if not ids: return {"deleted": 0, "message": f"无 {source_prefix} 文档"} collection.delete(ids=ids) logger.info("已删除 source=%s 的 %d 个文档", source_prefix, len(ids)) return {"deleted": len(ids), "source": source_prefix} except Exception as e: logger.error("删除文档失败: %s", e) raise HTTPException(status_code=500, detail=str(e)) @router.post("/sync") async def trigger_sync(): """触发一次完整知识库同步""" global _sync_task try: # 如果已有同步任务在运行,返回提示 if _sync_task and not _sync_task.done(): return {"status": "running", "message": "同步任务已在运行中"} _sync_task = asyncio.create_task(_run_sync()) return {"status": "started", "message": "已启动知识库同步任务"} except Exception as e: logger.error("触发同步失败: %s", e) raise HTTPException(status_code=500, detail=str(e)) async def _run_sync(): """执行完整同步""" try: from app.tasks.knowledge_sync import sync_knowledge_base await sync_knowledge_base() logger.info("知识库同步完成") except Exception as e: logger.error("知识库同步失败: %s", e) @router.put("/document/{doc_id}") async def update_document(doc_id: str, req: UpdateDocumentRequest): """更新单个文档内容(先获取原 metadata,再删除重建,保持 ID 不变)""" try: from app.rag.retriever import RagRetriever from app.rag.embeddings import get_embeddings from app.config import settings from langchain_chroma import Chroma retriever = RagRetriever() collection = retriever.vectorstore._collection # 先获取原文档 metadata,避免删除后丢失 old_result = collection.get(ids=[doc_id], include=["metadatas", "documents"]) old_metas = old_result.get("metadatas") or [] old_docs = old_result.get("documents") or [] if not old_metas: raise HTTPException(status_code=404, detail="文档不存在") original_meta = old_metas[0] original_content = old_docs[0] if old_docs else "" # 删除旧文档 collection.delete(ids=[doc_id]) logger.info("已删除旧文档: %s", doc_id) # 构建新文档:保留原有 metadata,仅覆盖 content/title meta = req.metadata_override or {} # 保留原有 source/type/title 等字段,除非显式覆盖 for k, v in original_meta.items(): if k not in meta: meta[k] = v if req.title: meta["title"] = req.title # 使用原 ID 重新写入(显式传 ids,避免生成新 ID) from langchain_core.documents import Document doc = Document(page_content=req.content, metadata=meta) await retriever.vectorstore.aadd_documents([doc], ids=[doc_id]) logger.info("已更新文档: %s", doc_id) return {"success": True, "message": "文档已更新", "doc_id": doc_id} except HTTPException: raise except Exception as e: logger.error("更新文档失败: %s", e) raise HTTPException(status_code=500, detail=str(e)) def _load_sync_state() -> str: """加载上次同步时间""" try: import json from app.config import settings import os sync_file = os.path.join(settings.chroma_db_path, ".sync_state") if os.path.exists(sync_file): with open(sync_file) as f: state = json.load(f) return state.get("last_sync", "未知") except Exception: pass return "未知" class VerifySentence(BaseModel): id: str text: str class VerifyRequest(BaseModel): sentences: List[VerifySentence] source: str = "microbiome" k: int = 3 threshold: float = 0.7 @router.post("/verify") async def verify(req: VerifyRequest): """知识库出处自检:对每个待检整句做 RAG 检索,返回最大 cosine 相似度。 度量说明:本库 ChromaDB collection 未配置 hnsw:space(默认 L2 距离), similarity_search_with_score 的 score 是 L2 距离(越小越相似),不是相似度, 因而返回前必须取候选向量与 query 向量独立计算 cosine 相似度, 不得把 L2 距离当相似度、也不得用 1 - d^2/2 近似(仅对归一化向量成立)。 """ try: store = Chroma( collection_name="cfc_knowledge", embedding_function=get_embeddings(), persist_directory=settings.chroma_db_path, ) embedder = get_embeddings() results = [] for s in req.sentences: text = (s.text or "").strip() if not text: results.append({"id": s.id, "max_similarity": 0.0, "passed": False, "matched_title": None, "matched_chunk": None}) continue try: candidates = store.similarity_search_with_score( text, k=req.k, filter={"source": req.source}) except Exception as e: logger.warning("verify 检索失败 sentence=%s: %s", s.id, e) results.append({"id": s.id, "max_similarity": 0.0, "passed": False, "matched_title": None, "matched_chunk": None}) continue query_vec = await asyncio.to_thread(embedder.embed_query, text) best = {"max_similarity": 0.0, "matched_title": None, "matched_chunk": None} for doc, _dist in candidates: meta = doc.metadata or {} doc_id = meta.get("id") if not doc_id: continue try: got = store._collection.get(ids=[doc_id], include=["embeddings"]) vecs = got.get("embeddings") # 真实 chromadb 返回 np.ndarray shape (1,dim);直接按长度判断, # 避免 对数组求真值抛 ValueError if vecs is None or len(vecs) == 0: continue cand_vec = vecs[0] except Exception as e: logger.warning("verify 取 embedding 失败 doc=%s: %s", doc_id, e) continue sim = _cosine(query_vec, cand_vec) if sim > best["max_similarity"]: best = { "max_similarity": sim, "matched_title": meta.get("title"), "matched_chunk": (doc.page_content[:120] if doc.page_content else None), } best["id"] = s.id best["passed"] = best["max_similarity"] >= req.threshold results.append(best) all_passed = all(r["passed"] for r in results) return {"code": 200, "data": { "source": req.source, "threshold": req.threshold, "results": results, "all_passed": all_passed, }} except Exception as e: logger.error("verify 端点异常: %s", e) raise HTTPException(status_code=500, detail=str(e)) def _cosine(a, b) -> float: """余弦相似度(对任意范数向量成立,不依赖归一化)。 兼容 numpy 数组:真实 chromadb collection.get(include=["embeddings"]) 返回 numpy.ndarray(shape (1,dim)),先 list() 序列化、再按长度判断, 避免对数组求真值(if not a)抛 ValueError。返回值为 pure float。 """ a = list(a) b = list(b) if not a or not b or len(a) != len(b): return 0.0 dot = sum(x * y for x, y in zip(a, b)) na = math.sqrt(sum(float(x) * float(x) for x in a)) nb = math.sqrt(sum(float(y) * float(y) for y in b)) if na == 0.0 or nb == 0.0: return 0.0 return float(dot) / (float(na) * float(nb))