cfc/cfc-langgraph/app/rag/loader.py

from app.tools.java_client import JavaClient
from typing import Optional
import logging

logger = logging.getLogger(__name__)


class KnowledgeLoader:
    """知识库加载器: 从 Java 侧拉取文章并格式化"""

    def __init__(self):
        self.java = JavaClient()

    async def load_all_articles(self) -> list[dict]:
        """获取所有已发布文章"""
        return await self.java.get_published_articles()

    async def load_updated_since(self, since: str) -> list[dict]:
        """增量获取: 获取某个时间后更新的文章"""
        try:
            client = await self.java._get_client()
            resp = await client.post("/api/article/updated-since", json={
                "since": since,
                "status": "published",
            })
            data = resp.json()
            if data.get("code") == 200:
                body = data.get("data", [])
                # 兼容两种响应：直接 list 或 {records:[...]} 分页结构
                if isinstance(body, dict):
                    body = body.get("records", []) or []
                return body
        except Exception as e:
            logger.warning("增量获取文章失败: %s", e)
        return []

    def format_for_indexing(self, articles: list[dict]) -> list[dict]:
        """将文章格式化为可索引的文档"""
        docs = []
        for article in articles:
            content = f"{article.get('title', '')}\n\n{article.get('summary', '')}\n\n{article.get('content', '')}"
            docs.append({
                "id": f"article_{article['id']}",
                "content": content,
                "metadata": {
                    "source": "article",
                    "article_id": article["id"],
                    "title": article.get("title", ""),
                    "tags": article.get("tags", ""),
                    "updated_at": article.get("updatedAt", ""),
                },
            })
        return docs