cfc/docs/参考资料/extract_all.py

"""
PDF 健康报告 → 完整内容提取脚本
================================
提取"个体化食物推荐表"全部内容，输出：
  1. 原始文本（debug用）
  2. 结构化CSV（每行一个食物，10个字段）
  3. 推荐指数汇总表（标准营养 + 每人推荐指数）

用法：python extract_all.py [PDF目录]

依赖：pip install PyPDF2
"""

import sys, os, csv, re, json
sys.stdout.reconfigure(encoding='utf-8')
sys.stderr.reconfigure(encoding='utf-8')
from PyPDF2 import PdfReader

# ── 字段定义 ──
COLUMNS = ['名称', '分类', '推荐指数', '能量KJ', '蛋白g', '脂肪g',
           '碳水化合物g', '淀粉g', '总膳食纤维g', '胆固醇mg']

NUM_FIELDS = ['能量KJ', '蛋白g', '脂肪g', '碳水化合物g', '淀粉g', '总膳食纤维g', '胆固醇mg']

KNOWN_CATS = [
    '主食', '乳制品', '干果', '坚果', '快餐', '水产品',
    '水果', '汤', '肉类', '蔬菜', '豆类及豆制品', '蛋类', '饮料'
]

# ── CJK 偏旁部首 → 标准汉字映射 ──
RADICAL_MAP = {
    '\u2f18': '卜', '\u2f1f': '土', '\u2f24': '大', '\u2f26': '子',
    '\u2f29': '小', '\u2f2d': '山', '\u2f32': '干', '\u2f3c': '心',
    '\u2f42': '文', '\u2f46': '无', '\u2f4a': '木', '\u2f50': '比',
    '\u2f54': '水', '\u2f55': '火', '\u2f5c': '牛', '\u2f5f': '玉',
    '\u2f60': '瓜', '\u2f62': '甘', '\u2f63': '生', '\u2f64': '用',
    '\u2f69': '白', '\u2f6a': '皮', '\u2f6c': '目', '\u2f6f': '石',
    '\u2f75': '竹', '\u2f76': '米', '\u2f7a': '羊', '\u2f7b': '羽',
    '\u2f7c': '老', '\u2f7f': '耳', '\u2f81': '肉', '\u2f90': '衣',
    '\u2f95': '谷', '\u2f96': '豆', '\u2f9d': '身', '\u2fa6': '金',
    '\u2faf': '面', '\u2fb2': '韭', '\u2fb9': '香', '\u2fca': '黑',
    '\u2ec9': '贝', '\u2edd': '食', '\u2ee2': '马', '\u2ee5': '鱼',
    '\u2ee8': '麦', '\u2ee9': '黄', '\u2ef0': '龙',
}

SKIP_TEXTS = [
    '根据您的肠道菌群', '分值从-100', '食物推荐考虑', '食物推荐是综合',
    '需要注意的是', '本饮食推荐', '该饮食推荐根据', '后续表格中的营养',
    '16S 高通量测序', '基于机器学习和', '肠道菌群健康检测报告说明',
    '检测方法及局限性', '数据分析及模型', '结果解读及使用',
    '影响因素说明', '建议将检测结果', '营养建议说明',
    '重要提示', '推荐食物清单', '实际食用时需结合', '如有特殊疾病',
    '免责声明', '本检测报告仅供', '以上模型预测', '正常范围的定义',
]


def normalize(text):
    """替换偏旁部首为标准汉字"""
    return ''.join(RADICAL_MAP.get(c, c) for c in text)


def parse_vertical(lines):
    """逐行格式：名称/分类/数值/... 每9行一组"""
    rows = []
    i = 0
    while i + 9 < len(lines):
        name = lines[i].strip()
        cat = lines[i + 1].strip()
        if cat not in KNOWN_CATS:
            i += 1
            continue
        nums = []
        ok = True
        for j in range(2, 10):
            v = lines[i + j].replace('\u2212', '-').replace('\u2014', '-').strip()
            try:
                int(v)
                nums.append(v)
            except ValueError:
                ok = False
                break
        if ok and len(nums) == 8:
            rows.append(dict(zip(COLUMNS, [name, cat] + nums)))
        i += 1
    return rows


def split_7_fields(s):
    """将7个连续数字串拆分为字段（回溯）"""
    results = []
    ranges = [(2, 4), (1, 2), (1, 2), (1, 2), (1, 2), (1, 2), (1, 4)]

    def backtrack(pos, idx, nums):
        if idx == 7:
            if pos == len(s):
                results.append(list(nums))
            return
        if pos >= len(s):
            return
        lo, hi = ranges[idx]
        for w in range(lo, min(hi + 1, len(s) - pos + 1)):
            chunk = s[pos:pos + w]
            if chunk.isdigit():
                backtrack(pos + w, idx + 1, nums + [int(chunk)])

    backtrack(0, 0, [])
    return results


def decode_compressed(name, num_str, ref_vals=None):
    """解码压缩格式数字串"""
    raw = num_str.lstrip('-')
    has_neg = num_str.startswith('-')
    results = []

    for rec_len in range(1, 3):
        if rec_len > len(raw):
            continue
        rec = ('-' if has_neg else '') + raw[:rec_len]
        try:
            rec_val = int(rec)
            if not (-100 <= rec_val <= 100):
                continue
        except ValueError:
            continue
        remain = raw[rec_len:]
        candidates = split_7_fields(remain)
        for nums in candidates:
            if ref_vals:
                matches = sum(1 for i in range(7) if ref_vals[i] == nums[i])
                if matches >= 6:
                    results.append([rec_val] + nums)
            else:
                results.append([rec_val] + nums)

    if not results:
        return None
    if ref_vals:
        results.sort(key=lambda r: (sum(1 for i in range(7) if ref_vals[i] == r[1:][i]),
                                    -len(str(abs(r[0])))), reverse=True)
        if sum(1 for i in range(7) if ref_vals[i] == results[0][1:][i]) < 6:
            return None
    return results[0]


def parse_compressed(text, ref_lookup=None):
    """压缩格式：所有字段在一行无分隔"""
    header = '名称分类推荐指数能量KJ蛋白g脂肪g碳水化合物g淀粉g总膳食纤维g胆固醇mg'
    rest = text
    if header in rest:
        rest = rest.replace(header, '')
    for kw in SKIP_TEXTS:
        rest = rest.replace(kw, '')

    rows = []
    while rest:
        best_cat, best_idx = None, len(rest)
        for cat in KNOWN_CATS:
            idx = rest.find(cat)
            if idx != -1 and idx < best_idx:
                best_idx, best_cat = idx, cat
        if best_cat is None:
            break

        name = rest[:best_idx]
        rest = rest[best_idx + len(best_cat):]

        num_str = ''
        while rest and (rest[0].isdigit() or rest[0] in '-\u2212\u2014'):
            c = '-' if rest[0] in '\u2212\u2014' else rest[0]
            num_str += c
            rest = rest[1:]

        if not name or not num_str:
            continue

        ref_vals = ref_lookup.get(name) if ref_lookup else None
        decoded = decode_compressed(name, num_str, ref_vals)
        if decoded:
            rows.append(dict(zip(COLUMNS, [name, best_cat] + [str(v) for v in decoded])))

    return rows


def detect_format(text):
    """自动检测格式：vertical(逐行) / compressed(压缩)"""
    lines = [l.strip() for l in text.split('\n') if l.strip() and not re.match(r'\d+/\d+', l)]
    if not lines:
        return 'unknown'
    # 一行很长 + 无换行 = compressed
    if len(lines) <= 3 and len(lines[0]) > 200:
        return 'compressed'
    long_count = sum(1 for l in lines[:10] if len(l) > 100)
    if long_count >= 3:
        return 'compressed'
    cat_count = sum(1 for l in lines[:80] if l in KNOWN_CATS)
    if cat_count >= 5:
        return 'vertical'
    return 'unknown'


def extract_food_table(pdf_path, ref_lookup=None):
    """从PDF提取食物推荐表，返回 (rows, fmt)"""
    reader = PdfReader(pdf_path)
    name = os.path.splitext(os.path.basename(pdf_path))[0]

    # 找到表格起始页
    food_start = None
    for i, page in enumerate(reader.pages):
        if '个体化食物推荐表' in page.extract_text():
            food_start = i
            break
    if food_start is None:
        return None, 'not_found'

    # 检测格式
    first_text = normalize(reader.pages[food_start + 1].extract_text())
    fmt = detect_format(first_text)

    rows = []
    if fmt == 'compressed':
        for i in range(food_start + 1, len(reader.pages)):
            text = normalize(reader.pages[i].extract_text())
            text = re.sub(r'\d+/\d+', '', text)
            rows.extend(parse_compressed(text, ref_lookup))
    else:
        all_lines = []
        for i in range(food_start + 1, len(reader.pages)):
            for line in normalize(reader.pages[i].extract_text()).split('\n'):
                lt = line.strip()
                if not lt or re.match(r'\d+/\d+', lt) or lt in COLUMNS:
                    continue
                if len(lt) > 60 and any(k in lt for k in SKIP_TEXTS):
                    continue
                all_lines.append(lt)
        rows = parse_vertical(all_lines)

    return rows, fmt


def main(pdf_dir):
    pdf_dir = pdf_dir or r'D:\workspace\cfc\docs\参考资料'
    outdir = pdf_dir
    all_reports = {}

    # ── Step 1: 提取每份PDF ──
    print('=' * 60)
    print('Step 1: 提取食物推荐表')
    print('=' * 60)

    pdf_files = sorted(f for f in os.listdir(pdf_dir) if f.lower().endswith('.pdf'))
    for pdf_file in pdf_files:
        pdf_path = os.path.join(pdf_dir, pdf_file)
        label = pdf_file.replace('.pdf', '')
        print(f'\n{label}...', end=' ')

        try:
            rows, fmt = extract_food_table(pdf_path)
        except Exception as e:
            print(f'ERROR: {e}')
            continue

        if rows is None:
            print('SKIP (未找到表格)')
            continue

        print(f'[{fmt}] {len(rows)} 条')
        all_reports[label] = rows

        # 保存原始文本
        raw_path = os.path.join(outdir, f'{label}-原始文本.txt')
        reader = PdfReader(pdf_path)
        with open(raw_path, 'w', encoding='utf-8') as f:
            for page in reader.pages:
                f.write(f'--- Page ---\n')
                f.write(normalize(page.extract_text()))
                f.write('\n')
        print(f'  raw -> {os.path.basename(raw_path)}')

        # 保存CSV
        csv_name = f'{label}-个体化食物推荐表.csv'
        csv_path = os.path.join(outdir, csv_name)
        with open(csv_path, 'w', newline='', encoding='utf-8-sig') as f:
            w = csv.DictWriter(f, fieldnames=COLUMNS)
            w.writeheader()
            w.writerows(rows)
        print(f'  csv -> {csv_name}')

    if not all_reports:
        print('\n未找到任何有效PDF')
        return

    # ── Step 2: 生成推荐指数汇总表 ──
    print('\n' + '=' * 60)
    print('Step 2: 生成推荐指数汇总表')
    print('=' * 60)

    # 标准营养参考（优先用侯报告，224条最完整）
    ref_report = None
    std_label = None
    for pref in ['530010234-侯', '侯']:
        for label in all_reports:
            if pref in label:
                ref_report = all_reports[label]
                std_label = label
                break
        if ref_report:
            break
    if not ref_report:
        # 用条目最多的
        std_label = max(all_reports, key=lambda k: len(all_reports[k]))
        ref_report = all_reports[std_label]
    print(f'标准参考: {std_label} ({len(ref_report)} 条)')

    std_lookup = {}
    for r in ref_report:
        std_lookup[r['名称']] = {k: r[k] for k in ['分类'] + NUM_FIELDS}

    # 收集推荐指数
    report_labels = {}
    for label in all_reports:
        if '某人' in label:
            report_labels[label] = '某人'
        elif '侯' in label:
            report_labels[label] = '侯'
        elif '547982403' in label:
            report_labels[label] = '547982403'
        elif '儿童' in label:
            report_labels[label] = '儿童示例'
        elif '朱' in label:
            report_labels[label] = '朱评估报告'
        else:
            report_labels[label] = label

    rec_all = {}
    for label, short in report_labels.items():
        rec_all[short] = {}
        for r in all_reports[label]:
            rec_all[short][r['名称']] = r['推荐指数']

    sum_cols = ['名称', '分类'] + NUM_FIELDS + list(report_labels.values())
    sum_path = os.path.join(outdir, '推荐指数汇总.csv')
    with open(sum_path, 'w', newline='', encoding='utf-8-sig') as f:
        w = csv.DictWriter(f, fieldnames=sum_cols)
        w.writeheader()
        for r in ref_report:
            name = r['名称']
            s = std_lookup[name]
            row = {'名称': name, '分类': s['分类']}
            for k in NUM_FIELDS:
                row[k] = s[k]
            for short in report_labels.values():
                row[short] = rec_all.get(short, {}).get(name, '')
            w.writerow(row)

    print(f'推荐指数汇总: {len(ref_report)} 行 -> {os.path.basename(sum_path)}')
    for short in report_labels.values():
        missing = sum(1 for r in ref_report if not rec_all.get(short, {}).get(r['名称'], ''))
        print(f'  {short}: {"完整" if missing == 0 else f"缺失 {missing} 条"}')

    print('\nDone!')


if __name__ == '__main__':
    pdf_dir = sys.argv[1] if len(sys.argv) > 1 else None
    main(pdf_dir)