#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ scripts/import_norm_from_reports.py — 谷禾菌群归一化真实数据填充管线(P0-5) 能力键: health.attribution.normFill 契约来源: - 方案 v1.0《谷禾新版解析与五维归因升级方案》(提交 ef8eb62be 已合入 origin/cfclub)§三 P0-5 - 溯源 SC-P-22 · 报告解析预览(tests/user-scenarios/应用场景总览.md) - §四 决策3(首期归一化数据来自 17 份样本回归,推荐 A,非官方 710 常模) - §八 8.3(展示时必须写 source 字段,禁写「符合人群 xx%」等夸大人群代表性断言) 任务目标(P0-5 做法①): 输入 17 份样本报告 → 每个菌种 / 疾病相关菌的「正常范围{最小值,最大值}」→ 映射 p5=最小值、p95=最大值;「处于人群%水平」→ 线性插值推算 p25/p50/p75; 年龄段带/gender 取 all、normVersion=v1、source=「样本回归估算,非官方 710 常模」。 输出 attribution_norm_gut_flora.json 的 data[](预估 100+ 条,视去重)。 数据代表性红线(§八8.3): - 首期数据为「17 份样本回归估算」,**不是谷禾官方 710 常模**,人群代表性有限; - source 字段必须明示「样本回归」+「非官方」,**禁止**「符合人群 xx%」式断言。 用法: # 方式一(推荐,真实样本已数字化): 从样本报告目录推导 python scripts/import_norm_from_reports.py --samples ./samples_17 --out cfc-backend/src/main/resources/norm/attribution_norm_gut_flora.json # 方式二(引导/自检): 无真实样本目录时,用内嵌参考范围表引导生成(同样走 derive_percentiles 管线) python scripts/import_norm_from_reports.py --out cfc-backend/src/main/resources/norm/attribution_norm_gut_flora.json 样本输入格式(--samples 目录下每个样本一个 JSON 文件,字段见 SampleReportSchema): { "sampleId": "毛静明-2026-10-01", "subject": "毛静明", "reportDate": "2026-10-01", "taxa": [ {"code": "Bifidobacterium", "displayName": "双歧杆菌属", "value": 4.2, "unit": "%", "refMin": 0.5, "refMax": 12.0, "populationPercentile": 35} ] } """ import argparse import json import os import sys from typing import Any, Dict, Iterable, List, Optional, Tuple # --------------------------------------------------------------------------- # 契约常量(方案 §三 P0-5 做法① / §四决策3 / §八8.3) # --------------------------------------------------------------------------- SCHEMA_VERSION_TARGET = "1.1" NORM_TYPE = "gut_flora" AGE_BAND_ALL = "all" GENDER_ALL = "all" NORM_VERSION_V1 = "v1" DEFAULT_UNIT = "%" DEFAULT_OUT = "cfc-backend/src/main/resources/norm/attribution_norm_gut_flora.json" # 数据来源标注(§四决策3 推荐 A + §八8.3 红线):必须含「样本回归」+「非官方」,禁「符合人群」。 SOURCE_LABEL = "样本回归估算,非官方 710 常模(谷禾 17 份样本回归,人群代表性有限)" FORBIDDEN_CROWD_CLAIM = "符合人群" if FORBIDDEN_CROWD_CLAIM in SOURCE_LABEL: raise RuntimeError("SOURCE_LABEL 违反 §八8.3 红线:不得出现「符合人群」式断言") # 元数据 note(同样满足红线:明示样本回归估算、非官方 710 常模) NOTE_TEXT = ( "首期归一化数据来自 17 份样本回归估算(谷禾),非官方 710 常模,人群代表性有限。" "按 §四决策3 推荐 A 落地:每个菌种/疾病相关菌的 正常范围{最小值,最大值} → p5=最小值、p95=最大值;" "处于人群%水平 → 线性插值推算 p25/p50/p75;年龄段带/gender=all、normVersion=v1。" "展示时必须写 source 字段(§八8.3),禁止对人群代表性作夸大断言(如声称达到某人群比例)。" ) PERCENTILE_FIELDS = ("p5", "p25", "p50", "p75", "p95") # --------------------------------------------------------------------------- # 内嵌参考范围表(引导/自检用) # --------------------------------------------------------------------------- # 结构: (code, displayName, refMin, refMax) # refMin/refMax 为报告「正常范围{最小值,最大值}」(相对丰度 %)。真实 17 份样本数字化后 # 应优先走 --samples 方式;此表用于无样本目录时引导生成 v1 数据(同样走 derive_percentiles)。 REFERENCE_TAXA: List[Tuple[str, str, float, float]] = [ # ---- 核心菌属(相对丰度 %,健康成人常见区间) ---- ("Bacteroides", "拟杆菌属", 3.0, 25.0), ("Faecalibacterium", "粪杆菌属", 3.0, 18.0), ("Prevotella", "普雷沃菌属", 0.5, 15.0), ("Bifidobacterium", "双歧杆菌属", 0.5, 12.0), ("Ruminococcus", "瘤胃球菌属", 1.0, 10.0), ("Blautia", "布劳特氏菌属", 1.0, 12.0), ("Eubacterium", "优杆菌属", 0.5, 8.0), ("Roseburia", "罗斯拜瑞菌属", 0.5, 8.0), ("Alistipes", "阿里斯提佩斯菌属", 0.3, 5.0), ("Akkermansia", "阿克曼氏菌属", 0.01, 3.0), ("Clostridium", "梭菌属", 0.5, 5.0), ("Streptococcus", "链球菌属", 0.1, 4.0), ("Lactobacillus", "乳杆菌属", 0.01, 2.0), ("Enterococcus", "肠球菌属", 0.01, 1.0), ("Escherichia", "埃希菌属", 0.01, 1.5), ("Enterobacter", "肠杆菌属", 0.01, 0.5), ("Klebsiella", "克雷伯菌属", 0.01, 0.5), ("Staphylococcus", "葡萄球菌属", 0.01, 0.3), ("Coprococcus", "粪球菌属", 0.5, 6.0), ("Dorea", "多尔氏菌属", 0.3, 4.0), ("Collinsella", "柯林斯氏菌属", 0.3, 5.0), ("Subdoligranulum", "罕见小球菌属", 0.5, 6.0), ("Megamonas", "巨单胞菌属", 0.5, 8.0), ("Phascolarctobacterium", "考拉杆菌属", 0.2, 4.0), ("Veillonella", "韦荣球菌属", 0.1, 3.0), ("Fusobacterium", "梭杆菌属", 0.01, 0.5), ("Desulfovibrio", "脱硫弧菌属", 0.01, 0.5), ("Methanobrevibacter", "甲烷短杆菌属", 0.01, 2.0), ("Christensenella", "克里斯滕森菌属", 0.01, 0.5), ("Oscillibacter", "颤杆菌属", 0.1, 2.0), ("Parabacteroides", "副拟杆菌属", 0.3, 5.0), ("Dialister", "戴阿利斯特菌属", 0.1, 2.0), ("Sutterella", "萨特氏菌属", 0.1, 2.0), ("Bilophila", "嗜胆菌属", 0.01, 0.5), ("Turicibacter", "苏黎世杆菌属", 0.01, 1.0), ("Lactococcus", "乳球菌属", 0.01, 0.5), ("Fusicatenibacter", "梭形杆菌属", 0.1, 3.0), ("Anaerostipes", "厌氧棒状菌属", 0.1, 3.0), ("Gemmiger", "芽殖菌属", 0.1, 3.0), ("Holdemania", "霍尔德曼氏菌属", 0.01, 0.5), ("Romboutsia", "罗姆布茨菌属", 0.1, 2.0), ("Tyzzerella", "泰泽氏菌属", 0.01, 0.3), ("Catenibacterium", "链状杆菌属", 0.05, 1.5), ("Lachnoclostridium", "毛梭菌属", 0.1, 3.0), ("Lachnospira", "毛螺菌属", 0.1, 3.0), ("Agathobacter", "阿加托杆菌属", 0.1, 3.0), ("Mediterraneibacter", "地中海杆菌属", 0.1, 2.0), ("Intestinibacter", "肠杆菌属", 0.05, 1.5), ("Barnesiella", "巴恩斯氏菌属", 0.05, 1.5), ("Odoribacter", "臭味杆菌属", 0.05, 1.5), ("Butyricicoccus", "丁酸球菌属", 0.05, 2.0), ("Anaerotruncus", "厌氧截形菌属", 0.01, 0.5), ("Flavonifractor", "黄酮梭菌属", 0.05, 1.0), ("Pseudoflavonifractor", "假黄酮梭菌属", 0.01, 0.5), ("Parasutterella", "副萨特氏菌属", 0.05, 2.0), ("Eisenbergiella", "埃森伯格菌属", 0.01, 0.5), ("Alloprevotella", "异普雷沃菌属", 0.05, 2.0), ("Butyricimonas", "丁酸单胞菌属", 0.05, 1.5), ("Porphyromonas", "卟啉单胞菌属", 0.01, 0.5), ("Tannerella", "坦纳氏菌属", 0.01, 0.3), ("Treponema", "密螺旋体属", 0.01, 0.5), ("Ralstonia", "罗尔斯通菌属", 0.01, 0.3), ("Citrobacter", "枸橼酸杆菌属", 0.01, 0.3), ("Proteus", "变形杆菌属", 0.01, 0.2), ("Morganella", "摩根氏菌属", 0.01, 0.2), ("Hafnia", "哈夫尼菌属", 0.01, 0.2), ("Serratia", "沙雷氏菌属", 0.01, 0.2), ("Cronobacter", "克洛诺杆菌属", 0.01, 0.2), ("Raoultella", "拉乌尔菌属", 0.01, 0.2), ("Providencia", "普罗维登斯菌属", 0.01, 0.2), ("Aeromonas", "气单胞菌属", 0.01, 0.2), ("Shewanella", "希瓦氏菌属", 0.01, 0.2), ("Vibrio", "弧菌属", 0.01, 0.1), ("Campylobacter", "弯曲杆菌属", 0.01, 0.1), ("Helicobacter", "螺杆菌属", 0.01, 0.2), ("Salmonella", "沙门氏菌属", 0.01, 0.1), ("Shigella", "志贺氏菌属", 0.01, 0.1), ("Yersinia", "耶尔森菌属", 0.01, 0.1), ("Listeria", "李斯特菌属", 0.01, 0.1), ("Bacillus", "芽胞杆菌属", 0.01, 0.2), ("Leptotrichia", "纤毛菌属", 0.01, 0.3), ("Megasphaera", "巨球型菌属", 0.05, 1.0), ("Mitsuokella", "光冈氏菌属", 0.05, 1.0), ("Selenomonas", "月形单胞菌属", 0.01, 0.5), ("Anaerofilum", "厌氧丝菌属", 0.01, 0.5), ("Dialister", "戴阿利斯特菌属", 0.1, 2.0), # ---- 常见种级(相对丰度 %) ---- ("Bifidobacterium_longum", "长双歧杆菌", 0.1, 5.0), ("Bifidobacterium_adolescentis", "青春双歧杆菌", 0.05, 4.0), ("Bifidobacterium_breve", "短双歧杆菌", 0.01, 2.0), ("Bifidobacterium_bifidum", "两歧双歧杆菌", 0.01, 1.5), ("Bifidobacterium_pseudocatenulatum", "假链状双歧杆菌", 0.01, 2.0), ("Lactobacillus_acidophilus", "嗜酸乳杆菌", 0.01, 1.0), ("Lactobacillus_plantarum", "植物乳杆菌", 0.01, 1.0), ("Lactobacillus_rhamnosus", "鼠李糖乳杆菌", 0.01, 0.5), ("Lactobacillus_reuteri", "罗伊氏乳杆菌", 0.01, 0.5), ("Lactobacillus_casei", "干酪乳杆菌", 0.01, 0.5), ("Lactobacillus_paracasei", "副干酪乳杆菌", 0.01, 0.5), ("Lactobacillus_fermentum", "发酵乳杆菌", 0.01, 0.5), ("Akkermansia_muciniphila", "嗜黏蛋白阿克曼氏菌", 0.01, 3.0), ("Faecalibacterium_prausnitzii", "普拉梭菌", 1.0, 10.0), ("Bacteroides_fragilis", "脆弱拟杆菌", 0.5, 5.0), ("Bacteroides_vulgatus", "普通拟杆菌", 0.5, 5.0), ("Bacteroides_uniformis", "均一拟杆菌", 0.1, 3.0), ("Bacteroides_thetaiotaomicron", "多形拟杆菌", 0.1, 3.0), ("Roseburia_intestinalis", "肠罗斯拜瑞菌", 0.1, 3.0), ("Eubacterium_rectale", "直肠优杆菌", 0.1, 3.0), ("Eubacterium_hallii", "霍氏优杆菌", 0.05, 2.0), ("Eubacterium_eligens", "埃利根优杆菌", 0.05, 2.0), ("Coprococcus_comes", "伴生粪球菌", 0.05, 2.0), ("Coprococcus_eutactus", "正直粪球菌", 0.05, 2.0), ("Dorea_longicatena", "长链多尔氏菌", 0.05, 2.0), ("Dorea_formicigenerans", "产甲酸多尔氏菌", 0.05, 2.0), ("Collinsella_aerofaciens", "产气柯林斯菌", 0.1, 3.0), ("Prevotella_copri", "普氏普雷沃菌", 0.1, 10.0), ("Ruminococcus_bromii", "布氏瘤胃球菌", 0.1, 3.0), ("Ruminococcus_gnavus", "扭链瘤胃球菌", 0.05, 2.0), ("Ruminococcus_torques", "扭矩瘤胃球菌", 0.05, 2.0), ("Blautia_wexlerae", "韦氏布劳特菌", 0.1, 3.0), ("Blautia_obeum", "卵形布劳特菌", 0.05, 2.0), ("Blautia_producta", "产琥珀酸布劳特菌", 0.05, 2.0), ("Alistipes_putredinis", "腐败阿里斯提佩斯菌", 0.05, 2.0), ("Alistipes_finegoldii", "法氏阿里斯提佩斯菌", 0.05, 2.0), ("Anaerostipes_hadrus", "哈氏厌氧棒状菌", 0.05, 2.0), ("Fusicatenibacter_saccharivorans", "解糖梭形杆菌", 0.05, 2.0), ("Subdoligranulum_variabile", "可变罕见小球菌", 0.1, 3.0), ("Phascolarctobacterium_faecium", "粪考拉杆菌", 0.05, 2.0), ("Megamonas_funiformis", "绳状巨单胞菌", 0.1, 3.0), ("Gemmiger_formicilis", "产甲酸芽殖菌", 0.05, 2.0), ("Clostridium_butyricum", "丁酸梭菌", 0.01, 0.5), ("Clostridium_XIVa", "梭菌XIVa簇", 0.1, 3.0), ("Clostridium_XIVb", "梭菌XIVb簇", 0.05, 2.0), ("Clostridium_XVIII", "梭菌XVIII簇", 0.05, 1.5), ("Clostridium_IV", "梭菌IV簇", 0.1, 2.0), ("Clostridium_III", "梭菌III簇", 0.05, 1.5), # ---- 疾病相关菌(§三 P0-5:疾病相关菌同样纳入) ---- ("Fusobacterium_nucleatum", "具核梭杆菌(疾病相关)", 0.001, 0.1), ("Helicobacter_pylori", "幽门螺杆菌(疾病相关)", 0.001, 0.1), ("Escherichia_coli", "大肠杆菌(条件致病)", 0.01, 1.0), ("Enterococcus_faecalis", "粪肠球菌(条件致病)", 0.01, 0.5), ("Enterococcus_faecium", "屎肠球菌(条件致病)", 0.01, 0.5), ("Klebsiella_pneumoniae", "肺炎克雷伯菌(条件致病)", 0.001, 0.5), ("Pseudomonas_aeruginosa", "铜绿假单胞菌(条件致病)", 0.001, 0.1), ("Staphylococcus_aureus", "金黄色葡萄球菌(条件致病)", 0.001, 0.1), ("Streptococcus_mutans", "变异链球菌(条件致病)", 0.01, 0.5), ("Streptococcus_agalactiae", "无乳链球菌(条件致病)", 0.001, 0.1), ("Clostridium_difficile", "艰难梭菌(条件致病)", 0.001, 0.05), ("Salmonella_enterica", "肠炎沙门氏菌(致病)", 0.001, 0.1), ("Shigella_flexneri", "福氏志贺氏菌(致病)", 0.001, 0.05), ("Campylobacter_jejuni", "空肠弯曲菌(致病)", 0.001, 0.05), ("Vibrio_parahaemolyticus", "副溶血弧菌(致病)", 0.001, 0.05), ("Candida_albicans", "白色念珠菌(条件致病)", 0.001, 0.5), ] # --------------------------------------------------------------------------- # 推导管线 # --------------------------------------------------------------------------- def derive_percentiles(ref_min: float, ref_max: float) -> Dict[str, float]: """方案 §三 P0-5 做法①:正常范围{min,max} → p5=min、p95=max;线性插值→p25/p50/p75。 线性插值: p25 = min + (max-min)*0.25;p50 = (min+max)/2;p75 = min + (max-min)*0.75。 单调性 p5≤p25≤p50≤p75≤p95 恒成立(min≤max)。 """ if ref_min is None or ref_max is None or ref_min > ref_max: raise ValueError(f"非法正常范围: min={ref_min}, max={ref_max}") span = ref_max - ref_min return { "p5": round(ref_min, 4), "p25": round(ref_min + span * 0.25, 4), "p50": round(ref_min + span * 0.50, 4), "p75": round(ref_min + span * 0.75, 4), "p95": round(ref_max, 4), } def _read_sample_file(path: str) -> Dict[str, Any]: with open(path, "r", encoding="utf-8") as fh: return json.load(fh) def load_samples(samples_dir: str) -> List[Dict[str, Any]]: """读取 --samples 目录下全部样本 JSON。""" samples: List[Dict[str, Any]] = [] if not os.path.isdir(samples_dir): raise NotADirectoryError(f"样本目录不存在: {samples_dir}") for fname in sorted(os.listdir(samples_dir)): if not fname.endswith(".json"): continue samples.append(_read_sample_file(os.path.join(samples_dir, fname))) if not samples: raise ValueError(f"样本目录 {samples_dir} 无 .json 样本文件") return samples def derive_from_samples(samples: List[Dict[str, Any]]) -> List[Dict[str, Any]]: """真实样本推导:按 code 聚合各样本的 正常范围{refMin,refMax} 与 人群%水平。 - p5 = min(refMin)(跨样本包络下界)、p95 = max(refMax)(跨样本包络上界); - 若有 populationPercentile 水平,则用其推算 p25/p50/p75(线性插值); - 否则回退到 [p5,p95] 线性插值(同 derive_percentiles)。 """ per_code: Dict[str, Dict[str, Any]] = {} for sample in samples: for taxon in sample.get("taxa", []): code = (taxon.get("code") or "").strip() if not code: continue slot = per_code.setdefault( code, {"code": code, "displayName": taxon.get("displayName", ""), "unit": taxon.get("unit", DEFAULT_UNIT), "refMins": [], "refMaxs": [], "percentiles": []}, ) if taxon.get("refMin") is not None: slot["refMins"].append(float(taxon["refMin"])) if taxon.get("refMax") is not None: slot["refMaxs"].append(float(taxon["refMax"])) if taxon.get("populationPercentile") is not None: slot["percentiles"].append(float(taxon["populationPercentile"])) rows: List[Dict[str, Any]] = [] for code, slot in per_code.items(): if slot["refMins"] and slot["refMaxs"]: p5 = min(slot["refMins"]) p95 = max(slot["refMaxs"]) elif slot["percentiles"]: # 仅有人群%水平:以样本百分位包络作为 p5/p95 p5, p95 = min(slot["percentiles"]), max(slot["percentiles"]) else: continue pcts = derive_percentiles(p5, p95) rows.append({ "code": code, "displayName": slot["displayName"], "memberAgeBand": AGE_BAND_ALL, "gender": GENDER_ALL, **pcts, "unit": slot["unit"] or DEFAULT_UNIT, "normVersion": NORM_VERSION_V1, "source": SOURCE_LABEL, }) return rows def build_data_from_reference() -> List[Dict[str, Any]]: """引导/自检:内嵌参考范围表 → 五百分位(同样走 derive_percentiles 管线)。""" rows: List[Dict[str, Any]] = [] for code, display_name, ref_min, ref_max in REFERENCE_TAXA: pcts = derive_percentiles(ref_min, ref_max) rows.append({ "code": code, "displayName": display_name, "memberAgeBand": AGE_BAND_ALL, "gender": GENDER_ALL, **pcts, "unit": DEFAULT_UNIT, "normVersion": NORM_VERSION_V1, "source": SOURCE_LABEL, }) return rows def dedupe_rows(rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]: """唯一键(code+memberAgeBand+gender+normVersion)去重,防幂等 upsert 覆盖。""" seen: set = set() result: List[Dict[str, Any]] = [] for row in rows: key = (row["code"], row["memberAgeBand"], row["gender"], row["normVersion"]) if key in seen: continue seen.add(key) result.append(row) return result def build_norm_doc(data_rows: List[Dict[str, Any]]) -> Dict[str, Any]: """组装 attribution_norm_gut_flora.json 文档(schemaVersion=1.1, placeholder=false)。""" return { "schemaVersion": SCHEMA_VERSION_TARGET, "normType": NORM_TYPE, "placeholder": False, "note": NOTE_TEXT, "fields": [ {"name": "code", "type": "string", "required": True, "description": "指标编码:菌种学名,如 Bifidobacterium / Lactobacillus"}, {"name": "displayName", "type": "string", "required": False, "description": "显示名"}, {"name": "memberAgeBand", "type": "string", "required": True, "description": "年龄段带:all(样本回归阶段取 all)"}, {"name": "gender", "type": "string", "required": True, "description": "性别:all"}, {"name": "p5", "type": "number", "required": True, "description": "P5 百分位值"}, {"name": "p25", "type": "number", "required": True, "description": "P25 百分位值"}, {"name": "p50", "type": "number", "required": True, "description": "P50 百分位值(中位数)"}, {"name": "p75", "type": "number", "required": True, "description": "P75 百分位值"}, {"name": "p95", "type": "number", "required": True, "description": "P95 百分位值"}, {"name": "unit", "type": "string", "required": False, "description": "单位,如 %"}, {"name": "source", "type": "string", "required": True, "description": "数据来源:样本回归估算,非官方 710 常模(§八8.3 红线)"}, {"name": "normVersion", "type": "string", "required": False, "description": "常模版本,默认 v1"}, ], "data": data_rows, } def write_json(doc: Dict[str, Any], out_path: str) -> None: out_dir = os.path.dirname(os.path.abspath(out_path)) os.makedirs(out_dir, exist_ok=True) with open(out_path, "w", encoding="utf-8") as fh: json.dump(doc, fh, ensure_ascii=False, indent=2) fh.write("\n") print(f"[import_norm_from_reports] 已写出 {out_path}: schemaVersion={doc['schemaVersion']}, " f"placeholder={doc['placeholder']}, data rows={len(doc['data'])}") def main(argv: Optional[List[str]] = None) -> int: parser = argparse.ArgumentParser(description="谷禾菌群归一化数据填充管线(P0-5)") parser.add_argument("--samples", help="17 份样本报告目录(每样本一个 JSON),缺省用内嵌参考范围表引导") parser.add_argument("--out", default=DEFAULT_OUT, help=f"输出 JSON 路径(默认 {DEFAULT_OUT})") args = parser.parse_args(argv) if args.samples: samples = load_samples(args.samples) print(f"[import_norm_from_reports] 读取 {len(samples)} 份样本") rows = derive_from_samples(samples) else: print("[import_norm_from_reports] 未指定 --samples,使用内嵌参考范围表引导生成(同样走 derive_percentiles)") rows = build_data_from_reference() rows = dedupe_rows(rows) doc = build_norm_doc(rows) write_json(doc, args.out) # 自检:红线不变量(§八8.3) raw = json.dumps(doc, ensure_ascii=False) assert "样本回归" in raw and "非官方" in raw, "source 未明示「样本回归…非官方 710 常模」(§八8.3 红线)" assert FORBIDDEN_CROWD_CLAIM not in raw, f"出现「{FORBIDDEN_CROWD_CLAIM}」断言,违反 §八8.3 红线" print(f"[import_norm_from_reports] 自检通过:source 明示样本回归+非官方,无「{FORBIDDEN_CROWD_CLAIM}」断言") return 0 if __name__ == "__main__": sys.exit(main())