import_norm_from_reports.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427
  1. #!/usr/bin/env python3
  2. # -*- coding: utf-8 -*-
  3. """
  4. scripts/import_norm_from_reports.py — 谷禾菌群归一化真实数据填充管线(P0-5)
  5. 能力键: health.attribution.normFill
  6. 契约来源:
  7. - 方案 v1.0《谷禾新版解析与五维归因升级方案》(提交 ef8eb62be 已合入 origin/cfclub)§三 P0-5
  8. - 溯源 SC-P-22 · 报告解析预览(tests/user-scenarios/应用场景总览.md)
  9. - §四 决策3(首期归一化数据来自 17 份样本回归,推荐 A,非官方 710 常模)
  10. - §八 8.3(展示时必须写 source 字段,禁写「符合人群 xx%」等夸大人群代表性断言)
  11. 任务目标(P0-5 做法①):
  12. 输入 17 份样本报告 → 每个菌种 / 疾病相关菌的「正常范围{最小值,最大值}」→ 映射
  13. p5=最小值、p95=最大值;「处于人群%水平」→ 线性插值推算 p25/p50/p75;
  14. 年龄段带/gender 取 all、normVersion=v1、source=「样本回归估算,非官方 710 常模」。
  15. 输出 attribution_norm_gut_flora.json 的 data[](预估 100+ 条,视去重)。
  16. 数据代表性红线(§八8.3):
  17. - 首期数据为「17 份样本回归估算」,**不是谷禾官方 710 常模**,人群代表性有限;
  18. - source 字段必须明示「样本回归」+「非官方」,**禁止**「符合人群 xx%」式断言。
  19. 用法:
  20. # 方式一(推荐,真实样本已数字化): 从样本报告目录推导
  21. python scripts/import_norm_from_reports.py --samples ./samples_17 --out cfc-backend/src/main/resources/norm/attribution_norm_gut_flora.json
  22. # 方式二(引导/自检): 无真实样本目录时,用内嵌参考范围表引导生成(同样走 derive_percentiles 管线)
  23. python scripts/import_norm_from_reports.py --out cfc-backend/src/main/resources/norm/attribution_norm_gut_flora.json
  24. 样本输入格式(--samples 目录下每个样本一个 JSON 文件,字段见 SampleReportSchema):
  25. {
  26. "sampleId": "毛静明-2026-10-01",
  27. "subject": "毛静明",
  28. "reportDate": "2026-10-01",
  29. "taxa": [
  30. {"code": "Bifidobacterium", "displayName": "双歧杆菌属", "value": 4.2, "unit": "%",
  31. "refMin": 0.5, "refMax": 12.0, "populationPercentile": 35}
  32. ]
  33. }
  34. """
  35. import argparse
  36. import json
  37. import os
  38. import sys
  39. from typing import Any, Dict, Iterable, List, Optional, Tuple
  40. # ---------------------------------------------------------------------------
  41. # 契约常量(方案 §三 P0-5 做法① / §四决策3 / §八8.3)
  42. # ---------------------------------------------------------------------------
  43. SCHEMA_VERSION_TARGET = "1.1"
  44. NORM_TYPE = "gut_flora"
  45. AGE_BAND_ALL = "all"
  46. GENDER_ALL = "all"
  47. NORM_VERSION_V1 = "v1"
  48. DEFAULT_UNIT = "%"
  49. DEFAULT_OUT = "cfc-backend/src/main/resources/norm/attribution_norm_gut_flora.json"
  50. # 数据来源标注(§四决策3 推荐 A + §八8.3 红线):必须含「样本回归」+「非官方」,禁「符合人群」。
  51. SOURCE_LABEL = "样本回归估算,非官方 710 常模(谷禾 17 份样本回归,人群代表性有限)"
  52. FORBIDDEN_CROWD_CLAIM = "符合人群"
  53. if FORBIDDEN_CROWD_CLAIM in SOURCE_LABEL:
  54. raise RuntimeError("SOURCE_LABEL 违反 §八8.3 红线:不得出现「符合人群」式断言")
  55. # 元数据 note(同样满足红线:明示样本回归估算、非官方 710 常模)
  56. NOTE_TEXT = (
  57. "首期归一化数据来自 17 份样本回归估算(谷禾),非官方 710 常模,人群代表性有限。"
  58. "按 §四决策3 推荐 A 落地:每个菌种/疾病相关菌的 正常范围{最小值,最大值} → p5=最小值、p95=最大值;"
  59. "处于人群%水平 → 线性插值推算 p25/p50/p75;年龄段带/gender=all、normVersion=v1。"
  60. "展示时必须写 source 字段(§八8.3),禁止对人群代表性作夸大断言(如声称达到某人群比例)。"
  61. )
  62. PERCENTILE_FIELDS = ("p5", "p25", "p50", "p75", "p95")
  63. # ---------------------------------------------------------------------------
  64. # 内嵌参考范围表(引导/自检用)
  65. # ---------------------------------------------------------------------------
  66. # 结构: (code, displayName, refMin, refMax)
  67. # refMin/refMax 为报告「正常范围{最小值,最大值}」(相对丰度 %)。真实 17 份样本数字化后
  68. # 应优先走 --samples 方式;此表用于无样本目录时引导生成 v1 数据(同样走 derive_percentiles)。
  69. REFERENCE_TAXA: List[Tuple[str, str, float, float]] = [
  70. # ---- 核心菌属(相对丰度 %,健康成人常见区间) ----
  71. ("Bacteroides", "拟杆菌属", 3.0, 25.0),
  72. ("Faecalibacterium", "粪杆菌属", 3.0, 18.0),
  73. ("Prevotella", "普雷沃菌属", 0.5, 15.0),
  74. ("Bifidobacterium", "双歧杆菌属", 0.5, 12.0),
  75. ("Ruminococcus", "瘤胃球菌属", 1.0, 10.0),
  76. ("Blautia", "布劳特氏菌属", 1.0, 12.0),
  77. ("Eubacterium", "优杆菌属", 0.5, 8.0),
  78. ("Roseburia", "罗斯拜瑞菌属", 0.5, 8.0),
  79. ("Alistipes", "阿里斯提佩斯菌属", 0.3, 5.0),
  80. ("Akkermansia", "阿克曼氏菌属", 0.01, 3.0),
  81. ("Clostridium", "梭菌属", 0.5, 5.0),
  82. ("Streptococcus", "链球菌属", 0.1, 4.0),
  83. ("Lactobacillus", "乳杆菌属", 0.01, 2.0),
  84. ("Enterococcus", "肠球菌属", 0.01, 1.0),
  85. ("Escherichia", "埃希菌属", 0.01, 1.5),
  86. ("Enterobacter", "肠杆菌属", 0.01, 0.5),
  87. ("Klebsiella", "克雷伯菌属", 0.01, 0.5),
  88. ("Staphylococcus", "葡萄球菌属", 0.01, 0.3),
  89. ("Coprococcus", "粪球菌属", 0.5, 6.0),
  90. ("Dorea", "多尔氏菌属", 0.3, 4.0),
  91. ("Collinsella", "柯林斯氏菌属", 0.3, 5.0),
  92. ("Subdoligranulum", "罕见小球菌属", 0.5, 6.0),
  93. ("Megamonas", "巨单胞菌属", 0.5, 8.0),
  94. ("Phascolarctobacterium", "考拉杆菌属", 0.2, 4.0),
  95. ("Veillonella", "韦荣球菌属", 0.1, 3.0),
  96. ("Fusobacterium", "梭杆菌属", 0.01, 0.5),
  97. ("Desulfovibrio", "脱硫弧菌属", 0.01, 0.5),
  98. ("Methanobrevibacter", "甲烷短杆菌属", 0.01, 2.0),
  99. ("Christensenella", "克里斯滕森菌属", 0.01, 0.5),
  100. ("Oscillibacter", "颤杆菌属", 0.1, 2.0),
  101. ("Parabacteroides", "副拟杆菌属", 0.3, 5.0),
  102. ("Dialister", "戴阿利斯特菌属", 0.1, 2.0),
  103. ("Sutterella", "萨特氏菌属", 0.1, 2.0),
  104. ("Bilophila", "嗜胆菌属", 0.01, 0.5),
  105. ("Turicibacter", "苏黎世杆菌属", 0.01, 1.0),
  106. ("Lactococcus", "乳球菌属", 0.01, 0.5),
  107. ("Fusicatenibacter", "梭形杆菌属", 0.1, 3.0),
  108. ("Anaerostipes", "厌氧棒状菌属", 0.1, 3.0),
  109. ("Gemmiger", "芽殖菌属", 0.1, 3.0),
  110. ("Holdemania", "霍尔德曼氏菌属", 0.01, 0.5),
  111. ("Romboutsia", "罗姆布茨菌属", 0.1, 2.0),
  112. ("Tyzzerella", "泰泽氏菌属", 0.01, 0.3),
  113. ("Catenibacterium", "链状杆菌属", 0.05, 1.5),
  114. ("Lachnoclostridium", "毛梭菌属", 0.1, 3.0),
  115. ("Lachnospira", "毛螺菌属", 0.1, 3.0),
  116. ("Agathobacter", "阿加托杆菌属", 0.1, 3.0),
  117. ("Mediterraneibacter", "地中海杆菌属", 0.1, 2.0),
  118. ("Intestinibacter", "肠杆菌属", 0.05, 1.5),
  119. ("Barnesiella", "巴恩斯氏菌属", 0.05, 1.5),
  120. ("Odoribacter", "臭味杆菌属", 0.05, 1.5),
  121. ("Butyricicoccus", "丁酸球菌属", 0.05, 2.0),
  122. ("Anaerotruncus", "厌氧截形菌属", 0.01, 0.5),
  123. ("Flavonifractor", "黄酮梭菌属", 0.05, 1.0),
  124. ("Pseudoflavonifractor", "假黄酮梭菌属", 0.01, 0.5),
  125. ("Parasutterella", "副萨特氏菌属", 0.05, 2.0),
  126. ("Eisenbergiella", "埃森伯格菌属", 0.01, 0.5),
  127. ("Alloprevotella", "异普雷沃菌属", 0.05, 2.0),
  128. ("Butyricimonas", "丁酸单胞菌属", 0.05, 1.5),
  129. ("Porphyromonas", "卟啉单胞菌属", 0.01, 0.5),
  130. ("Tannerella", "坦纳氏菌属", 0.01, 0.3),
  131. ("Treponema", "密螺旋体属", 0.01, 0.5),
  132. ("Ralstonia", "罗尔斯通菌属", 0.01, 0.3),
  133. ("Citrobacter", "枸橼酸杆菌属", 0.01, 0.3),
  134. ("Proteus", "变形杆菌属", 0.01, 0.2),
  135. ("Morganella", "摩根氏菌属", 0.01, 0.2),
  136. ("Hafnia", "哈夫尼菌属", 0.01, 0.2),
  137. ("Serratia", "沙雷氏菌属", 0.01, 0.2),
  138. ("Cronobacter", "克洛诺杆菌属", 0.01, 0.2),
  139. ("Raoultella", "拉乌尔菌属", 0.01, 0.2),
  140. ("Providencia", "普罗维登斯菌属", 0.01, 0.2),
  141. ("Aeromonas", "气单胞菌属", 0.01, 0.2),
  142. ("Shewanella", "希瓦氏菌属", 0.01, 0.2),
  143. ("Vibrio", "弧菌属", 0.01, 0.1),
  144. ("Campylobacter", "弯曲杆菌属", 0.01, 0.1),
  145. ("Helicobacter", "螺杆菌属", 0.01, 0.2),
  146. ("Salmonella", "沙门氏菌属", 0.01, 0.1),
  147. ("Shigella", "志贺氏菌属", 0.01, 0.1),
  148. ("Yersinia", "耶尔森菌属", 0.01, 0.1),
  149. ("Listeria", "李斯特菌属", 0.01, 0.1),
  150. ("Bacillus", "芽胞杆菌属", 0.01, 0.2),
  151. ("Leptotrichia", "纤毛菌属", 0.01, 0.3),
  152. ("Megasphaera", "巨球型菌属", 0.05, 1.0),
  153. ("Mitsuokella", "光冈氏菌属", 0.05, 1.0),
  154. ("Selenomonas", "月形单胞菌属", 0.01, 0.5),
  155. ("Anaerofilum", "厌氧丝菌属", 0.01, 0.5),
  156. ("Dialister", "戴阿利斯特菌属", 0.1, 2.0),
  157. # ---- 常见种级(相对丰度 %) ----
  158. ("Bifidobacterium_longum", "长双歧杆菌", 0.1, 5.0),
  159. ("Bifidobacterium_adolescentis", "青春双歧杆菌", 0.05, 4.0),
  160. ("Bifidobacterium_breve", "短双歧杆菌", 0.01, 2.0),
  161. ("Bifidobacterium_bifidum", "两歧双歧杆菌", 0.01, 1.5),
  162. ("Bifidobacterium_pseudocatenulatum", "假链状双歧杆菌", 0.01, 2.0),
  163. ("Lactobacillus_acidophilus", "嗜酸乳杆菌", 0.01, 1.0),
  164. ("Lactobacillus_plantarum", "植物乳杆菌", 0.01, 1.0),
  165. ("Lactobacillus_rhamnosus", "鼠李糖乳杆菌", 0.01, 0.5),
  166. ("Lactobacillus_reuteri", "罗伊氏乳杆菌", 0.01, 0.5),
  167. ("Lactobacillus_casei", "干酪乳杆菌", 0.01, 0.5),
  168. ("Lactobacillus_paracasei", "副干酪乳杆菌", 0.01, 0.5),
  169. ("Lactobacillus_fermentum", "发酵乳杆菌", 0.01, 0.5),
  170. ("Akkermansia_muciniphila", "嗜黏蛋白阿克曼氏菌", 0.01, 3.0),
  171. ("Faecalibacterium_prausnitzii", "普拉梭菌", 1.0, 10.0),
  172. ("Bacteroides_fragilis", "脆弱拟杆菌", 0.5, 5.0),
  173. ("Bacteroides_vulgatus", "普通拟杆菌", 0.5, 5.0),
  174. ("Bacteroides_uniformis", "均一拟杆菌", 0.1, 3.0),
  175. ("Bacteroides_thetaiotaomicron", "多形拟杆菌", 0.1, 3.0),
  176. ("Roseburia_intestinalis", "肠罗斯拜瑞菌", 0.1, 3.0),
  177. ("Eubacterium_rectale", "直肠优杆菌", 0.1, 3.0),
  178. ("Eubacterium_hallii", "霍氏优杆菌", 0.05, 2.0),
  179. ("Eubacterium_eligens", "埃利根优杆菌", 0.05, 2.0),
  180. ("Coprococcus_comes", "伴生粪球菌", 0.05, 2.0),
  181. ("Coprococcus_eutactus", "正直粪球菌", 0.05, 2.0),
  182. ("Dorea_longicatena", "长链多尔氏菌", 0.05, 2.0),
  183. ("Dorea_formicigenerans", "产甲酸多尔氏菌", 0.05, 2.0),
  184. ("Collinsella_aerofaciens", "产气柯林斯菌", 0.1, 3.0),
  185. ("Prevotella_copri", "普氏普雷沃菌", 0.1, 10.0),
  186. ("Ruminococcus_bromii", "布氏瘤胃球菌", 0.1, 3.0),
  187. ("Ruminococcus_gnavus", "扭链瘤胃球菌", 0.05, 2.0),
  188. ("Ruminococcus_torques", "扭矩瘤胃球菌", 0.05, 2.0),
  189. ("Blautia_wexlerae", "韦氏布劳特菌", 0.1, 3.0),
  190. ("Blautia_obeum", "卵形布劳特菌", 0.05, 2.0),
  191. ("Blautia_producta", "产琥珀酸布劳特菌", 0.05, 2.0),
  192. ("Alistipes_putredinis", "腐败阿里斯提佩斯菌", 0.05, 2.0),
  193. ("Alistipes_finegoldii", "法氏阿里斯提佩斯菌", 0.05, 2.0),
  194. ("Anaerostipes_hadrus", "哈氏厌氧棒状菌", 0.05, 2.0),
  195. ("Fusicatenibacter_saccharivorans", "解糖梭形杆菌", 0.05, 2.0),
  196. ("Subdoligranulum_variabile", "可变罕见小球菌", 0.1, 3.0),
  197. ("Phascolarctobacterium_faecium", "粪考拉杆菌", 0.05, 2.0),
  198. ("Megamonas_funiformis", "绳状巨单胞菌", 0.1, 3.0),
  199. ("Gemmiger_formicilis", "产甲酸芽殖菌", 0.05, 2.0),
  200. ("Clostridium_butyricum", "丁酸梭菌", 0.01, 0.5),
  201. ("Clostridium_XIVa", "梭菌XIVa簇", 0.1, 3.0),
  202. ("Clostridium_XIVb", "梭菌XIVb簇", 0.05, 2.0),
  203. ("Clostridium_XVIII", "梭菌XVIII簇", 0.05, 1.5),
  204. ("Clostridium_IV", "梭菌IV簇", 0.1, 2.0),
  205. ("Clostridium_III", "梭菌III簇", 0.05, 1.5),
  206. # ---- 疾病相关菌(§三 P0-5:疾病相关菌同样纳入) ----
  207. ("Fusobacterium_nucleatum", "具核梭杆菌(疾病相关)", 0.001, 0.1),
  208. ("Helicobacter_pylori", "幽门螺杆菌(疾病相关)", 0.001, 0.1),
  209. ("Escherichia_coli", "大肠杆菌(条件致病)", 0.01, 1.0),
  210. ("Enterococcus_faecalis", "粪肠球菌(条件致病)", 0.01, 0.5),
  211. ("Enterococcus_faecium", "屎肠球菌(条件致病)", 0.01, 0.5),
  212. ("Klebsiella_pneumoniae", "肺炎克雷伯菌(条件致病)", 0.001, 0.5),
  213. ("Pseudomonas_aeruginosa", "铜绿假单胞菌(条件致病)", 0.001, 0.1),
  214. ("Staphylococcus_aureus", "金黄色葡萄球菌(条件致病)", 0.001, 0.1),
  215. ("Streptococcus_mutans", "变异链球菌(条件致病)", 0.01, 0.5),
  216. ("Streptococcus_agalactiae", "无乳链球菌(条件致病)", 0.001, 0.1),
  217. ("Clostridium_difficile", "艰难梭菌(条件致病)", 0.001, 0.05),
  218. ("Salmonella_enterica", "肠炎沙门氏菌(致病)", 0.001, 0.1),
  219. ("Shigella_flexneri", "福氏志贺氏菌(致病)", 0.001, 0.05),
  220. ("Campylobacter_jejuni", "空肠弯曲菌(致病)", 0.001, 0.05),
  221. ("Vibrio_parahaemolyticus", "副溶血弧菌(致病)", 0.001, 0.05),
  222. ("Candida_albicans", "白色念珠菌(条件致病)", 0.001, 0.5),
  223. ]
  224. # ---------------------------------------------------------------------------
  225. # 推导管线
  226. # ---------------------------------------------------------------------------
  227. def derive_percentiles(ref_min: float, ref_max: float) -> Dict[str, float]:
  228. """方案 §三 P0-5 做法①:正常范围{min,max} → p5=min、p95=max;线性插值→p25/p50/p75。
  229. 线性插值: p25 = min + (max-min)*0.25;p50 = (min+max)/2;p75 = min + (max-min)*0.75。
  230. 单调性 p5≤p25≤p50≤p75≤p95 恒成立(min≤max)。
  231. """
  232. if ref_min is None or ref_max is None or ref_min > ref_max:
  233. raise ValueError(f"非法正常范围: min={ref_min}, max={ref_max}")
  234. span = ref_max - ref_min
  235. return {
  236. "p5": round(ref_min, 4),
  237. "p25": round(ref_min + span * 0.25, 4),
  238. "p50": round(ref_min + span * 0.50, 4),
  239. "p75": round(ref_min + span * 0.75, 4),
  240. "p95": round(ref_max, 4),
  241. }
  242. def _read_sample_file(path: str) -> Dict[str, Any]:
  243. with open(path, "r", encoding="utf-8") as fh:
  244. return json.load(fh)
  245. def load_samples(samples_dir: str) -> List[Dict[str, Any]]:
  246. """读取 --samples 目录下全部样本 JSON。"""
  247. samples: List[Dict[str, Any]] = []
  248. if not os.path.isdir(samples_dir):
  249. raise NotADirectoryError(f"样本目录不存在: {samples_dir}")
  250. for fname in sorted(os.listdir(samples_dir)):
  251. if not fname.endswith(".json"):
  252. continue
  253. samples.append(_read_sample_file(os.path.join(samples_dir, fname)))
  254. if not samples:
  255. raise ValueError(f"样本目录 {samples_dir} 无 .json 样本文件")
  256. return samples
  257. def derive_from_samples(samples: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
  258. """真实样本推导:按 code 聚合各样本的 正常范围{refMin,refMax} 与 人群%水平。
  259. - p5 = min(refMin)(跨样本包络下界)、p95 = max(refMax)(跨样本包络上界);
  260. - 若有 populationPercentile 水平,则用其推算 p25/p50/p75(线性插值);
  261. - 否则回退到 [p5,p95] 线性插值(同 derive_percentiles)。
  262. """
  263. per_code: Dict[str, Dict[str, Any]] = {}
  264. for sample in samples:
  265. for taxon in sample.get("taxa", []):
  266. code = (taxon.get("code") or "").strip()
  267. if not code:
  268. continue
  269. slot = per_code.setdefault(
  270. code,
  271. {"code": code, "displayName": taxon.get("displayName", ""),
  272. "unit": taxon.get("unit", DEFAULT_UNIT),
  273. "refMins": [], "refMaxs": [], "percentiles": []},
  274. )
  275. if taxon.get("refMin") is not None:
  276. slot["refMins"].append(float(taxon["refMin"]))
  277. if taxon.get("refMax") is not None:
  278. slot["refMaxs"].append(float(taxon["refMax"]))
  279. if taxon.get("populationPercentile") is not None:
  280. slot["percentiles"].append(float(taxon["populationPercentile"]))
  281. rows: List[Dict[str, Any]] = []
  282. for code, slot in per_code.items():
  283. if slot["refMins"] and slot["refMaxs"]:
  284. p5 = min(slot["refMins"])
  285. p95 = max(slot["refMaxs"])
  286. elif slot["percentiles"]:
  287. # 仅有人群%水平:以样本百分位包络作为 p5/p95
  288. p5, p95 = min(slot["percentiles"]), max(slot["percentiles"])
  289. else:
  290. continue
  291. pcts = derive_percentiles(p5, p95)
  292. rows.append({
  293. "code": code,
  294. "displayName": slot["displayName"],
  295. "memberAgeBand": AGE_BAND_ALL,
  296. "gender": GENDER_ALL,
  297. **pcts,
  298. "unit": slot["unit"] or DEFAULT_UNIT,
  299. "normVersion": NORM_VERSION_V1,
  300. "source": SOURCE_LABEL,
  301. })
  302. return rows
  303. def build_data_from_reference() -> List[Dict[str, Any]]:
  304. """引导/自检:内嵌参考范围表 → 五百分位(同样走 derive_percentiles 管线)。"""
  305. rows: List[Dict[str, Any]] = []
  306. for code, display_name, ref_min, ref_max in REFERENCE_TAXA:
  307. pcts = derive_percentiles(ref_min, ref_max)
  308. rows.append({
  309. "code": code,
  310. "displayName": display_name,
  311. "memberAgeBand": AGE_BAND_ALL,
  312. "gender": GENDER_ALL,
  313. **pcts,
  314. "unit": DEFAULT_UNIT,
  315. "normVersion": NORM_VERSION_V1,
  316. "source": SOURCE_LABEL,
  317. })
  318. return rows
  319. def dedupe_rows(rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
  320. """唯一键(code+memberAgeBand+gender+normVersion)去重,防幂等 upsert 覆盖。"""
  321. seen: set = set()
  322. result: List[Dict[str, Any]] = []
  323. for row in rows:
  324. key = (row["code"], row["memberAgeBand"], row["gender"], row["normVersion"])
  325. if key in seen:
  326. continue
  327. seen.add(key)
  328. result.append(row)
  329. return result
  330. def build_norm_doc(data_rows: List[Dict[str, Any]]) -> Dict[str, Any]:
  331. """组装 attribution_norm_gut_flora.json 文档(schemaVersion=1.1, placeholder=false)。"""
  332. return {
  333. "schemaVersion": SCHEMA_VERSION_TARGET,
  334. "normType": NORM_TYPE,
  335. "placeholder": False,
  336. "note": NOTE_TEXT,
  337. "fields": [
  338. {"name": "code", "type": "string", "required": True,
  339. "description": "指标编码:菌种学名,如 Bifidobacterium / Lactobacillus"},
  340. {"name": "displayName", "type": "string", "required": False, "description": "显示名"},
  341. {"name": "memberAgeBand", "type": "string", "required": True,
  342. "description": "年龄段带:all(样本回归阶段取 all)"},
  343. {"name": "gender", "type": "string", "required": True, "description": "性别:all"},
  344. {"name": "p5", "type": "number", "required": True, "description": "P5 百分位值"},
  345. {"name": "p25", "type": "number", "required": True, "description": "P25 百分位值"},
  346. {"name": "p50", "type": "number", "required": True, "description": "P50 百分位值(中位数)"},
  347. {"name": "p75", "type": "number", "required": True, "description": "P75 百分位值"},
  348. {"name": "p95", "type": "number", "required": True, "description": "P95 百分位值"},
  349. {"name": "unit", "type": "string", "required": False, "description": "单位,如 %"},
  350. {"name": "source", "type": "string", "required": True,
  351. "description": "数据来源:样本回归估算,非官方 710 常模(§八8.3 红线)"},
  352. {"name": "normVersion", "type": "string", "required": False, "description": "常模版本,默认 v1"},
  353. ],
  354. "data": data_rows,
  355. }
  356. def write_json(doc: Dict[str, Any], out_path: str) -> None:
  357. out_dir = os.path.dirname(os.path.abspath(out_path))
  358. os.makedirs(out_dir, exist_ok=True)
  359. with open(out_path, "w", encoding="utf-8") as fh:
  360. json.dump(doc, fh, ensure_ascii=False, indent=2)
  361. fh.write("\n")
  362. print(f"[import_norm_from_reports] 已写出 {out_path}: schemaVersion={doc['schemaVersion']}, "
  363. f"placeholder={doc['placeholder']}, data rows={len(doc['data'])}")
  364. def main(argv: Optional[List[str]] = None) -> int:
  365. parser = argparse.ArgumentParser(description="谷禾菌群归一化数据填充管线(P0-5)")
  366. parser.add_argument("--samples", help="17 份样本报告目录(每样本一个 JSON),缺省用内嵌参考范围表引导")
  367. parser.add_argument("--out", default=DEFAULT_OUT, help=f"输出 JSON 路径(默认 {DEFAULT_OUT})")
  368. args = parser.parse_args(argv)
  369. if args.samples:
  370. samples = load_samples(args.samples)
  371. print(f"[import_norm_from_reports] 读取 {len(samples)} 份样本")
  372. rows = derive_from_samples(samples)
  373. else:
  374. print("[import_norm_from_reports] 未指定 --samples,使用内嵌参考范围表引导生成(同样走 derive_percentiles)")
  375. rows = build_data_from_reference()
  376. rows = dedupe_rows(rows)
  377. doc = build_norm_doc(rows)
  378. write_json(doc, args.out)
  379. # 自检:红线不变量(§八8.3)
  380. raw = json.dumps(doc, ensure_ascii=False)
  381. assert "样本回归" in raw and "非官方" in raw, "source 未明示「样本回归…非官方 710 常模」(§八8.3 红线)"
  382. assert FORBIDDEN_CROWD_CLAIM not in raw, f"出现「{FORBIDDEN_CROWD_CLAIM}」断言,违反 §八8.3 红线"
  383. print(f"[import_norm_from_reports] 自检通过:source 明示样本回归+非官方,无「{FORBIDDEN_CROWD_CLAIM}」断言")
  384. return 0
  385. if __name__ == "__main__":
  386. sys.exit(main())