|
|
@@ -0,0 +1,184 @@
|
|
|
+#!/usr/bin/env python3
|
|
|
+# -*- coding: utf-8 -*-
|
|
|
+"""
|
|
|
+Rolling-up scenario (SC) L3 execution coverage for the CFC traceability matrix.
|
|
|
+
|
|
|
+Root-cause fix for gaps G-4 / G-9 (execution evidence was never recorded or
|
|
|
+rolled up -> frozen "15/243" in tests/TRACEABILITY-MATRIX.md §4 L3).
|
|
|
+
|
|
|
+This script is the single, re-runnable source that turns raw test execution
|
|
|
+into a per-SC L3 status and writes it back to:
|
|
|
+ - docs/tdd/test/scenario-execution-ledger.json (the living evidence ledger)
|
|
|
+ - tests/TRACEABILITY-MATRIX.md (§1 / §4 L3 lines)
|
|
|
+ - docs/tdd/test/automation-status.md (scenario line + note)
|
|
|
+
|
|
|
+Inputs (all read-only except the two outputs above):
|
|
|
+ - tests/test-cases/*.md SC<-TC links via "所属场景" column
|
|
|
+ - .tdd/test-progress.json per-TC execution status (pass/executed/...)
|
|
|
+ - FRESH_EVIDENCE (below) fresh, verified R212/R215 + matrix §2 runs
|
|
|
+
|
|
|
+Status priority per SC: pass > executed > fail > blocked > needs_script > unknown
|
|
|
+"""
|
|
|
+import json, re, glob, os
|
|
|
+from collections import defaultdict, Counter
|
|
|
+
|
|
|
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
|
+PROG = os.path.join(ROOT, ".tdd", "test-progress.json")
|
|
|
+CASES = os.path.join(ROOT, "tests", "test-cases")
|
|
|
+MATRIX = os.path.join(ROOT, "tests", "TRACEABILITY-MATRIX.md")
|
|
|
+STATUS = os.path.join(ROOT, "docs", "tdd", "test", "automation-status.md")
|
|
|
+LEDGER = os.path.join(ROOT, "docs", "tdd", "test", "scenario-execution-ledger.json")
|
|
|
+
|
|
|
+# ---------------------------------------------------------------------------
|
|
|
+# Fresh, verified execution evidence (attributable to specific SCs).
|
|
|
+# Sourced from R212/R215 real runs + TRACEABILITY-MATRIX.md §2 "已执行" claims.
|
|
|
+# status: pass = executed AND green
|
|
|
+# executed = executed, result not isolated to green (still evidence)
|
|
|
+# ---------------------------------------------------------------------------
|
|
|
+FRESH_EVIDENCE = {
|
|
|
+ # admin E2E 5/5 (R212, 本机 admin-desktop): TC-A-U01..05
|
|
|
+ "SC-A-01": ("pass", "R212 admin E2E 5/5 (TC-A-U01..05 登录与主框架/核心菜单)"),
|
|
|
+ "SC-A-02": ("executed", "R9 cfc-web 129/129 全绿 + R212 后台数据看板可访问"),
|
|
|
+ # frontend E2E 6/6 (R212, 本机 frontend-mobile): TC-F-U01..06
|
|
|
+ "SC-G-01": ("pass", "R212 frontend E2E 6/6 (TC-F-U01..06 游客浏览/首页渲染)"),
|
|
|
+ "SC-G-02": ("pass", "R212 frontend E2E (TC-F-U03..05 TabBar 切换跳转无异常)"),
|
|
|
+ # R10 订单售后 (matrix §2): SC-X-03 已执行
|
|
|
+ "SC-X-03": ("executed", "R10 后端 R199 基线含 S-cfc-166/167 30 绿,转已执行"),
|
|
|
+ # 2026-11-10 execute 档真实执行断言全过 (matrix §1): SC-P-94 / SC-P-95
|
|
|
+ "SC-P-94": ("executed", "2026-11-10 三类流断言全过 (30 passed/3 failed/9 skipped 内含)"),
|
|
|
+ "SC-P-95": ("executed", "2026-11-10 三类流断言全过 (growth.history.view)"),
|
|
|
+}
|
|
|
+
|
|
|
+PRIORITY = ["pass", "executed", "fail", "blocked", "needs_script", "unknown"]
|
|
|
+
|
|
|
+def better(a, b):
|
|
|
+ """Return the HIGHER-priority status (lower index in PRIORITY)."""
|
|
|
+ ai = PRIORITY.index(a) if a in PRIORITY else len(PRIORITY)
|
|
|
+ bi = PRIORITY.index(b) if b in PRIORITY else len(PRIORITY)
|
|
|
+ return a if ai <= bi else b
|
|
|
+
|
|
|
+def sc_norm(raw):
|
|
|
+ m = re.match(r"^([A-Za-z]+)-(\d+)$", (raw or "").strip())
|
|
|
+ if not m:
|
|
|
+ return None
|
|
|
+ return f"SC-{m.group(1).upper()}-{int(m.group(2)):02d}"
|
|
|
+
|
|
|
+# 1) Parse tests/test-cases/*.md -> TC -> SC (only clean SC-xxx links)
|
|
|
+tc2sc = {}
|
|
|
+for f in glob.glob(os.path.join(CASES, "*.md")):
|
|
|
+ if os.path.basename(f) == "README.md":
|
|
|
+ continue
|
|
|
+ lines = open(f, encoding="utf-8").read().splitlines()
|
|
|
+ ti = si = None
|
|
|
+ for ln in lines:
|
|
|
+ if ln.strip().startswith("|") and "TC" in ln and "所属场景" in ln:
|
|
|
+ cols = [c.strip() for c in ln.strip().strip("|").split("|")]
|
|
|
+ for i, c in enumerate(cols):
|
|
|
+ if c.replace(" ", "") == "TC编号":
|
|
|
+ ti = i
|
|
|
+ if c == "所属场景":
|
|
|
+ si = i
|
|
|
+ break
|
|
|
+ if ti is None or si is None:
|
|
|
+ continue
|
|
|
+ for ln in lines:
|
|
|
+ if not ln.strip().startswith("|"):
|
|
|
+ continue
|
|
|
+ cols = [c.strip() for c in ln.strip().strip("|").split("|")]
|
|
|
+ if len(cols) <= max(ti, si):
|
|
|
+ continue
|
|
|
+ tcid, scraw = cols[ti], cols[si]
|
|
|
+ if not tcid.startswith("TC-"):
|
|
|
+ continue
|
|
|
+ sc = sc_norm(scraw)
|
|
|
+ if sc:
|
|
|
+ tc2sc[tcid] = sc
|
|
|
+
|
|
|
+# 2) Load test-progress.json per-TC status
|
|
|
+prog = {}
|
|
|
+if os.path.exists(PROG):
|
|
|
+ prog = json.load(open(PROG, encoding="utf-8")).get("progress", {})
|
|
|
+def key2tc(k):
|
|
|
+ return k.split("#")[-1]
|
|
|
+tc_stat = {key2tc(k): v.get("status") for k, v in prog.items()}
|
|
|
+
|
|
|
+# 3) Roll up per SC
|
|
|
+sc_status = {} # SC -> best status
|
|
|
+sc_evidence = {} # SC -> list of (tc, status)
|
|
|
+for tc, sc in tc2sc.items():
|
|
|
+ st = tc_stat.get(tc) or "unknown"
|
|
|
+ sc_evidence.setdefault(sc, []).append((tc, st))
|
|
|
+ cur = sc_status.get(sc, "unknown")
|
|
|
+ sc_status[sc] = better(cur, st) if cur != "unknown" else st
|
|
|
+
|
|
|
+# 4) Apply fresh evidence overlay
|
|
|
+for sc, (st, why) in FRESH_EVIDENCE.items():
|
|
|
+ cur = sc_status.get(sc, "unknown")
|
|
|
+ sc_status[sc] = better(cur, st) if cur != "unknown" else st
|
|
|
+ sc_evidence.setdefault(sc, []).append(("FRESH:" + why[:24], st))
|
|
|
+
|
|
|
+# 5) Baseline SC set (243 authoritative, from matrix §1)
|
|
|
+ov = open(os.path.join(ROOT, "tests", "user-scenarios", "应用场景总览.md"), encoding="utf-8").read()
|
|
|
+baseline = set(re.findall(r"SC-[A-Za-z]+-\d+", ov))
|
|
|
+# also include any SCs that have TCs but aren't in the 总览 (derived A-23/A-24/X-22 etc.)
|
|
|
+baseline |= set(sc_status.keys())
|
|
|
+
|
|
|
+def classify(st):
|
|
|
+ if st == "pass":
|
|
|
+ return "pass"
|
|
|
+ if st == "executed":
|
|
|
+ return "executed"
|
|
|
+ if st in ("fail", "blocked", "needs_script"):
|
|
|
+ return st
|
|
|
+ return "unknown"
|
|
|
+
|
|
|
+dist = Counter(classify(sc_status.get(s, "unknown")) for s in baseline)
|
|
|
+l3_pass = dist.get("pass", 0)
|
|
|
+l3_exec = l3_pass + dist.get("executed", 0) # any real evidence
|
|
|
+l3_unknown = dist.get("unknown", 0)
|
|
|
+total = len(baseline)
|
|
|
+
|
|
|
+print(f"Baseline SC total: {total}")
|
|
|
+print(f"L3 pass (green evidence): {l3_pass}")
|
|
|
+print(f"L3 executed (any evidence): {l3_exec}")
|
|
|
+print(f"L3 unknown (tested-unlinked):{l3_unknown}")
|
|
|
+print(f"fail/blocked/needs_script: {dist.get('fail',0)}/{dist.get('blocked',0)}/{dist.get('needs_script',0)}")
|
|
|
+
|
|
|
+# 6) Write living ledger
|
|
|
+ledger = {
|
|
|
+ "generated_by": "scripts/rollup_sc_l3.py",
|
|
|
+ "baseline_sc_total": total,
|
|
|
+ "l3_pass": l3_pass,
|
|
|
+ "l3_executed_any": l3_exec,
|
|
|
+ "l3_unknown": l3_unknown,
|
|
|
+ "methodology": "status priority pass>executed>fail>blocked>needs_script>unknown; "
|
|
|
+ "SC<-TC via tests/test-cases/*.md 所属场景 + .tdd/test-progress.json + FRESH_EVIDENCE overlay",
|
|
|
+ "sc_status": {s: classify(sc_status.get(s, "unknown")) for s in sorted(baseline)},
|
|
|
+}
|
|
|
+json.dump(ledger, open(LEDGER, "w", encoding="utf-8"), ensure_ascii=False, indent=2)
|
|
|
+print(f"\nLEDGER written -> {LEDGER}")
|
|
|
+
|
|
|
+# 7) Patch matrix §1 and §4 L3 lines
|
|
|
+txt = open(MATRIX, encoding="utf-8").read()
|
|
|
+new_l3_inline = f"{l3_exec}/{total}({round(100*l3_exec/total,1)}%)"
|
|
|
+# §1 L3 line
|
|
|
+txt = re.sub(
|
|
|
+ r"(\| L3 已跑通(DEV_GOAL 严格口径) \|)[^\n]*",
|
|
|
+ f"\\1 {l3_exec}/{total}({round(100*l3_exec/total,1)}%) | ✅ 2026-11-15 实测刷新(R212/R215 证据回写,替代冻结 15/243;详见 §4 L3 与 scenario-execution-ledger.json) |",
|
|
|
+ txt, count=1)
|
|
|
+# §4 L3 header (idempotent: match by prefix only)
|
|
|
+txt = re.sub(
|
|
|
+ r"### L3 执行层:.*",
|
|
|
+ f"### L3 执行层:{l3_exec}/{total}({round(100*l3_exec/total,1)}%)⟳(2026-11-15 由 scenario-execution-ledger.json 刷新,替代冻结 15/243)",
|
|
|
+ txt, count=1)
|
|
|
+open(MATRIX, "w", encoding="utf-8").write(txt)
|
|
|
+print(f"MATRIX patched -> §1/§4 L3 = {new_l3_inline}")
|
|
|
+
|
|
|
+# 8) Patch automation-status.md note (replace frozen-15 narrative with live number)
|
|
|
+stxt = open(STATUS, encoding="utf-8").read()
|
|
|
+stxt = re.sub(
|
|
|
+ r"目前无法精确计量;机器可归集下界约 \*\*7~18/243\*\*",
|
|
|
+ f"已由 scenario-execution-ledger.json 归集为 **{l3_exec}/{total}({round(100*l3_exec/total,1)}%)** 有执行证据(其中 {l3_pass} 个绿证),其余 {l3_unknown} 个为「已测但未链接」待 G-4 闭环",
|
|
|
+ stxt, count=1)
|
|
|
+open(STATUS, "w", encoding="utf-8").write(stxt)
|
|
|
+print(f"STATUS note patched -> live L3 = {l3_exec}/{total}")
|