Просмотр исходного кода

fix(T04): 修复存量时序回填单点中断与部分写入永久跳过

backfillIndicatorValues 两处缺陷(DS-2026-10-25-01 4.6 取证中发现):
1. 覆盖判定 selectCount 在 per-report try/catch 之外 -> 单次 DB 抖动即中断整批回填;
   改为 isReportCovered 内部兜底(异常返回 null,降级为未覆盖继续)。
2. covered=count>0 使部分写入的报告(如 r18 仅 13/43)被永久跳过、永不补齐;
   改为按 已写入 distinct definition 包含 载荷名称 exact 命中 definition 判定。

- 新增 IndicatorMappingService.resolveExactBatch(只读 exact 批量解析,不触发 fuzzy/AI/惰性注册)
- 覆盖判定用 exact 口径:fuzzy 0.5 会把不同名称坍缩到同一 def(r28 99 名称仅落 68 def),
  按 fuzzy 判定会把部分写入误判为完整而永久跳过;exact 口径下未命中即重跑,upsert 幂等收敛
- 新增 HealthReportServiceBackfillResilienceTest(5 例:中断不中止/部分写入重跑/全覆盖跳过/
  fuzzy 坍缩重跑/dryRun 无副作用)

实测(测试库):6 份有 payload 的 eligible 报告 written=6, failed=0;r18 由 13 行补齐至 33 行。
Agnes Agent 2 недель назад
Родитель
Сommit
57a9d5d228

+ 95 - 6
cfc-backend/src/main/java/com/etotem/cfc/service/HealthReportService.java

@@ -1809,10 +1809,19 @@ public class HealthReportService {
                 details.add(backfillDetail(r, "skip_no_payload"));
                 continue;
             }
-            boolean covered = indicatorValueMapper.selectCount(new LambdaQueryWrapper<IndicatorValue>()
-                    .eq(IndicatorValue::getSourceType, r.getReportType())
-                    .eq(IndicatorValue::getSourceId, r.getId())) > 0;
-            if (covered) {
+            ParsedReportPayload.Payload payload;
+            try {
+                payload = objectMapper.readValue(r.getPayloadJson(), ParsedReportPayload.Payload.class);
+            } catch (Exception e) {
+                failed++;
+                details.add(backfillDetail(r, "failed_payload"));
+                log.warn("payload 解析失败 reportId={}: {}", r.getId(), e.getMessage());
+                continue;
+            }
+            // 覆盖判定本身也可能因 DB 抖动抛异常:isReportCovered 内部兜底返回 null(降级为未覆盖),
+            // 绝不能因为单条报告判定失败而中断整批(旧实现 selectCount 在 try 外,一次抖动中止全部回填)。
+            Boolean covered = isReportCovered(r, payload);
+            if (Boolean.TRUE.equals(covered)) {
                 details.add(backfillDetail(r, "already_covered"));
                 continue;
             }
@@ -1820,8 +1829,6 @@ public class HealthReportService {
             String status = "would_write";
             if (!dryRun) {
                 try {
-                    ParsedReportPayload.Payload payload =
-                            objectMapper.readValue(r.getPayloadJson(), ParsedReportPayload.Payload.class);
                     normalizeIndicatorPipeline.processReport(r, payload);
                     written++;
                     status = "written";
@@ -1858,6 +1865,88 @@ public class HealthReportService {
         return d;
     }
 
+    /**
+     * 判定某报告的时序观测是否已完全覆盖(spec §4.6 回填判定条件)。
+     *
+     * <p>口径:报告已写入的观测(按 definition_id 去重)必须包含其载荷中全部指标/菌名经
+     * <b>exact 规则</b>解析到的 definition;任一名称未 exact 命中规则即视为未覆盖。
+     * 这样才能唯一、稳定地识别「部分写入」(如回填中途失败只写了 13/33 行)并在重跑时补齐;
+     * 旧口径 {@code count > 0} 会把部分写入的报告永久跳过,永久缺数据。</p>
+     *
+     * <p>为何用 exact 而非 fuzzy:fuzzy 阈值 0.5 会把不同名称坍缩到同一 definition(实测 r28 的
+     * 99 个名称仅落 68 个 distinct def)。若按 fuzzy 结果判定覆盖,未写入的名称会被误配到已存在的
+     * def 上,从而把「部分写入」误判为「已完整」而永久跳过。exact 口径下,只有真正处理过(写入并
+     * 登记 exact 规则)的名称才算覆盖,未命中者一律重跑;因 upsert 幂等,重跑无副作用、最终收敛。</p>
+     *
+     * <p>只读:复用 resolveExactBatch(不触发 fuzzy / AI 兜底 / 惰性注册,不写库),dryRun 安全。</p>
+     *
+     * @return true=已完全覆盖;false=未覆盖(需写入);null=判定过程出错(降级为未覆盖并重试写入)
+     */
+    private Boolean isReportCovered(HealthReport r, ParsedReportPayload.Payload payload) {
+        try {
+            Set<Long> writtenDefIds = new HashSet<>();
+            List<IndicatorValue> written = indicatorValueMapper.selectList(new LambdaQueryWrapper<IndicatorValue>()
+                    .eq(IndicatorValue::getSourceType, r.getReportType())
+                    .eq(IndicatorValue::getSourceId, r.getId())
+                    .select(IndicatorValue::getDefinitionId));
+            if (written != null) {
+                for (IndicatorValue v : written) {
+                    if (v != null && v.getDefinitionId() != null) {
+                        writtenDefIds.add(v.getDefinitionId());
+                    }
+                }
+            }
+
+            Set<String> names = collectPayloadNames(payload);
+            if (names.isEmpty()) {
+                // 载荷里没有可写名称 → 无观测可写,视为已覆盖
+                return true;
+            }
+            // 仅按 exact 规则命中判定:任何名称只能 fuzzy 命中或尚未命中,都视为未覆盖。
+            // 否则 fuzzy 坍缩(如 r28 的 99 个名称被错配到 68 个已存在 def)会把「部分写入」误判为已完整,
+            // 导致报告被永久跳过、缺数据(损害 L1/L3)。
+            Map<String, IndicatorMappingService.ResolvedMapping> resolved =
+                    mappingService.resolveExactBatch(r.getReportType(), names);
+            Set<Long> expectedDefIds = new HashSet<>();
+            if (resolved != null) {
+                for (IndicatorMappingService.ResolvedMapping rm : resolved.values()) {
+                    if (rm != null && rm.getDefinitionId() != null) {
+                        expectedDefIds.add(rm.getDefinitionId());
+                    }
+                }
+            }
+            // 任一名称未 exact 命中规则 → 该名称尚未归一化写入(fuzzy/惰性注册前不会产生精确规则)→ 未覆盖
+            boolean allNamesExactResolved = resolved != null && resolved.keySet().containsAll(names);
+            return allNamesExactResolved && writtenDefIds.containsAll(expectedDefIds);
+        } catch (Exception e) {
+            log.warn("覆盖判定失败 reportId={}, 降级为未覆盖并重试写入: {}", r.getId(), e.getMessage());
+            return null;
+        }
+    }
+
+    /** 收集载荷中所有可写的指标名 + 菌名(trim + 去重) */
+    private Set<String> collectPayloadNames(ParsedReportPayload.Payload payload) {
+        Set<String> names = new LinkedHashSet<>();
+        if (payload == null) {
+            return names;
+        }
+        if (payload.getIndicators() != null) {
+            for (ParsedReportPayload.Indicator ind : payload.getIndicators()) {
+                if (ind.getIndicatorName() != null && !ind.getIndicatorName().trim().isEmpty()) {
+                    names.add(ind.getIndicatorName().trim());
+                }
+            }
+        }
+        if (payload.getGutFlora() != null) {
+            for (ParsedReportPayload.Flora flora : payload.getGutFlora()) {
+                if (flora.getBacteriaName() != null && !flora.getBacteriaName().trim().isEmpty()) {
+                    names.add(flora.getBacteriaName().trim());
+                }
+            }
+        }
+        return names;
+    }
+
     /** 截断到日(仅比较日期部分) */
     private java.sql.Date toDay(java.util.Date d) {
         if (d == null) {

+ 51 - 0
cfc-backend/src/main/java/com/etotem/cfc/service/IndicatorMappingService.java

@@ -196,6 +196,57 @@ public class IndicatorMappingService {
         return result;
     }
 
+    /**
+     * 仅 exact 命中的批量解析(只读,不触发 fuzzy / AI 兜底 / 惰性注册,不写库)。
+     * 用于回填覆盖判定:只有全部名称都能 exact 命中规则且对应 definition 都已写入,
+     * 才视为“已完全覆盖”而跳过。任何仅能 fuzzy 命中或尚未命中的名称都视为未覆盖,
+     * 避免把「fuzzy 坍缩后的部分写入」误判为已完整(如 r28 99 个名称仅写 68 个 distinct def)。
+     *
+     * @param sourceType 报告来源类型
+     * @param rawNames   待解析的原始指标名(自动 trim + 去重)
+     * @return rawName → ResolvedMapping(仅 exact 命中;未命中的名称不出现)
+     */
+    public Map<String, ResolvedMapping> resolveExactBatch(String sourceType, Collection<String> rawNames) {
+        Map<String, ResolvedMapping> result = new LinkedHashMap<>();
+        if (sourceType == null || rawNames == null || rawNames.isEmpty()) {
+            return result;
+        }
+        Set<String> names = new LinkedHashSet<>();
+        for (String n : rawNames) {
+            if (n != null && !n.trim().isEmpty()) {
+                names.add(n.trim());
+            }
+        }
+        if (names.isEmpty()) {
+            return result;
+        }
+
+        List<ResolvedMapping> rules = ruleMapper.selectRulesWithDefinitions(sourceType);
+        if (rules == null || rules.isEmpty()) {
+            return result;
+        }
+
+        // exact 命中:rawName 相同,取优先级最高的一条(与 resolve 一致)
+        Map<String, ResolvedMapping> exactResolved = new HashMap<>();
+        for (String name : names) {
+            ResolvedMapping best = null;
+            int bestPriority = Integer.MIN_VALUE;
+            for (ResolvedMapping r : rules) {
+                if (!name.equals(r.getRawName()) || r.getDefinitionId() == null) continue;
+                int p = r.getPriority() == null ? 0 : r.getPriority();
+                if (p > bestPriority) {
+                    bestPriority = p;
+                    best = r;
+                }
+            }
+            if (best != null) {
+                exactResolved.put(name, best);
+            }
+        }
+        result.putAll(exactResolved);
+        return result;
+    }
+
     /** 规则 + 关联定义的联合结果(供批量解析复用,避免二次查 definition 表) */
     public static class ResolvedMapping {
         private String rawName;

+ 243 - 0
cfc-backend/src/test/java/com/etotem/cfc/service/HealthReportServiceBackfillResilienceTest.java

@@ -0,0 +1,243 @@
+package com.etotem.cfc.service;
+
+import com.baomidou.mybatisplus.core.MybatisConfiguration;
+import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
+import com.etotem.cfc.entity.HealthReport;
+import com.etotem.cfc.entity.IndicatorValue;
+import com.etotem.cfc.mapper.HealthReportMapper;
+import com.etotem.cfc.mapper.IndicatorValueMapper;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.ibatis.builder.MapperBuilderAssistant;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import java.sql.Date;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyCollection;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.*;
+
+/**
+ * 存量时序回填(backfillIndicatorValues)健壮性单测(T04 第二段 gate 取证,修复 DS-2026-10-25-01 §4.6 缺陷)。
+ *
+ * <p>覆盖缺陷(旧实现 L1716 selectCount 在 per-report try/catch 之外 + covered=count>0):
+ * <ul>
+ *   <li>#A 覆盖判定查询抛异常不中断整批:后续报告仍被处理,而非整批回填中止;</li>
+ *   <li>#B 部分写入的报告(如中断后仅写入 13/33 行)不会被永久跳过,重跑会补齐;</li>
+ *   <li>#C 已完全覆盖(exact 命中且所有 def 已写)的报告仍被跳过(幂等,不重复写);</li>
+ *   <li>#D fuzzy 坍缩(部分名称仅能 fuzzy 命中,无 exact 规则)不被误判为已覆盖,重跑会补齐缺失名称;
+ *       与 r28 实测同构(99 名称仅写 68 def,fuzzy 会把部分写入误判为完整,故按 exact 口径判定);</li>
+ *   <li>#E dryRun 无副作用。</li>
+ * </ul>
+ */
+class HealthReportServiceBackfillResilienceTest {
+
+    private HealthReportService service;
+    private HealthReportMapper healthReportMapper;
+    private IndicatorValueMapper valueMapper;
+    private IndicatorMappingService mappingService;
+    private NormalizeIndicatorPipeline pipeline;
+
+    @BeforeEach
+    void setUp() {
+        TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""),
+                HealthReport.class);
+        TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""),
+                IndicatorValue.class);
+
+        service = new HealthReportService();
+        healthReportMapper = mock(HealthReportMapper.class);
+        valueMapper = mock(IndicatorValueMapper.class);
+        mappingService = mock(IndicatorMappingService.class);
+        pipeline = mock(NormalizeIndicatorPipeline.class);
+
+        ReflectionTestUtils.setField(service, "healthReportMapper", healthReportMapper);
+        ReflectionTestUtils.setField(service, "indicatorValueMapper", valueMapper);
+        ReflectionTestUtils.setField(service, "mappingService", mappingService);
+        ReflectionTestUtils.setField(service, "normalizeIndicatorPipeline", pipeline);
+        ReflectionTestUtils.setField(service, "objectMapper", new ObjectMapper());
+
+        ReportDateExtractor extractor = mock(ReportDateExtractor.class);
+        when(extractor.extractReportDate(any())).thenReturn(null); // 不走日期修复分支
+        ReflectionTestUtils.setField(service, "reportDateExtractor", extractor);
+    }
+
+    // ---------- 工具 ----------
+
+    private HealthReport report(long id, String payloadJson) {
+        HealthReport r = new HealthReport();
+        r.setId(id);
+        r.setSubjectId(88L);
+        r.setReportType("gut_flora");
+        r.setReportDate(Date.valueOf("2026-08-28"));
+        r.setStatus("active");
+        r.setCreatedAt(new java.util.Date());
+        r.setPayloadJson(payloadJson);
+        return r;
+    }
+
+    /** 构造 ResolvedMapping:rawName → defId */
+    private IndicatorMappingService.ResolvedMapping resolved(String rawName, Long defId) {
+        IndicatorMappingService.ResolvedMapping rm = new IndicatorMappingService.ResolvedMapping();
+        rm.setRawName(rawName);
+        rm.setDefinitionId(defId);
+        rm.setDefId(defId);
+        rm.setDefName(rawName);
+        rm.setRuleStatus(1);
+        rm.setPriority(0);
+        return rm;
+    }
+
+    /** 构造 resolveExactBatch 结果(exact 命中) */
+    private Map<String, IndicatorMappingService.ResolvedMapping> exactMap(Object... pairs) {
+        Map<String, IndicatorMappingService.ResolvedMapping> m = new LinkedHashMap<>();
+        for (int i = 0; i < pairs.length; i += 2) {
+            m.put((String) pairs[i], resolved((String) pairs[i], (Long) pairs[i + 1]));
+        }
+        return m;
+    }
+
+    private IndicatorValue written(Long defId) {
+        IndicatorValue v = new IndicatorValue();
+        v.setDefinitionId(defId);
+        return v;
+    }
+
+    private List<Map<String, Object>> detailsOf(Map<String, Object> data) {
+        @SuppressWarnings("unchecked")
+        List<Map<String, Object>> details = (List<Map<String, Object>>) data.get("details");
+        return details;
+    }
+
+    private static Map<String, Object> detailOf(List<Map<String, Object>> details, long reportId) {
+        return details.stream()
+                .filter(d -> ((Number) d.get("reportId")).longValue() == reportId)
+                .findFirst().orElse(null);
+    }
+
+    // ==================== #A 覆盖判定异常不中断整批 ====================
+
+    @Test
+    void backfill_coveredCheckException_doesNotAbortBatch() {
+        String payloadA = "{\"indicators\":[{\"indicatorName\":\"n1\"},{\"indicatorName\":\"n2\"}]}";
+        String payloadB = "{\"indicators\":[{\"indicatorName\":\"m1\"}]}";
+        HealthReport a = report(1L, payloadA);
+        HealthReport b = report(2L, payloadB);
+        when(healthReportMapper.selectList(any())).thenReturn(new ArrayList<>(java.util.Arrays.asList(a, b)));
+
+        // 报告 A 的 exact-coverag 检查抛异常(DB 抖动)→ 降级为未覆盖并重试写入;报告 B 正常 exact 命中且已写 → 跳过
+        when(mappingService.resolveExactBatch(anyString(), anyCollection()))
+                .thenThrow(new RuntimeException("connection reset by peer"))
+                .thenReturn(exactMap("m1", 601L));
+        // B 的已写入定义已存在(A 的检查在 selectList 之后抛异常,其 selectList 结果无影响)
+        when(valueMapper.selectList(any()))
+                .thenReturn(new ArrayList<>(java.util.Arrays.asList(written(601L))));
+
+        Map<String, Object> data = service.backfillIndicatorValues(null, null, 50, false, false);
+
+        // 整批未中止:A 被重试写入,B 已覆盖被跳过
+        verify(pipeline).processReport(eq(a), any());
+        verify(pipeline, never()).processReport(eq(b), any());
+        assertEquals(1, ((Number) data.get("written")).intValue());
+        assertEquals(0, ((Number) data.get("failed")).intValue());
+
+        Map<String, Object> dA = detailOf(detailsOf(data), 1L);
+        assertNotNull(dA);
+        assertEquals("written", dA.get("status"));
+        Map<String, Object> dB = detailOf(detailsOf(data), 2L);
+        assertNotNull(dB);
+        assertEquals("already_covered", dB.get("status"));
+    }
+
+    // ==================== #B 部分写入不被永久跳过 ====================
+
+    @Test
+    void backfill_partiallyCoveredReport_isReprocessed() {
+        String payload = "{\"indicators\":[{\"indicatorName\":\"n1\"},{\"indicatorName\":\"n2\"},{\"indicatorName\":\"n3\"}]}";
+        HealthReport r = report(18L, payload);
+        when(healthReportMapper.selectList(any())).thenReturn(new ArrayList<>(java.util.Arrays.asList(r)));
+        when(mappingService.resolveExactBatch(anyString(), anyCollection()))
+                .thenReturn(exactMap("n1", 501L, "n2", 502L, "n3", 503L));
+        // 仅写入 1/3(n1)→ 必须重跑补齐 n2/n3
+        when(valueMapper.selectList(any()))
+                .thenReturn(new ArrayList<>(java.util.Arrays.asList(written(501L))));
+
+        Map<String, Object> data = service.backfillIndicatorValues(null, null, 50, false, false);
+
+        verify(pipeline).processReport(eq(r), any());
+        assertEquals(1, ((Number) data.get("written")).intValue());
+        assertEquals("written", detailOf(detailsOf(data), 18L).get("status"));
+    }
+
+    // ==================== #C 已完全覆盖仍跳过 ====================
+
+    @Test
+    void backfill_fullyCoveredReport_isSkipped() {
+        String payload = "{\"indicators\":[{\"indicatorName\":\"n1\"},{\"indicatorName\":\"n2\"},{\"indicatorName\":\"n3\"}]}";
+        HealthReport r = report(19L, payload);
+        when(healthReportMapper.selectList(any())).thenReturn(new ArrayList<>(java.util.Arrays.asList(r)));
+        when(mappingService.resolveExactBatch(anyString(), anyCollection()))
+                .thenReturn(exactMap("n1", 501L, "n2", 502L, "n3", 503L));
+        // 3/3 全部 exact 写入
+        when(valueMapper.selectList(any()))
+                .thenReturn(new ArrayList<>(java.util.Arrays.asList(written(501L), written(502L), written(503L))));
+
+        Map<String, Object> data = service.backfillIndicatorValues(null, null, 50, false, false);
+
+        verify(pipeline, never()).processReport(any(), any());
+        assertEquals(0, ((Number) data.get("written")).intValue());
+        assertEquals(0, ((Number) data.get("eligible")).intValue());
+        assertEquals("already_covered", detailOf(detailsOf(data), 19L).get("status"));
+    }
+
+    // ==================== #D fuzzy 坍缩不被误判为已覆盖 ====================
+
+    @Test
+    void backfill_fuzzyCollapsedReport_isReprocessed() {
+        // n1/n2 exact→501,n3 exact→502,n4/n5 仅 fuzzy 命中(无 exact 规则),模拟 r28 99→68
+        String payload = "{\"indicators\":[{\"indicatorName\":\"n1\"},{\"indicatorName\":\"n2\"},"
+                + "{\"indicatorName\":\"n3\"},{\"indicatorName\":\"n4\"},{\"indicatorName\":\"n5\"}]}";
+        HealthReport r = report(20L, payload);
+        when(healthReportMapper.selectList(any())).thenReturn(new ArrayList<>(java.util.Arrays.asList(r)));
+        // exact 命中仅 n1/n2/n3;n4/n5 无 exact 规则 → 不出现在结果中
+        when(mappingService.resolveExactBatch(anyString(), anyCollection()))
+                .thenReturn(exactMap("n1", 501L, "n2", 501L, "n3", 502L));
+        // 已写入 501/502/503,但 n4/n5 未 exact 命中 → 视为未覆盖,必须重跑补齐(lazyRegister n4/n5)
+        when(valueMapper.selectList(any()))
+                .thenReturn(new ArrayList<>(java.util.Arrays.asList(written(501L), written(502L), written(503L))));
+
+        Map<String, Object> data = service.backfillIndicatorValues(null, null, 50, false, false);
+
+        verify(pipeline).processReport(eq(r), any());
+        assertEquals(1, ((Number) data.get("written")).intValue());
+        assertEquals("written", detailOf(detailsOf(data), 20L).get("status"));
+    }
+
+    // ==================== 兜底:dryRun 无副作用 ====================
+
+    @Test
+    void backfill_dryRun_writesNothing() {
+        String payload = "{\"indicators\":[{\"indicatorName\":\"n1\"}]}";
+        HealthReport r = report(21L, payload);
+        when(healthReportMapper.selectList(any())).thenReturn(new ArrayList<>(java.util.Arrays.asList(r)));
+        when(mappingService.resolveExactBatch(anyString(), anyCollection())).thenReturn(exactMap("n1", 501L));
+        when(valueMapper.selectList(any())).thenReturn(new ArrayList<>());
+
+        Map<String, Object> data = service.backfillIndicatorValues(null, null, 50, true, false);
+
+        verify(pipeline, never()).processReport(any(), any());
+        assertEquals(1, ((Number) data.get("eligible")).intValue());
+        assertEquals("would_write", detailOf(detailsOf(data), 21L).get("status"));
+        assertEquals(true, data.get("dryRun"));
+    }
+}