|
|
@@ -190,6 +190,60 @@ public class AiSmartCleanService {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ /**
|
|
|
+ * 推一条「清洗日志」（cmd 104）：只做流水账，不携带进度/统计，前端直接追加到处理日志。
|
|
|
+ *
|
|
|
+ * <p>与 cmd 101（进度）/103（状态快照）分开：日志行不参与进度环与终态判定，
|
|
|
+ * 避免「一行日志把进度跳到 0」这类副作用。消息统一按
|
|
|
+ * {@code [阶段] 动作 · 对象} 组织，对象带上文件名与 sheet 名，保证一条日志能独立看懂。
|
|
|
+ */
|
|
|
+ private void pushLog(String message) {
|
|
|
+ try {
|
|
|
+ Long userId = CaseContextHolder.getUserId();
|
|
|
+ if (userId == null) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ sseService.sendSee(userId, new SseDTO(104, 0, message, new SeeDataDTO(0, 0)));
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.debug("清洗日志推送失败: {}", e.getMessage());
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /** 文件名显示口径：去掉落盘时的雪花 ID 前缀（与 AiCleanService 的建表命名规则一致） */
|
|
|
+ private static String fileLabel(FileInfo root) {
|
|
|
+ String name = root == null ? null : root.getFileName();
|
|
|
+ return StrUtil.nullToEmpty(name).replaceFirst("^\\d{15,}_", "");
|
|
|
+ }
|
|
|
+
|
|
|
+ /** 日志里的对象标签：`「文件名」·sheet 名`，便于把并发判定的日志与具体表对上 */
|
|
|
+ private static String sheetLabel(PendingAi p) {
|
|
|
+ String file = fileLabel(p.root());
|
|
|
+ String sheetName = p.sheet() == null ? null : p.sheet().getSheetName();
|
|
|
+ if (StrUtil.isBlank(file)) {
|
|
|
+ return StrUtil.isBlank(sheetName) ? "" : "「" + sheetName + "」";
|
|
|
+ }
|
|
|
+ return StrUtil.isBlank(sheetName) ? "「" + file + "」" : "「" + file + "」·" + sheetName;
|
|
|
+ }
|
|
|
+
|
|
|
+ /** 命中途径的一句话说明：同表头指纹 / 表头打分 / 模型判定（带自评置信度） */
|
|
|
+ private static String matchReason(AiCleanJob job) {
|
|
|
+ String by = StrUtil.nullToEmpty(job.getMatchedBy());
|
|
|
+ String how = switch (by) {
|
|
|
+ case "HEADER_CACHE" -> "同表头指纹";
|
|
|
+ case "SCORE" -> "表头打分";
|
|
|
+ case "LLM" -> "模型判定";
|
|
|
+ case "SELF_HEADER" -> "按表头自建";
|
|
|
+ default -> null;
|
|
|
+ };
|
|
|
+ if (how == null) {
|
|
|
+ return "";
|
|
|
+ }
|
|
|
+ java.math.BigDecimal conf = job.getConfidence();
|
|
|
+ return conf != null && "模型判定".equals(how)
|
|
|
+ ? "（" + how + " " + conf.setScale(2, java.math.RoundingMode.HALF_UP).toPlainString() + "）"
|
|
|
+ : "（" + how + "）";
|
|
|
+ }
|
|
|
+
|
|
|
/**
|
|
|
* 批次终态的 SSE 收口（口径与 {@code DmService.doClean} 完成时一致）：
|
|
|
* 统计（cmd 101，100%）→ 状态快照（cmd 103）→ 治理启动宣告（cmd 102，开启治理时）。
|
|
|
@@ -279,7 +333,8 @@ public class AiSmartCleanService {
|
|
|
long failed = Math.max(0, existingFailed + aiRejectedRows);
|
|
|
long unprocessed = Math.max(0, aiUnprocessedRows);
|
|
|
log.info("AI 清洗行数统计: batchId={}, 成功={}, 失败={}, 未处理={}", batchId, completed, failed, unprocessed);
|
|
|
- sseService.sendSee(userId, new SseDTO(101, 100, "[数据清洗]所有文件清洗完成！",
|
|
|
+ sseService.sendSee(userId, new SseDTO(101, 100,
|
|
|
+ "[清洗] 全部完成：成功 " + completed + " 行，失败 " + failed + " 行",
|
|
|
new SeeDataDTO((int) Math.min(completed, Integer.MAX_VALUE),
|
|
|
(int) Math.min(failed, Integer.MAX_VALUE))));
|
|
|
SmartCleanStatusVO vo = status(caseId, batchId);
|
|
|
@@ -306,6 +361,9 @@ public class AiSmartCleanService {
|
|
|
Stats st = new Stats();
|
|
|
try {
|
|
|
stage(batchId, AiCleanStageEnum.JUDGING);
|
|
|
+ int batchSheetTotal = roots.stream()
|
|
|
+ .mapToInt(r -> r.getChildren() == null ? 0 : r.getChildren().size()).sum();
|
|
|
+ pushLog("[清洗] 开始处理：" + roots.size() + " 个文件 / " + batchSheetTotal + " 个 sheet");
|
|
|
// (root, sheetNode, job) 三元组：判定阶段只收集，不动数据
|
|
|
List<PendingAi> pending = new ArrayList<>();
|
|
|
for (FileInfo root : roots) {
|
|
|
@@ -381,6 +439,10 @@ public class AiSmartCleanService {
|
|
|
|
|
|
stage(batchId, AiCleanStageEnum.LOADING_AI);
|
|
|
pushProgress(caseId, batchId);
|
|
|
+ int aiPlanCount = (int) pending.stream().filter(p -> plans.containsKey(p.job().getId())).count();
|
|
|
+ if (aiPlanCount > 0) {
|
|
|
+ pushLog("[入库] 开始 AI 动态表入库：" + aiPlanCount + " 个 sheet");
|
|
|
+ }
|
|
|
for (PendingAi p : pending) {
|
|
|
AiJudgeResult.AiPlan plan = plans.get(p.job().getId());
|
|
|
if (plan == null) {
|
|
|
@@ -438,8 +500,14 @@ public class AiSmartCleanService {
|
|
|
p.job().setExitCode(com.zsjz.ai.module.aiclean.enums.AiCleanExitEnum.A.name());
|
|
|
aiCleanService.finish(p.job(), AiCleanStatusEnum.DISPATCHED_A, null, System.currentTimeMillis());
|
|
|
bump(batchId, "existingHit");
|
|
|
+ pushLog(sheetLabel(p) + " → 命中既有模板「" + hit.nameCn() + "」，转入既有清洗");
|
|
|
+ }
|
|
|
+ case AiJudgeResult.AiPlan plan -> {
|
|
|
+ plans.put(p.job().getId(), plan);
|
|
|
+ String target = plan.candidate() == null ? "未匹配到模板，按表头新建表"
|
|
|
+ : "命中模板「" + plan.candidate().nameCn() + "」" + matchReason(p.job()) + "，准备入库";
|
|
|
+ pushLog(sheetLabel(p) + " → " + target);
|
|
|
}
|
|
|
- case AiJudgeResult.AiPlan plan -> plans.put(p.job().getId(), plan);
|
|
|
case AiJudgeResult.NeedReview r -> {
|
|
|
aiCleanService.finish(p.job(), AiCleanStatusEnum.NEED_REVIEW, r.reason(),
|
|
|
System.currentTimeMillis());
|
|
|
@@ -447,6 +515,7 @@ public class AiSmartCleanService {
|
|
|
st.needReview.incrementAndGet();
|
|
|
bump(batchId, "needReview");
|
|
|
bumpRows(batchId, 0, 0, nz(p.sheet().getDataNum()));
|
|
|
+ pushLog(sheetLabel(p) + " → 需人工确认：" + StrUtil.maxLength(r.reason(), 120));
|
|
|
}
|
|
|
case AiJudgeResult.Skip r -> {
|
|
|
aiCleanService.finish(p.job(), AiCleanStatusEnum.SKIP, r.reason(), System.currentTimeMillis());
|
|
|
@@ -456,6 +525,7 @@ public class AiSmartCleanService {
|
|
|
st.needReview.incrementAndGet();
|
|
|
bump(batchId, "needReview");
|
|
|
bumpRows(batchId, 0, 0, nz(p.sheet().getDataNum()));
|
|
|
+ pushLog(sheetLabel(p) + " → 已跳过：" + StrUtil.maxLength(r.reason(), 120));
|
|
|
}
|
|
|
case AiJudgeResult.Failed r -> fail(batchId, p, r.reason(), st);
|
|
|
case null -> fail(batchId, p, "AI 判定无结论", st);
|
|
|
@@ -470,6 +540,7 @@ public class AiSmartCleanService {
|
|
|
st.failed.incrementAndGet();
|
|
|
bump(batchId, "failed");
|
|
|
bumpRows(batchId, 0, 0, nz(p.sheet().getDataNum()));
|
|
|
+ pushLog(sheetLabel(p) + " → 失败：" + StrUtil.maxLength(reason, 120));
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
@@ -518,9 +589,12 @@ public class AiSmartCleanService {
|
|
|
applied++;
|
|
|
}
|
|
|
}
|
|
|
- log.info("智能清洗：按降级顺序串行清洗 1级表={} 个 → 2级表={} 个",
|
|
|
- l1ByRoot.values().stream().mapToInt(List::size).sum(),
|
|
|
- l2ByRoot.values().stream().mapToInt(List::size).sum());
|
|
|
+ int l1Count = l1ByRoot.values().stream().mapToInt(List::size).sum();
|
|
|
+ int l2Count = l2ByRoot.values().stream().mapToInt(List::size).sum();
|
|
|
+ log.info("智能清洗：按降级顺序串行清洗 1级表={} 个 → 2级表={} 个", l1Count, l2Count);
|
|
|
+ if (l1Count + l2Count > 0) {
|
|
|
+ pushLog("[清洗] 开始既有模板清洗：" + l1Count + " 个已匹配 sheet + " + l2Count + " 个 AI 命中 sheet");
|
|
|
+ }
|
|
|
// 两趟都静默：收口消息由本类在 3级入库完成后统一推送（run → pushTerminal），
|
|
|
// 保证「所有文件清洗完成 / 开始数据治理」出现在全部清洗动作之后
|
|
|
runCleanPass(roots, l1ByRoot, true);
|
|
|
@@ -565,6 +639,7 @@ public class AiSmartCleanService {
|
|
|
|
|
|
private void loadPlan(Long batchId, PendingAi p, AiJudgeResult.AiPlan plan, Stats st) {
|
|
|
long start = System.currentTimeMillis();
|
|
|
+ String target = plan.candidate() == null ? "新建表" : "表「" + plan.candidate().nameCn() + "」";
|
|
|
try {
|
|
|
long written = aiCleanService.loadViaAiTemplate(p.job(), p.root(), p.sheet(), plan);
|
|
|
if (written < 0) {
|
|
|
@@ -572,6 +647,7 @@ public class AiSmartCleanService {
|
|
|
st.failed.incrementAndGet();
|
|
|
bump(batchId, "failed");
|
|
|
bumpRows(batchId, 0, nz(p.job().getRowRejected()), nz(p.job().getRowUnprocessed()));
|
|
|
+ pushLog(sheetLabel(p) + " → 入库失败：" + target);
|
|
|
return;
|
|
|
}
|
|
|
aiCleanService.finish(p.job(), AiCleanStatusEnum.SUCCESS, null, start);
|
|
|
@@ -581,12 +657,14 @@ public class AiSmartCleanService {
|
|
|
bump(batchId, "selfBuilt");
|
|
|
}
|
|
|
bumpRows(batchId, nz(p.job().getRowLoaded()), nz(p.job().getRowRejected()), 0);
|
|
|
+ pushLog(sheetLabel(p) + " → 已写入 " + written + " 行（" + target + "）");
|
|
|
} catch (Exception e) {
|
|
|
log.error("AI 入库异常: jobId={}", p.job().getId(), e);
|
|
|
aiCleanService.finish(p.job(), AiCleanStatusEnum.FAIL, StrUtil.maxLength(e.getMessage(), 900), start);
|
|
|
st.failed.incrementAndGet();
|
|
|
bump(batchId, "failed");
|
|
|
bumpRows(batchId, 0, nz(p.job().getRowRejected()), nz(p.job().getRowUnprocessed()));
|
|
|
+ pushLog(sheetLabel(p) + " → 入库异常：" + StrUtil.maxLength(e.getMessage(), 120));
|
|
|
}
|
|
|
}
|
|
|
|
