|
|
@@ -17,6 +17,7 @@ import com.zsjz.ai.module.aiclean.enums.AiCleanStatusEnum;
|
|
|
import com.zsjz.ai.module.aiclean.mapper.AiCleanBatchMapper;
|
|
|
import com.zsjz.ai.module.aiclean.mapper.AiCleanJobMapper;
|
|
|
import com.zsjz.ai.module.aiclean.match.BatchMatchCache;
|
|
|
+import com.zsjz.ai.module.dm.clean.SheetCompleteCallback;
|
|
|
import com.zsjz.ai.module.dm.mapper.FileInfoMapper;
|
|
|
import com.zsjz.ai.module.dm.service.DmService;
|
|
|
import com.zsjz.ai.module.govern.serivce.GovernConfService;
|
|
|
@@ -443,6 +444,8 @@ public class AiSmartCleanService {
|
|
|
int batchSheetTotal = roots.stream()
|
|
|
.mapToInt(r -> r.getChildren() == null ? 0 : r.getChildren().size()).sum();
|
|
|
pushLog("[清洗] 开始处理:" + roots.size() + " 个文件 / " + batchSheetTotal + " 个 sheet");
|
|
|
+ // 匹配阶段第一段:上传时已匹配模板的 1 级 sheet 先落日志,AI 判定结果随后到达
|
|
|
+ logSheetMatches(roots);
|
|
|
// (root, sheetNode, job) 三元组:判定阶段只收集,不动数据
|
|
|
List<PendingAi> pending = new ArrayList<>();
|
|
|
for (FileInfo root : roots) {
|
|
|
@@ -560,11 +563,20 @@ public class AiSmartCleanService {
|
|
|
// 按降级顺序串行清洗(需求 2026-10-11):1级(上传已匹配模板)先洗并等它
|
|
|
// 全部落库,再洗 2级(AI 命中既有模板),然后才进 3级(AI 自建表入库)。
|
|
|
// 4级(无法清洗)在判定阶段就只标记、不产生清洗任务。
|
|
|
- int toExisting = dispatchExistingByLevel(batchId, roots, existingHits);
|
|
|
+ ExistingPlan existingPlan = planExistingByLevel(roots, existingHits);
|
|
|
+ int aiPlanCount = (int) pending.stream().filter(p -> plans.containsKey(p.job().getId())).count();
|
|
|
+ // 提取阶段的分母:1级 + 2级 + 3级 AI 计划;整批共用一个计数器,
|
|
|
+ // 序号只出现在 [结果] 行,按 sheet 完成顺序连续推进。
|
|
|
+ ExtractionReporter reporter = new ExtractionReporter(
|
|
|
+ existingPlan.l1Count() + existingPlan.l2Count() + aiPlanCount);
|
|
|
+ SheetCompleteCallback sheetCallback = (root, sheet) -> reportSheetResult(root, sheet, reporter);
|
|
|
+ int toExisting = dispatchExistingByLevel(batchId, roots, existingPlan, sheetCallback, reporter);
|
|
|
|
|
|
if (stopRequested.contains(batchId)) {
|
|
|
stage(batchId, AiCleanStageEnum.STOPPED);
|
|
|
finishInterruptedJobs(batchId, false);
|
|
|
+ // 停止也要把已清洗部分的人员/对象落库,不能因为中途停止丢掉人员库
|
|
|
+ flushPersonCacheAtEnd();
|
|
|
pushStoppedTerminal(caseId, batchId);
|
|
|
log.info("智能清洗批次已停止(既有清洗阶段): batchId={}, 交既有清洗={} 条", batchId, toExisting);
|
|
|
return;
|
|
|
@@ -572,7 +584,6 @@ public class AiSmartCleanService {
|
|
|
|
|
|
stage(batchId, AiCleanStageEnum.LOADING_AI);
|
|
|
pushProgress(caseId, batchId);
|
|
|
- int aiPlanCount = (int) pending.stream().filter(p -> plans.containsKey(p.job().getId())).count();
|
|
|
if (aiPlanCount > 0) {
|
|
|
pushLog("[入库] 开始写入自动新建的数据表:" + aiPlanCount + " 个 sheet");
|
|
|
}
|
|
|
@@ -587,9 +598,12 @@ public class AiSmartCleanService {
|
|
|
System.currentTimeMillis());
|
|
|
continue;
|
|
|
}
|
|
|
- loadPlan(batchId, p, plan, st);
|
|
|
+ loadPlan(batchId, p, plan, st, reporter);
|
|
|
pushProgress(caseId, batchId);
|
|
|
}
|
|
|
+ // 3 级入库也属于提取:所有提取动作结束后再合并人员库/对象,
|
|
|
+ // 保证日志顺序是「匹配 → 提取结果 → 人员库合并 → 终态」。
|
|
|
+ flushPersonCacheAtEnd();
|
|
|
// 终态看本地计数,不回读 DB:并发写的行刚被别人累加过,读回来的值不代表这一轮
|
|
|
boolean stopped = stopRequested.contains(batchId);
|
|
|
AiCleanStageEnum finalStage = stopped ? AiCleanStageEnum.STOPPED
|
|
|
@@ -609,6 +623,8 @@ public class AiSmartCleanService {
|
|
|
st.needReview.get(), st.failed.get(), System.currentTimeMillis() - start);
|
|
|
} catch (Exception e) {
|
|
|
log.error("智能清洗批次异常: batchId={}", batchId, e);
|
|
|
+ // 异常中断也要把已缓存的人员/对象落库,避免残留到同一案件下一批普通清洗
|
|
|
+ flushPersonCacheAtEnd();
|
|
|
AiCleanBatch update = new AiCleanBatch();
|
|
|
update.setStage(AiCleanStageEnum.FAILED);
|
|
|
update.setLastError(StrUtil.maxLength(e.getMessage(), 900));
|
|
|
@@ -651,7 +667,7 @@ public class AiSmartCleanService {
|
|
|
p.job().setExitCode(com.zsjz.ai.module.aiclean.enums.AiCleanExitEnum.A.name());
|
|
|
aiCleanService.finish(p.job(), AiCleanStatusEnum.DISPATCHED_A, null, System.currentTimeMillis());
|
|
|
bump(batchId, "existingHit");
|
|
|
- pushLog(sheetLabel(p) + " → 已匹配既有模板「" + hit.nameCn() + "」,按该模板提取数据并写入业务表");
|
|
|
+ pushLog("[匹配] " + sheetLabel(p) + " → 已匹配既有模板「" + hit.nameCn() + "」,按该模板提取数据并写入业务表");
|
|
|
}
|
|
|
case AiJudgeResult.AiPlan plan -> {
|
|
|
plans.put(p.job().getId(), plan);
|
|
|
@@ -659,7 +675,7 @@ public class AiSmartCleanService {
|
|
|
// 日志说明这一点,避免用户以为第二个文件又新建了一张表。
|
|
|
String headerKey = plan.ev() == null ? null : plan.ev().headerMd5();
|
|
|
boolean repeated = StrUtil.isNotBlank(headerKey) && !plannedHeaderKeys.add(headerKey);
|
|
|
- pushLog(sheetLabel(p) + " → " + (repeated
|
|
|
+ pushLog("[匹配] " + sheetLabel(p) + " → " + (repeated
|
|
|
? "与同批次已识别的表结构相同,跳过重复识别,写入同一数据表"
|
|
|
: aiPlanLabel(plan) + matchReason(p.job())));
|
|
|
}
|
|
|
@@ -670,7 +686,7 @@ public class AiSmartCleanService {
|
|
|
st.needReview.incrementAndGet();
|
|
|
bump(batchId, "needReview");
|
|
|
bumpRows(batchId, 0, 0, nz(p.sheet().getDataNum()));
|
|
|
- pushLog(sheetLabel(p) + " → 需人工确认:" + StrUtil.maxLength(r.reason(), 120));
|
|
|
+ pushLog("[匹配] " + sheetLabel(p) + " → 需人工确认:" + StrUtil.maxLength(r.reason(), 120));
|
|
|
}
|
|
|
case AiJudgeResult.Skip r -> {
|
|
|
aiCleanService.finish(p.job(), AiCleanStatusEnum.SKIP, r.reason(), System.currentTimeMillis());
|
|
|
@@ -680,7 +696,7 @@ public class AiSmartCleanService {
|
|
|
st.needReview.incrementAndGet();
|
|
|
bump(batchId, "needReview");
|
|
|
bumpRows(batchId, 0, 0, nz(p.sheet().getDataNum()));
|
|
|
- pushLog(sheetLabel(p) + " → 已跳过:" + StrUtil.maxLength(r.reason(), 120));
|
|
|
+ pushLog("[匹配] " + sheetLabel(p) + " → 已跳过:" + StrUtil.maxLength(r.reason(), 120));
|
|
|
}
|
|
|
case AiJudgeResult.Failed r -> fail(batchId, p, r.reason(), st);
|
|
|
case null -> fail(batchId, p, "AI 判定无结论", st);
|
|
|
@@ -695,26 +711,33 @@ public class AiSmartCleanService {
|
|
|
st.failed.incrementAndGet();
|
|
|
bump(batchId, "failed");
|
|
|
bumpRows(batchId, 0, 0, nz(p.sheet().getDataNum()));
|
|
|
- pushLog(sheetLabel(p) + " → 失败:" + StrUtil.maxLength(reason, 120));
|
|
|
+ pushLog("[匹配] " + sheetLabel(p) + " → 失败:" + StrUtil.maxLength(reason, 120));
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
- * 按降级顺序串行执行既有链路清洗:先 1级(前端已匹配模板的表),等它全部落库后再
|
|
|
- * 2级(AI 命中既有模板的表)。两级各自的根树只放本级的 sheet ——
|
|
|
- * NEED_REVIEW / Skip 的表不进任何一趟,doClean 不再为它们记「未配置模板」噪音日志。
|
|
|
+ * 既有链路两趟清洗的分组结果(只分组,不发清洗)。
|
|
|
*
|
|
|
- * <p>趟间用 {@link DmService#doClean(DoCleanDTO, boolean)} 返回的 Latch 严格排队。
|
|
|
- * 两趟都静默:收口消息(统计/终态快照/治理宣告)由本类在 3级入库完成后统一推送
|
|
|
- * (见 {@link #pushTerminal}),保证「治理启动」出现在全部清洗动作之后。
|
|
|
+ * @param l2TemplateNames 2级 sheet 的模板展示名(sheetId → nameCn):只用于日志,
|
|
|
+ * 不改写 {@code childTemplateName},避免影响建表命名
|
|
|
+ */
|
|
|
+ private record ExistingPlan(Map<FileInfo, List<FileInfo>> l1ByRoot,
|
|
|
+ Map<FileInfo, List<FileInfo>> l2ByRoot,
|
|
|
+ Map<Long, String> l2TemplateNames,
|
|
|
+ int l1Count, int l2Count, int applied) {
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 按降级顺序编排既有链路清洗:1级(前端已匹配模板)先洗并等它全部落库,
|
|
|
+ * 再洗 2级(AI 命中既有模板);NEED_REVIEW / Skip 的表不进任何一趟。
|
|
|
*
|
|
|
- * @return 并入既有链路的 AI 命中表数(仅日志用)
|
|
|
+ * <p>分组与执行拆开:编排层要先拿到 1/2 级 sheet 数,才能定下提取阶段的分母
|
|
|
+ * (1级 + 2级 + 3级 AI 计划),并让整批共用一个 {@link ExtractionReporter}。
|
|
|
*/
|
|
|
- private int dispatchExistingByLevel(Long batchId, List<FileInfo> roots, List<AiJudgeResult.ExistingHit> hits) {
|
|
|
+ private ExistingPlan planExistingByLevel(List<FileInfo> roots, List<AiJudgeResult.ExistingHit> hits) {
|
|
|
int applied = 0;
|
|
|
- // 先在原树上把 1级 / 2级 sheet 分组:templateId 非空的是前端已匹配(1级),
|
|
|
- // 空的按命中结果补挂模板并归入 2级;都没命中的(3/4级)不进既有链路
|
|
|
Map<FileInfo, List<FileInfo>> l1ByRoot = new IdentityHashMap<>();
|
|
|
Map<FileInfo, List<FileInfo>> l2ByRoot = new IdentityHashMap<>();
|
|
|
+ Map<Long, String> l2TemplateNames = new java.util.HashMap<>();
|
|
|
for (FileInfo root : roots) {
|
|
|
if (root == null || root.getChildren() == null) {
|
|
|
continue;
|
|
|
@@ -741,67 +764,156 @@ public class AiSmartCleanService {
|
|
|
// 切「表头行/数据行」,缺了首行就 NPE(2026-10-11 实测零落库事故)
|
|
|
sheet.setLineNo(hit.headerRow());
|
|
|
l2ByRoot.computeIfAbsent(root, _ -> new ArrayList<>()).add(sheet);
|
|
|
+ if (sheet.getId() != null) {
|
|
|
+ l2TemplateNames.put(sheet.getId(), hit.nameCn());
|
|
|
+ }
|
|
|
applied++;
|
|
|
}
|
|
|
}
|
|
|
int l1Count = l1ByRoot.values().stream().mapToInt(List::size).sum();
|
|
|
int l2Count = l2ByRoot.values().stream().mapToInt(List::size).sum();
|
|
|
+ return new ExistingPlan(l1ByRoot, l2ByRoot, l2TemplateNames, l1Count, l2Count, applied);
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 按降级顺序串行执行既有链路清洗,逐表结果由 {@link SheetCompleteCallback} 实时上报。
|
|
|
+ *
|
|
|
+ * <p>两趟都静默:收口消息(统计/终态快照/治理宣告)由本类在 3级入库完成后统一推送
|
|
|
+ * (见 {@link #pushTerminal}),保证「治理启动」出现在全部清洗动作之后。
|
|
|
+ *
|
|
|
+ * @return 并入既有链路的 AI 命中表数(仅日志用)
|
|
|
+ */
|
|
|
+ private int dispatchExistingByLevel(Long batchId, List<FileInfo> roots, ExistingPlan plan,
|
|
|
+ SheetCompleteCallback sheetCallback, ExtractionReporter reporter) {
|
|
|
+ int l1Count = plan.l1Count();
|
|
|
+ int l2Count = plan.l2Count();
|
|
|
log.info("智能清洗:按降级顺序串行清洗 1级表={} 个 → 2级表={} 个", l1Count, l2Count);
|
|
|
if (l1Count + l2Count > 0) {
|
|
|
pushLog("[清洗] 开始按模板清洗:" + l1Count + " 个已匹配 sheet + " + l2Count + " 个 AI 识别 sheet");
|
|
|
}
|
|
|
- // 1级(上传时已匹配模板)的表此前没有任何逐表日志,用户在进度页只能看到 AI 的判定日志,
|
|
|
- // 会以为这些表没被提取。这里补齐「用哪个模板开始提取」与「提取了多少行」两条流水账。
|
|
|
- logSheetStart(roots, l1ByRoot);
|
|
|
- // 两趟都静默:收口消息由本类在 3级入库完成后统一推送(run → pushTerminal),
|
|
|
- // 保证「所有文件清洗完成 / 开始数据治理」出现在全部清洗动作之后
|
|
|
- runCleanPass(roots, l1ByRoot, true);
|
|
|
- logSheetResults(roots, l1ByRoot);
|
|
|
+ logSheetStart(roots, plan.l1ByRoot(), null);
|
|
|
+ runCleanPass(roots, plan.l1ByRoot(), true, sheetCallback);
|
|
|
+ // 空表 / 文件读取失败不会触发逐 sheet 回调,趟末补齐,保证每张表都有 [结果]
|
|
|
+ logMissingSheetResults(roots, plan.l1ByRoot(), reporter);
|
|
|
if (stopRequested.contains(batchId)) {
|
|
|
// 停止:1 级趟已跑完(在途文件都已结束),2 级不再启动
|
|
|
pushLog("[清洗] 已停止:2级既有清洗不再执行");
|
|
|
- return applied;
|
|
|
+ return plan.applied();
|
|
|
}
|
|
|
- runCleanPass(roots, l2ByRoot, true);
|
|
|
- logSheetResults(roots, l2ByRoot);
|
|
|
- return applied;
|
|
|
+ logSheetStart(roots, plan.l2ByRoot(), plan.l2TemplateNames());
|
|
|
+ runCleanPass(roots, plan.l2ByRoot(), true, sheetCallback);
|
|
|
+ logMissingSheetResults(roots, plan.l2ByRoot(), reporter);
|
|
|
+ return plan.applied();
|
|
|
}
|
|
|
|
|
|
- /** 既有链路开始清洗前的逐表日志:说明这张表按哪个模板提取数据 */
|
|
|
- private void logSheetStart(List<FileInfo> roots, Map<FileInfo, List<FileInfo>> sheetsByRoot) {
|
|
|
+ /** 匹配阶段第一段:上传时已匹配模板的 1 级 sheet,先于 AI 判定结果输出 */
|
|
|
+ private void logSheetMatches(List<FileInfo> roots) {
|
|
|
for (FileInfo root : roots) {
|
|
|
- for (FileInfo sheet : sheetsByRoot.getOrDefault(root, List.of())) {
|
|
|
- pushLog("[清洗] " + sheetLabel(root, sheet) + " → 已匹配模板「"
|
|
|
- + templateLabel(sheet) + "」,开始提取表格数据");
|
|
|
+ if (root == null || root.getChildren() == null) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ for (FileInfo sheet : root.getChildren()) {
|
|
|
+ if (sheet == null || sheet.getTemplateId() == null) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ pushLog("[匹配] " + sheetLabel(root, sheet) + " → 已匹配模板「" + templateLabel(sheet) + "」");
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
- * 既有链路一趟跑完后的逐表结果日志。
|
|
|
+ * 既有链路开始提取前的逐表日志:说明这张表按哪个模板提取数据。
|
|
|
*
|
|
|
- * <p>行数直接取内存里的 sheet 对象:{@code StreamingEtlListener} 清洗完成时会把
|
|
|
- * {@code dataNum/failNum} 回写到传进去的同一批对象上,无需再查库。
|
|
|
+ * @param templateNames 2级 sheet 的模板展示名(1级传 null,直接用 templateLabel)
|
|
|
*/
|
|
|
- private void logSheetResults(List<FileInfo> roots, Map<FileInfo, List<FileInfo>> sheetsByRoot) {
|
|
|
+ private void logSheetStart(List<FileInfo> roots, Map<FileInfo, List<FileInfo>> sheetsByRoot,
|
|
|
+ Map<Long, String> templateNames) {
|
|
|
for (FileInfo root : roots) {
|
|
|
for (FileInfo sheet : sheetsByRoot.getOrDefault(root, List.of())) {
|
|
|
- if (sheet.getFileStatus() != FileSheetStateEnum.FILE_CLEAN_COMPLETE) {
|
|
|
- pushLog("[清洗] " + sheetLabel(root, sheet) + " → 清洗未完成,详情见失败明细");
|
|
|
- continue;
|
|
|
- }
|
|
|
- int total = nz(sheet.getDataNum());
|
|
|
- int failed = nz(sheet.getFailNum());
|
|
|
- pushLog("[清洗] " + sheetLabel(root, sheet) + " → 提取完成:成功 "
|
|
|
- + Math.max(0, total - failed) + " 行,失败 " + failed + " 行");
|
|
|
+ String name = templateNames != null && sheet.getId() != null
|
|
|
+ ? templateNames.getOrDefault(sheet.getId(), templateLabel(sheet))
|
|
|
+ : templateLabel(sheet);
|
|
|
+ pushLog("[清洗] " + sheetLabel(root, sheet) + " → 已匹配模板「"
|
|
|
+ + name + "」,开始提取表格数据");
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /** 一趟结束后补齐未触发回调的 sheet(空表、读取失败等) */
|
|
|
+ private void logMissingSheetResults(List<FileInfo> roots, Map<FileInfo, List<FileInfo>> sheetsByRoot,
|
|
|
+ ExtractionReporter reporter) {
|
|
|
+ for (FileInfo root : roots) {
|
|
|
+ for (FileInfo sheet : sheetsByRoot.getOrDefault(root, List.of())) {
|
|
|
+ reportSheetResult(root, sheet, reporter);
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ /**
|
|
|
+ * 逐表结果上报:同一张 sheet 只报一次(回调 / 趟末补齐 / AI 入库共用一套去重)。
|
|
|
+ *
|
|
|
+ * <p>多文件并发清洗时,取号与推送必须互斥:否则线程 A 取到 5、线程 B 取到 6 后
|
|
|
+ * 先推出去,日志里就会出现 6/93 排在 5/93 前面。
|
|
|
+ */
|
|
|
+ private void reportSheetResult(FileInfo root, FileInfo sheet, ExtractionReporter reporter) {
|
|
|
+ if (sheet == null) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ synchronized (reporter) {
|
|
|
+ if (!reporter.markReported(sheet)) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ int done = reporter.finished();
|
|
|
+ pushLog("[结果] (" + done + "/" + reporter.total() + ") " + sheetLabel(root, sheet) + " → "
|
|
|
+ + sheetResultText(sheet));
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /** 逐表结果文案:行数取内存终值(StreamingEtlListener 已回写 dataNum/failNum) */
|
|
|
+ private static String sheetResultText(FileInfo sheet) {
|
|
|
+ if (sheet.getFileStatus() != FileSheetStateEnum.FILE_CLEAN_COMPLETE) {
|
|
|
+ return "清洗未完成,详情见失败明细";
|
|
|
+ }
|
|
|
+ int total = nz(sheet.getDataNum());
|
|
|
+ int failed = nz(sheet.getFailNum());
|
|
|
+ return "提取完成:成功 " + Math.max(0, total - failed) + " 行,失败 " + failed + " 行";
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 提取阶段的公共计数器:分母在判定结束后一次算定(1级 + 2级 + 3级 AI 计划),
|
|
|
+ * 分子按 sheet 完成实时累加;{@code reported} 跨回调/补齐/AI 入库去重。
|
|
|
+ */
|
|
|
+ private static final class ExtractionReporter {
|
|
|
+ private final int total;
|
|
|
+ private final AtomicInteger extracted = new AtomicInteger();
|
|
|
+ private final Set<Long> reported = ConcurrentHashMap.newKeySet();
|
|
|
+
|
|
|
+ private ExtractionReporter(int total) {
|
|
|
+ this.total = Math.max(0, total);
|
|
|
+ }
|
|
|
+
|
|
|
+ int total() {
|
|
|
+ return total;
|
|
|
+ }
|
|
|
+
|
|
|
+ int finished() {
|
|
|
+ return extracted.incrementAndGet();
|
|
|
+ }
|
|
|
+
|
|
|
+ /** @return true = 首次上报,需要输出日志 */
|
|
|
+ boolean markReported(FileInfo sheet) {
|
|
|
+ if (sheet == null || sheet.getId() == null) {
|
|
|
+ return true;
|
|
|
+ }
|
|
|
+ return reported.add(sheet.getId());
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
/**
|
|
|
* 把本级的 sheet 挂到根节点<b>浅拷贝</b>上发一趟既有清洗,并等它全部落库
|
|
|
*/
|
|
|
- private void runCleanPass(List<FileInfo> roots, Map<FileInfo, List<FileInfo>> sheetsByRoot, boolean silent) {
|
|
|
+ private void runCleanPass(List<FileInfo> roots, Map<FileInfo, List<FileInfo>> sheetsByRoot, boolean silent,
|
|
|
+ SheetCompleteCallback sheetCallback) {
|
|
|
// 根节点用浅拷贝:不改动前端提交的原始树;两趟各持各的根,互不覆盖
|
|
|
// (sheet 对象本身仍是同一批引用,瞬态字段只在分组时写一次)
|
|
|
List<FileInfo> passRoots = new ArrayList<>();
|
|
|
@@ -824,7 +936,7 @@ public class AiSmartCleanService {
|
|
|
DoCleanDTO dto = new DoCleanDTO();
|
|
|
dto.setFileInfos(passRoots);
|
|
|
dto.setReClean(0);
|
|
|
- CountDownLatch latch = dmService.doClean(dto, silent);
|
|
|
+ CountDownLatch latch = dmService.doClean(dto, silent, sheetCallback);
|
|
|
try {
|
|
|
latch.await();
|
|
|
} catch (InterruptedException e) {
|
|
|
@@ -833,6 +945,36 @@ public class AiSmartCleanService {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ /**
|
|
|
+ * 批次级人员库/对象落库:所有提取动作(含 3 级 AI 入库)结束后统一执行一次。
|
|
|
+ *
|
|
|
+ * <p>静默趟不再各自落库,保证日志顺序是「匹配 → 提取结果 → 人员库合并 → 终态」。
|
|
|
+ * 缓存为空(纯 AI 批次)时直接跳过,不输出「0 条」噪音。
|
|
|
+ */
|
|
|
+ private void flushPersonCacheAtEnd() {
|
|
|
+ try {
|
|
|
+ DmService.PersonFlushResult result = dmService.flushPersonCache((libCount, personCount) -> {
|
|
|
+ pushLog("[清洗] 正在合并人员库(" + libCount + " 条)");
|
|
|
+ pushLog("[清洗] 正在创建对象(" + personCount + " 个)");
|
|
|
+ });
|
|
|
+ if (result.libCount() + result.personCount() == 0) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ if (result.failed()) {
|
|
|
+ log.error("人员库/对象落库失败: {}", result.error());
|
|
|
+ pushLog("[清洗] 人员库/对象落库失败:" + result.error());
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ pushLog("[清洗] 人员库合并完成:" + result.libCount() + " 条,耗时 "
|
|
|
+ + (result.elapsedMs() / 1000) + " 秒");
|
|
|
+ pushLog("[清洗] 对象创建完成:" + result.personCount() + " 个,耗时 "
|
|
|
+ + (result.elapsedMs() / 1000) + " 秒");
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("人员库/对象落库异常", e);
|
|
|
+ pushLog("[清洗] 人员库/对象落库失败:" + StrUtil.maxLength(e.getMessage(), 120));
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
/**
|
|
|
* 出口 B 的目标说明:模型明明选中了既有模板却最终建了新表时,必须把原因写进处理日志,
|
|
|
* 否则用户只看到「命中模板」、却在结果页发现数据进了一张新表。
|
|
|
@@ -851,11 +993,13 @@ public class AiSmartCleanService {
|
|
|
return "与既有模板「" + cand.nameCn() + "」相似,但字段无法完全对应,已按当前表头新建数据表";
|
|
|
}
|
|
|
|
|
|
- private void loadPlan(Long batchId, PendingAi p, AiJudgeResult.AiPlan plan, Stats st) {
|
|
|
+ private void loadPlan(Long batchId, PendingAi p, AiJudgeResult.AiPlan plan, Stats st,
|
|
|
+ ExtractionReporter reporter) {
|
|
|
long start = System.currentTimeMillis();
|
|
|
String source = plan.candidate() == null ? "按表头识别"
|
|
|
: plan.candidate().fromAiLib() ? "历史模板「" + plan.candidate().nameCn() + "」"
|
|
|
: "参考既有模板「" + plan.candidate().nameCn() + "」";
|
|
|
+ pushLog("[清洗] " + sheetLabel(p) + " → 开始提取表格数据(" + source + ")");
|
|
|
try {
|
|
|
long written = aiCleanService.loadViaAiTemplate(p.job(), p.root(), p.sheet(), plan);
|
|
|
if (written < 0) {
|
|
|
@@ -863,7 +1007,7 @@ public class AiSmartCleanService {
|
|
|
st.failed.incrementAndGet();
|
|
|
bump(batchId, "failed");
|
|
|
bumpRows(batchId, 0, nz(p.job().getRowRejected()), nz(p.job().getRowUnprocessed()));
|
|
|
- pushLog(sheetLabel(p) + " → 写入失败(" + source + ")");
|
|
|
+ reportAiResult(p, reporter, "写入失败(" + source + ")");
|
|
|
return;
|
|
|
}
|
|
|
aiCleanService.finish(p.job(), AiCleanStatusEnum.SUCCESS, null, start);
|
|
|
@@ -873,14 +1017,25 @@ public class AiSmartCleanService {
|
|
|
bump(batchId, "selfBuilt");
|
|
|
}
|
|
|
bumpRows(batchId, nz(p.job().getRowLoaded()), nz(p.job().getRowRejected()), 0);
|
|
|
- pushLog(sheetLabel(p) + " → 提取完成:已写入 " + written + " 行(" + source + ")");
|
|
|
+ reportAiResult(p, reporter, "提取完成:已写入 " + written + " 行(" + source + ")");
|
|
|
} catch (Exception e) {
|
|
|
log.error("AI 入库异常: jobId={}", p.job().getId(), e);
|
|
|
aiCleanService.finish(p.job(), AiCleanStatusEnum.FAIL, StrUtil.maxLength(e.getMessage(), 900), start);
|
|
|
st.failed.incrementAndGet();
|
|
|
bump(batchId, "failed");
|
|
|
bumpRows(batchId, 0, nz(p.job().getRowRejected()), nz(p.job().getRowUnprocessed()));
|
|
|
- pushLog(sheetLabel(p) + " → 写入异常:" + StrUtil.maxLength(e.getMessage(), 120));
|
|
|
+ reportAiResult(p, reporter, "写入异常:" + StrUtil.maxLength(e.getMessage(), 120));
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /** AI 入库的逐表结果:与模板清洗共用同一个 [结果] x/N 计数器 */
|
|
|
+ private void reportAiResult(PendingAi p, ExtractionReporter reporter, String text) {
|
|
|
+ synchronized (reporter) {
|
|
|
+ if (!reporter.markReported(p.sheet())) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ int done = reporter.finished();
|
|
|
+ pushLog("[结果] (" + done + "/" + reporter.total() + ") " + sheetLabel(p) + " → " + text);
|
|
|
}
|
|
|
}
|
|
|
|