|
|
@@ -103,6 +103,8 @@ public class DmService {
|
|
|
private com.zsjz.ai.module.aiclean.mapper.AiCleanJobMapper aiCleanJobMapper;
|
|
|
@Autowired
|
|
|
private com.zsjz.ai.module.aiclean.mapper.AiCleanTemplateMapper aiCleanTemplateMapper;
|
|
|
+ @Autowired
|
|
|
+ private com.zsjz.ai.module.aiclean.tree.AiGovernService aiGovernService;
|
|
|
|
|
|
/**
|
|
|
* 预览文件内容
|
|
|
@@ -623,6 +625,14 @@ public class DmService {
|
|
|
failNum += Objects.requireNonNullElse(child.getFailNum(), 0);
|
|
|
}
|
|
|
int completedCount = dataNum - failNum;
|
|
|
+ // 所有清洗任务已完成:预览阶段动态建的 raw_{sheetId} 临时表不再需要,整表清掉;
|
|
|
+ // 不清理会随每次上传在案件库里无限累积(AI 微批还会再叠一份),拖慢备份与开案。
|
|
|
+ try {
|
|
|
+ int droppedRaw = com.zsjz.ai.module.aiclean.exec.AiDuckDb.dropTablesByPrefix("raw_");
|
|
|
+ log.info("清洗完成,已清理预览临时表 raw_*: {} 张", droppedRaw);
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.warn("清洗完成后清理 raw_ 临时表失败(不影响清洗结果): {}", e.getMessage());
|
|
|
+ }
|
|
|
if (silent) {
|
|
|
// 静默趟:收口消息(终态/关会话/同时治理)由智能清洗批次统一决定时机,
|
|
|
// 但事实性清理(全文检索刷盘/临时文件)与统计照常执行
|
|
|
@@ -872,6 +882,13 @@ public class DmService {
|
|
|
if (CollUtil.isEmpty(fileInfos)) {
|
|
|
return List.of();
|
|
|
}
|
|
|
+ // sheet 的 AI 任务记录(每 sheet 取最新一条):清洗方式/备注 + AI 动态表的行数口径都靠它
|
|
|
+ List<Long> sheetIds = fileInfos.stream()
|
|
|
+ .filter(info -> info.getPid() != null && info.getPid() != 0L)
|
|
|
+ .map(FileInfo::getId)
|
|
|
+ .filter(Objects::nonNull)
|
|
|
+ .toList();
|
|
|
+ Map<Long, com.zsjz.ai.module.aiclean.entity.AiCleanJob> latestJobs = loadLatestAiJobs(sheetIds);
|
|
|
Map<Long, List<FileInfo>> fileMap = fileInfos.stream().collect(Collectors.groupingBy(FileInfo::getPid));
|
|
|
List<FileInfo> rootFiles = Lists.newArrayList();
|
|
|
for (FileInfo fileInfo : fileInfos) {
|
|
|
@@ -888,14 +905,33 @@ public class DmService {
|
|
|
int dataNum = 0;
|
|
|
int failNum = 0;
|
|
|
for (FileInfo child : childrens) {
|
|
|
- dataNum += Objects.requireNonNullElse(child.getDataNum(), 0);
|
|
|
- failNum += Objects.requireNonNullElse(child.getFailNum(), 0);
|
|
|
+ com.zsjz.ai.module.aiclean.entity.AiCleanJob job = latestJobs.get(child.getId());
|
|
|
+ if (job != null && job.getAiTemplateId() != null) {
|
|
|
+ // AI 动态表(出口 B)的 sheet:file_info 里是前置解析的原始行数(含标题/表头行),
|
|
|
+ // 真实入库量看 AI 任务 —— 否则会重复累加,结果页数字虚高
|
|
|
+ int rowTotal = Objects.requireNonNullElse(job.getRowTotal(), 0);
|
|
|
+ int rowLoaded = Objects.requireNonNullElse(job.getRowLoaded(), 0);
|
|
|
+ int aiFail = Math.max(0, rowTotal - rowLoaded);
|
|
|
+ child.setDataNum(rowTotal);
|
|
|
+ child.setFailNum(aiFail);
|
|
|
+ child.setSuccessNum(rowLoaded);
|
|
|
+ dataNum += rowTotal;
|
|
|
+ failNum += aiFail;
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ int childData = Objects.requireNonNullElse(child.getDataNum(), 0);
|
|
|
+ int childFail = Objects.requireNonNullElse(child.getFailNum(), 0);
|
|
|
+ // sheet 行自身的成功数 = 数据量 - 失败数(库里只写入 data_num/fail_num,
|
|
|
+ // 不回填的话列表 sheet 行的「导入成功」永远是 0)
|
|
|
+ child.setSuccessNum(Math.max(0, childData - childFail));
|
|
|
+ dataNum += childData;
|
|
|
+ failNum += childFail;
|
|
|
}
|
|
|
fileInfo.setDataNum(dataNum);
|
|
|
fileInfo.setFailNum(failNum);
|
|
|
- fileInfo.setSuccessNum(dataNum - failNum);
|
|
|
+ fileInfo.setSuccessNum(Math.max(0, dataNum - failNum));
|
|
|
// sheet 行展示「清洗方式 / 备注」:按 AI 任务记录回填瞬态字段
|
|
|
- fillSheetCleanInfo(childrens);
|
|
|
+ fillSheetCleanInfo(childrens, latestJobs);
|
|
|
}
|
|
|
return rootFiles
|
|
|
.stream()
|
|
|
@@ -904,6 +940,24 @@ public class DmService {
|
|
|
.toList();
|
|
|
}
|
|
|
|
|
|
+ /**
|
|
|
+ * 批量取每个 sheet 的最新一条 AI 任务(雪花 id 降序首个):
|
|
|
+ * 同一 sheet 可能被清洗多次,展示与统计都只认最新一次。
|
|
|
+ */
|
|
|
+ private Map<Long, com.zsjz.ai.module.aiclean.entity.AiCleanJob> loadLatestAiJobs(List<Long> sheetIds) {
|
|
|
+ Map<Long, com.zsjz.ai.module.aiclean.entity.AiCleanJob> latestJob = new HashMap<>();
|
|
|
+ if (CollUtil.isEmpty(sheetIds)) {
|
|
|
+ return latestJob;
|
|
|
+ }
|
|
|
+ for (com.zsjz.ai.module.aiclean.entity.AiCleanJob job : aiCleanJobMapper.selectList(
|
|
|
+ Wrappers.<com.zsjz.ai.module.aiclean.entity.AiCleanJob>lambdaQuery()
|
|
|
+ .in(com.zsjz.ai.module.aiclean.entity.AiCleanJob::getSheetId, sheetIds)
|
|
|
+ .orderByDesc(com.zsjz.ai.module.aiclean.entity.AiCleanJob::getId))) {
|
|
|
+ latestJob.putIfAbsent(job.getSheetId(), job);
|
|
|
+ }
|
|
|
+ return latestJob;
|
|
|
+ }
|
|
|
+
|
|
|
/**
|
|
|
* 回填 sheet 的清洗方式与备注(瞬态字段,供「导入文件管理」列表展示):
|
|
|
* <ul>
|
|
|
@@ -912,22 +966,14 @@ public class DmService {
|
|
|
* <li>无 AI 任务但已配模板(上传直配或手动清洗)→ 「系统清洗」</li>
|
|
|
* <li>都没有 → 留空(前端显示 -)</li>
|
|
|
* </ul>
|
|
|
- * sheet 可能被多次清洗(多条 job),取最新一条(雪花 id 降序首个)。
|
|
|
+ *
|
|
|
+ * @param latestJob 由 {@link #loadLatestAiJobs} 预取的 sheet 最新任务,避免每层重复查库
|
|
|
*/
|
|
|
- private void fillSheetCleanInfo(List<FileInfo> sheets) {
|
|
|
+ private void fillSheetCleanInfo(List<FileInfo> sheets,
|
|
|
+ Map<Long, com.zsjz.ai.module.aiclean.entity.AiCleanJob> latestJob) {
|
|
|
if (CollUtil.isEmpty(sheets)) {
|
|
|
return;
|
|
|
}
|
|
|
- List<Long> sheetIds = sheets.stream().map(FileInfo::getId).filter(Objects::nonNull).toList();
|
|
|
- Map<Long, com.zsjz.ai.module.aiclean.entity.AiCleanJob> latestJob = new HashMap<>();
|
|
|
- if (CollUtil.isNotEmpty(sheetIds)) {
|
|
|
- for (com.zsjz.ai.module.aiclean.entity.AiCleanJob job : aiCleanJobMapper.selectList(
|
|
|
- Wrappers.<com.zsjz.ai.module.aiclean.entity.AiCleanJob>lambdaQuery()
|
|
|
- .in(com.zsjz.ai.module.aiclean.entity.AiCleanJob::getSheetId, sheetIds)
|
|
|
- .orderByDesc(com.zsjz.ai.module.aiclean.entity.AiCleanJob::getId))) {
|
|
|
- latestJob.putIfAbsent(job.getSheetId(), job);
|
|
|
- }
|
|
|
- }
|
|
|
Map<Long, String> aiTableNames = new HashMap<>();
|
|
|
for (FileInfo sheet : sheets) {
|
|
|
com.zsjz.ai.module.aiclean.entity.AiCleanJob job = latestJob.get(sheet.getId());
|
|
|
@@ -959,7 +1005,10 @@ public class DmService {
|
|
|
if ("B".equals(job.getExitCode())) {
|
|
|
yield "AI 新建表:" + StrUtil.blankToDefault(
|
|
|
aiTableName(job.getAiTemplateId(), aiTableNames),
|
|
|
- job.getAiTemplateId() == null ? "AI 表" : "ai_t" + job.getAiTemplateId());
|
|
|
+ job.getAiTemplateId() == null
|
|
|
+ ? "AI 表"
|
|
|
+ : com.zsjz.ai.module.aiclean.entity.AiCleanTemplate
|
|
|
+ .physicalTableName(job.getAiTemplateId()));
|
|
|
}
|
|
|
yield ""; // 判定进行中,暂无备注
|
|
|
}
|
|
|
@@ -991,6 +1040,22 @@ public class DmService {
|
|
|
// 业务表一旦失败就再无重试入口)。失败时保留全部原数据,可直接重试删除
|
|
|
// (deleteDataByFileIds 按文件 ID 删,重试幂等)
|
|
|
deleteBusinessTablesConcurrently(tableInfoList, ids);
|
|
|
+ // AI 动态表不在 table_info 里,上面的业务表清理覆盖不到;先把要删的「文件根 / sheet」归类,
|
|
|
+ // 再按 file_id(或 file_id+sheet_id)清掉动态表里的行和对应 AI 任务,避免删完还能查到数据
|
|
|
+ List<FileInfo> affectedFiles = fileInfoMapper.selectList(Wrappers.<FileInfo>lambdaQuery()
|
|
|
+ .in(FileInfo::getId, ids)
|
|
|
+ .or()
|
|
|
+ .in(FileInfo::getPid, ids));
|
|
|
+ Set<Long> rootIds = new HashSet<>(ids);
|
|
|
+ List<FileInfo> sheetTargets = new ArrayList<>();
|
|
|
+ for (FileInfo row : affectedFiles) {
|
|
|
+ if (row.getPid() != null && row.getPid() != 0L && ids.contains(row.getId())) {
|
|
|
+ // 传进来的是工作表:从「整文件删除」里摘出来,按 sheet 粒度删
|
|
|
+ rootIds.remove(row.getId());
|
|
|
+ sheetTargets.add(row);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ cleanupAiDynamicRows(rootIds, sheetTargets);
|
|
|
// 业务数据删净后再删文件登记行与 AI 识别档案(主库)
|
|
|
fileInfoMapper.delete(Wrappers.lambdaQuery(FileInfo.class)
|
|
|
.in(FileInfo::getId, ids)
|
|
|
@@ -1006,6 +1071,70 @@ public class DmService {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ /**
|
|
|
+ * 删除导入文件时清掉 AI 动态表里的对应行 + 对应 AI 任务,并刷新 AI 治理节点计数。
|
|
|
+ *
|
|
|
+ * @param rootIds 整文件删除的文件根 id
|
|
|
+ * @param sheetTargets 只删工作表的记录(按 file_id + sheet_id 精确删)
|
|
|
+ */
|
|
|
+ private void cleanupAiDynamicRows(Set<Long> rootIds, List<FileInfo> sheetTargets) {
|
|
|
+ List<Long> sheetIds = sheetTargets.stream().map(FileInfo::getId).filter(Objects::nonNull).toList();
|
|
|
+ if (rootIds.isEmpty() && sheetIds.isEmpty()) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ List<com.zsjz.ai.module.aiclean.entity.AiCleanJob> jobs = aiCleanJobMapper.selectList(
|
|
|
+ aiJobScopeQuery(rootIds, sheetIds));
|
|
|
+ if (CollUtil.isEmpty(jobs)) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ Set<Long> templateIds = jobs.stream()
|
|
|
+ .map(com.zsjz.ai.module.aiclean.entity.AiCleanJob::getAiTemplateId)
|
|
|
+ .filter(Objects::nonNull)
|
|
|
+ .collect(Collectors.toSet());
|
|
|
+ for (Long templateId : templateIds) {
|
|
|
+ com.zsjz.ai.module.aiclean.entity.AiCleanTemplate tpl = aiCleanTemplateMapper.selectById(templateId);
|
|
|
+ if (tpl == null || StrUtil.isBlank(tpl.getTableNameEn())) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ long deleted = 0L;
|
|
|
+ if (!rootIds.isEmpty()) {
|
|
|
+ deleted += com.zsjz.ai.module.aiclean.exec.AiDuckDb.deleteRowsByFileIds(
|
|
|
+ tpl.getTableNameEn(), new ArrayList<>(rootIds));
|
|
|
+ }
|
|
|
+ for (FileInfo sheet : sheetTargets) {
|
|
|
+ if (sheet.getId() != null && sheet.getPid() != null && sheet.getPid() != 0L) {
|
|
|
+ deleted += com.zsjz.ai.module.aiclean.exec.AiDuckDb.deleteByFileSheet(
|
|
|
+ tpl.getTableNameEn(), sheet.getPid(), sheet.getId());
|
|
|
+ }
|
|
|
+ }
|
|
|
+ log.info("删除文件时清理 AI 动态表行: table={}, fileIds={}, sheetIds={}, rows={}",
|
|
|
+ tpl.getTableNameEn(), rootIds, sheetIds, deleted);
|
|
|
+ }
|
|
|
+ // 任务行同步清掉,避免 job 指向已不存在的文件(模板共享,不删模板本身)
|
|
|
+ aiCleanJobMapper.delete(aiJobScopeQuery(rootIds, sheetIds));
|
|
|
+ // 行数变了要重建 AI 治理节点;失败只记日志,不影响文件删除结果
|
|
|
+ try {
|
|
|
+ aiGovernService.calcAiTree();
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.warn("删除文件后刷新 AI 治理节点失败(不影响删除结果): {}", e.getMessage());
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper<com.zsjz.ai.module.aiclean.entity.AiCleanJob> aiJobScopeQuery(
|
|
|
+ Set<Long> rootIds, List<Long> sheetIds) {
|
|
|
+ var query = Wrappers.<com.zsjz.ai.module.aiclean.entity.AiCleanJob>lambdaQuery();
|
|
|
+ if (!rootIds.isEmpty()) {
|
|
|
+ query.in(com.zsjz.ai.module.aiclean.entity.AiCleanJob::getFileId, rootIds);
|
|
|
+ }
|
|
|
+ if (!sheetIds.isEmpty()) {
|
|
|
+ if (!rootIds.isEmpty()) {
|
|
|
+ query.or();
|
|
|
+ }
|
|
|
+ query.in(com.zsjz.ai.module.aiclean.entity.AiCleanJob::getSheetId, sheetIds);
|
|
|
+ }
|
|
|
+ return query;
|
|
|
+ }
|
|
|
+
|
|
|
private void deleteBusinessTablesConcurrently(List<String> tableNames, List<Long> ids) {
|
|
|
if (CollUtil.isEmpty(tableNames)) {
|
|
|
return;
|