|
|
@@ -11,6 +11,7 @@ import com.zsjz.ai.common.context.CaseAccessGuard;
|
|
|
import com.zsjz.ai.common.enums.FileCategoryEnum;
|
|
|
import com.zsjz.ai.common.utils.Json;
|
|
|
import com.zsjz.ai.common.model.dm.dto.TableRuleDTO;
|
|
|
+import com.zsjz.ai.common.model.dm.entity.CleanErrorLog;
|
|
|
import com.zsjz.ai.common.model.dm.entity.FileInfo;
|
|
|
import com.zsjz.ai.common.model.plat.entity.TableInfo;
|
|
|
import com.zsjz.ai.module.aiclean.entity.AiCleanField;
|
|
|
@@ -19,6 +20,7 @@ import com.zsjz.ai.module.aiclean.entity.AiCleanRule;
|
|
|
import com.zsjz.ai.module.aiclean.entity.AiCleanTemplate;
|
|
|
import com.zsjz.ai.module.aiclean.enums.AiCleanExitEnum;
|
|
|
import com.zsjz.ai.module.aiclean.enums.AiCleanStatusEnum;
|
|
|
+import com.zsjz.ai.module.aiclean.enums.AiRejectReason;
|
|
|
import com.zsjz.ai.module.aiclean.exec.AiDuckDb;
|
|
|
import com.zsjz.ai.module.aiclean.exec.AiRecordCleaner;
|
|
|
import com.zsjz.ai.module.aiclean.exec.AiRejectCollector;
|
|
|
@@ -30,6 +32,7 @@ import com.zsjz.ai.module.aiclean.match.AiTemplateMatcher;
|
|
|
import com.zsjz.ai.module.aiclean.model.AiMatchOutcome;
|
|
|
import com.zsjz.ai.module.aiclean.model.AiRowCleanResult;
|
|
|
import com.zsjz.ai.module.aiclean.model.AiRuleDraft;
|
|
|
+import com.zsjz.ai.module.aiclean.model.ColumnError;
|
|
|
import com.zsjz.ai.module.aiclean.model.RuleBind;
|
|
|
import com.zsjz.ai.module.aiclean.model.SheetEvidence;
|
|
|
import com.zsjz.ai.module.aiclean.model.TemplateCandidate;
|
|
|
@@ -41,6 +44,7 @@ import com.zsjz.ai.module.aiclean.rule.CompiledColumn;
|
|
|
import com.zsjz.ai.module.aiclean.store.AiCleanStore;
|
|
|
import com.zsjz.ai.module.aiclean.support.HeaderFingerprint;
|
|
|
import com.zsjz.ai.module.aiclean.verify.AiRuleVerifier;
|
|
|
+import com.zsjz.ai.module.dm.mapper.CleanErrorLogMapper;
|
|
|
import com.zsjz.ai.module.dm.mapper.FileInfoMapper;
|
|
|
import com.zsjz.ai.module.agent.llm.LlmResult;
|
|
|
import lombok.RequiredArgsConstructor;
|
|
|
@@ -85,6 +89,8 @@ public class AiCleanService {
|
|
|
|
|
|
private final FileInfoMapper fileInfoMapper;
|
|
|
|
|
|
+ private final CleanErrorLogMapper cleanErrorLogMapper;
|
|
|
+
|
|
|
private final AiTemplateMatcher matcher;
|
|
|
|
|
|
private final ExistingTemplateDispatcher dispatcher;
|
|
|
@@ -540,6 +546,8 @@ public class AiCleanService {
|
|
|
AiRejectCollector rejects = new AiRejectCollector(job.getId(), job.getCaseId(), job.getFileId(),
|
|
|
job.getSheetId(), ev.primaryBlock().blockNo(), columnNameCn,
|
|
|
rejectLogSample, rejectLogAll);
|
|
|
+ // 与手动清洗共用 clean_error_log:导入文件页的「导入失败」弹框直接按批次/文件查这张表
|
|
|
+ List<CleanErrorLog> errorLogs = new ArrayList<>();
|
|
|
long written = -1L;
|
|
|
try (AiDuckDb.BatchWriter writer = AiDuckDb.openWriter(tpl.getTableNameEn(), cols)) {
|
|
|
// 行数据优先取自上传预览表:不用为了入库再把整份源文件重新解析一遍。
|
|
|
@@ -550,6 +558,10 @@ public class AiCleanService {
|
|
|
AiRowCleanResult result = AiRecordCleaner.cleanRowDetailed(row, aiPlan.columns());
|
|
|
rejects.record(lineNo, result);
|
|
|
if (result.rejected()) {
|
|
|
+ errorLogs.add(buildErrorLog(job, lineNo, result.errors(), columnNameCn));
|
|
|
+ if (errorLogs.size() >= 500) {
|
|
|
+ flushErrorLogs(errorLogs);
|
|
|
+ }
|
|
|
return;
|
|
|
}
|
|
|
Map<String, Object> cleaned = new LinkedHashMap<>(result.values());
|
|
|
@@ -569,6 +581,7 @@ public class AiCleanService {
|
|
|
writer.add(cleaned);
|
|
|
} catch (RuntimeException e) {
|
|
|
rejects.recordWriterError(lineNo, e.getMessage());
|
|
|
+ errorLogs.add(buildWriterErrorLog(job, lineNo, e.getMessage()));
|
|
|
throw e;
|
|
|
}
|
|
|
};
|
|
|
@@ -594,10 +607,12 @@ public class AiCleanService {
|
|
|
AiDuckDb.dropTable(tpl.getTableNameEn());
|
|
|
}
|
|
|
rejects.flush();
|
|
|
+ flushErrorLogs(errorLogs);
|
|
|
fillLoadStats(job, rejects, 0, true);
|
|
|
return -1L;
|
|
|
}
|
|
|
rejects.flush();
|
|
|
+ flushErrorLogs(errorLogs);
|
|
|
if (written <= 0) {
|
|
|
// 一行没落库不能算成功(含「读到 0 行」):全自动链路没有人看日志,
|
|
|
// 写成 0 行会被上层当成清洗完成。新建表删掉不留幽灵;复用表里还有
|
|
|
@@ -648,6 +663,77 @@ public class AiCleanService {
|
|
|
return written;
|
|
|
}
|
|
|
|
|
|
+ /**
|
|
|
+ * 构造手动清洗同款失败日志:写进 {@code clean_error_log},由导入文件页的
|
|
|
+ * 「导入失败」弹框统一展示,不再为 AI 清洗单独准备弹框。
|
|
|
+ */
|
|
|
+ private CleanErrorLog buildErrorLog(AiCleanJob job, long lineNo, List<ColumnError> errors,
|
|
|
+ Map<String, String> columnNameCn) {
|
|
|
+ CleanErrorLog log = new CleanErrorLog();
|
|
|
+ log.setFileId(job.getFileId());
|
|
|
+ log.setSheetId(job.getSheetId());
|
|
|
+ log.setBatchId(job.getBatchId());
|
|
|
+ log.setFileName(job.getFileName());
|
|
|
+ log.setSheetName(job.getSheetName());
|
|
|
+ log.setContent(formatRowErrors(lineNo, errors, columnNameCn));
|
|
|
+ log.setCreateDateTime(LocalDateTime.now());
|
|
|
+ return log;
|
|
|
+ }
|
|
|
+
|
|
|
+ private CleanErrorLog buildWriterErrorLog(AiCleanJob job, long lineNo, String message) {
|
|
|
+ CleanErrorLog log = new CleanErrorLog();
|
|
|
+ log.setFileId(job.getFileId());
|
|
|
+ log.setSheetId(job.getSheetId());
|
|
|
+ log.setBatchId(job.getBatchId());
|
|
|
+ log.setFileName(job.getFileName());
|
|
|
+ log.setSheetName(job.getSheetName());
|
|
|
+ log.setContent("第 " + lineNo + " 行:写入失败"
|
|
|
+ + (StrUtil.isBlank(message) ? "" : ":" + message));
|
|
|
+ log.setCreateDateTime(LocalDateTime.now());
|
|
|
+ return log;
|
|
|
+ }
|
|
|
+
|
|
|
+ /** 例:第 2 行:c1(身份证号)必填列类型不合法,原值 110105194912310021 */
|
|
|
+ private String formatRowErrors(long lineNo, List<ColumnError> errors, Map<String, String> columnNameCn) {
|
|
|
+ List<String> parts = new ArrayList<>();
|
|
|
+ if (errors != null) {
|
|
|
+ for (ColumnError error : errors) {
|
|
|
+ if (error == null) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ String column = StrUtil.trim(error.columnName());
|
|
|
+ if (column == null) {
|
|
|
+ column = "";
|
|
|
+ }
|
|
|
+ String columnCn = column.isEmpty() || columnNameCn == null ? null : columnNameCn.get(column);
|
|
|
+ String columnText = column.isEmpty() ? ""
|
|
|
+ : StrUtil.isBlank(columnCn) ? column : column + "(" + columnCn + ")";
|
|
|
+ AiRejectReason reason = error.reason();
|
|
|
+ String reasonText = reason == null ? "未知原因" : reason.label();
|
|
|
+ String valueText = StrUtil.isBlank(error.valuePreview())
|
|
|
+ ? "" : ",原值 " + error.valuePreview();
|
|
|
+ parts.add((columnText.isEmpty() ? "" : columnText + ":") + reasonText + valueText);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ String detail = String.join(";", parts);
|
|
|
+ return "第 " + lineNo + " 行" + (detail.isEmpty() ? "" : ":" + detail);
|
|
|
+ }
|
|
|
+
|
|
|
+ /** 日志写失败不能影响入库,只记录应用日志。 */
|
|
|
+ private void flushErrorLogs(List<CleanErrorLog> logs) {
|
|
|
+ if (logs.isEmpty()) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ try {
|
|
|
+ cleanErrorLogMapper.insert(logs);
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("AI 清洗失败日志写入 clean_error_log 失败(不影响入库): size={}, err={}",
|
|
|
+ logs.size(), e.getMessage());
|
|
|
+ } finally {
|
|
|
+ logs.clear();
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
/**
|
|
|
* 把行级统计写回 job,并合并进 {@code detail.load}。
|
|
|
*
|