|
|
@@ -4,6 +4,7 @@ import cn.hutool.core.io.FileUtil;
|
|
|
import cn.hutool.core.util.IdUtil;
|
|
|
import cn.hutool.core.util.StrUtil;
|
|
|
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
|
|
|
+import com.fasterxml.jackson.core.type.TypeReference;
|
|
|
import com.zsjz.ai.common.base.BasicColumn;
|
|
|
import com.zsjz.ai.common.cache.GlobalCache;
|
|
|
import com.zsjz.ai.common.context.CaseAccessGuard;
|
|
|
@@ -20,12 +21,14 @@ import com.zsjz.ai.module.aiclean.enums.AiCleanExitEnum;
|
|
|
import com.zsjz.ai.module.aiclean.enums.AiCleanStatusEnum;
|
|
|
import com.zsjz.ai.module.aiclean.exec.AiDuckDb;
|
|
|
import com.zsjz.ai.module.aiclean.exec.AiRecordCleaner;
|
|
|
+import com.zsjz.ai.module.aiclean.exec.AiRejectCollector;
|
|
|
import com.zsjz.ai.module.aiclean.exec.ExistingTemplateDispatcher;
|
|
|
import com.zsjz.ai.module.aiclean.mapper.AiCleanFieldMapper;
|
|
|
import com.zsjz.ai.module.aiclean.mapper.AiCleanJobMapper;
|
|
|
import com.zsjz.ai.module.aiclean.mapper.AiCleanTemplateMapper;
|
|
|
import com.zsjz.ai.module.aiclean.match.AiTemplateMatcher;
|
|
|
import com.zsjz.ai.module.aiclean.model.AiMatchOutcome;
|
|
|
+import com.zsjz.ai.module.aiclean.model.AiRowCleanResult;
|
|
|
import com.zsjz.ai.module.aiclean.model.AiRuleDraft;
|
|
|
import com.zsjz.ai.module.aiclean.model.RuleBind;
|
|
|
import com.zsjz.ai.module.aiclean.model.SheetEvidence;
|
|
|
@@ -34,12 +37,14 @@ import com.zsjz.ai.module.aiclean.model.VerifyReport;
|
|
|
import com.zsjz.ai.module.aiclean.probe.AiRowReader;
|
|
|
import com.zsjz.ai.module.aiclean.probe.SheetProbe;
|
|
|
import com.zsjz.ai.module.aiclean.rule.AiRulePlan;
|
|
|
+import com.zsjz.ai.module.aiclean.rule.CompiledColumn;
|
|
|
import com.zsjz.ai.module.aiclean.store.AiCleanStore;
|
|
|
import com.zsjz.ai.module.aiclean.verify.AiRuleVerifier;
|
|
|
import com.zsjz.ai.module.dm.mapper.FileInfoMapper;
|
|
|
import com.zsjz.ai.module.agent.llm.LlmResult;
|
|
|
import lombok.RequiredArgsConstructor;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
+import org.springframework.beans.factory.annotation.Value;
|
|
|
import org.springframework.stereotype.Service;
|
|
|
|
|
|
import java.io.File;
|
|
|
@@ -89,6 +94,14 @@ public class AiCleanService {
|
|
|
|
|
|
private final CaseAccessGuard caseAccessGuard;
|
|
|
|
|
|
+ /** 每张表最多打多少条「拒绝行」应用日志(完整明细在 ai_clean_reject 表里) */
|
|
|
+ @Value("${zsjz.ai-clean.reject-log-sample:20}")
|
|
|
+ private int rejectLogSample;
|
|
|
+
|
|
|
+ /** 调试开关:每一行拒绝都打应用日志(默认只打前 N 条,避免刷爆) */
|
|
|
+ @Value("${zsjz.ai-clean.reject-log-all:false}")
|
|
|
+ private boolean rejectLogAll;
|
|
|
+
|
|
|
/** 手动重建当前案件的 AI 治理节点(幂等;正常情况由治理任务与 apply 后自动触发) */
|
|
|
public int refreshTree() {
|
|
|
return aiGovernService.calcAiTree();
|
|
|
@@ -264,16 +277,46 @@ public class AiCleanService {
|
|
|
if (selfBinds.isEmpty()) {
|
|
|
return new AiJudgeResult.NeedReview("表头为空,无法按文件自建表");
|
|
|
}
|
|
|
+ // 自建表照常入库,但匹配率口径必须能把它与真实模板命中分开
|
|
|
+ job.setMatchedBy(AiMatchOutcome.BY_SELF_HEADER);
|
|
|
return new AiJudgeResult.AiPlan(ev, selfBinds, null, null, headers);
|
|
|
}
|
|
|
// 指纹/打分命中 AI 模板库的表:复用存档规则与已有动态表,零模型调用 ——
|
|
|
// header_md5 是唯一键,再建同指纹模板会撞 uk_ai_clean_tpl_md5(2026-10-11 实测)
|
|
|
if (!outcome.exit().writesExistingTable() && outcome.candidate().aiTemplateId() != null) {
|
|
|
- List<RuleBind> stored = store.loadBinds(outcome.candidate().aiTemplateId());
|
|
|
- if (!stored.isEmpty()) {
|
|
|
- return new AiJudgeResult.AiPlan(ev, stored, outcome.candidate(), null, headers);
|
|
|
+ AiCleanTemplate tpl = templateMapper.selectById(outcome.candidate().aiTemplateId());
|
|
|
+ if (tpl != null && !"ONLINE".equals(tpl.getStatus())) {
|
|
|
+ // OFFLINE/DRAFT 模板仍占着 header_md5 唯一键,继续新建会撞键并留下幽灵表,
|
|
|
+ // 明确转人工比静默失败好;Phase B 再做重新激活/晋升。
|
|
|
+ return new AiJudgeResult.NeedReview("命中 AI 模板但状态为 " + tpl.getStatus()
|
|
|
+ + ",需人工确认后再清洗");
|
|
|
}
|
|
|
- // 存档规则缺失(历史异常数据)→ 落回原路:按本次表头问模型补绑定
|
|
|
+ if (tpl != null) {
|
|
|
+ List<RuleBind> stored = store.loadBinds(tpl.getId());
|
|
|
+ List<AiCleanField> storedFields = stored.isEmpty() ? List.of()
|
|
|
+ : fieldMapper.selectList(Wrappers.<AiCleanField>lambdaQuery()
|
|
|
+ .eq(AiCleanField::getAiTemplateId, tpl.getId())
|
|
|
+ .orderByAsc(AiCleanField::getFieldSort));
|
|
|
+ if (!stored.isEmpty() && !storedFields.isEmpty()) {
|
|
|
+ // 复用路径也要抽样校验:存档类型/必填可能已经不适配当前文件,
|
|
|
+ // 不校验就直接写,会把错列/错类型的数据静默落库。
|
|
|
+ mark(job, AiCleanStatusEnum.VERIFYING);
|
|
|
+ List<CompiledColumn> columns = AiRulePlan.forStoredTemplate(storedFields, stored).columns();
|
|
|
+ List<Map<Integer, String>> sample = loadSample(root, sheet, ev, job);
|
|
|
+ VerifyReport report = verify(sample, columns, outcome.candidate(), outcome, ev);
|
|
|
+ job.setConfidence(java.math.BigDecimal.valueOf(report.confidence().overall()));
|
|
|
+ job.setDetail(detailJson(report));
|
|
|
+ if (requiredRejected(columns, report)) {
|
|
|
+ return new AiJudgeResult.NeedReview("复用模板的必填列抽样错误率过高,需人工确认");
|
|
|
+ }
|
|
|
+ if (!report.confidence().pass(AiCleanExitEnum.B)) {
|
|
|
+ return new AiJudgeResult.NeedReview(StrUtil.format(
|
|
|
+ "复用模板置信度未达出口 B 的放行线: {}", report.confidence()));
|
|
|
+ }
|
|
|
+ return new AiJudgeResult.AiPlan(ev, stored, outcome.candidate(), null, headers);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ // 存档规则/字段缺失(历史异常数据)→ 落回原路:按本次表头问模型补绑定
|
|
|
}
|
|
|
// 选模板那次调用已经把列绑定一起返回了,复用它:单次模型调用实测就要几十秒,
|
|
|
// 为了拿 binds 再打一次等于把延迟和 token 直接翻倍
|
|
|
@@ -305,19 +348,15 @@ public class AiCleanService {
|
|
|
}
|
|
|
|
|
|
mark(job, AiCleanStatusEnum.VERIFYING);
|
|
|
- int headerRow = ev.primaryBlock().headerRow();
|
|
|
- // 抽样优先读上传预览表 raw_{sheetId}:与源文件同一套读值口径(Fesod 同配置 + 同款转换器),
|
|
|
- // 但不用再把整份文件重新解析一遍。老预览表 / 预览写入失败时回退读源文件。
|
|
|
- List<Map<Integer, String>> sample = AiDuckDb.readRawSample(job.getSheetId(), headerRow, SAMPLE,
|
|
|
- sheet.getDataNum() == null ? 0 : sheet.getDataNum());
|
|
|
- if (sample == null) {
|
|
|
- log.info("预览表不可用,AI 抽样回退读源文件: jobId={}, sheetId={}", job.getId(), job.getSheetId());
|
|
|
- sample = AiRowReader.read(new File(root.getFilePath()), sheet.getSheetName(), headerRow, SAMPLE);
|
|
|
- }
|
|
|
- VerifyReport report = verify(binds, outcome.candidate(), headers, sample, outcome);
|
|
|
+ List<Map<Integer, String>> sample = loadSample(root, sheet, ev, job);
|
|
|
+ List<CompiledColumn> verifyColumns = resolveColumns(binds, ev, outcome.candidate(), outcome);
|
|
|
+ VerifyReport report = verify(sample, verifyColumns, outcome.candidate(), outcome, ev);
|
|
|
job.setConfidence(java.math.BigDecimal.valueOf(report.confidence().overall()));
|
|
|
// 抽样报告落库(预览行截到 5 行防膨胀):排障时能看到「哪个列错误率多少、为什么被拒」
|
|
|
job.setDetail(detailJson(report));
|
|
|
+ if (requiredRejected(verifyColumns, report)) {
|
|
|
+ return new AiJudgeResult.NeedReview("必填列抽样错误率过高,整行拒绝比例过大,需人工确认");
|
|
|
+ }
|
|
|
if (report.rejectedCols() != null && !report.rejectedCols().isEmpty()) {
|
|
|
binds = binds.stream().filter(b -> !report.rejectedCols().contains(b.getFileColIndex())).toList();
|
|
|
}
|
|
|
@@ -432,7 +471,7 @@ public class AiCleanService {
|
|
|
aiPlan = AiRulePlan.forStoredTemplate(storedFields, binds);
|
|
|
log.info("AI 表指纹命中,复用已有动态表: table={}, templateId={}", tpl.getTableNameEn(), tpl.getId());
|
|
|
} else {
|
|
|
- aiPlan = AiRulePlan.forAi(binds, headers);
|
|
|
+ aiPlan = AiRulePlan.forAi(binds, headers, ev.colStats(), plan.candidate());
|
|
|
// 模板 id 先行:物理表名依赖它;元数据登记挪到写数成功之后
|
|
|
tpl = new AiCleanTemplate();
|
|
|
tpl.setId(IdUtil.getSnowflakeNextId());
|
|
|
@@ -470,35 +509,32 @@ public class AiCleanService {
|
|
|
ruleRows.add(r);
|
|
|
}
|
|
|
}
|
|
|
- String err = reuse ? null : AiDuckDb.createAiTable(tpl.getTableNameEn(), aiPlan.fields());
|
|
|
+ // 复用模板时物理表可能在当前案件不存在(跨案件共享/被手工删过):
|
|
|
+ // CREATE TABLE IF NOT EXISTS 是幂等的,复用也走一次,避免 openWriter 直接失败
|
|
|
+ String err = AiDuckDb.createAiTable(tpl.getTableNameEn(), aiPlan.fields());
|
|
|
if (err != null) {
|
|
|
log.error("AI 动态表建表失败,跳过写入: table={}, err={}", tpl.getTableNameEn(), err);
|
|
|
return -1L;
|
|
|
}
|
|
|
- // 同一份文件的同一 sheet 重新清洗:先清掉它上一轮写进这张动态表的行,
|
|
|
- // 否则「重新清洗/改了模板再跑」会把整表数据重复追加(AI 表按 file_id+sheet_id 隔离,不会误删别的文件)
|
|
|
- long purged = AiDuckDb.deleteByFileSheet(tpl.getTableNameEn(), job.getFileId(), job.getSheetId());
|
|
|
- if (purged > 0) {
|
|
|
- log.info("AI 重清洗,已清理该 sheet 旧行: table={}, fileId={}, sheetId={}, rows={}",
|
|
|
- tpl.getTableNameEn(), job.getFileId(), job.getSheetId(), purged);
|
|
|
- }
|
|
|
- // lambda 要求事实最终变量:tpl/reuse 在上面按复用/新建二选一赋值后不再变
|
|
|
+ // lambda 要求事实最终变量:tpl 在上面按复用/新建二选一赋值后不再变
|
|
|
final AiCleanTemplate tplF = tpl;
|
|
|
- final boolean reuseF = reuse;
|
|
|
List<String> cols = aiPlan.fields().stream().map(AiCleanField::getColumnName).toList();
|
|
|
- long written;
|
|
|
- java.util.concurrent.atomic.AtomicLong total = new java.util.concurrent.atomic.AtomicLong();
|
|
|
int headerRow = ev.primaryBlock().headerRow();
|
|
|
+ AiRejectCollector rejects = new AiRejectCollector(job.getId(), job.getCaseId(), job.getFileId(),
|
|
|
+ job.getSheetId(), ev.primaryBlock().blockNo(), rejectLogSample, rejectLogAll);
|
|
|
+ long written = -1L;
|
|
|
try (AiDuckDb.BatchWriter writer = AiDuckDb.openWriter(tpl.getTableNameEn(), cols)) {
|
|
|
// 行数据优先取自上传预览表:不用为了入库再把整份源文件重新解析一遍。
|
|
|
// 注意这里仍然逐行走 AiRecordCleaner —— 预览表存的是原值(全 VARCHAR),
|
|
|
// 特殊字符剔除、函数链、类型规整都还没做,纯 SQL 的 CREATE TABLE AS 会把这些全部丢掉。
|
|
|
AiDuckDb.RawRowHandler handler = (row, lineNo) -> {
|
|
|
- Map<String, String> errors = new LinkedHashMap<>();
|
|
|
- Map<String, Object> cleaned = AiRecordCleaner.cleanRow(row, aiPlan.columns(), errors);
|
|
|
- if (cleaned == null) {
|
|
|
+ rejects.countSource();
|
|
|
+ AiRowCleanResult result = AiRecordCleaner.cleanRowDetailed(row, aiPlan.columns());
|
|
|
+ rejects.record(lineNo, result);
|
|
|
+ if (result.rejected()) {
|
|
|
return;
|
|
|
}
|
|
|
+ Map<String, Object> cleaned = new LinkedHashMap<>(result.values());
|
|
|
cleaned.put("id", IdUtil.getSnowflakeNextId());
|
|
|
cleaned.put("case_id", job.getCaseId());
|
|
|
cleaned.put("file_id", job.getFileId());
|
|
|
@@ -510,9 +546,13 @@ public class AiCleanService {
|
|
|
cleaned.put("ai_job_id", job.getId());
|
|
|
cleaned.put("ai_rule_ver", tplF.getVer() == null ? 1 : tplF.getVer());
|
|
|
cleaned.put("category", tplF.getCategory());
|
|
|
- // 写失败会在这里抛出,中断流式读取 —— 半截数据绝不能被当成成功
|
|
|
- writer.add(cleaned);
|
|
|
- total.incrementAndGet();
|
|
|
+ try {
|
|
|
+ // 写失败会在这里抛出,中断流式读取 —— 半截数据绝不能被当成成功
|
|
|
+ writer.add(cleaned);
|
|
|
+ } catch (RuntimeException e) {
|
|
|
+ rejects.recordWriterError(lineNo, e.getMessage());
|
|
|
+ throw e;
|
|
|
+ }
|
|
|
};
|
|
|
AiDuckDb.RawSheet raw = AiDuckDb.rawSheet(job.getSheetId(), expectedRawRows(sheet, job));
|
|
|
boolean ok;
|
|
|
@@ -530,31 +570,115 @@ public class AiCleanService {
|
|
|
written = ok ? writer.finish() : -1L;
|
|
|
} catch (Exception e) {
|
|
|
log.error("AI 流式入库失败: jobId={}, table={}, err={}", job.getId(), tpl.getTableNameEn(), e.getMessage());
|
|
|
+ // 半截新行按 job 清掉;旧行此时还没 purge,保持原样
|
|
|
+ AiDuckDb.deleteByJob(tpl.getTableNameEn(), job.getId());
|
|
|
+ if (!reuse) {
|
|
|
+ AiDuckDb.dropTable(tpl.getTableNameEn());
|
|
|
+ }
|
|
|
+ rejects.flush();
|
|
|
+ fillLoadStats(job, rejects, 0, true);
|
|
|
return -1L;
|
|
|
}
|
|
|
+ rejects.flush();
|
|
|
if (written <= 0) {
|
|
|
// 一行没落库不能算成功(含「读到 0 行」):全自动链路没有人看日志,
|
|
|
// 写成 0 行会被上层当成清洗完成。新建表删掉不留幽灵;复用表里还有
|
|
|
- // 此前批次的数据,绝不能连带删
|
|
|
+ // 此前批次的数据,只清本次 job 的半截行,绝不连带删旧数据
|
|
|
+ AiDuckDb.deleteByJob(tpl.getTableNameEn(), job.getId());
|
|
|
if (!reuse) {
|
|
|
AiDuckDb.dropTable(tpl.getTableNameEn());
|
|
|
}
|
|
|
+ fillLoadStats(job, rejects, 0, true);
|
|
|
return -1L;
|
|
|
}
|
|
|
- // 数据已落盘,现在才登记元数据为 ONLINE 并刷新治理节点(写序收口);
|
|
|
- // 复用模板只刷计数,不重复登记
|
|
|
- if (!reuse) {
|
|
|
+ // 数据已写成功,复用表现在才清旧行:先写后删,写入失败不会丢旧数据。
|
|
|
+ if (reuse) {
|
|
|
+ long purged = AiDuckDb.deleteByFileSheetExcludingJob(tpl.getTableNameEn(),
|
|
|
+ job.getFileId(), job.getSheetId(), job.getId());
|
|
|
+ if (purged < 0) {
|
|
|
+ // 旧行清不掉就回滚本次写入,避免同一 sheet 出现两份数据
|
|
|
+ AiDuckDb.deleteByJob(tpl.getTableNameEn(), job.getId());
|
|
|
+ fillLoadStats(job, rejects, 0, true);
|
|
|
+ return -1L;
|
|
|
+ }
|
|
|
+ if (purged > 0) {
|
|
|
+ log.info("AI 重清洗完成,已清理该 sheet 旧行: table={}, fileId={}, sheetId={}, rows={}",
|
|
|
+ tpl.getTableNameEn(), job.getFileId(), job.getSheetId(), purged);
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ // 数据已落盘,现在才登记元数据为 ONLINE 并刷新治理节点(写序收口)
|
|
|
tpl.setStatus("ONLINE");
|
|
|
- store.register(tpl, aiPlan.fields(), ruleRows);
|
|
|
+ try {
|
|
|
+ store.register(tpl, aiPlan.fields(), ruleRows);
|
|
|
+ } catch (Exception e) {
|
|
|
+ // 登记失败不能留下「有物理表、没有元数据」的幽灵表
|
|
|
+ log.error("AI 模板登记失败,清理新建动态表: jobId={}, table={}, err={}",
|
|
|
+ job.getId(), tpl.getTableNameEn(), e.getMessage());
|
|
|
+ AiDuckDb.dropTable(tpl.getTableNameEn());
|
|
|
+ fillLoadStats(job, rejects, 0, true);
|
|
|
+ return -1L;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ try {
|
|
|
+ store.refreshTree();
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("AI 清洗后刷新治理节点失败(数据已入库): {}", e.getMessage());
|
|
|
}
|
|
|
- store.refreshTree();
|
|
|
job.setAiTemplateId(tpl.getId());
|
|
|
job.setExitCode(AiCleanExitEnum.B.name());
|
|
|
- job.setRowTotal((int) Math.min(total.get(), Integer.MAX_VALUE));
|
|
|
- job.setRowLoaded((int) Math.min(written, Integer.MAX_VALUE));
|
|
|
+ fillLoadStats(job, rejects, written, false);
|
|
|
return written;
|
|
|
}
|
|
|
|
|
|
+ /**
|
|
|
+ * 把行级统计写回 job,并合并进 {@code detail.load}。
|
|
|
+ *
|
|
|
+ * <p>成功任务:{@code rowTotal = rowLoaded + rowRejected},{@code rowUnprocessed=0}。
|
|
|
+ * 失败任务:半截新行已按 job 清掉,最终表没有本次新增行,所以 {@code rowLoaded=0};
|
|
|
+ * 未被拒绝的行都算未完成(含已写但被回滚的行)。
|
|
|
+ */
|
|
|
+ static void fillLoadStats(AiCleanJob job, AiRejectCollector rejects, long written, boolean failed) {
|
|
|
+ int source = (int) Math.min(rejects.source(), Integer.MAX_VALUE);
|
|
|
+ int rejected = (int) Math.min(rejects.rejected(), Integer.MAX_VALUE);
|
|
|
+ int loaded = failed ? 0 : (int) Math.min(Math.max(0, written), Integer.MAX_VALUE);
|
|
|
+ int unprocessed = failed ? Math.max(0, source - rejected) : 0;
|
|
|
+ job.setRowTotal(source);
|
|
|
+ job.setRowLoaded(loaded);
|
|
|
+ job.setRowRejected(rejected);
|
|
|
+ job.setRowWarned((int) Math.min(rejects.warned(), Integer.MAX_VALUE));
|
|
|
+ job.setRowUnprocessed(unprocessed);
|
|
|
+ job.setDetail(mergeLoadDetail(job.getDetail(), rejects.summaryJson(loaded, unprocessed)));
|
|
|
+ }
|
|
|
+
|
|
|
+ /** 保留 verify 的 detail,同时把行级统计挂到 {@code load} 节点下 */
|
|
|
+ private static String mergeLoadDetail(String existing, String loadJson) {
|
|
|
+ String fallback = "{\"load\":" + loadJson + "}";
|
|
|
+ if (StrUtil.isBlank(existing)) {
|
|
|
+ return StrUtil.maxLength(fallback, 8000);
|
|
|
+ }
|
|
|
+ try {
|
|
|
+ Map<String, Object> root = Json.toCls(existing, new TypeReference<Map<String, Object>>() {
|
|
|
+ });
|
|
|
+ root.put("load", Json.toCls(loadJson, new TypeReference<Map<String, Object>>() {
|
|
|
+ }));
|
|
|
+ return StrUtil.maxLength(Json.toStr(root), 8000);
|
|
|
+ } catch (Exception e) {
|
|
|
+ return StrUtil.maxLength(fallback, 8000);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 分页查询某个 job 的行级失败明细;跨案件拒绝。
|
|
|
+ */
|
|
|
+ public Map<String, Object> rejectPage(Long jobId, long page, long size, String reason) {
|
|
|
+ AiCleanJob job = jobId == null ? null : jobMapper.selectById(jobId);
|
|
|
+ if (job == null) {
|
|
|
+ return Map.of("records", List.of(), "total", 0L, "size", size, "current", Math.max(1, page));
|
|
|
+ }
|
|
|
+ caseAccessGuard.requireOwnedCurrentCase(job.getCaseId());
|
|
|
+ return AiDuckDb.rejectPage(jobId, reason, page, size);
|
|
|
+ }
|
|
|
+
|
|
|
/**
|
|
|
* 预览表完整性校验用的行数:上传解析时 {@code FileInfo.dataNum} 记的就是写入 raw 的行数。
|
|
|
*
|
|
|
@@ -655,6 +779,7 @@ public class AiCleanService {
|
|
|
return 0L;
|
|
|
}
|
|
|
long n = store.revert(tpl.getTableNameEn(), jobId);
|
|
|
+ AiDuckDb.deleteRejectsByJob(jobId);
|
|
|
job.setStatus(AiCleanStatusEnum.SKIP);
|
|
|
job.setUpdateTime(LocalDateTime.now());
|
|
|
job.setFailReason("已按 job 撤销,删除行数 " + n);
|
|
|
@@ -662,12 +787,52 @@ public class AiCleanService {
|
|
|
return n;
|
|
|
}
|
|
|
|
|
|
- private VerifyReport verify(List<RuleBind> binds, TemplateCandidate cand, List<String> headers,
|
|
|
- List<Map<Integer, String>> sample, AiMatchOutcome outcome) {
|
|
|
- AiRulePlan.Plan plan = AiRulePlan.forAi(binds, headers);
|
|
|
- long total = sample.size();
|
|
|
- return AiRuleVerifier.verify(sample, plan.columns(), outcome.candidate().score(),
|
|
|
- cand.score(), total);
|
|
|
+ /**
|
|
|
+ * 抽样:优先后传预览表(不重读源文件),不可用时回退源文件。
|
|
|
+ */
|
|
|
+ private List<Map<Integer, String>> loadSample(FileInfo root, FileInfo sheet, SheetEvidence ev,
|
|
|
+ AiCleanJob job) {
|
|
|
+ int headerRow = ev.primaryBlock().headerRow();
|
|
|
+ List<Map<Integer, String>> sample = AiDuckDb.readRawSample(job.getSheetId(), headerRow, SAMPLE,
|
|
|
+ sheet.getDataNum() == null ? 0 : sheet.getDataNum());
|
|
|
+ if (sample == null) {
|
|
|
+ log.info("预览表不可用,AI 抽样回退读源文件: jobId={}, sheetId={}", job.getId(), job.getSheetId());
|
|
|
+ sample = AiRowReader.read(new File(root.getFilePath()), sheet.getSheetName(), headerRow, SAMPLE);
|
|
|
+ }
|
|
|
+ return sample;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 与真正执行相同的列计划:出口 A 用 table_field,出口 B 用模板/形状推断。
|
|
|
+ * verify 与 load 共用这一处,避免「校验一套、执行另一套」。
|
|
|
+ */
|
|
|
+ private List<CompiledColumn> resolveColumns(List<RuleBind> binds, SheetEvidence ev,
|
|
|
+ TemplateCandidate cand, AiMatchOutcome outcome) {
|
|
|
+ List<String> headers = ev.primaryBlock().headers();
|
|
|
+ if (cand != null && !cand.fromAiLib() && outcome.exit() == AiCleanExitEnum.A) {
|
|
|
+ return AiRulePlan.forExistingColumns(binds, cand, headers);
|
|
|
+ }
|
|
|
+ return AiRulePlan.forAi(binds, headers, ev.colStats(), cand).columns();
|
|
|
+ }
|
|
|
+
|
|
|
+ private VerifyReport verify(List<Map<Integer, String>> sample, List<CompiledColumn> columns,
|
|
|
+ TemplateCandidate cand, AiMatchOutcome outcome, SheetEvidence ev) {
|
|
|
+ return AiRuleVerifier.verify(sample, columns, cand == null ? 0d : cand.score(),
|
|
|
+ draftConfidence(outcome), ev.totalRows());
|
|
|
+ }
|
|
|
+
|
|
|
+ private static double draftConfidence(AiMatchOutcome outcome) {
|
|
|
+ return outcome.draft() != null && outcome.draft().getConfidence() != null
|
|
|
+ ? outcome.draft().getConfidence() : 0d;
|
|
|
+ }
|
|
|
+
|
|
|
+ /** 必填列被抽样拒绝时不能静默删列 —— 关键列缺失比整体转人工更糟 */
|
|
|
+ private static boolean requiredRejected(List<CompiledColumn> columns, VerifyReport report) {
|
|
|
+ if (report.rejectedCols() == null || report.rejectedCols().isEmpty()) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ return columns.stream().anyMatch(c -> c.required()
|
|
|
+ && report.rejectedCols().contains(c.colIndex()));
|
|
|
}
|
|
|
|
|
|
/**
|