|
|
@@ -1,64 +1,284 @@
|
|
|
package com.zsjz.ai.module.aiclean.match;
|
|
|
|
|
|
+import cn.hutool.core.util.StrUtil;
|
|
|
+import com.zsjz.ai.module.aiclean.model.ColStat;
|
|
|
import com.zsjz.ai.module.aiclean.model.TemplateCandidate;
|
|
|
import com.zsjz.ai.module.aiclean.support.HeaderFingerprint;
|
|
|
import com.zsjz.ai.module.aiclean.support.HeaderNormalizer;
|
|
|
|
|
|
+import java.util.ArrayList;
|
|
|
+import java.util.HashMap;
|
|
|
+import java.util.HashSet;
|
|
|
import java.util.List;
|
|
|
+import java.util.Locale;
|
|
|
+import java.util.Map;
|
|
|
+import java.util.Set;
|
|
|
|
|
|
/**
|
|
|
* 归一化后的表头打分。
|
|
|
*
|
|
|
* <p>与既有打分的区别:既有是「交集数 + 必填数」的<b>绝对值</b>(PreDataListener:386),
|
|
|
* 模板字段越多越容易得高分,且没有阈值概念 —— 分不清「像」和「就是」。
|
|
|
- * 这里换成两个归一化比值相加:
|
|
|
+ *
|
|
|
+ * <p>2026-10-22 起不再只看表头集合:
|
|
|
* <ul>
|
|
|
- * <li>覆盖率:命中了模板里多少比例的字段(防漏);</li>
|
|
|
- * <li>Jaccard:两边列名整体有多重合(防串)。</li>
|
|
|
+ * <li>表头集合:覆盖率 + Jaccard(修复重复列把分母撑大的问题);</li>
|
|
|
+ * <li>字段名:候选的字段中文名/英文名是否出现在文件表头里(同义改写场景);</li>
|
|
|
+ * <li>列序:同序加分,乱序只给弱分(但不会因此丢掉候选);</li>
|
|
|
+ * <li>形状:{@link ColStat#shape()} 与候选字段类型是否矛盾;</li>
|
|
|
+ * <li>必填覆盖:候选的 required 字段是否都能在文件表头里找到。</li>
|
|
|
* </ul>
|
|
|
- * 于是分数落在 0~1,档位阈值才有意义。
|
|
|
+ * 分数落在 0~1,出口 A 还必须额外通过「确定性分数 + 必填覆盖」闸门,
|
|
|
+ * 模型自评置信度不能单独把候选升进出口 A。
|
|
|
*
|
|
|
* @author cc
|
|
|
* @since 2026/9/21
|
|
|
*/
|
|
|
public final class HeaderScorer {
|
|
|
|
|
|
- /** 强匹配线:必填全覆盖之外,整体分数还要过这道 */
|
|
|
- public static final double STRONG = 0.90;
|
|
|
+ /** 强匹配线:确定性分数过这道,且必填覆盖通过,才允许出口 A */
|
|
|
+ public static final double STRONG = 0.85;
|
|
|
+
|
|
|
+ /** 弱匹配线:低于它说明表头几乎没有共同点 */
|
|
|
+ public static final double WEAK = 0.55;
|
|
|
|
|
|
- /** 弱匹配线:低于它直接不匹配(走出口 B) */
|
|
|
- public static final double WEAK = 0.50;
|
|
|
+ /** 常见同义表头(只做评分归一,不参与指纹,避免指纹碰撞) */
|
|
|
+ private static final Map<String, String> SYNONYMS = Map.ofEntries(
|
|
|
+ Map.entry("账户名称", "户名"),
|
|
|
+ Map.entry("客户名称", "户名"),
|
|
|
+ Map.entry("户名", "户名"),
|
|
|
+ Map.entry("姓名", "户名"),
|
|
|
+ Map.entry("发生额", "交易金额"),
|
|
|
+ Map.entry("交易额", "交易金额"),
|
|
|
+ Map.entry("金额", "交易金额"),
|
|
|
+ Map.entry("交易日期", "日期"),
|
|
|
+ Map.entry("日期", "日期"),
|
|
|
+ Map.entry("交易时间", "时间"),
|
|
|
+ Map.entry("时间", "时间"),
|
|
|
+ Map.entry("卡号", "账号"),
|
|
|
+ Map.entry("银行账号", "账号"),
|
|
|
+ Map.entry("账号", "账号"),
|
|
|
+ Map.entry("账户", "账号"),
|
|
|
+ Map.entry("手机号", "电话"),
|
|
|
+ Map.entry("手机", "电话"),
|
|
|
+ Map.entry("联系电话", "电话"),
|
|
|
+ Map.entry("电话", "电话"),
|
|
|
+ Map.entry("身份证", "身份证号"),
|
|
|
+ Map.entry("身份证号码", "身份证号"),
|
|
|
+ Map.entry("证件号码", "身份证号"),
|
|
|
+ Map.entry("身份证号", "身份证号"));
|
|
|
|
|
|
private HeaderScorer() {
|
|
|
}
|
|
|
|
|
|
+ public static double score(List<String> headers, TemplateCandidate cand) {
|
|
|
+ return score(headers, cand, List.of());
|
|
|
+ }
|
|
|
+
|
|
|
/**
|
|
|
- * @param headers 文件表头(原始)
|
|
|
- * @param cand 候选模板
|
|
|
+ * @param headers 文件表头(原始)
|
|
|
+ * @param cand 候选模板
|
|
|
+ * @param colStats 文件列统计(主块),可为空;用于形状兼容性
|
|
|
* @return 0~1 的分数
|
|
|
*/
|
|
|
- public static double score(List<String> headers, TemplateCandidate cand) {
|
|
|
- List<String> fileCols = HeaderNormalizer.normalizeAll(nonBlank(headers));
|
|
|
- List<String> tplCols = HeaderNormalizer.normalizeAll(nonBlank(cand.headers()));
|
|
|
- if (fileCols.isEmpty() || tplCols.isEmpty()) {
|
|
|
+ public static double score(List<String> headers, TemplateCandidate cand, List<ColStat> colStats) {
|
|
|
+ List<String> fileNames = canonicalNames(headers);
|
|
|
+ List<String> tplNames = canonicalNames(cand.headers());
|
|
|
+ if (fileNames.isEmpty() || tplNames.isEmpty()) {
|
|
|
return 0d;
|
|
|
}
|
|
|
- long hit = fileCols.stream().distinct().filter(tplCols::contains).count();
|
|
|
- double coverage = (double) hit / tplCols.size();
|
|
|
- double jaccard = HeaderFingerprint.jaccard(fileCols, tplCols);
|
|
|
- return 0.7 * Math.min(1d, coverage) + 0.3 * jaccard;
|
|
|
+ Set<String> fileSet = new HashSet<>(fileNames);
|
|
|
+ Set<String> tplSet = new HashSet<>(tplNames);
|
|
|
+ long hit = fileSet.stream().filter(tplSet::contains).count();
|
|
|
+ double coverage = (double) hit / tplSet.size();
|
|
|
+ double jaccard = HeaderFingerprint.jaccard(fileSet, tplSet);
|
|
|
+ double header = 0.7 * Math.min(1d, coverage) + 0.3 * jaccard;
|
|
|
+
|
|
|
+ double field = fieldCoverage(fileSet, cand.fields());
|
|
|
+ double order = orderScore(fileNames, tplNames);
|
|
|
+ double shape = shapeScore(headers, cand.fields(), colStats);
|
|
|
+ double required = requiredCoverage(fileSet, cand.fields());
|
|
|
+ return 0.55 * header + 0.15 * field + 0.10 * order + 0.10 * shape + 0.10 * required;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 出口 A 的确定性闸门:短模板要求更高分,且必填字段必须都能在文件表头里找到;
|
|
|
+ * 明显的类型矛盾(如 DATE 字段对上一列日期形状为金额)也不许出口 A。
|
|
|
+ */
|
|
|
+ public static boolean passesStrong(List<String> headers, TemplateCandidate cand, double score,
|
|
|
+ List<ColStat> colStats) {
|
|
|
+ if (!requiredCovered(headers, cand)) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ if (!shapeSafe(headers, cand, colStats)) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ int cols = canonicalNames(cand.headers()).size();
|
|
|
+ return cols <= 3 ? score >= 0.90 : score >= STRONG;
|
|
|
+ }
|
|
|
+
|
|
|
+ /** 形状与字段类型没有明显矛盾;没有样本/形状为 TEXT 时视为安全 */
|
|
|
+ public static boolean shapeSafe(List<String> headers, TemplateCandidate cand, List<ColStat> colStats) {
|
|
|
+ return shapeScore(headers, cand.fields(), colStats) >= 0.5;
|
|
|
+ }
|
|
|
+
|
|
|
+ /** 候选里所有 required 字段是否都能在文件表头中找到 */
|
|
|
+ public static boolean requiredCovered(List<String> headers, TemplateCandidate cand) {
|
|
|
+ Set<String> fileSet = new HashSet<>(canonicalNames(headers));
|
|
|
+ return requiredCoverage(fileSet, cand.fields()) >= 1d;
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* 模板里有多少列在文件表头中出现了(用于「必填全覆盖」判断)。
|
|
|
*/
|
|
|
public static long matchedCount(List<String> headers, List<String> needCols) {
|
|
|
- List<String> fileCols = HeaderNormalizer.normalizeAll(nonBlank(headers));
|
|
|
- return nonBlank(needCols).stream().map(HeaderNormalizer::normalize)
|
|
|
- .filter(fileCols::contains).count();
|
|
|
+ Set<String> fileSet = new HashSet<>(canonicalNames(headers));
|
|
|
+ return canonicalNames(needCols).stream().filter(fileSet::contains).count();
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 评分归一:去括号/空白/后缀 + 同义词映射。指纹仍用 {@link HeaderNormalizer#normalize},
|
|
|
+ * 不受同义词影响,避免不同表头被折叠成同一个 md5。
|
|
|
+ */
|
|
|
+ static String canonicalForScore(String raw) {
|
|
|
+ String n = HeaderNormalizer.normalize(raw);
|
|
|
+ return SYNONYMS.getOrDefault(n, n);
|
|
|
}
|
|
|
|
|
|
- private static List<String> nonBlank(List<String> in) {
|
|
|
- return in == null ? List.of() : in.stream().filter(s -> s != null && !s.isBlank()).toList();
|
|
|
+ private static List<String> canonicalNames(List<String> headers) {
|
|
|
+ if (headers == null) {
|
|
|
+ return List.of();
|
|
|
+ }
|
|
|
+ List<String> out = new ArrayList<>(headers.size());
|
|
|
+ for (String h : headers) {
|
|
|
+ String n = canonicalForScore(h);
|
|
|
+ if (!n.isEmpty()) {
|
|
|
+ out.add(n);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return out;
|
|
|
+ }
|
|
|
+
|
|
|
+ private static double fieldCoverage(Set<String> fileSet, List<TemplateCandidate.FieldBrief> fields) {
|
|
|
+ if (fields == null || fields.isEmpty()) {
|
|
|
+ // 历史无字段模板:不给正向加分,也不重罚
|
|
|
+ return 0.5d;
|
|
|
+ }
|
|
|
+ int hit = 0;
|
|
|
+ for (TemplateCandidate.FieldBrief f : fields) {
|
|
|
+ if (f == null) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ String cn = canonicalForScore(f.cn());
|
|
|
+ String en = canonicalForScore(f.en());
|
|
|
+ if ((!cn.isEmpty() && fileSet.contains(cn)) || (!en.isEmpty() && fileSet.contains(en))) {
|
|
|
+ hit++;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return (double) hit / fields.size();
|
|
|
+ }
|
|
|
+
|
|
|
+ private static double orderScore(List<String> fileName, List<String> tplNames) {
|
|
|
+ int last = -1;
|
|
|
+ int found = 0;
|
|
|
+ boolean ordered = true;
|
|
|
+ for (String t : tplNames) {
|
|
|
+ int idx = fileName.indexOf(t);
|
|
|
+ if (idx < 0) {
|
|
|
+ ordered = false;
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ found++;
|
|
|
+ if (idx <= last) {
|
|
|
+ ordered = false;
|
|
|
+ }
|
|
|
+ last = idx;
|
|
|
+ }
|
|
|
+ if (found == 0) {
|
|
|
+ return 0d;
|
|
|
+ }
|
|
|
+ return ordered ? 1d : 0.4d;
|
|
|
+ }
|
|
|
+
|
|
|
+ static double shapeScore(List<String> headers,
|
|
|
+ List<TemplateCandidate.FieldBrief> fields, List<ColStat> colStats) {
|
|
|
+ if (fields == null || fields.isEmpty() || colStats == null || colStats.isEmpty()) {
|
|
|
+ return 0.5d;
|
|
|
+ }
|
|
|
+ Map<Integer, ColStat> statByIndex = new HashMap<>();
|
|
|
+ for (ColStat s : colStats) {
|
|
|
+ if (s != null) {
|
|
|
+ statByIndex.put(s.idx(), s);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ int known = 0;
|
|
|
+ int compatible = 0;
|
|
|
+ for (TemplateCandidate.FieldBrief f : fields) {
|
|
|
+ if (f == null) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ int pos = indexOfHeader(headers, f);
|
|
|
+ if (pos < 0) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ ColStat stat = statByIndex.get(pos);
|
|
|
+ String shape = stat == null ? "" : StrUtil.nullToEmpty(stat.shape()).toUpperCase(Locale.ROOT);
|
|
|
+ if (shape.isEmpty() || "TEXT".equals(shape) || "EMPTY".equals(shape)) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ known++;
|
|
|
+ if (compatible(f.type(), shape)) {
|
|
|
+ compatible++;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return known == 0 ? 0.5d : (double) compatible / known;
|
|
|
+ }
|
|
|
+
|
|
|
+ private static int indexOfHeader(List<String> headers, TemplateCandidate.FieldBrief field) {
|
|
|
+ String cn = canonicalForScore(field.cn());
|
|
|
+ String en = canonicalForScore(field.en());
|
|
|
+ for (int i = 0; i < headers.size(); i++) {
|
|
|
+ String n = canonicalForScore(headers.get(i));
|
|
|
+ if (!n.isEmpty() && ((!cn.isEmpty() && n.equals(cn)) || (!en.isEmpty() && n.equals(en)))) {
|
|
|
+ return i;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return -1;
|
|
|
+ }
|
|
|
+
|
|
|
+ private static double requiredCoverage(Set<String> fileSet, List<TemplateCandidate.FieldBrief> fields) {
|
|
|
+ if (fields == null || fields.isEmpty()) {
|
|
|
+ return 1d;
|
|
|
+ }
|
|
|
+ int required = 0;
|
|
|
+ int hit = 0;
|
|
|
+ for (TemplateCandidate.FieldBrief f : fields) {
|
|
|
+ if (f == null || !f.required()) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ required++;
|
|
|
+ String cn = canonicalForScore(f.cn());
|
|
|
+ String en = canonicalForScore(f.en());
|
|
|
+ if ((!cn.isEmpty() && fileSet.contains(cn)) || (!en.isEmpty() && fileSet.contains(en))) {
|
|
|
+ hit++;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return required == 0 ? 1d : (double) hit / required;
|
|
|
+ }
|
|
|
+
|
|
|
+ /** 形状与字段类型是否明显矛盾;不确定(TEXT/空)一律视为兼容 */
|
|
|
+ private static boolean compatible(String fieldType, String shape) {
|
|
|
+ String type = StrUtil.nullToEmpty(fieldType).trim().toUpperCase(Locale.ROOT);
|
|
|
+ if (type.isEmpty() || "TEXT".equals(type)) {
|
|
|
+ return true;
|
|
|
+ }
|
|
|
+ return switch (type) {
|
|
|
+ case "DATE", "DATE_TIME", "TIME" -> !Set.of("MONEY", "PHONE", "IDCARD", "ACCT").contains(shape);
|
|
|
+ case "MONEY", "NUMBER" -> !Set.of("DATE", "DATETIME", "TIME", "PHONE", "IDCARD", "ACCT").contains(shape);
|
|
|
+ // 电话/卡号/身份证可能被 Excel 存成数字,NUMBER 形状不判矛盾
|
|
|
+ case "PHONE", "ACCT_CARD_NO", "ID_CARD_NUMBER" ->
|
|
|
+ !Set.of("MONEY", "DATE", "DATETIME", "TIME").contains(shape);
|
|
|
+ default -> true;
|
|
|
+ };
|
|
|
}
|
|
|
}
|