|
|
@@ -13,19 +13,15 @@ import com.zsjz.ai.module.aiclean.entity.AiCleanTemplate;
|
|
|
import com.zsjz.ai.module.aiclean.mapper.AiCleanFieldMapper;
|
|
|
import com.zsjz.ai.module.aiclean.mapper.AiCleanTemplateMapper;
|
|
|
import com.zsjz.ai.module.agent.mapper.RagVectorMapper;
|
|
|
-import com.zsjz.ai.module.agent.rag.EmbeddingModelFactory;
|
|
|
-import com.zsjz.ai.module.agent.rag.EmbeddingModelFactory.EmbeddingSpec;
|
|
|
+import com.zsjz.ai.module.agent.rag.EmbeddingProvider;
|
|
|
import com.zsjz.ai.module.plat.mapper.TableFieldMapper;
|
|
|
import com.zsjz.ai.module.plat.mapper.TableInfoMapper;
|
|
|
-import io.agentscope.core.embedding.EmbeddingModel;
|
|
|
-import io.agentscope.core.message.TextBlock;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
|
|
import org.springframework.context.event.EventListener;
|
|
|
import org.springframework.stereotype.Service;
|
|
|
import org.springframework.util.StringUtils;
|
|
|
|
|
|
-import java.time.Duration;
|
|
|
import java.util.ArrayList;
|
|
|
import java.util.LinkedHashMap;
|
|
|
import java.util.List;
|
|
|
@@ -72,8 +68,8 @@ public class AiCleanTemplateVectorIndex {
|
|
|
/** AI 模板库文档类型 */
|
|
|
public static final String KIND_AI = "AI";
|
|
|
|
|
|
- /** 向量表名前缀;完整表名 = rag_tpl_d{dims}(按维度隔离,换模型后自然重建) */
|
|
|
- static final String VECTOR_TABLE_PREFIX = "rag_tpl_d";
|
|
|
+ /** 向量表名前缀;完整表名 = rag_tpl{tag}_d{dims}(按维度/模型隔离,换模型后自然重建) */
|
|
|
+ static final String VECTOR_TABLE_PREFIX = "rag_tpl";
|
|
|
|
|
|
/**
|
|
|
* 全量重建时「连续失败」熔断阈值。
|
|
|
@@ -89,22 +85,8 @@ public class AiCleanTemplateVectorIndex {
|
|
|
/** 文档 id 前缀:AI 模板 a:{ai_clean_template.id} */
|
|
|
private static final String DOC_ID_AI_PREFIX = "a:";
|
|
|
|
|
|
- /**
|
|
|
- * 嵌入调用超时:与 {@code RagSchemaService} 同口径,单条嵌入最多等 60s。
|
|
|
- * 超时按「这条文档没写成功」处理,绝不让异步线程或请求线程无限挂住。
|
|
|
- */
|
|
|
- private static final Duration EMBED_TIMEOUT = Duration.ofSeconds(60);
|
|
|
-
|
|
|
- /**
|
|
|
- * EmbeddingSpec 解析缓存时长。
|
|
|
- *
|
|
|
- * <p>{@code resolveDefault()} 每次都要查 {@code agent_model} 表并构建 provider 实例;召回发生在每次 AI 识别上,
|
|
|
- * 不缓存就是白查一次库。30s 足够短:换模型 / 停用模型最多半分钟后生效,不需要重启。
|
|
|
- */
|
|
|
- private static final long SPEC_CACHE_MILLIS = 30_000L;
|
|
|
-
|
|
|
private final AiCleanVectorProperties properties;
|
|
|
- private final EmbeddingModelFactory embeddingModelFactory;
|
|
|
+ private final EmbeddingProvider embeddingProvider;
|
|
|
private final RagVectorMapper ragVectorMapper;
|
|
|
private final TableInfoMapper tableInfoMapper;
|
|
|
private final TableFieldMapper tableFieldMapper;
|
|
|
@@ -131,18 +113,15 @@ public class AiCleanTemplateVectorIndex {
|
|
|
*/
|
|
|
private final Set<String> ensuredVectorTables = ConcurrentHashMap.newKeySet();
|
|
|
|
|
|
- /** 嵌入模型规格缓存(含 null 不缓存:没配模型时每次重查,配置模型后无需等缓存过期) */
|
|
|
- private volatile CachedSpec cachedSpec;
|
|
|
-
|
|
|
public AiCleanTemplateVectorIndex(AiCleanVectorProperties properties,
|
|
|
- EmbeddingModelFactory embeddingModelFactory,
|
|
|
+ EmbeddingProvider embeddingProvider,
|
|
|
RagVectorMapper ragVectorMapper,
|
|
|
TableInfoMapper tableInfoMapper,
|
|
|
TableFieldMapper tableFieldMapper,
|
|
|
AiCleanTemplateMapper aiCleanTemplateMapper,
|
|
|
AiCleanFieldMapper aiCleanFieldMapper) {
|
|
|
this.properties = properties;
|
|
|
- this.embeddingModelFactory = embeddingModelFactory;
|
|
|
+ this.embeddingProvider = embeddingProvider;
|
|
|
this.ragVectorMapper = ragVectorMapper;
|
|
|
this.tableInfoMapper = tableInfoMapper;
|
|
|
this.tableFieldMapper = tableFieldMapper;
|
|
|
@@ -151,22 +130,29 @@ public class AiCleanTemplateVectorIndex {
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
- * 向量索引是否可用:开关打开且配置了 type='embedding' 的活跃模型。
|
|
|
+ * 向量索引是否可用:开关打开且当前向量模型(本地 / 在线由 zsjz.embedding.enabled 决定)可用。
|
|
|
*/
|
|
|
public boolean isAvailable() {
|
|
|
if (!properties.isEnabled()) {
|
|
|
return false;
|
|
|
}
|
|
|
try {
|
|
|
- return resolveSpec() != null;
|
|
|
+ return embeddingProvider.isAvailable();
|
|
|
} catch (Exception e) {
|
|
|
- // 解析模型这一步理论上已 fail-open(EmbeddingModelFactory 内部吞异常返回 null),
|
|
|
+ // 路由层理论上已 fail-open(在线解析内部吞异常返回 null),
|
|
|
// 这里再兜一层是因为 mock / 未来实现都可能抛,召回链路上不允许它冒泡
|
|
|
log.error("判断 AI 模板向量索引可用性失败(按不可用处理): {}", e.getMessage());
|
|
|
return false;
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ /**
|
|
|
+ * 当前向量模型不可用的原因(供接口层给出可操作提示)。
|
|
|
+ */
|
|
|
+ public String unavailableMessage() {
|
|
|
+ return embeddingProvider.unavailableMessage();
|
|
|
+ }
|
|
|
+
|
|
|
/**
|
|
|
* 增量重建一份既有模板的向量(异步,模板新增/更新后调用)。
|
|
|
*/
|
|
|
@@ -238,14 +224,15 @@ public class AiCleanTemplateVectorIndex {
|
|
|
}
|
|
|
try {
|
|
|
if (!isAvailable()) {
|
|
|
- log.info("启动自举跳过模板向量索引:未启用或未配置 type='embedding' 的活跃模型");
|
|
|
+ log.info("启动自举跳过模板向量索引:{}", embeddingProvider.unavailableMessage());
|
|
|
return;
|
|
|
}
|
|
|
- EmbeddingSpec spec = resolveSpec();
|
|
|
- if (spec == null) {
|
|
|
+ int dimensions = embeddingProvider.dimensions();
|
|
|
+ if (dimensions <= 0) {
|
|
|
+ log.info("启动自举跳过模板向量索引:嵌入模型维度无效");
|
|
|
return;
|
|
|
}
|
|
|
- String vectorTable = vectorTableName(spec.dimensions());
|
|
|
+ String vectorTable = vectorTableName(embeddingProvider.tableTag(), dimensions);
|
|
|
boolean empty = !vectorTableExists(vectorTable)
|
|
|
|| ragVectorMapper.countVectorDocs(vectorTable) == 0;
|
|
|
if (!empty) {
|
|
|
@@ -265,19 +252,22 @@ public class AiCleanTemplateVectorIndex {
|
|
|
* <p>先构建完整文档清单再动表:读库失败时旧的向量表还在,索引不会因为一次失败被清空。
|
|
|
*
|
|
|
* @return 实际写入的文档数(嵌入失败 / 维度不符的文档被跳过,不计入)
|
|
|
- * @throws ServerException 开关关闭、未配置嵌入模型或建表 / 写库失败时
|
|
|
+ * @throws ServerException 开关关闭、向量模型不可用或建表 / 写库失败时
|
|
|
*/
|
|
|
public int rebuildNow() {
|
|
|
if (!properties.isEnabled()) {
|
|
|
throw new ServerException(400, "AI 模板向量索引未启用(zsjz.ai-clean.vector.enabled=false)");
|
|
|
}
|
|
|
- EmbeddingSpec spec = resolveSpec();
|
|
|
- if (spec == null) {
|
|
|
- throw new ServerException(400, "未配置嵌入模型(agent_model 表需存在 type='embedding' 的活跃记录)");
|
|
|
+ if (!embeddingProvider.isAvailable()) {
|
|
|
+ throw new ServerException(400, embeddingProvider.unavailableMessage());
|
|
|
+ }
|
|
|
+ int dimensions = embeddingProvider.dimensions();
|
|
|
+ if (dimensions <= 0) {
|
|
|
+ throw new ServerException(500, "嵌入模型维度无效: " + dimensions);
|
|
|
}
|
|
|
synchronized (rebuildLock) {
|
|
|
try {
|
|
|
- String vectorTable = vectorTableName(spec.dimensions());
|
|
|
+ String vectorTable = vectorTableName(embeddingProvider.tableTag(), dimensions);
|
|
|
List<VectorDoc> docs = loadAllDocuments();
|
|
|
int total = docs.size();
|
|
|
int max = properties.getMaxDocumentsPerRebuild();
|
|
|
@@ -287,25 +277,25 @@ public class AiCleanTemplateVectorIndex {
|
|
|
docs = docs.subList(0, max);
|
|
|
}
|
|
|
ragVectorMapper.dropVectorTable(vectorTable);
|
|
|
- ragVectorMapper.createVectorTableIfNotExists(vectorTable, spec.dimensions());
|
|
|
+ ragVectorMapper.createVectorTableIfNotExists(vectorTable, dimensions);
|
|
|
ensuredVectorTables.add(vectorTable);
|
|
|
int written = 0;
|
|
|
// 连续失败熔断:embedding 连续失败到阈值就中止本次重建(返回已写入数),
|
|
|
- // 通常是模型 / Key 不可用,继续逐条重试只会刷屏且拖长重建时间
|
|
|
+ // 通常是模型不可用,继续逐条重试只会刷屏且拖长重建时间
|
|
|
int consecutiveFailures = 0;
|
|
|
for (VectorDoc doc : docs) {
|
|
|
- if (writeDoc(vectorTable, spec, doc)) {
|
|
|
+ if (writeDoc(vectorTable, dimensions, doc)) {
|
|
|
written++;
|
|
|
consecutiveFailures = 0;
|
|
|
} else if (++consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) {
|
|
|
log.error("AI 模板向量全量重建连续 {} 次写入失败,已中止(已写入 {} 条)。"
|
|
|
- + "请检查 agent_model 中 type='embedding' 模型的 Key / 端点,或 pgvector 是否可用",
|
|
|
+ + "请检查嵌入模型(本地模型文件 / 在线模型 Key 与端点)或 pgvector 是否可用",
|
|
|
consecutiveFailures, written);
|
|
|
break;
|
|
|
}
|
|
|
}
|
|
|
log.info("AI 模板向量全量重建完成: table={}, dims={}, docs={}/{}(截断前 {})",
|
|
|
- vectorTable, spec.dimensions(), written, docs.size(), total);
|
|
|
+ vectorTable, dimensions, written, docs.size(), total);
|
|
|
return written;
|
|
|
} catch (ServerException e) {
|
|
|
throw e;
|
|
|
@@ -329,20 +319,23 @@ public class AiCleanTemplateVectorIndex {
|
|
|
return List.of();
|
|
|
}
|
|
|
try {
|
|
|
- EmbeddingSpec spec = resolveSpec();
|
|
|
- if (spec == null) {
|
|
|
+ if (!embeddingProvider.isAvailable()) {
|
|
|
return List.of();
|
|
|
}
|
|
|
int limit = topK > 0 ? topK : properties.getTopK();
|
|
|
if (limit <= 0) {
|
|
|
return List.of();
|
|
|
}
|
|
|
- String vectorTable = vectorTableName(spec.dimensions());
|
|
|
+ int dimensions = embeddingProvider.dimensions();
|
|
|
+ if (dimensions <= 0) {
|
|
|
+ return List.of();
|
|
|
+ }
|
|
|
+ String vectorTable = vectorTableName(embeddingProvider.tableTag(), dimensions);
|
|
|
if (!vectorTableExists(vectorTable)) {
|
|
|
// 还没索引过(刚部署 / 从没 rebuild):直接按无召回处理,别让调用方等一次空查询异常
|
|
|
return List.of();
|
|
|
}
|
|
|
- double[] queryVector = embed(spec, queryText);
|
|
|
+ double[] queryVector = embed(queryText, dimensions);
|
|
|
if (queryVector == null) {
|
|
|
return List.of();
|
|
|
}
|
|
|
@@ -389,20 +382,23 @@ public class AiCleanTemplateVectorIndex {
|
|
|
return List.of();
|
|
|
}
|
|
|
try {
|
|
|
- EmbeddingSpec spec = resolveSpec();
|
|
|
- if (spec == null) {
|
|
|
+ if (!embeddingProvider.isAvailable()) {
|
|
|
return List.of();
|
|
|
}
|
|
|
int limit = topK > 0 ? topK : properties.getTopK();
|
|
|
if (limit <= 0) {
|
|
|
return List.of();
|
|
|
}
|
|
|
- String vectorTable = vectorTableName(spec.dimensions());
|
|
|
+ int dimensions = embeddingProvider.dimensions();
|
|
|
+ if (dimensions <= 0) {
|
|
|
+ return List.of();
|
|
|
+ }
|
|
|
+ String vectorTable = vectorTableName(embeddingProvider.tableTag(), dimensions);
|
|
|
if (!vectorTableExists(vectorTable)) {
|
|
|
// 还没建过索引(新部署 / 从没 rebuild):按无结果处理,别让调用方等一次空查询异常
|
|
|
return List.of();
|
|
|
}
|
|
|
- double[] queryVector = embed(spec, queryText);
|
|
|
+ double[] queryVector = embed(queryText, dimensions);
|
|
|
if (queryVector == null) {
|
|
|
return List.of();
|
|
|
}
|
|
|
@@ -435,11 +431,14 @@ public class AiCleanTemplateVectorIndex {
|
|
|
*/
|
|
|
public IndexStatus status() {
|
|
|
try {
|
|
|
- EmbeddingSpec spec = resolveSpec();
|
|
|
- if (spec == null) {
|
|
|
+ if (!embeddingProvider.isAvailable()) {
|
|
|
return new IndexStatus(false, null, null, 0, 0L);
|
|
|
}
|
|
|
- String vectorTable = vectorTableName(spec.dimensions());
|
|
|
+ int dimensions = embeddingProvider.dimensions();
|
|
|
+ if (dimensions <= 0) {
|
|
|
+ return new IndexStatus(false, null, embeddingProvider.modelName(), 0, 0L);
|
|
|
+ }
|
|
|
+ String vectorTable = vectorTableName(embeddingProvider.tableTag(), dimensions);
|
|
|
long docCount = 0L;
|
|
|
try {
|
|
|
if (vectorTableExists(vectorTable)) {
|
|
|
@@ -448,8 +447,8 @@ public class AiCleanTemplateVectorIndex {
|
|
|
} catch (Exception e) {
|
|
|
log.debug("读取模板向量表行数失败(状态按 0 处理): {}", e.getMessage());
|
|
|
}
|
|
|
- return new IndexStatus(properties.isEnabled(), vectorTable, spec.modelName(),
|
|
|
- spec.dimensions(), docCount);
|
|
|
+ return new IndexStatus(properties.isEnabled(), vectorTable, embeddingProvider.modelName(),
|
|
|
+ dimensions, docCount);
|
|
|
} catch (Exception e) {
|
|
|
log.error("读取 AI 模板向量索引状态失败: {}", e.getMessage());
|
|
|
return new IndexStatus(false, null, null, 0, 0L);
|
|
|
@@ -502,24 +501,28 @@ public class AiCleanTemplateVectorIndex {
|
|
|
*/
|
|
|
int indexTableNow(Integer tableId) {
|
|
|
try {
|
|
|
- EmbeddingSpec spec = resolveSpec();
|
|
|
- if (spec == null) {
|
|
|
- log.debug("未配置嵌入模型,跳过既有模板向量索引: tableId={}", tableId);
|
|
|
+ if (!embeddingProvider.isAvailable()) {
|
|
|
+ log.debug("嵌入模型不可用,跳过既有模板向量索引: tableId={}", tableId);
|
|
|
return 0;
|
|
|
}
|
|
|
+ int dimensions = embeddingProvider.dimensions();
|
|
|
+ if (dimensions <= 0) {
|
|
|
+ return 0;
|
|
|
+ }
|
|
|
+ String docId = DOC_ID_TABLE_PREFIX + tableId;
|
|
|
+ String vectorTable = vectorTableName(embeddingProvider.tableTag(), dimensions);
|
|
|
TableInfo table = tableInfoMapper.selectById(tableId);
|
|
|
if (table == null) {
|
|
|
- removeDoc(spec, DOC_ID_TABLE_PREFIX + tableId);
|
|
|
+ removeDoc(vectorTable, docId);
|
|
|
return 0;
|
|
|
}
|
|
|
List<TableField> fields = loadTableFields(table);
|
|
|
if (fields.isEmpty()) {
|
|
|
- removeDoc(spec, DOC_ID_TABLE_PREFIX + tableId);
|
|
|
+ removeDoc(vectorTable, docId);
|
|
|
return 0;
|
|
|
}
|
|
|
- String vectorTable = vectorTableName(spec.dimensions());
|
|
|
- ensureVectorTable(vectorTable, spec.dimensions());
|
|
|
- return writeDoc(vectorTable, spec, buildTableDoc(table, fields)) ? 1 : 0;
|
|
|
+ ensureVectorTable(vectorTable, dimensions);
|
|
|
+ return writeDoc(vectorTable, dimensions, buildTableDoc(table, fields)) ? 1 : 0;
|
|
|
} catch (Exception e) {
|
|
|
log.error("增量索引既有模板失败(已降级,不影响清洗主流程): tableId={}, error={}",
|
|
|
tableId, e.getMessage());
|
|
|
@@ -537,14 +540,19 @@ public class AiCleanTemplateVectorIndex {
|
|
|
*/
|
|
|
int indexAiTemplateNow(Long aiTemplateId) {
|
|
|
try {
|
|
|
- EmbeddingSpec spec = resolveSpec();
|
|
|
- if (spec == null) {
|
|
|
- log.debug("未配置嵌入模型,跳过 AI 模板向量索引: aiTemplateId={}", aiTemplateId);
|
|
|
+ if (!embeddingProvider.isAvailable()) {
|
|
|
+ log.debug("嵌入模型不可用,跳过 AI 模板向量索引: aiTemplateId={}", aiTemplateId);
|
|
|
return 0;
|
|
|
}
|
|
|
+ int dimensions = embeddingProvider.dimensions();
|
|
|
+ if (dimensions <= 0) {
|
|
|
+ return 0;
|
|
|
+ }
|
|
|
+ String docId = DOC_ID_AI_PREFIX + aiTemplateId;
|
|
|
+ String vectorTable = vectorTableName(embeddingProvider.tableTag(), dimensions);
|
|
|
AiCleanTemplate template = aiCleanTemplateMapper.selectById(aiTemplateId);
|
|
|
if (template == null) {
|
|
|
- removeDoc(spec, DOC_ID_AI_PREFIX + aiTemplateId);
|
|
|
+ removeDoc(vectorTable, docId);
|
|
|
return 0;
|
|
|
}
|
|
|
List<AiCleanField> fields = aiCleanFieldMapper.selectList(
|
|
|
@@ -554,12 +562,11 @@ public class AiCleanTemplateVectorIndex {
|
|
|
.orderByAsc(AiCleanField::getId));
|
|
|
if (fields == null || fields.isEmpty()) {
|
|
|
// 无字段的模板既不能匹配也不能被选表,按删除处理(与全量重建口径一致)
|
|
|
- removeDoc(spec, DOC_ID_AI_PREFIX + aiTemplateId);
|
|
|
+ removeDoc(vectorTable, docId);
|
|
|
return 0;
|
|
|
}
|
|
|
- String vectorTable = vectorTableName(spec.dimensions());
|
|
|
- ensureVectorTable(vectorTable, spec.dimensions());
|
|
|
- return writeDoc(vectorTable, spec, buildAiDoc(template, fields)) ? 1 : 0;
|
|
|
+ ensureVectorTable(vectorTable, dimensions);
|
|
|
+ return writeDoc(vectorTable, dimensions, buildAiDoc(template, fields)) ? 1 : 0;
|
|
|
} catch (Exception e) {
|
|
|
log.error("增量索引 AI 模板失败(已降级,不影响清洗主流程): aiTemplateId={}, error={}",
|
|
|
aiTemplateId, e.getMessage());
|
|
|
@@ -572,18 +579,20 @@ public class AiCleanTemplateVectorIndex {
|
|
|
*/
|
|
|
private void removeDoc(String docId) {
|
|
|
try {
|
|
|
- EmbeddingSpec spec = resolveSpec();
|
|
|
- if (spec == null) {
|
|
|
+ if (!embeddingProvider.isAvailable()) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ int dimensions = embeddingProvider.dimensions();
|
|
|
+ if (dimensions <= 0) {
|
|
|
return;
|
|
|
}
|
|
|
- removeDoc(spec, docId);
|
|
|
+ removeDoc(vectorTableName(embeddingProvider.tableTag(), dimensions), docId);
|
|
|
} catch (Exception e) {
|
|
|
log.error("删除模板向量失败(已降级): docId={}, error={}", docId, e.getMessage());
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- private void removeDoc(EmbeddingSpec spec, String docId) {
|
|
|
- String vectorTable = vectorTableName(spec.dimensions());
|
|
|
+ private void removeDoc(String vectorTable, String docId) {
|
|
|
if (!vectorTableExists(vectorTable)) {
|
|
|
return;
|
|
|
}
|
|
|
@@ -961,53 +970,32 @@ public class AiCleanTemplateVectorIndex {
|
|
|
// ==================== 嵌入与向量存取 ====================
|
|
|
|
|
|
/**
|
|
|
- * 解析默认嵌入模型(带 30s 缓存)。
|
|
|
- *
|
|
|
- * @return 规格;未配置 / 解析失败返回 null(不抛)
|
|
|
+ * 嵌入单段文本并校验维度;维度不符 / 调用失败都返回 null(由调用方按「跳过该文档」处理)。
|
|
|
*/
|
|
|
- private EmbeddingSpec resolveSpec() {
|
|
|
- CachedSpec cached = cachedSpec;
|
|
|
- long now = System.currentTimeMillis();
|
|
|
- if (cached != null && cached.expireAtMillis() > now) {
|
|
|
- return cached.spec();
|
|
|
- }
|
|
|
- EmbeddingSpec spec;
|
|
|
+ private double[] embed(String content, int dimensions) {
|
|
|
try {
|
|
|
- spec = embeddingModelFactory.resolveDefault();
|
|
|
+ double[] vector = embeddingProvider.embed(content);
|
|
|
+ if (vector == null || vector.length != dimensions) {
|
|
|
+ // 维度不符说明模型配置与实际输出对不上:这一行写进去会让整表向量检索崩掉
|
|
|
+ // (pgvector 的 vector(n) 列强校验维度),所以宁可跳过并告警
|
|
|
+ log.info("嵌入向量维度不一致,跳过该模板文档(期望 {},实际 {}): model={}",
|
|
|
+ dimensions, vector == null ? 0 : vector.length, embeddingProvider.modelName());
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+ return vector;
|
|
|
} catch (Exception e) {
|
|
|
- log.error("解析默认嵌入模型失败(按不可用处理): {}", e.getMessage());
|
|
|
- spec = null;
|
|
|
- }
|
|
|
- cachedSpec = spec == null ? null : new CachedSpec(spec, now + SPEC_CACHE_MILLIS);
|
|
|
- return spec;
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 嵌入单段文本并校验维度;维度不符 / 调用失败都返回 null(由调用方按「跳过该文档」处理)。
|
|
|
- */
|
|
|
- private double[] embed(EmbeddingSpec spec, String content) {
|
|
|
- EmbeddingModel model = spec.model();
|
|
|
- if (model == null) {
|
|
|
- log.info("嵌入模型实例为空,跳过本次模板向量计算: model={}", spec.modelName());
|
|
|
- return null;
|
|
|
- }
|
|
|
- double[] vector = model.embed(TextBlock.builder().text(content).build()).block(EMBED_TIMEOUT);
|
|
|
- if (vector == null || vector.length != spec.dimensions()) {
|
|
|
- // 维度不符说明 model.config.dimensions 与实际模型对不上:这一行写进去会让整表向量检索崩掉
|
|
|
- // (pgvector 的 vector(n) 列强校验维度),所以宁可跳过并告警
|
|
|
- log.info("嵌入向量维度不一致,跳过该模板文档(期望 {},实际 {}): model={}",
|
|
|
- spec.dimensions(), vector == null ? 0 : vector.length, spec.modelName());
|
|
|
+ log.error("向量计算失败(跳过该模板文档): model={}, error={}",
|
|
|
+ embeddingProvider.modelName(), e.getMessage());
|
|
|
return null;
|
|
|
}
|
|
|
- return vector;
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* 写入一篇文档;失败只告警不抛(单条失败不该拖垮整批增量/重建)。
|
|
|
*/
|
|
|
- private boolean writeDoc(String vectorTable, EmbeddingSpec spec, VectorDoc doc) {
|
|
|
+ private boolean writeDoc(String vectorTable, int dimensions, VectorDoc doc) {
|
|
|
try {
|
|
|
- double[] vector = embed(spec, doc.content());
|
|
|
+ double[] vector = embed(doc.content(), dimensions);
|
|
|
if (vector == null) {
|
|
|
return false;
|
|
|
}
|
|
|
@@ -1044,13 +1032,15 @@ public class AiCleanTemplateVectorIndex {
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
- * 向量表名:rag_tpl_d{dims}。
|
|
|
+ * 向量表名:rag_tpl{tag}_d{dims}。
|
|
|
*
|
|
|
- * <p>维度进表名:换嵌入模型且维度变化时自然落到新表,不会出现「旧表 vector(1024) 塞 1536 维」的写失败;
|
|
|
+ * <p>维度进表名:换嵌入模型且维度变化时自然落到新表,不会出现「旧表 vector(1024) 塞 1536 维」的写失败。
|
|
|
+ * 本地模型额外带模型名标签(如 {@code rag_tpl_bge_m3_d1024}),与在线模型的历史向量物理隔离 ——
|
|
|
+ * 本地/在线开关切换后不会把另一套向量空间的结果当成当前索引继续检索。
|
|
|
* 旧表按设计留待人工清理(见设计文档 §7)。
|
|
|
*/
|
|
|
- static String vectorTableName(int dimensions) {
|
|
|
- return VECTOR_TABLE_PREFIX + dimensions;
|
|
|
+ static String vectorTableName(String tag, int dimensions) {
|
|
|
+ return VECTOR_TABLE_PREFIX + (tag == null ? "" : tag) + "_d" + dimensions;
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
@@ -1088,12 +1078,6 @@ public class AiCleanTemplateVectorIndex {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- /**
|
|
|
- * 嵌入模型规格缓存项。
|
|
|
- */
|
|
|
- private record CachedSpec(EmbeddingSpec spec, long expireAtMillis) {
|
|
|
- }
|
|
|
-
|
|
|
/**
|
|
|
* 一条召回命中。
|
|
|
*
|