|
|
@@ -0,0 +1,365 @@
|
|
|
+package com.zsjz.ai.module.document;
|
|
|
+
|
|
|
+import cn.hutool.core.util.StrUtil;
|
|
|
+import cn.hutool.json.JSONObject;
|
|
|
+import cn.hutool.json.JSONUtil;
|
|
|
+import jakarta.annotation.PreDestroy;
|
|
|
+import lombok.RequiredArgsConstructor;
|
|
|
+import lombok.extern.slf4j.Slf4j;
|
|
|
+import org.springframework.stereotype.Component;
|
|
|
+import org.vosk.LibVosk;
|
|
|
+import org.vosk.LogLevel;
|
|
|
+import org.vosk.Model;
|
|
|
+import org.vosk.Recognizer;
|
|
|
+
|
|
|
+import java.io.BufferedReader;
|
|
|
+import java.io.IOException;
|
|
|
+import java.io.InputStream;
|
|
|
+import java.io.InputStreamReader;
|
|
|
+import java.nio.charset.StandardCharsets;
|
|
|
+import java.nio.file.Files;
|
|
|
+import java.nio.file.Path;
|
|
|
+import java.nio.file.Paths;
|
|
|
+import java.util.ArrayDeque;
|
|
|
+import java.util.ArrayList;
|
|
|
+import java.util.Deque;
|
|
|
+import java.util.List;
|
|
|
+import java.util.Locale;
|
|
|
+import java.util.concurrent.TimeUnit;
|
|
|
+import java.util.regex.Matcher;
|
|
|
+import java.util.regex.Pattern;
|
|
|
+
|
|
|
+/**
|
|
|
+ * 音视频转写:外部 ffmpeg 解码成 16k 单声道 PCM,流式喂 Vosk。
|
|
|
+ *
|
|
|
+ * <h3>为什么自己调 ffmpeg</h3>
|
|
|
+ * Tika 3.3.1 已经没有 ffmpeg 解析器,也没有任何本地语音识别实现(其转写桥只有云端 Amazon Transcribe);
|
|
|
+ * 它对音频只给容器/ID3 元数据。要拿到可用文本只能自己解码 + 本地识别。
|
|
|
+ *
|
|
|
+ * <h3>实现要点</h3>
|
|
|
+ * <ul>
|
|
|
+ * <li>PCM 走 stdout 管道,不落临时文件;stderr 单独线程排空(不排空会让 ffmpeg 写满缓冲卡死);</li>
|
|
|
+ * <li>时长上限用 {@code -t} 交给 ffmpeg 截断,避免长视频把上传请求挂住;</li>
|
|
|
+ * <li>Vosk {@link Model} 单实例懒加载(加载慢、占内存),{@link Recognizer} 每文件一个;</li>
|
|
|
+ * <li>进程超时 {@code destroyForcibly},坏文件不会一直占着线程。</li>
|
|
|
+ * </ul>
|
|
|
+ */
|
|
|
+@Slf4j
|
|
|
+@Component
|
|
|
+@RequiredArgsConstructor
|
|
|
+public class MediaTranscriber {
|
|
|
+
|
|
|
+ /** ffmpeg 输入信息里的时长行,如 "Duration: 00:03:21.12" */
|
|
|
+ private static final Pattern DURATION = Pattern.compile("Duration:\\s*(\\d+):(\\d+):(\\d+(?:\\.\\d+)?)");
|
|
|
+
|
|
|
+ private static final int STDERR_KEEP_LINES = 20;
|
|
|
+
|
|
|
+ private final AsrProperties props;
|
|
|
+
|
|
|
+ private volatile Path ffmpeg;
|
|
|
+
|
|
|
+ private volatile Model model;
|
|
|
+
|
|
|
+ private volatile String unavailableReason;
|
|
|
+
|
|
|
+ private volatile boolean probed;
|
|
|
+
|
|
|
+ /**
|
|
|
+ * @param text 转写文本(每个 final 段一行)
|
|
|
+ * @param durationSeconds 媒体时长(秒);解析不到为 0
|
|
|
+ * @param truncated 是否因时长/字符上限截断
|
|
|
+ */
|
|
|
+ public record Transcript(String text, int durationSeconds, boolean truncated) {
|
|
|
+ }
|
|
|
+
|
|
|
+ public boolean isAvailable() {
|
|
|
+ probe();
|
|
|
+ return ffmpeg != null && model != null;
|
|
|
+ }
|
|
|
+
|
|
|
+ public String unavailableReason() {
|
|
|
+ probe();
|
|
|
+ return unavailableReason;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 转写一个音视频文件。
|
|
|
+ *
|
|
|
+ * @throws IllegalStateException 未启用 / 未就绪 / 无音频轨道 / 解码失败(消息可直接展示给用户)
|
|
|
+ */
|
|
|
+ public Transcript transcribe(Path file) {
|
|
|
+ if (!props.isEnabled()) {
|
|
|
+ throw new IllegalStateException("音视频转写未启用(zsjz.asr.enabled=false)");
|
|
|
+ }
|
|
|
+ if (!isAvailable()) {
|
|
|
+ throw new IllegalStateException("本地语音转写未就绪:" + unavailableReason
|
|
|
+ + "(见 docs/design/ai-media-ocr-asr.md)");
|
|
|
+ }
|
|
|
+ int maxDuration = Math.max(1, props.getMaxDurationSeconds());
|
|
|
+ List<String> command = new ArrayList<>(List.of(
|
|
|
+ ffmpeg.toString(), "-hide_banner", "-nostdin",
|
|
|
+ "-i", file.toAbsolutePath().toString(),
|
|
|
+ "-vn", "-sn", "-dn",
|
|
|
+ "-ac", "1", "-ar", String.valueOf(props.getSampleRate()),
|
|
|
+ "-acodec", "pcm_s16le", "-f", "s16le",
|
|
|
+ "-t", String.valueOf(maxDuration), "pipe:1"));
|
|
|
+
|
|
|
+ StringBuilder text = new StringBuilder();
|
|
|
+ boolean truncated = false;
|
|
|
+ Process process = null;
|
|
|
+ StderrSink stderr = null;
|
|
|
+ try {
|
|
|
+ process = new ProcessBuilder(command).redirectErrorStream(false).start();
|
|
|
+ stderr = new StderrSink(process.getErrorStream());
|
|
|
+ Thread stderrThread = Thread.ofVirtual().name("asr-ffmpeg-stderr").start(stderr);
|
|
|
+
|
|
|
+ try (Recognizer recognizer = new Recognizer(model, props.getSampleRate());
|
|
|
+ InputStream pcm = process.getInputStream()) {
|
|
|
+ recognizer.setWords(false);
|
|
|
+ byte[] buffer = new byte[8192];
|
|
|
+ long deadline = System.currentTimeMillis()
|
|
|
+ + Math.max(30, props.getFfmpegTimeoutSeconds()) * 1000L;
|
|
|
+ int read;
|
|
|
+ while ((read = pcm.read(buffer)) > 0) {
|
|
|
+ if (System.currentTimeMillis() > deadline) {
|
|
|
+ truncated = true;
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ if (recognizer.acceptWaveForm(buffer, read)) {
|
|
|
+ appendSegment(text, recognizer.getResult());
|
|
|
+ if (text.length() >= props.getMaxChars()) {
|
|
|
+ truncated = true;
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ appendSegment(text, recognizer.getFinalResult());
|
|
|
+ }
|
|
|
+
|
|
|
+ if (!process.waitFor(10, TimeUnit.SECONDS)) {
|
|
|
+ process.destroyForcibly();
|
|
|
+ // destroyForcibly 是异步的:不等它落地就 exitValue() 会抛 IllegalThreadStateException
|
|
|
+ process.waitFor(5, TimeUnit.SECONDS);
|
|
|
+ }
|
|
|
+ stderrThread.join(3000);
|
|
|
+
|
|
|
+ int exitCode = process.isAlive() ? -1 : process.exitValue();
|
|
|
+ String result = text.toString().strip();
|
|
|
+ int duration = stderr.durationSeconds();
|
|
|
+ if (duration > maxDuration) {
|
|
|
+ truncated = true;
|
|
|
+ }
|
|
|
+ if (result.isEmpty()) {
|
|
|
+ // 视频没有音轨、编码不支持等都在这里:把 ffmpeg 的错误尾巴交给用户可比「空文本」有用得多
|
|
|
+ String detail = stderr.tail();
|
|
|
+ if (exitCode != 0 || !detail.isBlank()) {
|
|
|
+ throw new IllegalStateException("音视频未识别出语音内容"
|
|
|
+ + (detail.isBlank() ? "" : "(ffmpeg: " + StrUtil.maxLength(detail, 300) + ")"));
|
|
|
+ }
|
|
|
+ throw new IllegalStateException("音视频未识别出语音内容");
|
|
|
+ }
|
|
|
+ if (result.length() > props.getMaxChars()) {
|
|
|
+ result = result.substring(0, props.getMaxChars());
|
|
|
+ truncated = true;
|
|
|
+ }
|
|
|
+ return new Transcript(result, Math.max(duration, 0), truncated);
|
|
|
+ } catch (IOException e) {
|
|
|
+ throw new IllegalStateException("ffmpeg 启动失败: " + e.getMessage(), e);
|
|
|
+ } catch (InterruptedException e) {
|
|
|
+ Thread.currentThread().interrupt();
|
|
|
+ throw new IllegalStateException("音视频转写被中断");
|
|
|
+ } finally {
|
|
|
+ if (process != null && process.isAlive()) {
|
|
|
+ process.destroyForcibly();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ @PreDestroy
|
|
|
+ void shutdown() {
|
|
|
+ Model current = model;
|
|
|
+ model = null;
|
|
|
+ if (current != null) {
|
|
|
+ try {
|
|
|
+ current.close();
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.debug("关闭 Vosk 模型失败(忽略): {}", e.getMessage());
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // ==================== 初始化 ====================
|
|
|
+
|
|
|
+ private synchronized void probe() {
|
|
|
+ if (probed) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ probed = true;
|
|
|
+ try {
|
|
|
+ // Vosk 的 JNA 绑定把 native 返回的 UTF-8 JSON 转成 Java String;JNA 默认用
|
|
|
+ // sun.jnu.encoding(中文 Windows 是 GBK)解码,中文会变成「杞﹁締」这类乱码。
|
|
|
+ // jna.encoding 是每次转换现读的系统属性,这里钉成 UTF-8(不覆盖运维已有的设置)。
|
|
|
+ if (System.getProperty("jna.encoding") == null) {
|
|
|
+ System.setProperty("jna.encoding", "UTF-8");
|
|
|
+ }
|
|
|
+ ffmpeg = resolveFfmpeg();
|
|
|
+ verifyFfmpeg(ffmpeg);
|
|
|
+ Path modelDir = modelDir();
|
|
|
+ LibVosk.setLogLevel(LogLevel.WARNINGS);
|
|
|
+ model = new Model(modelDir.toString());
|
|
|
+ log.info("[ASR] Vosk 就绪: model={}, ffmpeg={}", modelDir, ffmpeg);
|
|
|
+ } catch (Throwable e) {
|
|
|
+ // UnsatisfiedLinkError(原生库与平台不匹配)同样要转成可读原因
|
|
|
+ unavailableReason = e.getClass().getSimpleName()
|
|
|
+ + (StrUtil.isBlank(e.getMessage()) ? "" : ": " + e.getMessage());
|
|
|
+ log.warn("[ASR] 本地语音转写不可用: {}", unavailableReason);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private Path resolveFfmpeg() throws IOException {
|
|
|
+ if (StrUtil.isNotBlank(props.getFfmpegPath())) {
|
|
|
+ Path configured = resolve(props.getFfmpegPath());
|
|
|
+ if (!Files.isRegularFile(configured)) {
|
|
|
+ throw new IOException("zsjz.asr.ffmpeg-path 指向的文件不存在: " + configured);
|
|
|
+ }
|
|
|
+ return configured;
|
|
|
+ }
|
|
|
+ String exe = System.getProperty("os.name", "").toLowerCase(Locale.ROOT).contains("win")
|
|
|
+ ? "ffmpeg.exe" : "ffmpeg";
|
|
|
+ Path bundled = Paths.get(System.getProperty("user.dir"), "tools", "ffmpeg", exe)
|
|
|
+ .toAbsolutePath().normalize();
|
|
|
+ if (Files.isRegularFile(bundled)) {
|
|
|
+ return bundled;
|
|
|
+ }
|
|
|
+ Path onPath = findOnPath(exe);
|
|
|
+ if (onPath != null) {
|
|
|
+ return onPath;
|
|
|
+ }
|
|
|
+ throw new IOException("未找到 ffmpeg(已检查 zsjz.asr.ffmpeg-path、tools/ffmpeg/ 与 PATH)");
|
|
|
+ }
|
|
|
+
|
|
|
+ private static Path findOnPath(String exe) {
|
|
|
+ String pathEnv = System.getenv("PATH");
|
|
|
+ if (pathEnv == null) {
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+ for (String dir : pathEnv.split(java.io.File.pathSeparator)) {
|
|
|
+ if (dir.isBlank()) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ Path candidate = Paths.get(dir, exe);
|
|
|
+ if (Files.isRegularFile(candidate)) {
|
|
|
+ return candidate.toAbsolutePath().normalize();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+
|
|
|
+ /** 确认挑中的文件真的能执行:PATH 上同名但损坏/不是 ffmpeg 的情况并不罕见 */
|
|
|
+ private static void verifyFfmpeg(Path ffmpeg) throws IOException, InterruptedException {
|
|
|
+ Process process = new ProcessBuilder(ffmpeg.toString(), "-version")
|
|
|
+ .redirectErrorStream(true).start();
|
|
|
+ try (InputStream in = process.getInputStream()) {
|
|
|
+ in.readAllBytes();
|
|
|
+ }
|
|
|
+ if (!process.waitFor(10, TimeUnit.SECONDS) || process.exitValue() != 0) {
|
|
|
+ process.destroyForcibly();
|
|
|
+ throw new IOException("不是可用的 ffmpeg 可执行文件: " + ffmpeg);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private Path modelDir() throws IOException {
|
|
|
+ Path dir = resolve(props.getModelPath());
|
|
|
+ if (!Files.isDirectory(dir)) {
|
|
|
+ throw new IOException("Vosk 模型目录不存在: " + dir + "(应包含 am/ 与 conf/)");
|
|
|
+ }
|
|
|
+ if (!Files.isDirectory(dir.resolve("am"))) {
|
|
|
+ throw new IOException("Vosk 模型目录结构不完整: " + dir + "(缺 am/)");
|
|
|
+ }
|
|
|
+ return dir;
|
|
|
+ }
|
|
|
+
|
|
|
+ private static Path resolve(String path) {
|
|
|
+ Path p = Paths.get(path);
|
|
|
+ return p.isAbsolute() ? p.normalize()
|
|
|
+ : Paths.get(System.getProperty("user.dir")).resolve(p).toAbsolutePath().normalize();
|
|
|
+ }
|
|
|
+
|
|
|
+ // ==================== 工具 ====================
|
|
|
+
|
|
|
+ /** Vosk 每段结果都是 {"text": "..."},空段跳过,其余一行一段(模型不输出标点,换行能明显提升可读性) */
|
|
|
+ static void appendSegment(StringBuilder sb, String json) {
|
|
|
+ if (StrUtil.isBlank(json)) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ try {
|
|
|
+ JSONObject obj = JSONUtil.parseObj(json);
|
|
|
+ String line = StrUtil.trimToEmpty(obj.getStr("text"));
|
|
|
+ if (!line.isEmpty()) {
|
|
|
+ sb.append(line).append('\n');
|
|
|
+ }
|
|
|
+ } catch (Exception e) {
|
|
|
+ // 解析不出来说明库的输出格式变了:宁可丢掉这一段,也不把 JSON 原文塞进正文
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /** 从 ffmpeg 的 "Duration: HH:MM:SS.ss" 行解析秒数;解析不到返回 0 */
|
|
|
+ static int parseDuration(String line) {
|
|
|
+ if (line == null) {
|
|
|
+ return 0;
|
|
|
+ }
|
|
|
+ Matcher matcher = DURATION.matcher(line);
|
|
|
+ if (!matcher.find()) {
|
|
|
+ return 0;
|
|
|
+ }
|
|
|
+ int hours = Integer.parseInt(matcher.group(1));
|
|
|
+ int minutes = Integer.parseInt(matcher.group(2));
|
|
|
+ double seconds = Double.parseDouble(matcher.group(3));
|
|
|
+ return (int) Math.round(hours * 3600 + minutes * 60 + seconds);
|
|
|
+ }
|
|
|
+
|
|
|
+ /** 排空 ffmpeg stderr:保留尾部若干行(错误原因)并解析时长 */
|
|
|
+ private static final class StderrSink implements Runnable {
|
|
|
+
|
|
|
+ private final InputStream err;
|
|
|
+
|
|
|
+ private final Deque<String> lines = new ArrayDeque<>();
|
|
|
+
|
|
|
+ private volatile int durationSeconds;
|
|
|
+
|
|
|
+ private StderrSink(InputStream err) {
|
|
|
+ this.err = err;
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void run() {
|
|
|
+ try (BufferedReader reader = new BufferedReader(
|
|
|
+ new InputStreamReader(err, StandardCharsets.UTF_8))) {
|
|
|
+ String line;
|
|
|
+ while ((line = reader.readLine()) != null) {
|
|
|
+ synchronized (lines) {
|
|
|
+ if (durationSeconds == 0) {
|
|
|
+ durationSeconds = parseDuration(line);
|
|
|
+ }
|
|
|
+ lines.addLast(line);
|
|
|
+ if (lines.size() > STDERR_KEEP_LINES) {
|
|
|
+ lines.removeFirst();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ } catch (Exception e) {
|
|
|
+ // 进程被杀时流会异常关闭,忽略
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ int durationSeconds() {
|
|
|
+ return durationSeconds;
|
|
|
+ }
|
|
|
+
|
|
|
+ String tail() {
|
|
|
+ synchronized (lines) {
|
|
|
+ return String.join(" | ", lines);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|