|
|
@@ -100,6 +100,13 @@ public class PythonExecutor {
|
|
|
/** Python 输出目录的执行归属文件:第一行 userId,第二行 sessionId */
|
|
|
private static final String EXEC_OWNER_FILE = ".exec-owner";
|
|
|
|
|
|
+ /** 内联输出上限的硬上限(与 ZCode 的 BASH_MAX_OUTPUT_LENGTH 上限一致) */
|
|
|
+ private static final int MAX_INLINE_OUTPUT_CHARS = 150_000;
|
|
|
+
|
|
|
+ /** 截断时完整输出落盘的文件名(放在 py-out/{execId} 下,不参与产物白名单) */
|
|
|
+ private static final String STDOUT_LOG = "stdout.log";
|
|
|
+ private static final String STDERR_LOG = "stderr.log";
|
|
|
+
|
|
|
/**
|
|
|
* 关系图辅助模块源码。
|
|
|
*
|
|
|
@@ -260,6 +267,18 @@ public class PythonExecutor {
|
|
|
return available;
|
|
|
}
|
|
|
|
|
|
+ /**
|
|
|
+ * 单路输出内联字符上限:配置值钳到 [1, 150000],非法值回退 30000。
|
|
|
+ * 超出后由 {@link OutputCollector} 保留头尾并把完整输出落盘。
|
|
|
+ */
|
|
|
+ private int resolveMaxOutputChars() {
|
|
|
+ int configured = etlProperties.getPython().getMaxOutputChars();
|
|
|
+ if (configured <= 0) {
|
|
|
+ configured = 30_000;
|
|
|
+ }
|
|
|
+ return Math.min(configured, MAX_INLINE_OUTPUT_CHARS);
|
|
|
+ }
|
|
|
+
|
|
|
/**
|
|
|
* 执行 Python 代码(子进程沙箱)
|
|
|
*
|
|
|
@@ -328,10 +347,13 @@ public class PythonExecutor {
|
|
|
pb.environment().put("MPLCONFIGDIR", mplDir.toAbsolutePath().toString());
|
|
|
|
|
|
Process process = pb.start();
|
|
|
+ int maxInlineChars = resolveMaxOutputChars();
|
|
|
+ long maxFullChars = etlProperties.getPython().getMaxFullOutputChars();
|
|
|
+ Path outDir = storageLayout.pythonOutDir(workspaceId, execId);
|
|
|
OutputCollector stdout = new OutputCollector(process.getInputStream(),
|
|
|
- etlProperties.getPython().getMaxOutputChars());
|
|
|
+ maxInlineChars, maxFullChars, outDir.resolve(STDOUT_LOG));
|
|
|
OutputCollector stderr = new OutputCollector(process.getErrorStream(),
|
|
|
- etlProperties.getPython().getMaxOutputChars());
|
|
|
+ maxInlineChars, maxFullChars, outDir.resolve(STDERR_LOG));
|
|
|
stdout.start();
|
|
|
stderr.start();
|
|
|
|
|
|
@@ -349,12 +371,21 @@ public class PythonExecutor {
|
|
|
RelationGraphFile relationGraph = readRelationGraph(sandbox);
|
|
|
List<String> imageUrls = collectGeneratedFiles(
|
|
|
workspaceId, execId, sandbox, userId, sessionId);
|
|
|
+ // 只有截断时才落盘输出日志;若本目录没有产物,collectGeneratedFiles 不会触发 LRU 清理
|
|
|
+ if (stdout.isPersisted() || stderr.isPersisted()) {
|
|
|
+ evictOldExecs(storageLayout.pythonOutRoot(workspaceId));
|
|
|
+ }
|
|
|
long durationMs = System.currentTimeMillis() - start;
|
|
|
- log.info("Python 执行完成: workspaceId={}, execId={}, exitCode={}, images={}, relationGraph={}, costMs={}",
|
|
|
+ log.info("Python 执行完成: workspaceId={}, execId={}, exitCode={}, images={}, relationGraph={}, "
|
|
|
+ + "stdoutChars={}, stderrChars={}, stdoutTruncated={}, stderrTruncated={}, costMs={}",
|
|
|
workspaceId, execId, process.exitValue(), imageUrls.size(),
|
|
|
- relationGraph.json() != null, durationMs);
|
|
|
+ relationGraph.json() != null, stdout.getTotalChars(), stderr.getTotalChars(),
|
|
|
+ stdout.isTruncated(), stderr.isTruncated(), durationMs);
|
|
|
return new PythonResult(process.exitValue(), stdout.getContent(), stderr.getContent(),
|
|
|
- imageUrls, durationMs, null, relationGraph.json(), relationGraph.error());
|
|
|
+ imageUrls, durationMs, null, relationGraph.json(), relationGraph.error(),
|
|
|
+ stdout.isTruncated(), stderr.isTruncated(),
|
|
|
+ stdout.getTotalChars(), stderr.getTotalChars(),
|
|
|
+ stdout.getPersistedPath(), stderr.getPersistedPath());
|
|
|
} catch (IOException e) {
|
|
|
return PythonResult.error("Python 进程启动失败: " + e.getMessage());
|
|
|
} catch (InterruptedException e) {
|
|
|
@@ -398,10 +429,31 @@ public class PythonExecutor {
|
|
|
* <p>{@code relationGraphJson} / {@code relationGraphError} 对应沙箱里的
|
|
|
* {@code relation_graph.json}:前者是模型调用 {@code emit_relation_graph} 写出的原文,
|
|
|
* 后者是读取失败的原因,两者互斥;都为 null 表示本次没有生成关系图。
|
|
|
+ *
|
|
|
+ * <p><b>输出截断</b>:{@code stdout/stderr} 已按内联上限保留头尾(中间有截断说明);
|
|
|
+ * {@code *Truncated} 为真实截断标志,{@code *Chars} 是原始总字符数,
|
|
|
+ * {@code *FullOutputPath} 是完整输出落盘路径(未截断或落盘失败时为 null)。
|
|
|
*/
|
|
|
public record PythonResult(int exitCode, String stdout, String stderr,
|
|
|
List<String> imageUrls, long durationMs, String error,
|
|
|
- String relationGraphJson, String relationGraphError) {
|
|
|
+ String relationGraphJson, String relationGraphError,
|
|
|
+ boolean stdoutTruncated, boolean stderrTruncated,
|
|
|
+ long stdoutChars, long stderrChars,
|
|
|
+ String stdoutFullOutputPath, String stderrFullOutputPath) {
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 兼容旧调用/单测:没有截断元数据时按「未截断」处理。
|
|
|
+ */
|
|
|
+ public PythonResult(int exitCode, String stdout, String stderr,
|
|
|
+ List<String> imageUrls, long durationMs, String error,
|
|
|
+ String relationGraphJson, String relationGraphError) {
|
|
|
+ this(exitCode, stdout, stderr, imageUrls, durationMs, error,
|
|
|
+ relationGraphJson, relationGraphError,
|
|
|
+ false, false,
|
|
|
+ stdout == null ? 0L : stdout.length(),
|
|
|
+ stderr == null ? 0L : stderr.length(),
|
|
|
+ null, null);
|
|
|
+ }
|
|
|
|
|
|
static PythonResult error(String message) {
|
|
|
return new PythonResult(-1, "", "", List.of(), 0, message, null, null);
|
|
|
@@ -616,37 +668,262 @@ public class PythonExecutor {
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
- * 进程输出流采集线程(截断到 maxChars)
|
|
|
+ * 进程输出流采集线程:内联只保留头尾,超出上限时把完整输出落盘。
|
|
|
+ *
|
|
|
+ * <p>只留前 N 个字符会把错误栈/汇总行(通常在最后)丢掉;这里把内联预算对半分成
|
|
|
+ * 头/尾两段,中间用截断说明代替。第一次超出上限时把已读前缀写盘,后续输出继续追加,
|
|
|
+ * 模型可从结果里的 {@code *_full_output_path} 取全文(用 execute_python 读取)。
|
|
|
+ *
|
|
|
+ * <p>落盘只在截断时发生:短输出既不需要额外文件,也不会改变返回内容。
|
|
|
*/
|
|
|
- private static class OutputCollector extends Thread {
|
|
|
+ static class OutputCollector extends Thread {
|
|
|
|
|
|
private final java.io.InputStream in;
|
|
|
- private final int maxChars;
|
|
|
- private final StringBuilder sb = new StringBuilder();
|
|
|
-
|
|
|
- OutputCollector(java.io.InputStream in, int maxChars) {
|
|
|
+ private final int maxInlineChars;
|
|
|
+ private final int headChars;
|
|
|
+ private final int tailChars;
|
|
|
+ private final long maxPersistedChars;
|
|
|
+ private final Path outputPath;
|
|
|
+
|
|
|
+ /** 未截断时的全文缓冲 */
|
|
|
+ private final StringBuilder inline = new StringBuilder();
|
|
|
+ /** 截断后的头段 */
|
|
|
+ private final StringBuilder head = new StringBuilder();
|
|
|
+ /** 截断后的尾段(最多 tailChars + 1,保持有界) */
|
|
|
+ private final StringBuilder tail = new StringBuilder();
|
|
|
+
|
|
|
+ private long totalChars;
|
|
|
+ private boolean truncated;
|
|
|
+ private boolean persisted;
|
|
|
+ private boolean persistedTruncated;
|
|
|
+ private boolean persistFailed;
|
|
|
+ private java.io.BufferedWriter writer;
|
|
|
+ private long persistedChars;
|
|
|
+
|
|
|
+ OutputCollector(java.io.InputStream in, int maxInlineChars, long maxPersistedChars,
|
|
|
+ Path outputPath) {
|
|
|
this.in = in;
|
|
|
- this.maxChars = maxChars;
|
|
|
+ this.maxInlineChars = Math.max(1, maxInlineChars);
|
|
|
+ this.headChars = this.maxInlineChars / 2;
|
|
|
+ this.tailChars = this.maxInlineChars - this.headChars;
|
|
|
+ this.maxPersistedChars = maxPersistedChars;
|
|
|
+ this.outputPath = outputPath;
|
|
|
setDaemon(true);
|
|
|
}
|
|
|
|
|
|
@Override
|
|
|
public void run() {
|
|
|
try (java.io.BufferedReader reader = new java.io.BufferedReader(
|
|
|
- new java.io.InputStreamReader(in, java.nio.charset.StandardCharsets.UTF_8))) {
|
|
|
- int c;
|
|
|
- while ((c = reader.read()) != -1) {
|
|
|
- if (sb.length() < maxChars) {
|
|
|
- sb.append((char) c);
|
|
|
- }
|
|
|
+ new java.io.InputStreamReader(in, StandardCharsets.UTF_8))) {
|
|
|
+ char[] buf = new char[8192];
|
|
|
+ int n;
|
|
|
+ while ((n = reader.read(buf)) != -1) {
|
|
|
+ append(buf, n);
|
|
|
}
|
|
|
} catch (IOException ignored) {
|
|
|
// 进程终止导致的流关闭,忽略
|
|
|
+ } finally {
|
|
|
+ closeWriter();
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private void append(char[] buf, int len) {
|
|
|
+ if (len <= 0) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ totalChars += len;
|
|
|
+ if (!truncated) {
|
|
|
+ int room = maxInlineChars - inline.length();
|
|
|
+ if (len <= room) {
|
|
|
+ inline.append(buf, 0, len);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ if (room > 0) {
|
|
|
+ inline.append(buf, 0, room);
|
|
|
+ }
|
|
|
+ truncated = true;
|
|
|
+ // 此时 inline 恰好是截断前的完整前缀,先落盘再拆成头/尾
|
|
|
+ startPersistence();
|
|
|
+ splitInline();
|
|
|
+ appendTailAndPersist(buf, room, len - room);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ appendTailAndPersist(buf, 0, len);
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 把「前 maxInlineChars 个字符」拆成头段 + 尾段;不拆开代理对(emoji 等)。
|
|
|
+ *
|
|
|
+ * <p>拆分点落在高代理上时(代理对可能完整在 prefix 内,也可能低代理还在下一个 chunk),
|
|
|
+ * 把高代理让给尾段:尾段后续会拼上低代理,头段绝不能留下半个字符。
|
|
|
+ */
|
|
|
+ private void splitInline() {
|
|
|
+ String prefix = inline.toString();
|
|
|
+ inline.setLength(0);
|
|
|
+ int split = Math.min(headChars, prefix.length());
|
|
|
+ if (split > 0 && Character.isHighSurrogate(prefix.charAt(split - 1))) {
|
|
|
+ split--;
|
|
|
+ }
|
|
|
+ head.append(prefix, 0, split);
|
|
|
+ tail.append(prefix, split, prefix.length());
|
|
|
+ trimTail();
|
|
|
+ }
|
|
|
+
|
|
|
+ private void appendTailAndPersist(char[] buf, int off, int len) {
|
|
|
+ if (len <= 0) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ if (tailChars > 0) {
|
|
|
+ int take = Math.min(len, tailChars);
|
|
|
+ tail.append(buf, off + len - take, take);
|
|
|
+ trimTail();
|
|
|
+ }
|
|
|
+ persist(buf, off, len);
|
|
|
+ }
|
|
|
+
|
|
|
+ /** 尾段只保留最后 tailChars 个字符;超过两倍再裁剪,均摊 O(1)。 */
|
|
|
+ private void trimTail() {
|
|
|
+ if (tailChars <= 0) {
|
|
|
+ tail.setLength(0);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ if (tail.length() <= tailChars * 2) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ int cut = tail.length() - tailChars;
|
|
|
+ if (cut > 0 && Character.isLowSurrogate(tail.charAt(cut))) {
|
|
|
+ cut--;
|
|
|
+ }
|
|
|
+ tail.delete(0, cut);
|
|
|
+ }
|
|
|
+
|
|
|
+ private String tailText() {
|
|
|
+ if (tail.length() <= tailChars) {
|
|
|
+ return tail.toString();
|
|
|
+ }
|
|
|
+ int start = tail.length() - tailChars;
|
|
|
+ if (start > 0 && Character.isLowSurrogate(tail.charAt(start))) {
|
|
|
+ start--;
|
|
|
+ }
|
|
|
+ return tail.substring(start);
|
|
|
+ }
|
|
|
+
|
|
|
+ private void startPersistence() {
|
|
|
+ if (outputPath == null || persisted || persistFailed) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ try {
|
|
|
+ Path parent = outputPath.getParent();
|
|
|
+ if (parent != null) {
|
|
|
+ Files.createDirectories(parent);
|
|
|
+ }
|
|
|
+ writer = Files.newBufferedWriter(outputPath, StandardCharsets.UTF_8,
|
|
|
+ java.nio.file.StandardOpenOption.CREATE,
|
|
|
+ java.nio.file.StandardOpenOption.TRUNCATE_EXISTING,
|
|
|
+ java.nio.file.StandardOpenOption.WRITE);
|
|
|
+ persisted = true;
|
|
|
+ String prefix = inline.toString();
|
|
|
+ int allow = persistedAllowance(prefix.length());
|
|
|
+ if (allow > 0) {
|
|
|
+ writer.write(prefix, 0, allow);
|
|
|
+ persistedChars += allow;
|
|
|
+ }
|
|
|
+ if (allow < prefix.length()) {
|
|
|
+ persistedTruncated = true;
|
|
|
+ }
|
|
|
+ } catch (IOException e) {
|
|
|
+ persistFailed = true;
|
|
|
+ persistedTruncated = true;
|
|
|
+ log.warn("Python 输出落盘失败(仅返回内联头尾): path={}, err={}",
|
|
|
+ outputPath, e.getMessage());
|
|
|
+ closeWriter();
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ private void persist(char[] buf, int off, int len) {
|
|
|
+ if (writer == null || persistFailed || len <= 0) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ int allow = persistedAllowance(len);
|
|
|
+ if (allow <= 0) {
|
|
|
+ persistedTruncated = true;
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ try {
|
|
|
+ writer.write(buf, off, allow);
|
|
|
+ persistedChars += allow;
|
|
|
+ if (allow < len) {
|
|
|
+ persistedTruncated = true;
|
|
|
+ }
|
|
|
+ } catch (IOException e) {
|
|
|
+ persistFailed = true;
|
|
|
+ persistedTruncated = true;
|
|
|
+ log.warn("Python 输出落盘中断(仅返回内联头尾): path={}, err={}",
|
|
|
+ outputPath, e.getMessage());
|
|
|
+ closeWriter();
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /** 本次允许写入的字符数;{@code maxPersistedChars <= 0} 表示不限。 */
|
|
|
+ private int persistedAllowance(int requested) {
|
|
|
+ if (maxPersistedChars <= 0) {
|
|
|
+ return requested;
|
|
|
+ }
|
|
|
+ long remaining = maxPersistedChars - persistedChars;
|
|
|
+ if (remaining <= 0) {
|
|
|
+ return 0;
|
|
|
+ }
|
|
|
+ return (int) Math.min(requested, remaining);
|
|
|
+ }
|
|
|
+
|
|
|
+ private void closeWriter() {
|
|
|
+ if (writer == null) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ try {
|
|
|
+ writer.close();
|
|
|
+ } catch (IOException ignored) {
|
|
|
+ // 关闭失败不影响已写内容
|
|
|
+ }
|
|
|
+ writer = null;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 模型可见的输出:未截断返回全文;截断返回「头段 + 截断说明 + 尾段」。
|
|
|
+ */
|
|
|
String getContent() {
|
|
|
- return sb.toString();
|
|
|
+ if (!truncated) {
|
|
|
+ return inline.toString();
|
|
|
+ }
|
|
|
+ String tailText = tailText();
|
|
|
+ long truncatedChars = Math.max(0, totalChars - head.length() - tailText.length());
|
|
|
+ StringBuilder out = new StringBuilder(head);
|
|
|
+ out.append("\n\n... [").append(truncatedChars).append(" characters truncated");
|
|
|
+ if (persisted) {
|
|
|
+ out.append("; full output");
|
|
|
+ if (persistedTruncated) {
|
|
|
+ out.append(" (first ").append(persistedChars).append(" chars)");
|
|
|
+ }
|
|
|
+ out.append(": ").append(outputPath);
|
|
|
+ }
|
|
|
+ out.append("] ...\n\n").append(tailText);
|
|
|
+ return out.toString();
|
|
|
+ }
|
|
|
+
|
|
|
+ long getTotalChars() {
|
|
|
+ return totalChars;
|
|
|
+ }
|
|
|
+
|
|
|
+ boolean isTruncated() {
|
|
|
+ return truncated;
|
|
|
+ }
|
|
|
+
|
|
|
+ boolean isPersisted() {
|
|
|
+ return persisted;
|
|
|
+ }
|
|
|
+
|
|
|
+ String getPersistedPath() {
|
|
|
+ return persisted ? outputPath.toString() : null;
|
|
|
}
|
|
|
}
|
|
|
}
|