发布 v1.10 #5

Merged
czm merged 147 commits from develop into main 2026-08-20 11:36:27 +08:00
11 changed files with 588 additions and 66 deletions
Showing only changes of commit 5c29ca9407 - Show all commits

View File

@@ -43,6 +43,7 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic
private static final Logger LOG = LoggerFactory.getLogger(DocumentParseBridgeServiceImpl.class); private static final Logger LOG = LoggerFactory.getLogger(DocumentParseBridgeServiceImpl.class);
private static final String DEFAULT_DOCUMENT_PARSE_SERVICE_BEAN_NAME = "documentParseService"; private static final String DEFAULT_DOCUMENT_PARSE_SERVICE_BEAN_NAME = "documentParseService";
private static final long WORKFLOW_TEXT_MAX_BYTES = 100L * 1024L * 1024L;
@Nullable @Nullable
private final DocumentParseService defaultDocumentParseService; private final DocumentParseService defaultDocumentParseService;
@@ -80,7 +81,7 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic
@Override @Override
public DocumentParsedResult parse(DocumentSourceRef source, DocumentParseScenario scenario) { public DocumentParsedResult parse(DocumentSourceRef source, DocumentParseScenario scenario) {
try { try {
LoadedDocumentSource loadedSource = prepareSupportedSource(source); LoadedDocumentSource loadedSource = prepareSupportedSource(source, scenario);
LOG.info("桥接服务开始同步解析文档: fileName={}, contentType={}, scenario={}", LOG.info("桥接服务开始同步解析文档: fileName={}, contentType={}, scenario={}",
loadedSource.getFileName(), loadedSource.getContentType(), scenario); loadedSource.getFileName(), loadedSource.getContentType(), scenario);
DocumentParseService parseService = resolveService(loadedSource); DocumentParseService parseService = resolveService(loadedSource);
@@ -106,7 +107,7 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic
@Override @Override
public DocumentParseTaskStatus submit(DocumentSourceRef source, DocumentParseScenario scenario) { public DocumentParseTaskStatus submit(DocumentSourceRef source, DocumentParseScenario scenario) {
try { try {
LoadedDocumentSource loadedSource = prepareSupportedSource(source); LoadedDocumentSource loadedSource = prepareSupportedSource(source, scenario);
LOG.info("桥接服务开始提交异步解析任务: fileName={}, contentType={}, scenario={}", LOG.info("桥接服务开始提交异步解析任务: fileName={}, contentType={}, scenario={}",
loadedSource.getFileName(), loadedSource.getContentType(), scenario); loadedSource.getFileName(), loadedSource.getContentType(), scenario);
DocumentParseService parseService = resolveService(loadedSource); DocumentParseService parseService = resolveService(loadedSource);
@@ -204,8 +205,12 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic
return text == null ? 0 : text.length(); return text == null ? 0 : text.length();
} }
private LoadedDocumentSource prepareSupportedSource(DocumentSourceRef source) { private LoadedDocumentSource prepareSupportedSource(DocumentSourceRef source,
LoadedDocumentSource loadedSource = documentSourceLoader.load(source); DocumentParseScenario scenario) {
long maxBytes = scenario == DocumentParseScenario.WORKFLOW_TEXT
? WORKFLOW_TEXT_MAX_BYTES
: 0L;
LoadedDocumentSource loadedSource = documentSourceLoader.load(source, maxBytes);
if (!isSupportedByBridge(loadedSource)) { if (!isSupportedByBridge(loadedSource)) {
throw DocumentParseBridgeException.unsupportedSource("统一文档解析桥接当前仅支持 PDF、DOCX、PPTX、XLSX 文件"); throw DocumentParseBridgeException.unsupportedSource("统一文档解析桥接当前仅支持 PDF、DOCX、PPTX、XLSX 文件");
} }

View File

@@ -1,6 +1,8 @@
package tech.easyflow.ai.document.support; package tech.easyflow.ai.document.support;
import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Qualifier;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils; import org.springframework.util.StringUtils;
import tech.easyflow.ai.document.exception.DocumentParseBridgeException; import tech.easyflow.ai.document.exception.DocumentParseBridgeException;
@@ -23,6 +25,9 @@ import java.net.URLConnection;
@Component @Component
public class DocumentSourceLoader { public class DocumentSourceLoader {
private static final Logger LOG =
LoggerFactory.getLogger(DocumentSourceLoader.class);
private final FileStorageService fileStorageService; private final FileStorageService fileStorageService;
public DocumentSourceLoader(@Qualifier("default") FileStorageService fileStorageService) { public DocumentSourceLoader(@Qualifier("default") FileStorageService fileStorageService) {
@@ -36,38 +41,52 @@ public class DocumentSourceLoader {
* @return 内部已加载文档对象 * @return 内部已加载文档对象
*/ */
public LoadedDocumentSource load(DocumentSourceRef sourceRef) { public LoadedDocumentSource load(DocumentSourceRef sourceRef) {
return load(sourceRef, 0L);
}
/**
* 在实际字节数保护下加载文档源。
*
* @param sourceRef easyflow 文档源
* @param maxBytes 最大允许字节数;小于等于 0 时保持原有不限大小语义
* @return 内部已加载文档对象
*/
public LoadedDocumentSource load(DocumentSourceRef sourceRef, long maxBytes) {
if (sourceRef == null) { if (sourceRef == null) {
throw DocumentParseBridgeException.unsupportedSource("文档源不能为空"); throw DocumentParseBridgeException.unsupportedSource("文档源不能为空");
} }
if (hasContentBytes(sourceRef)) { if (hasContentBytes(sourceRef)) {
long actualBytes = sourceRef.getContentBytes().length;
assertWithinLimit(actualBytes, maxBytes);
logSizeMismatch(sourceRef, actualBytes);
return buildLoadedSource( return buildLoadedSource(
resolveFileName(sourceRef), resolveFileName(sourceRef),
resolveContentType(sourceRef, resolveFileName(sourceRef)), resolveContentType(sourceRef, resolveFileName(sourceRef)),
resolveSize(sourceRef, sourceRef.getContentBytes().length), actualBytes,
sourceRef.getContentBytes() sourceRef.getContentBytes()
); );
} }
if (StringUtils.hasText(sourceRef.getFilePath())) { if (StringUtils.hasText(sourceRef.getFilePath())) {
if (isRemoteUrl(sourceRef.getFilePath())) { if (isRemoteUrl(sourceRef.getFilePath())) {
return loadFromRemoteValue(sourceRef, sourceRef.getFilePath()); return loadFromRemoteValue(sourceRef, sourceRef.getFilePath(), maxBytes);
} }
return loadFromFilePath(sourceRef); return loadFromFilePath(sourceRef, maxBytes);
} }
if (StringUtils.hasText(sourceRef.getUrl())) { if (StringUtils.hasText(sourceRef.getUrl())) {
return loadFromUrl(sourceRef); return loadFromUrl(sourceRef, maxBytes);
} }
throw DocumentParseBridgeException.unsupportedSource("文档源缺少 filePath、url 或 contentBytes"); throw DocumentParseBridgeException.unsupportedSource("文档源缺少 filePath、url 或 contentBytes");
} }
private LoadedDocumentSource loadFromFilePath(DocumentSourceRef sourceRef) { private LoadedDocumentSource loadFromFilePath(DocumentSourceRef sourceRef, long maxBytes) {
String fileName = resolveFileName(sourceRef); String fileName = resolveFileName(sourceRef);
try (InputStream inputStream = fileStorageService.readStream(sourceRef.getFilePath())) { try (InputStream inputStream = fileStorageService.readStream(sourceRef.getFilePath())) {
byte[] contentBytes = inputStream.readAllBytes(); byte[] contentBytes = DocumentInputStreamSupport.readBytes(inputStream, maxBytes);
long actualSize = sourceRef.getSize() != null ? sourceRef.getSize() : fileStorageService.getFileSize(sourceRef.getFilePath()); logSizeMismatch(sourceRef, contentBytes.length);
return buildLoadedSource( return buildLoadedSource(
fileName, fileName,
resolveContentType(sourceRef, fileName), resolveContentType(sourceRef, fileName),
resolveSize(sourceRef, actualSize), (long) contentBytes.length,
contentBytes contentBytes
); );
} catch (IOException e) { } catch (IOException e) {
@@ -78,19 +97,21 @@ public class DocumentSourceLoader {
} }
} }
private LoadedDocumentSource loadFromUrl(DocumentSourceRef sourceRef) { private LoadedDocumentSource loadFromUrl(DocumentSourceRef sourceRef, long maxBytes) {
return loadFromRemoteValue(sourceRef, sourceRef.getUrl()); return loadFromRemoteValue(sourceRef, sourceRef.getUrl(), maxBytes);
} }
private LoadedDocumentSource loadFromRemoteValue(DocumentSourceRef sourceRef, String remoteUrl) { private LoadedDocumentSource loadFromRemoteValue(DocumentSourceRef sourceRef,
String remoteUrl,
long maxBytes) {
String fileName = resolveFileName(sourceRef); String fileName = resolveFileName(sourceRef);
try (InputStream inputStream = try (InputStream inputStream = DocumentInputStreamSupport.openRemote(remoteUrl, maxBytes)) {
DocumentInputStreamSupport.openRemote(remoteUrl, 0L)) { byte[] contentBytes = DocumentInputStreamSupport.readBytes(inputStream, maxBytes);
byte[] contentBytes = DocumentInputStreamSupport.readBytes(inputStream, 0L); logSizeMismatch(sourceRef, contentBytes.length);
return buildLoadedSource( return buildLoadedSource(
fileName, fileName,
resolveContentType(sourceRef, fileName), resolveContentType(sourceRef, fileName),
resolveSize(sourceRef, contentBytes.length), (long) contentBytes.length,
contentBytes contentBytes
); );
} catch (Exception e) { } catch (Exception e) {
@@ -132,10 +153,6 @@ public class DocumentSourceLoader {
return URLConnection.guessContentTypeFromName(fileName); return URLConnection.guessContentTypeFromName(fileName);
} }
private Long resolveSize(DocumentSourceRef sourceRef, long fallbackSize) {
return sourceRef.getSize() != null ? sourceRef.getSize() : fallbackSize;
}
private boolean hasContentBytes(DocumentSourceRef sourceRef) { private boolean hasContentBytes(DocumentSourceRef sourceRef) {
return sourceRef.getContentBytes() != null && sourceRef.getContentBytes().length > 0; return sourceRef.getContentBytes() != null && sourceRef.getContentBytes().length > 0;
} }
@@ -143,4 +160,40 @@ public class DocumentSourceLoader {
private boolean isRemoteUrl(String value) { private boolean isRemoteUrl(String value) {
return value.startsWith("http://") || value.startsWith("https://"); return value.startsWith("http://") || value.startsWith("https://");
} }
/**
* 记录上传声明大小与实际读取量不一致的情况。
*
* @param sourceRef 文档源
* @param actualBytes 实际读取字节数
*/
private void logSizeMismatch(DocumentSourceRef sourceRef, long actualBytes) {
Long declaredBytes = sourceRef.getSize();
if (declaredBytes == null || declaredBytes < 0L
|| declaredBytes == actualBytes) {
return;
}
LOG.warn(
"文档声明大小与实际读取量不一致: fileName={}, "
+ "declaredBytes={}, actualBytes={}",
resolveFileName(sourceRef),
declaredBytes,
actualBytes);
}
/**
* 校验已确认的实际字节数。
*
* @param actualBytes 实际字节数
* @param maxBytes 最大允许字节数
*/
private void assertWithinLimit(long actualBytes, long maxBytes) {
if (maxBytes > 0L && actualBytes > maxBytes) {
throw DocumentParseBridgeException.sourceLoadFailed(
"文档实际大小超过限制: " + maxBytes + " bytes",
new DocumentInputStreamSupport.SizeLimitExceededException(
maxBytes,
actualBytes));
}
}
} }

View File

@@ -36,8 +36,8 @@ public class WorkflowRunningParameterResolver {
private static final String DEFAULT_START_FORM_DESCRIPTION = "请先补充必要信息,再开始执行工作流。"; private static final String DEFAULT_START_FORM_DESCRIPTION = "请先补充必要信息,再开始执行工作流。";
private static final String DEFAULT_START_FORM_SUBMIT_TEXT = "开始"; private static final String DEFAULT_START_FORM_SUBMIT_TEXT = "开始";
private static final int FILE_MAX_COUNT = 10; private static final int FILE_MAX_COUNT = 10;
private static final long FILE_MAX_SINGLE_SIZE = 20L * 1024 * 1024; private static final long FILE_MAX_SINGLE_SIZE = 100L * 1024 * 1024;
private static final long FILE_MAX_TOTAL_SIZE = 50L * 1024 * 1024; private static final long FILE_MAX_TOTAL_SIZE = 100L * 1024 * 1024;
private static final long IMAGE_MAX_SINGLE_SIZE = 10L * 1024 * 1024; private static final long IMAGE_MAX_SINGLE_SIZE = 10L * 1024 * 1024;
@Resource @Resource
@@ -524,7 +524,7 @@ public class WorkflowRunningParameterResolver {
} }
Long size = parseLong(fileMap.get("size")); Long size = parseLong(fileMap.get("size"));
if (size != null && size > FILE_MAX_SINGLE_SIZE) { if (size != null && size > FILE_MAX_SINGLE_SIZE) {
throw new BusinessException("文件参数 " + parameterName + " 中单个文件不能超过 20MB"); throw new BusinessException("文件参数 " + parameterName + " 中单个文件不能超过 100MB");
} }
if (size != null && size > 0) { if (size != null && size > 0) {
totalSize += size; totalSize += size;
@@ -536,7 +536,7 @@ public class WorkflowRunningParameterResolver {
throw new BusinessException("文件参数 " + parameterName + " 最多上传 10 个文件"); throw new BusinessException("文件参数 " + parameterName + " 最多上传 10 个文件");
} }
if (totalSize > FILE_MAX_TOTAL_SIZE) { if (totalSize > FILE_MAX_TOTAL_SIZE) {
throw new BusinessException("文件参数 " + parameterName + " 的文件总大小不能超过 50MB"); throw new BusinessException("文件参数 " + parameterName + " 的文件总大小不能超过 100MB");
} }
return normalized; return normalized;
} }

View File

@@ -1,12 +1,16 @@
package tech.easyflow.ai.node; package tech.easyflow.ai.node;
import cn.hutool.http.HttpUtil; import com.easyagents.flow.core.util.IoBulkhead;
import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Qualifier;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import tech.easyflow.ai.document.model.DocumentParseScenario; import tech.easyflow.ai.document.model.DocumentParseScenario;
import tech.easyflow.ai.document.model.DocumentParsedResult; import tech.easyflow.ai.document.model.DocumentParsedResult;
import tech.easyflow.ai.document.model.DocumentSourceRef; import tech.easyflow.ai.document.model.DocumentSourceRef;
import tech.easyflow.ai.document.exception.DocumentParseBridgeException;
import tech.easyflow.ai.document.service.DocumentParseBridgeService; import tech.easyflow.ai.document.service.DocumentParseBridgeService;
import tech.easyflow.ai.document.support.DocumentInputStreamSupport;
import tech.easyflow.ai.document.support.DocumentParseSourceType; import tech.easyflow.ai.document.support.DocumentParseSourceType;
import tech.easyflow.common.filestorage.FileStorageService; import tech.easyflow.common.filestorage.FileStorageService;
import tech.easyflow.common.util.StringUtil; import tech.easyflow.common.util.StringUtil;
@@ -14,6 +18,9 @@ import tech.easyflow.common.web.exceptions.BusinessException;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
@@ -33,9 +40,11 @@ import java.util.Set;
*/ */
@Component @Component
public class DocNodeFileContentExtractor { public class DocNodeFileContentExtractor {
private static final Logger LOG =
LoggerFactory.getLogger(DocNodeFileContentExtractor.class);
private static final int FILE_MAX_COUNT = 10; private static final int FILE_MAX_COUNT = 10;
private static final long FILE_MAX_SINGLE_SIZE = 20L * 1024 * 1024; private static final long FILE_MAX_SINGLE_SIZE = 100L * 1024 * 1024;
private static final long FILE_MAX_TOTAL_SIZE = 50L * 1024 * 1024; private static final long FILE_MAX_TOTAL_SIZE = 100L * 1024 * 1024;
private final DocumentParseBridgeService documentParseBridgeService; private final DocumentParseBridgeService documentParseBridgeService;
private final FileStorageService fileStorageService; private final FileStorageService fileStorageService;
@@ -145,7 +154,7 @@ public class DocNodeFileContentExtractor {
} }
Long size = sourceRef.getSize(); Long size = sourceRef.getSize();
if (size != null && size > FILE_MAX_SINGLE_SIZE) { if (size != null && size > FILE_MAX_SINGLE_SIZE) {
throw new BusinessException("单个文件不能超过 20MB: " + sourceRef.getFileName()); throw new BusinessException("单个文件不能超过 100MB: " + sourceRef.getFileName());
} }
if (size != null && size > 0) { if (size != null && size > 0) {
totalSize += size; totalSize += size;
@@ -156,7 +165,7 @@ public class DocNodeFileContentExtractor {
throw new BusinessException("最多上传 10 个文件"); throw new BusinessException("最多上传 10 个文件");
} }
if (totalSize > FILE_MAX_TOTAL_SIZE) { if (totalSize > FILE_MAX_TOTAL_SIZE) {
throw new BusinessException("文件总大小不能超过 50MB"); throw new BusinessException("文件总大小不能超过 100MB");
} }
if (sourceRefs.isEmpty()) { if (sourceRefs.isEmpty()) {
throw new BusinessException("文件输入不能为空"); throw new BusinessException("文件输入不能为空");
@@ -207,7 +216,24 @@ public class DocNodeFileContentExtractor {
* @return 桥接提取出的主文本 * @return 桥接提取出的主文本
*/ */
private String extractBridgeContent(DocumentSourceRef sourceRef) { private String extractBridgeContent(DocumentSourceRef sourceRef) {
DocumentParsedResult parsedResult = documentParseBridgeService.parse(sourceRef, DocumentParseScenario.WORKFLOW_TEXT); DocumentParsedResult parsedResult;
try (IoBulkhead.Permit ignored =
IoBulkhead.documentParse().acquire(
"document:"
+ DocumentParseSourceType.resolve(
sourceRef.getFileName(),
sourceRef.getContentType()))) {
parsedResult = documentParseBridgeService.parse(
sourceRef,
DocumentParseScenario.WORKFLOW_TEXT);
} catch (DocumentParseBridgeException error) {
DocumentInputStreamSupport.SizeLimitExceededException sizeError =
findSizeLimitExceeded(error);
if (sizeError != null) {
throw actualSizeLimitException(sourceRef, sizeError);
}
throw error;
}
String preferredText = parsedResult == null ? null : parsedResult.getPreferredText(); String preferredText = parsedResult == null ? null : parsedResult.getPreferredText();
if (StringUtil.hasText(preferredText)) { if (StringUtil.hasText(preferredText)) {
return preferredText; return preferredText;
@@ -222,29 +248,155 @@ public class DocNodeFileContentExtractor {
} }
private String extractDefaultContent(DocumentSourceRef sourceRef) { private String extractDefaultContent(DocumentSourceRef sourceRef) {
try (InputStream inputStream = openInputStream(sourceRef)) { Path temporaryFile = null;
return readerManager.getReader().read(sourceRef.getFileName(), inputStream); try {
temporaryFile = Files.createTempFile(
"easyflow-doc-node-", ".content");
copySourceToTemporaryFile(sourceRef, temporaryFile);
/*
* 源 HTTP/对象存储流在进入解析器前已经关闭,解析过程中不会嵌套占用
* 网络或存储 lane避免慢解析耗尽上游连接许可。
*/
try (IoBulkhead.Permit ignored =
IoBulkhead.documentParse().acquire(
"document:default-reader");
InputStream inputStream =
Files.newInputStream(temporaryFile)) {
return readerManager.getReader().read(
sourceRef.getFileName(), inputStream);
}
} catch (IOException e) { } catch (IOException e) {
DocumentInputStreamSupport.SizeLimitExceededException sizeError =
findSizeLimitExceeded(e);
if (sizeError != null) {
throw actualSizeLimitException(sourceRef, sizeError);
}
throw new RuntimeException("读取文件内容失败: " + sourceRef.getFilePath(), e); throw new RuntimeException("读取文件内容失败: " + sourceRef.getFilePath(), e);
} catch (RuntimeException e) {
DocumentInputStreamSupport.SizeLimitExceededException sizeError =
findSizeLimitExceeded(e);
if (sizeError != null) {
throw actualSizeLimitException(sourceRef, sizeError);
}
throw e;
} finally {
if (temporaryFile != null) {
try {
Files.deleteIfExists(temporaryFile);
} catch (IOException cleanupError) {
// 临时文件由系统临时目录托管;清理失败只记录路径,不改变解析结果。
LOG.warn(
"清理文档解析临时文件失败path={}",
temporaryFile,
cleanupError);
}
}
}
}
/**
* 将受限源流复制到临时文件并在返回前关闭上游连接。
*
* @param sourceRef 文档源
* @param target 临时文件
* @throws IOException 读取、限额或写入失败
*/
private void copySourceToTemporaryFile(
DocumentSourceRef sourceRef, Path target) throws IOException {
String filePath = sourceRef.getFilePath();
boolean localStorage = StringUtil.hasText(filePath)
&& !isRemoteUrl(filePath);
if (localStorage) {
try (IoBulkhead.Permit ignored =
IoBulkhead.storage().acquire("storage:document-read");
InputStream inputStream = openInputStream(sourceRef);
OutputStream outputStream = Files.newOutputStream(target)) {
copy(inputStream, outputStream);
}
return;
}
try (InputStream inputStream = openInputStream(sourceRef);
OutputStream outputStream = Files.newOutputStream(target)) {
copy(inputStream, outputStream);
}
}
/**
* 使用固定缓冲区复制流。
*
* @param inputStream 输入
* @param outputStream 输出
* @throws IOException 复制失败
*/
private void copy(
InputStream inputStream, OutputStream outputStream)
throws IOException {
byte[] buffer = new byte[64 * 1024];
int read;
while ((read = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, read);
} }
} }
private InputStream openInputStream(DocumentSourceRef sourceRef) throws IOException { private InputStream openInputStream(DocumentSourceRef sourceRef) throws IOException {
String filePath = sourceRef.getFilePath(); String filePath = sourceRef.getFilePath();
if (StringUtil.hasText(filePath) && isRemoteUrl(filePath)) { if (StringUtil.hasText(filePath) && isRemoteUrl(filePath)) {
byte[] bytes = HttpUtil.downloadBytes(filePath); return DocumentInputStreamSupport.openRemote(filePath, FILE_MAX_SINGLE_SIZE);
return new java.io.ByteArrayInputStream(bytes);
} }
if (StringUtil.hasText(filePath)) { if (StringUtil.hasText(filePath)) {
return fileStorageService.readStream(filePath); return DocumentInputStreamSupport.limit(
fileStorageService.readStream(filePath),
FILE_MAX_SINGLE_SIZE);
} }
if (StringUtil.hasText(sourceRef.getUrl())) { if (StringUtil.hasText(sourceRef.getUrl())) {
byte[] bytes = HttpUtil.downloadBytes(sourceRef.getUrl()); return DocumentInputStreamSupport.openRemote(
return new java.io.ByteArrayInputStream(bytes); sourceRef.getUrl(),
FILE_MAX_SINGLE_SIZE);
} }
throw new IOException("文件输入缺少可读取路径"); throw new IOException("文件输入缺少可读取路径");
} }
/**
* 从异常链中查找实际读取量超限异常。
*
* @param error 原始异常
* @return 超限异常;不存在时返回 null
*/
private DocumentInputStreamSupport.SizeLimitExceededException
findSizeLimitExceeded(Throwable error) {
Throwable current = error;
while (current != null) {
if (current instanceof
DocumentInputStreamSupport.SizeLimitExceededException
sizeError) {
return sizeError;
}
current = current.getCause();
}
return null;
}
/**
* 记录实际读取量超限并生成面向工作流用户的业务异常。
*
* @param sourceRef 文档源
* @param sizeError 实际读取量超限异常
* @return 业务异常
*/
private BusinessException actualSizeLimitException(
DocumentSourceRef sourceRef,
DocumentInputStreamSupport.SizeLimitExceededException sizeError) {
LOG.warn(
"工作流文档实际读取量超过限制: fileName={}, "
+ "declaredBytes={}, actualBytes={}, maxBytes={}",
sourceRef.getFileName(),
sourceRef.getSize(),
sizeError.getActualBytes(),
sizeError.getMaxBytes());
return new BusinessException(
"文件实际读取大小超过 100MB: " + sourceRef.getFileName());
}
private boolean isRemoteUrl(String value) { private boolean isRemoteUrl(String value) {
return value.startsWith("http://") || value.startsWith("https://"); return value.startsWith("http://") || value.startsWith("https://");
} }

View File

@@ -50,6 +50,31 @@ public class DocumentParseBridgeServiceImplTest {
Assert.assertFalse(parseService.lastParseRequest.getReturnImages()); Assert.assertFalse(parseService.lastParseRequest.getReturnImages());
} }
/**
* 验证工作流同步解析场景向文档加载器传递 100 MiB 上限。
*/
@Test
public void shouldApplyHundredMiBWorkflowTextLimit() {
FakePdfDocumentParseService parseService = new FakePdfDocumentParseService();
RecordingDocumentSourceLoader sourceLoader =
new RecordingDocumentSourceLoader();
DocumentParseBridgeServiceImpl bridgeService =
buildBridgeService(
parseService,
null,
null,
parseService,
sourceLoader);
bridgeService.parse(
buildSource(),
DocumentParseScenario.WORKFLOW_TEXT);
Assert.assertEquals(
100L * 1024L * 1024L,
sourceLoader.maxBytes);
}
/** /**
* 验证异步提交、状态查询和结果查询链路可用。 * 验证异步提交、状态查询和结果查询链路可用。
*/ */
@@ -171,12 +196,36 @@ public class DocumentParseBridgeServiceImplTest {
PptxDocumentParseService pptxDocumentParseService, PptxDocumentParseService pptxDocumentParseService,
XlsxDocumentParseService xlsxDocumentParseService, XlsxDocumentParseService xlsxDocumentParseService,
DocumentParseService parseService) { DocumentParseService parseService) {
return buildBridgeService(
pdfDocumentParseService,
pptxDocumentParseService,
xlsxDocumentParseService,
parseService,
new DocumentSourceLoader(new InMemoryFileStorageService()));
}
/**
* 使用指定文档加载器创建桥接服务。
*
* @param pdfDocumentParseService PDF 解析服务
* @param pptxDocumentParseService PPTX 解析服务
* @param xlsxDocumentParseService XLSX 解析服务
* @param parseService 默认解析服务
* @param sourceLoader 文档加载器
* @return 文档解析桥接服务
*/
private DocumentParseBridgeServiceImpl buildBridgeService(
PdfDocumentParseService pdfDocumentParseService,
PptxDocumentParseService pptxDocumentParseService,
XlsxDocumentParseService xlsxDocumentParseService,
DocumentParseService parseService,
DocumentSourceLoader sourceLoader) {
return new DocumentParseBridgeServiceImpl( return new DocumentParseBridgeServiceImpl(
parseService, parseService,
pdfDocumentParseService, pdfDocumentParseService,
pptxDocumentParseService, pptxDocumentParseService,
xlsxDocumentParseService, xlsxDocumentParseService,
new DocumentSourceLoader(new InMemoryFileStorageService()), sourceLoader,
new DocumentParseRequestFactory(), new DocumentParseRequestFactory(),
new DocumentParseResultMapper() new DocumentParseResultMapper()
); );
@@ -215,6 +264,33 @@ public class DocumentParseBridgeServiceImplTest {
} }
} }
/**
* 记录桥接服务传入的文档大小上限。
*/
private static class RecordingDocumentSourceLoader extends DocumentSourceLoader {
private long maxBytes;
private RecordingDocumentSourceLoader() {
super(new InMemoryFileStorageService());
}
/**
* 记录大小上限后执行真实的小文件加载。
*
* @param sourceRef 文档源
* @param maxBytes 最大允许字节数
* @return 已加载文档
*/
@Override
public tech.easyflow.ai.document.support.LoadedDocumentSource load(
DocumentSourceRef sourceRef,
long maxBytes) {
this.maxBytes = maxBytes;
return super.load(sourceRef, maxBytes);
}
}
private static class FakePdfDocumentParseService implements PdfDocumentParseService { private static class FakePdfDocumentParseService implements PdfDocumentParseService {
private ParseRequest lastParseRequest; private ParseRequest lastParseRequest;

View File

@@ -81,6 +81,64 @@ public class DocumentSourceLoaderTest {
} }
} }
/**
* 验证远端响应声明值不可信时仍按实际读取字节数拒绝超限内容。
*
* @throws IOException 测试服务启动失败时抛出
*/
@Test
public void shouldRejectRemoteContentThatExceedsActualByteLimit() throws IOException {
DocumentSourceLoader loader = new DocumentSourceLoader(new FailingFileStorageService());
HttpServer server = HttpServer.create(new InetSocketAddress(0), 0);
byte[] body = "123456789".getBytes(StandardCharsets.UTF_8);
server.createContext("/oversized.pdf", exchange -> {
exchange.sendResponseHeaders(200, 0);
exchange.getResponseBody().write(body);
exchange.close();
});
server.start();
try {
DocumentSourceRef sourceRef = new DocumentSourceRef();
sourceRef.setFileName("oversized.pdf");
sourceRef.setFilePath(
"http://127.0.0.1:" + server.getAddress().getPort() + "/oversized.pdf");
sourceRef.setSize(1L);
try {
loader.load(sourceRef, 8L);
Assert.fail("expected DocumentParseBridgeException");
} catch (DocumentParseBridgeException exception) {
Assert.assertEquals("source_load_failed", exception.getCode());
Assert.assertTrue(exception.getCause()
instanceof DocumentInputStreamSupport.SizeLimitExceededException);
}
} finally {
server.stop(0);
}
}
/**
* 验证存储元数据虚高时按实际读取量接受小文件。
*/
@Test
public void shouldUseActualBytesWhenStorageMetadataIsIncorrect() {
byte[] content = new byte[58 * 1024];
DocumentSourceLoader loader = new DocumentSourceLoader(
new IncorrectSizeFileStorageService(content));
DocumentSourceRef sourceRef = DocumentSourceRef.ofPath(
"/attachment/document.docx");
sourceRef.setSize((long) content.length);
LoadedDocumentSource loadedSource = loader.load(
sourceRef,
20L * 1024L * 1024L);
Assert.assertEquals(
content.length,
loadedSource.getSize().longValue());
Assert.assertArrayEquals(content, loadedSource.getContentBytes());
}
private static class FakeFileStorageService implements FileStorageService { private static class FakeFileStorageService implements FileStorageService {
private final byte[] content; private final byte[] content;
@@ -109,6 +167,22 @@ public class DocumentSourceLoaderTest {
} }
} }
/**
* 返回错误大小元数据的存储测试替身。
*/
private static class IncorrectSizeFileStorageService
extends FakeFileStorageService {
private IncorrectSizeFileStorageService(byte[] content) {
super(content);
}
@Override
public long getFileSize(String path) {
return 20L * 1024L * 1024L + 1L;
}
}
private static class FailingFileStorageService implements FileStorageService { private static class FailingFileStorageService implements FileStorageService {
@Override @Override

View File

@@ -203,18 +203,18 @@ public class WorkflowRunningParameterResolverTest {
} }
/** /**
* 文件参数应允许 20MB 边界值,并拒绝超过边界的文件。 * 文件参数应允许 100MB 边界值,并拒绝超过边界的文件。
* *
* @throws Exception 反射注入失败 * @throws Exception 反射注入失败
*/ */
@Test @Test
public void testNormalizeRuntimeVariablesShouldEnforceTwentyMbSingleFileLimit() throws Exception { public void testNormalizeRuntimeVariablesShouldEnforceHundredMbSingleFileLimit() throws Exception {
WorkflowRunningParameterResolver resolver = newResolver(); WorkflowRunningParameterResolver resolver = newResolver();
Map<String, Object> variables = new LinkedHashMap<>(); Map<String, Object> variables = new LinkedHashMap<>();
variables.put("attachments", fileValue( variables.put("attachments", fileValue(
"accepted.pdf", "accepted.pdf",
"/files/accepted.pdf", "/files/accepted.pdf",
20L * 1024L * 1024L 100L * 1024L * 1024L
)); ));
Map<String, Object> normalized = resolver.normalizeRuntimeVariables( Map<String, Object> normalized = resolver.normalizeRuntimeVariables(
@@ -226,13 +226,47 @@ public class WorkflowRunningParameterResolverTest {
variables.put("attachments", fileValue( variables.put("attachments", fileValue(
"oversized.pdf", "oversized.pdf",
"/files/oversized.pdf", "/files/oversized.pdf",
20L * 1024L * 1024L + 1L 100L * 1024L * 1024L + 1L
)); ));
try { try {
resolver.normalizeRuntimeVariables(workflowContentWithStartParameters(), variables); resolver.normalizeRuntimeVariables(workflowContentWithStartParameters(), variables);
Assert.fail("expected BusinessException"); Assert.fail("expected BusinessException");
} catch (BusinessException exception) { } catch (BusinessException exception) {
Assert.assertEquals("文件参数 attachments 中单个文件不能超过 20MB", exception.getMessage()); Assert.assertEquals("文件参数 attachments 中单个文件不能超过 100MB", exception.getMessage());
}
}
/**
* 文件参数总大小应允许 100MB 边界值,并拒绝超过边界的文件列表。
*
* @throws Exception 反射注入失败
*/
@Test
public void testNormalizeRuntimeVariablesShouldEnforceHundredMbTotalFileLimit() throws Exception {
WorkflowRunningParameterResolver resolver = newResolver();
Map<String, Object> variables = new LinkedHashMap<>();
variables.put("attachments", List.of(
fileValue("first.pdf", "/files/first.pdf", 50L * 1024L * 1024L),
fileValue("second.pdf", "/files/second.pdf", 50L * 1024L * 1024L)
));
Map<String, Object> normalized = resolver.normalizeRuntimeVariables(
workflowContentWithStartParameters(),
variables
);
Assert.assertEquals(2, ((List<?>) normalized.get("attachments")).size());
variables.put("attachments", List.of(
fileValue("first.pdf", "/files/first.pdf", 50L * 1024L * 1024L),
fileValue("second.pdf", "/files/second.pdf", 50L * 1024L * 1024L + 1L)
));
try {
resolver.normalizeRuntimeVariables(workflowContentWithStartParameters(), variables);
Assert.fail("expected BusinessException");
} catch (BusinessException exception) {
Assert.assertEquals(
"文件参数 attachments 的文件总大小不能超过 100MB",
exception.getMessage());
} }
} }

View File

@@ -264,10 +264,10 @@ public class DocNodeFileContentExtractorTest {
} }
/** /**
* 验证文档节点允许 20MB 边界值,并拒绝超过边界的文件。 * 验证文档节点允许 100MB 边界值,并拒绝超过边界的文件。
*/ */
@Test @Test
public void shouldEnforceTwentyMbSingleFileLimit() { public void shouldEnforceHundredMbSingleFileLimit() {
DocNodeFileContentExtractor extractor = new DocNodeFileContentExtractor( DocNodeFileContentExtractor extractor = new DocNodeFileContentExtractor(
new RecordingDocumentParseBridgeService(), new RecordingDocumentParseBridgeService(),
new FakeFileStorageService(), new FakeFileStorageService(),
@@ -278,7 +278,7 @@ public class DocNodeFileContentExtractorTest {
"/files/accepted.pdf", "/files/accepted.pdf",
"application/pdf" "application/pdf"
); );
accepted.put("size", 20L * 1024L * 1024L); accepted.put("size", 100L * 1024L * 1024L);
Assert.assertEquals(1, extractor.toDocumentSourceRefs(accepted).size()); Assert.assertEquals(1, extractor.toDocumentSourceRefs(accepted).size());
Map<String, Object> oversized = buildFileValue( Map<String, Object> oversized = buildFileValue(
@@ -286,12 +286,72 @@ public class DocNodeFileContentExtractorTest {
"/files/oversized.pdf", "/files/oversized.pdf",
"application/pdf" "application/pdf"
); );
oversized.put("size", 20L * 1024L * 1024L + 1L); oversized.put("size", 100L * 1024L * 1024L + 1L);
try { try {
extractor.toDocumentSourceRefs(oversized); extractor.toDocumentSourceRefs(oversized);
Assert.fail("expected BusinessException"); Assert.fail("expected BusinessException");
} catch (BusinessException exception) { } catch (BusinessException exception) {
Assert.assertEquals("单个文件不能超过 20MB: oversized.pdf", exception.getMessage()); Assert.assertEquals("单个文件不能超过 100MB: oversized.pdf", exception.getMessage());
}
}
/**
* 验证文档节点允许总大小 100MB 边界值,并拒绝超过边界的文件列表。
*/
@Test
public void shouldEnforceHundredMbTotalFileLimit() {
DocNodeFileContentExtractor extractor = new DocNodeFileContentExtractor(
new RecordingDocumentParseBridgeService(),
new FakeFileStorageService(),
new FakeReaderManager("plain text")
);
Map<String, Object> first = buildFileValue(
"first.pdf",
"/files/first.pdf",
"application/pdf"
);
first.put("size", 50L * 1024L * 1024L);
Map<String, Object> second = buildFileValue(
"second.pdf",
"/files/second.pdf",
"application/pdf"
);
second.put("size", 50L * 1024L * 1024L);
Assert.assertEquals(2, extractor.toDocumentSourceRefs(List.of(first, second)).size());
second.put("size", 50L * 1024L * 1024L + 1L);
try {
extractor.toDocumentSourceRefs(List.of(first, second));
Assert.fail("expected BusinessException");
} catch (BusinessException exception) {
Assert.assertEquals("文件总大小不能超过 100MB", exception.getMessage());
}
}
/**
* 验证文件元数据偏小时仍按实际读取量拒绝超过 100MB 的内容。
*/
@Test
public void shouldEnforceActualStreamSizeLimit() {
DocNodeFileContentExtractor extractor = new DocNodeFileContentExtractor(
new RecordingDocumentParseBridgeService(),
new OversizedFileStorageService(),
new DrainingReaderManager()
);
Map<String, Object> input = buildFileValue(
"oversized.txt",
"/files/oversized.txt",
"text/plain"
);
input.put("size", 1L);
try {
extractor.extract(input);
Assert.fail("expected BusinessException");
} catch (BusinessException exception) {
Assert.assertEquals(
"文件实际读取大小超过 100MB: oversized.txt",
exception.getMessage());
} }
} }
@@ -398,6 +458,74 @@ public class DocNodeFileContentExtractorTest {
} }
} }
private static class DrainingReaderManager extends ReaderManager {
@Override
public ReadDocService getReader() {
return (fileName, is) -> {
byte[] buffer = new byte[8192];
try {
while (is.read(buffer) >= 0) {
// 仅消费输入流,用于验证实际字节数保护,不保留大对象。
}
return "unreachable";
} catch (IOException e) {
throw new RuntimeException(e);
}
};
}
}
private static class OversizedFileStorageService implements FileStorageService {
@Override
public String save(org.springframework.web.multipart.MultipartFile file) {
return null;
}
@Override
public void delete(String path) {
}
@Override
public String save(File file, String prePath) {
return null;
}
@Override
public InputStream readStream(String path) {
return new InputStream() {
private long remaining = 100L * 1024L * 1024L + 1L;
@Override
public int read() {
if (remaining <= 0L) {
return -1;
}
remaining--;
return 1;
}
@Override
public int read(byte[] buffer, int offset, int length) {
if (remaining <= 0L) {
return -1;
}
int count = (int) Math.min(remaining, length);
java.util.Arrays.fill(buffer, offset, offset + count, (byte) 1);
remaining -= count;
return count;
}
};
}
@Override
public long getFileSize(String path) {
// 模拟存储元数据不准确,实际流大小必须成为最终保护边界。
return 1L;
}
}
private static class FailingFileStorageService implements FileStorageService { private static class FailingFileStorageService implements FileStorageService {
@Override @Override

View File

@@ -127,20 +127,20 @@ describe('workflowFileValue', () => {
}); });
it('上传前校验单文件大小限制', () => { it('上传前校验单文件大小限制', () => {
const acceptedFile = new File( const acceptedFile = new File([], 'accepted.pdf');
[new Uint8Array(20 * 1024 * 1024)], const oversizedFile = new File([], 'large.pdf');
'accepted.pdf', Object.defineProperty(acceptedFile, 'size', {
); value: 100 * 1024 * 1024,
const oversizedFile = new File( });
[new Uint8Array(20 * 1024 * 1024 + 1)], Object.defineProperty(oversizedFile, 'size', {
'large.pdf', value: 100 * 1024 * 1024 + 1,
); });
expect(() => expect(() =>
validateWorkflowFileSelection([], [acceptedFile]), validateWorkflowFileSelection([], [acceptedFile]),
).not.toThrow(); ).not.toThrow();
expect(() => validateWorkflowFileSelection([], [oversizedFile])).toThrow( expect(() => validateWorkflowFileSelection([], [oversizedFile])).toThrow(
'单个文件不能超过 20.0 MB', '单个文件不能超过 100.0 MB',
); );
}); });
@@ -149,13 +149,13 @@ describe('workflowFileValue', () => {
{ {
fileName: 'existing.pdf', fileName: 'existing.pdf',
filePath: 'https://example.com/existing.pdf', filePath: 'https://example.com/existing.pdf',
size: 49 * 1024 * 1024, size: 99 * 1024 * 1024,
}, },
]; ];
const incomingFile = new File([new Uint8Array(2 * 1024 * 1024)], 'new.pdf'); const incomingFile = new File([new Uint8Array(2 * 1024 * 1024)], 'new.pdf');
expect(() => expect(() =>
validateWorkflowFileSelection(currentFiles, [incomingFile]), validateWorkflowFileSelection(currentFiles, [incomingFile]),
).toThrow('文件总大小不能超过 50.0 MB'); ).toThrow('文件总大小不能超过 100.0 MB');
}); });
}); });

View File

@@ -21,8 +21,8 @@ export interface WorkflowResourceLike {
export const WORKFLOW_FILE_LIMITS = { export const WORKFLOW_FILE_LIMITS = {
maxCount: 10, maxCount: 10,
maxSingleSize: 20 * 1024 * 1024, maxSingleSize: 100 * 1024 * 1024,
maxTotalSize: 50 * 1024 * 1024, maxTotalSize: 100 * 1024 * 1024,
} as const; } as const;
/** /**

View File

@@ -21,8 +21,8 @@ export interface WorkflowResourceLike {
export const WORKFLOW_FILE_LIMITS = { export const WORKFLOW_FILE_LIMITS = {
maxCount: 10, maxCount: 10,
maxSingleSize: 20 * 1024 * 1024, maxSingleSize: 100 * 1024 * 1024,
maxTotalSize: 50 * 1024 * 1024, maxTotalSize: 100 * 1024 * 1024,
} as const; } as const;
/** /**