fix: 将工作流文档解析上限统一为 100MiB
- 前后端统一单文件与总大小限制为 100MiB - 按实际读取字节数拦截超限文档并记录大小差异 - 补充存储、远程流与边界场景测试
This commit is contained in:
@@ -43,6 +43,7 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(DocumentParseBridgeServiceImpl.class);
|
||||
private static final String DEFAULT_DOCUMENT_PARSE_SERVICE_BEAN_NAME = "documentParseService";
|
||||
private static final long WORKFLOW_TEXT_MAX_BYTES = 100L * 1024L * 1024L;
|
||||
|
||||
@Nullable
|
||||
private final DocumentParseService defaultDocumentParseService;
|
||||
@@ -80,7 +81,7 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic
|
||||
@Override
|
||||
public DocumentParsedResult parse(DocumentSourceRef source, DocumentParseScenario scenario) {
|
||||
try {
|
||||
LoadedDocumentSource loadedSource = prepareSupportedSource(source);
|
||||
LoadedDocumentSource loadedSource = prepareSupportedSource(source, scenario);
|
||||
LOG.info("桥接服务开始同步解析文档: fileName={}, contentType={}, scenario={}",
|
||||
loadedSource.getFileName(), loadedSource.getContentType(), scenario);
|
||||
DocumentParseService parseService = resolveService(loadedSource);
|
||||
@@ -106,7 +107,7 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic
|
||||
@Override
|
||||
public DocumentParseTaskStatus submit(DocumentSourceRef source, DocumentParseScenario scenario) {
|
||||
try {
|
||||
LoadedDocumentSource loadedSource = prepareSupportedSource(source);
|
||||
LoadedDocumentSource loadedSource = prepareSupportedSource(source, scenario);
|
||||
LOG.info("桥接服务开始提交异步解析任务: fileName={}, contentType={}, scenario={}",
|
||||
loadedSource.getFileName(), loadedSource.getContentType(), scenario);
|
||||
DocumentParseService parseService = resolveService(loadedSource);
|
||||
@@ -204,8 +205,12 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic
|
||||
return text == null ? 0 : text.length();
|
||||
}
|
||||
|
||||
private LoadedDocumentSource prepareSupportedSource(DocumentSourceRef source) {
|
||||
LoadedDocumentSource loadedSource = documentSourceLoader.load(source);
|
||||
private LoadedDocumentSource prepareSupportedSource(DocumentSourceRef source,
|
||||
DocumentParseScenario scenario) {
|
||||
long maxBytes = scenario == DocumentParseScenario.WORKFLOW_TEXT
|
||||
? WORKFLOW_TEXT_MAX_BYTES
|
||||
: 0L;
|
||||
LoadedDocumentSource loadedSource = documentSourceLoader.load(source, maxBytes);
|
||||
if (!isSupportedByBridge(loadedSource)) {
|
||||
throw DocumentParseBridgeException.unsupportedSource("统一文档解析桥接当前仅支持 PDF、DOCX、PPTX、XLSX 文件");
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package tech.easyflow.ai.document.support;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.StringUtils;
|
||||
import tech.easyflow.ai.document.exception.DocumentParseBridgeException;
|
||||
@@ -23,6 +25,9 @@ import java.net.URLConnection;
|
||||
@Component
|
||||
public class DocumentSourceLoader {
|
||||
|
||||
private static final Logger LOG =
|
||||
LoggerFactory.getLogger(DocumentSourceLoader.class);
|
||||
|
||||
private final FileStorageService fileStorageService;
|
||||
|
||||
public DocumentSourceLoader(@Qualifier("default") FileStorageService fileStorageService) {
|
||||
@@ -36,38 +41,52 @@ public class DocumentSourceLoader {
|
||||
* @return 内部已加载文档对象
|
||||
*/
|
||||
public LoadedDocumentSource load(DocumentSourceRef sourceRef) {
|
||||
return load(sourceRef, 0L);
|
||||
}
|
||||
|
||||
/**
|
||||
* 在实际字节数保护下加载文档源。
|
||||
*
|
||||
* @param sourceRef easyflow 文档源
|
||||
* @param maxBytes 最大允许字节数;小于等于 0 时保持原有不限大小语义
|
||||
* @return 内部已加载文档对象
|
||||
*/
|
||||
public LoadedDocumentSource load(DocumentSourceRef sourceRef, long maxBytes) {
|
||||
if (sourceRef == null) {
|
||||
throw DocumentParseBridgeException.unsupportedSource("文档源不能为空");
|
||||
}
|
||||
if (hasContentBytes(sourceRef)) {
|
||||
long actualBytes = sourceRef.getContentBytes().length;
|
||||
assertWithinLimit(actualBytes, maxBytes);
|
||||
logSizeMismatch(sourceRef, actualBytes);
|
||||
return buildLoadedSource(
|
||||
resolveFileName(sourceRef),
|
||||
resolveContentType(sourceRef, resolveFileName(sourceRef)),
|
||||
resolveSize(sourceRef, sourceRef.getContentBytes().length),
|
||||
actualBytes,
|
||||
sourceRef.getContentBytes()
|
||||
);
|
||||
}
|
||||
if (StringUtils.hasText(sourceRef.getFilePath())) {
|
||||
if (isRemoteUrl(sourceRef.getFilePath())) {
|
||||
return loadFromRemoteValue(sourceRef, sourceRef.getFilePath());
|
||||
return loadFromRemoteValue(sourceRef, sourceRef.getFilePath(), maxBytes);
|
||||
}
|
||||
return loadFromFilePath(sourceRef);
|
||||
return loadFromFilePath(sourceRef, maxBytes);
|
||||
}
|
||||
if (StringUtils.hasText(sourceRef.getUrl())) {
|
||||
return loadFromUrl(sourceRef);
|
||||
return loadFromUrl(sourceRef, maxBytes);
|
||||
}
|
||||
throw DocumentParseBridgeException.unsupportedSource("文档源缺少 filePath、url 或 contentBytes");
|
||||
}
|
||||
|
||||
private LoadedDocumentSource loadFromFilePath(DocumentSourceRef sourceRef) {
|
||||
private LoadedDocumentSource loadFromFilePath(DocumentSourceRef sourceRef, long maxBytes) {
|
||||
String fileName = resolveFileName(sourceRef);
|
||||
try (InputStream inputStream = fileStorageService.readStream(sourceRef.getFilePath())) {
|
||||
byte[] contentBytes = inputStream.readAllBytes();
|
||||
long actualSize = sourceRef.getSize() != null ? sourceRef.getSize() : fileStorageService.getFileSize(sourceRef.getFilePath());
|
||||
byte[] contentBytes = DocumentInputStreamSupport.readBytes(inputStream, maxBytes);
|
||||
logSizeMismatch(sourceRef, contentBytes.length);
|
||||
return buildLoadedSource(
|
||||
fileName,
|
||||
resolveContentType(sourceRef, fileName),
|
||||
resolveSize(sourceRef, actualSize),
|
||||
(long) contentBytes.length,
|
||||
contentBytes
|
||||
);
|
||||
} catch (IOException e) {
|
||||
@@ -78,19 +97,21 @@ public class DocumentSourceLoader {
|
||||
}
|
||||
}
|
||||
|
||||
private LoadedDocumentSource loadFromUrl(DocumentSourceRef sourceRef) {
|
||||
return loadFromRemoteValue(sourceRef, sourceRef.getUrl());
|
||||
private LoadedDocumentSource loadFromUrl(DocumentSourceRef sourceRef, long maxBytes) {
|
||||
return loadFromRemoteValue(sourceRef, sourceRef.getUrl(), maxBytes);
|
||||
}
|
||||
|
||||
private LoadedDocumentSource loadFromRemoteValue(DocumentSourceRef sourceRef, String remoteUrl) {
|
||||
private LoadedDocumentSource loadFromRemoteValue(DocumentSourceRef sourceRef,
|
||||
String remoteUrl,
|
||||
long maxBytes) {
|
||||
String fileName = resolveFileName(sourceRef);
|
||||
try (InputStream inputStream =
|
||||
DocumentInputStreamSupport.openRemote(remoteUrl, 0L)) {
|
||||
byte[] contentBytes = DocumentInputStreamSupport.readBytes(inputStream, 0L);
|
||||
try (InputStream inputStream = DocumentInputStreamSupport.openRemote(remoteUrl, maxBytes)) {
|
||||
byte[] contentBytes = DocumentInputStreamSupport.readBytes(inputStream, maxBytes);
|
||||
logSizeMismatch(sourceRef, contentBytes.length);
|
||||
return buildLoadedSource(
|
||||
fileName,
|
||||
resolveContentType(sourceRef, fileName),
|
||||
resolveSize(sourceRef, contentBytes.length),
|
||||
(long) contentBytes.length,
|
||||
contentBytes
|
||||
);
|
||||
} catch (Exception e) {
|
||||
@@ -132,10 +153,6 @@ public class DocumentSourceLoader {
|
||||
return URLConnection.guessContentTypeFromName(fileName);
|
||||
}
|
||||
|
||||
private Long resolveSize(DocumentSourceRef sourceRef, long fallbackSize) {
|
||||
return sourceRef.getSize() != null ? sourceRef.getSize() : fallbackSize;
|
||||
}
|
||||
|
||||
private boolean hasContentBytes(DocumentSourceRef sourceRef) {
|
||||
return sourceRef.getContentBytes() != null && sourceRef.getContentBytes().length > 0;
|
||||
}
|
||||
@@ -143,4 +160,40 @@ public class DocumentSourceLoader {
|
||||
private boolean isRemoteUrl(String value) {
|
||||
return value.startsWith("http://") || value.startsWith("https://");
|
||||
}
|
||||
|
||||
/**
|
||||
* 记录上传声明大小与实际读取量不一致的情况。
|
||||
*
|
||||
* @param sourceRef 文档源
|
||||
* @param actualBytes 实际读取字节数
|
||||
*/
|
||||
private void logSizeMismatch(DocumentSourceRef sourceRef, long actualBytes) {
|
||||
Long declaredBytes = sourceRef.getSize();
|
||||
if (declaredBytes == null || declaredBytes < 0L
|
||||
|| declaredBytes == actualBytes) {
|
||||
return;
|
||||
}
|
||||
LOG.warn(
|
||||
"文档声明大小与实际读取量不一致: fileName={}, "
|
||||
+ "declaredBytes={}, actualBytes={}",
|
||||
resolveFileName(sourceRef),
|
||||
declaredBytes,
|
||||
actualBytes);
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验已确认的实际字节数。
|
||||
*
|
||||
* @param actualBytes 实际字节数
|
||||
* @param maxBytes 最大允许字节数
|
||||
*/
|
||||
private void assertWithinLimit(long actualBytes, long maxBytes) {
|
||||
if (maxBytes > 0L && actualBytes > maxBytes) {
|
||||
throw DocumentParseBridgeException.sourceLoadFailed(
|
||||
"文档实际大小超过限制: " + maxBytes + " bytes",
|
||||
new DocumentInputStreamSupport.SizeLimitExceededException(
|
||||
maxBytes,
|
||||
actualBytes));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,8 +36,8 @@ public class WorkflowRunningParameterResolver {
|
||||
private static final String DEFAULT_START_FORM_DESCRIPTION = "请先补充必要信息,再开始执行工作流。";
|
||||
private static final String DEFAULT_START_FORM_SUBMIT_TEXT = "开始";
|
||||
private static final int FILE_MAX_COUNT = 10;
|
||||
private static final long FILE_MAX_SINGLE_SIZE = 20L * 1024 * 1024;
|
||||
private static final long FILE_MAX_TOTAL_SIZE = 50L * 1024 * 1024;
|
||||
private static final long FILE_MAX_SINGLE_SIZE = 100L * 1024 * 1024;
|
||||
private static final long FILE_MAX_TOTAL_SIZE = 100L * 1024 * 1024;
|
||||
private static final long IMAGE_MAX_SINGLE_SIZE = 10L * 1024 * 1024;
|
||||
|
||||
@Resource
|
||||
@@ -524,7 +524,7 @@ public class WorkflowRunningParameterResolver {
|
||||
}
|
||||
Long size = parseLong(fileMap.get("size"));
|
||||
if (size != null && size > FILE_MAX_SINGLE_SIZE) {
|
||||
throw new BusinessException("文件参数 " + parameterName + " 中单个文件不能超过 20MB");
|
||||
throw new BusinessException("文件参数 " + parameterName + " 中单个文件不能超过 100MB");
|
||||
}
|
||||
if (size != null && size > 0) {
|
||||
totalSize += size;
|
||||
@@ -536,7 +536,7 @@ public class WorkflowRunningParameterResolver {
|
||||
throw new BusinessException("文件参数 " + parameterName + " 最多上传 10 个文件");
|
||||
}
|
||||
if (totalSize > FILE_MAX_TOTAL_SIZE) {
|
||||
throw new BusinessException("文件参数 " + parameterName + " 的文件总大小不能超过 50MB");
|
||||
throw new BusinessException("文件参数 " + parameterName + " 的文件总大小不能超过 100MB");
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
@@ -1,12 +1,16 @@
|
||||
package tech.easyflow.ai.node;
|
||||
|
||||
import cn.hutool.http.HttpUtil;
|
||||
import com.easyagents.flow.core.util.IoBulkhead;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.ai.document.model.DocumentParseScenario;
|
||||
import tech.easyflow.ai.document.model.DocumentParsedResult;
|
||||
import tech.easyflow.ai.document.model.DocumentSourceRef;
|
||||
import tech.easyflow.ai.document.exception.DocumentParseBridgeException;
|
||||
import tech.easyflow.ai.document.service.DocumentParseBridgeService;
|
||||
import tech.easyflow.ai.document.support.DocumentInputStreamSupport;
|
||||
import tech.easyflow.ai.document.support.DocumentParseSourceType;
|
||||
import tech.easyflow.common.filestorage.FileStorageService;
|
||||
import tech.easyflow.common.util.StringUtil;
|
||||
@@ -14,6 +18,9 @@ import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.LinkedHashMap;
|
||||
@@ -33,9 +40,11 @@ import java.util.Set;
|
||||
*/
|
||||
@Component
|
||||
public class DocNodeFileContentExtractor {
|
||||
private static final Logger LOG =
|
||||
LoggerFactory.getLogger(DocNodeFileContentExtractor.class);
|
||||
private static final int FILE_MAX_COUNT = 10;
|
||||
private static final long FILE_MAX_SINGLE_SIZE = 20L * 1024 * 1024;
|
||||
private static final long FILE_MAX_TOTAL_SIZE = 50L * 1024 * 1024;
|
||||
private static final long FILE_MAX_SINGLE_SIZE = 100L * 1024 * 1024;
|
||||
private static final long FILE_MAX_TOTAL_SIZE = 100L * 1024 * 1024;
|
||||
|
||||
private final DocumentParseBridgeService documentParseBridgeService;
|
||||
private final FileStorageService fileStorageService;
|
||||
@@ -145,7 +154,7 @@ public class DocNodeFileContentExtractor {
|
||||
}
|
||||
Long size = sourceRef.getSize();
|
||||
if (size != null && size > FILE_MAX_SINGLE_SIZE) {
|
||||
throw new BusinessException("单个文件不能超过 20MB: " + sourceRef.getFileName());
|
||||
throw new BusinessException("单个文件不能超过 100MB: " + sourceRef.getFileName());
|
||||
}
|
||||
if (size != null && size > 0) {
|
||||
totalSize += size;
|
||||
@@ -156,7 +165,7 @@ public class DocNodeFileContentExtractor {
|
||||
throw new BusinessException("最多上传 10 个文件");
|
||||
}
|
||||
if (totalSize > FILE_MAX_TOTAL_SIZE) {
|
||||
throw new BusinessException("文件总大小不能超过 50MB");
|
||||
throw new BusinessException("文件总大小不能超过 100MB");
|
||||
}
|
||||
if (sourceRefs.isEmpty()) {
|
||||
throw new BusinessException("文件输入不能为空");
|
||||
@@ -207,7 +216,24 @@ public class DocNodeFileContentExtractor {
|
||||
* @return 桥接提取出的主文本
|
||||
*/
|
||||
private String extractBridgeContent(DocumentSourceRef sourceRef) {
|
||||
DocumentParsedResult parsedResult = documentParseBridgeService.parse(sourceRef, DocumentParseScenario.WORKFLOW_TEXT);
|
||||
DocumentParsedResult parsedResult;
|
||||
try (IoBulkhead.Permit ignored =
|
||||
IoBulkhead.documentParse().acquire(
|
||||
"document:"
|
||||
+ DocumentParseSourceType.resolve(
|
||||
sourceRef.getFileName(),
|
||||
sourceRef.getContentType()))) {
|
||||
parsedResult = documentParseBridgeService.parse(
|
||||
sourceRef,
|
||||
DocumentParseScenario.WORKFLOW_TEXT);
|
||||
} catch (DocumentParseBridgeException error) {
|
||||
DocumentInputStreamSupport.SizeLimitExceededException sizeError =
|
||||
findSizeLimitExceeded(error);
|
||||
if (sizeError != null) {
|
||||
throw actualSizeLimitException(sourceRef, sizeError);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
String preferredText = parsedResult == null ? null : parsedResult.getPreferredText();
|
||||
if (StringUtil.hasText(preferredText)) {
|
||||
return preferredText;
|
||||
@@ -222,29 +248,155 @@ public class DocNodeFileContentExtractor {
|
||||
}
|
||||
|
||||
private String extractDefaultContent(DocumentSourceRef sourceRef) {
|
||||
try (InputStream inputStream = openInputStream(sourceRef)) {
|
||||
return readerManager.getReader().read(sourceRef.getFileName(), inputStream);
|
||||
Path temporaryFile = null;
|
||||
try {
|
||||
temporaryFile = Files.createTempFile(
|
||||
"easyflow-doc-node-", ".content");
|
||||
copySourceToTemporaryFile(sourceRef, temporaryFile);
|
||||
/*
|
||||
* 源 HTTP/对象存储流在进入解析器前已经关闭,解析过程中不会嵌套占用
|
||||
* 网络或存储 lane,避免慢解析耗尽上游连接许可。
|
||||
*/
|
||||
try (IoBulkhead.Permit ignored =
|
||||
IoBulkhead.documentParse().acquire(
|
||||
"document:default-reader");
|
||||
InputStream inputStream =
|
||||
Files.newInputStream(temporaryFile)) {
|
||||
return readerManager.getReader().read(
|
||||
sourceRef.getFileName(), inputStream);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
DocumentInputStreamSupport.SizeLimitExceededException sizeError =
|
||||
findSizeLimitExceeded(e);
|
||||
if (sizeError != null) {
|
||||
throw actualSizeLimitException(sourceRef, sizeError);
|
||||
}
|
||||
throw new RuntimeException("读取文件内容失败: " + sourceRef.getFilePath(), e);
|
||||
} catch (RuntimeException e) {
|
||||
DocumentInputStreamSupport.SizeLimitExceededException sizeError =
|
||||
findSizeLimitExceeded(e);
|
||||
if (sizeError != null) {
|
||||
throw actualSizeLimitException(sourceRef, sizeError);
|
||||
}
|
||||
throw e;
|
||||
} finally {
|
||||
if (temporaryFile != null) {
|
||||
try {
|
||||
Files.deleteIfExists(temporaryFile);
|
||||
} catch (IOException cleanupError) {
|
||||
// 临时文件由系统临时目录托管;清理失败只记录路径,不改变解析结果。
|
||||
LOG.warn(
|
||||
"清理文档解析临时文件失败,path={}",
|
||||
temporaryFile,
|
||||
cleanupError);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 将受限源流复制到临时文件并在返回前关闭上游连接。
|
||||
*
|
||||
* @param sourceRef 文档源
|
||||
* @param target 临时文件
|
||||
* @throws IOException 读取、限额或写入失败
|
||||
*/
|
||||
private void copySourceToTemporaryFile(
|
||||
DocumentSourceRef sourceRef, Path target) throws IOException {
|
||||
String filePath = sourceRef.getFilePath();
|
||||
boolean localStorage = StringUtil.hasText(filePath)
|
||||
&& !isRemoteUrl(filePath);
|
||||
if (localStorage) {
|
||||
try (IoBulkhead.Permit ignored =
|
||||
IoBulkhead.storage().acquire("storage:document-read");
|
||||
InputStream inputStream = openInputStream(sourceRef);
|
||||
OutputStream outputStream = Files.newOutputStream(target)) {
|
||||
copy(inputStream, outputStream);
|
||||
}
|
||||
return;
|
||||
}
|
||||
try (InputStream inputStream = openInputStream(sourceRef);
|
||||
OutputStream outputStream = Files.newOutputStream(target)) {
|
||||
copy(inputStream, outputStream);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用固定缓冲区复制流。
|
||||
*
|
||||
* @param inputStream 输入
|
||||
* @param outputStream 输出
|
||||
* @throws IOException 复制失败
|
||||
*/
|
||||
private void copy(
|
||||
InputStream inputStream, OutputStream outputStream)
|
||||
throws IOException {
|
||||
byte[] buffer = new byte[64 * 1024];
|
||||
int read;
|
||||
while ((read = inputStream.read(buffer)) != -1) {
|
||||
outputStream.write(buffer, 0, read);
|
||||
}
|
||||
}
|
||||
|
||||
private InputStream openInputStream(DocumentSourceRef sourceRef) throws IOException {
|
||||
String filePath = sourceRef.getFilePath();
|
||||
if (StringUtil.hasText(filePath) && isRemoteUrl(filePath)) {
|
||||
byte[] bytes = HttpUtil.downloadBytes(filePath);
|
||||
return new java.io.ByteArrayInputStream(bytes);
|
||||
return DocumentInputStreamSupport.openRemote(filePath, FILE_MAX_SINGLE_SIZE);
|
||||
}
|
||||
if (StringUtil.hasText(filePath)) {
|
||||
return fileStorageService.readStream(filePath);
|
||||
return DocumentInputStreamSupport.limit(
|
||||
fileStorageService.readStream(filePath),
|
||||
FILE_MAX_SINGLE_SIZE);
|
||||
}
|
||||
if (StringUtil.hasText(sourceRef.getUrl())) {
|
||||
byte[] bytes = HttpUtil.downloadBytes(sourceRef.getUrl());
|
||||
return new java.io.ByteArrayInputStream(bytes);
|
||||
return DocumentInputStreamSupport.openRemote(
|
||||
sourceRef.getUrl(),
|
||||
FILE_MAX_SINGLE_SIZE);
|
||||
}
|
||||
throw new IOException("文件输入缺少可读取路径");
|
||||
}
|
||||
|
||||
/**
|
||||
* 从异常链中查找实际读取量超限异常。
|
||||
*
|
||||
* @param error 原始异常
|
||||
* @return 超限异常;不存在时返回 null
|
||||
*/
|
||||
private DocumentInputStreamSupport.SizeLimitExceededException
|
||||
findSizeLimitExceeded(Throwable error) {
|
||||
Throwable current = error;
|
||||
while (current != null) {
|
||||
if (current instanceof
|
||||
DocumentInputStreamSupport.SizeLimitExceededException
|
||||
sizeError) {
|
||||
return sizeError;
|
||||
}
|
||||
current = current.getCause();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 记录实际读取量超限并生成面向工作流用户的业务异常。
|
||||
*
|
||||
* @param sourceRef 文档源
|
||||
* @param sizeError 实际读取量超限异常
|
||||
* @return 业务异常
|
||||
*/
|
||||
private BusinessException actualSizeLimitException(
|
||||
DocumentSourceRef sourceRef,
|
||||
DocumentInputStreamSupport.SizeLimitExceededException sizeError) {
|
||||
LOG.warn(
|
||||
"工作流文档实际读取量超过限制: fileName={}, "
|
||||
+ "declaredBytes={}, actualBytes={}, maxBytes={}",
|
||||
sourceRef.getFileName(),
|
||||
sourceRef.getSize(),
|
||||
sizeError.getActualBytes(),
|
||||
sizeError.getMaxBytes());
|
||||
return new BusinessException(
|
||||
"文件实际读取大小超过 100MB: " + sourceRef.getFileName());
|
||||
}
|
||||
|
||||
private boolean isRemoteUrl(String value) {
|
||||
return value.startsWith("http://") || value.startsWith("https://");
|
||||
}
|
||||
|
||||
@@ -50,6 +50,31 @@ public class DocumentParseBridgeServiceImplTest {
|
||||
Assert.assertFalse(parseService.lastParseRequest.getReturnImages());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证工作流同步解析场景向文档加载器传递 100 MiB 上限。
|
||||
*/
|
||||
@Test
|
||||
public void shouldApplyHundredMiBWorkflowTextLimit() {
|
||||
FakePdfDocumentParseService parseService = new FakePdfDocumentParseService();
|
||||
RecordingDocumentSourceLoader sourceLoader =
|
||||
new RecordingDocumentSourceLoader();
|
||||
DocumentParseBridgeServiceImpl bridgeService =
|
||||
buildBridgeService(
|
||||
parseService,
|
||||
null,
|
||||
null,
|
||||
parseService,
|
||||
sourceLoader);
|
||||
|
||||
bridgeService.parse(
|
||||
buildSource(),
|
||||
DocumentParseScenario.WORKFLOW_TEXT);
|
||||
|
||||
Assert.assertEquals(
|
||||
100L * 1024L * 1024L,
|
||||
sourceLoader.maxBytes);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证异步提交、状态查询和结果查询链路可用。
|
||||
*/
|
||||
@@ -171,12 +196,36 @@ public class DocumentParseBridgeServiceImplTest {
|
||||
PptxDocumentParseService pptxDocumentParseService,
|
||||
XlsxDocumentParseService xlsxDocumentParseService,
|
||||
DocumentParseService parseService) {
|
||||
return buildBridgeService(
|
||||
pdfDocumentParseService,
|
||||
pptxDocumentParseService,
|
||||
xlsxDocumentParseService,
|
||||
parseService,
|
||||
new DocumentSourceLoader(new InMemoryFileStorageService()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用指定文档加载器创建桥接服务。
|
||||
*
|
||||
* @param pdfDocumentParseService PDF 解析服务
|
||||
* @param pptxDocumentParseService PPTX 解析服务
|
||||
* @param xlsxDocumentParseService XLSX 解析服务
|
||||
* @param parseService 默认解析服务
|
||||
* @param sourceLoader 文档加载器
|
||||
* @return 文档解析桥接服务
|
||||
*/
|
||||
private DocumentParseBridgeServiceImpl buildBridgeService(
|
||||
PdfDocumentParseService pdfDocumentParseService,
|
||||
PptxDocumentParseService pptxDocumentParseService,
|
||||
XlsxDocumentParseService xlsxDocumentParseService,
|
||||
DocumentParseService parseService,
|
||||
DocumentSourceLoader sourceLoader) {
|
||||
return new DocumentParseBridgeServiceImpl(
|
||||
parseService,
|
||||
pdfDocumentParseService,
|
||||
pptxDocumentParseService,
|
||||
xlsxDocumentParseService,
|
||||
new DocumentSourceLoader(new InMemoryFileStorageService()),
|
||||
sourceLoader,
|
||||
new DocumentParseRequestFactory(),
|
||||
new DocumentParseResultMapper()
|
||||
);
|
||||
@@ -215,6 +264,33 @@ public class DocumentParseBridgeServiceImplTest {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 记录桥接服务传入的文档大小上限。
|
||||
*/
|
||||
private static class RecordingDocumentSourceLoader extends DocumentSourceLoader {
|
||||
|
||||
private long maxBytes;
|
||||
|
||||
private RecordingDocumentSourceLoader() {
|
||||
super(new InMemoryFileStorageService());
|
||||
}
|
||||
|
||||
/**
|
||||
* 记录大小上限后执行真实的小文件加载。
|
||||
*
|
||||
* @param sourceRef 文档源
|
||||
* @param maxBytes 最大允许字节数
|
||||
* @return 已加载文档
|
||||
*/
|
||||
@Override
|
||||
public tech.easyflow.ai.document.support.LoadedDocumentSource load(
|
||||
DocumentSourceRef sourceRef,
|
||||
long maxBytes) {
|
||||
this.maxBytes = maxBytes;
|
||||
return super.load(sourceRef, maxBytes);
|
||||
}
|
||||
}
|
||||
|
||||
private static class FakePdfDocumentParseService implements PdfDocumentParseService {
|
||||
|
||||
private ParseRequest lastParseRequest;
|
||||
|
||||
@@ -81,6 +81,64 @@ public class DocumentSourceLoaderTest {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证远端响应声明值不可信时仍按实际读取字节数拒绝超限内容。
|
||||
*
|
||||
* @throws IOException 测试服务启动失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void shouldRejectRemoteContentThatExceedsActualByteLimit() throws IOException {
|
||||
DocumentSourceLoader loader = new DocumentSourceLoader(new FailingFileStorageService());
|
||||
HttpServer server = HttpServer.create(new InetSocketAddress(0), 0);
|
||||
byte[] body = "123456789".getBytes(StandardCharsets.UTF_8);
|
||||
server.createContext("/oversized.pdf", exchange -> {
|
||||
exchange.sendResponseHeaders(200, 0);
|
||||
exchange.getResponseBody().write(body);
|
||||
exchange.close();
|
||||
});
|
||||
server.start();
|
||||
try {
|
||||
DocumentSourceRef sourceRef = new DocumentSourceRef();
|
||||
sourceRef.setFileName("oversized.pdf");
|
||||
sourceRef.setFilePath(
|
||||
"http://127.0.0.1:" + server.getAddress().getPort() + "/oversized.pdf");
|
||||
sourceRef.setSize(1L);
|
||||
|
||||
try {
|
||||
loader.load(sourceRef, 8L);
|
||||
Assert.fail("expected DocumentParseBridgeException");
|
||||
} catch (DocumentParseBridgeException exception) {
|
||||
Assert.assertEquals("source_load_failed", exception.getCode());
|
||||
Assert.assertTrue(exception.getCause()
|
||||
instanceof DocumentInputStreamSupport.SizeLimitExceededException);
|
||||
}
|
||||
} finally {
|
||||
server.stop(0);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证存储元数据虚高时按实际读取量接受小文件。
|
||||
*/
|
||||
@Test
|
||||
public void shouldUseActualBytesWhenStorageMetadataIsIncorrect() {
|
||||
byte[] content = new byte[58 * 1024];
|
||||
DocumentSourceLoader loader = new DocumentSourceLoader(
|
||||
new IncorrectSizeFileStorageService(content));
|
||||
DocumentSourceRef sourceRef = DocumentSourceRef.ofPath(
|
||||
"/attachment/document.docx");
|
||||
sourceRef.setSize((long) content.length);
|
||||
|
||||
LoadedDocumentSource loadedSource = loader.load(
|
||||
sourceRef,
|
||||
20L * 1024L * 1024L);
|
||||
|
||||
Assert.assertEquals(
|
||||
content.length,
|
||||
loadedSource.getSize().longValue());
|
||||
Assert.assertArrayEquals(content, loadedSource.getContentBytes());
|
||||
}
|
||||
|
||||
private static class FakeFileStorageService implements FileStorageService {
|
||||
|
||||
private final byte[] content;
|
||||
@@ -109,6 +167,22 @@ public class DocumentSourceLoaderTest {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 返回错误大小元数据的存储测试替身。
|
||||
*/
|
||||
private static class IncorrectSizeFileStorageService
|
||||
extends FakeFileStorageService {
|
||||
|
||||
private IncorrectSizeFileStorageService(byte[] content) {
|
||||
super(content);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getFileSize(String path) {
|
||||
return 20L * 1024L * 1024L + 1L;
|
||||
}
|
||||
}
|
||||
|
||||
private static class FailingFileStorageService implements FileStorageService {
|
||||
|
||||
@Override
|
||||
|
||||
@@ -203,18 +203,18 @@ public class WorkflowRunningParameterResolverTest {
|
||||
}
|
||||
|
||||
/**
|
||||
* 文件参数应允许 20MB 边界值,并拒绝超过边界的文件。
|
||||
* 文件参数应允许 100MB 边界值,并拒绝超过边界的文件。
|
||||
*
|
||||
* @throws Exception 反射注入失败
|
||||
*/
|
||||
@Test
|
||||
public void testNormalizeRuntimeVariablesShouldEnforceTwentyMbSingleFileLimit() throws Exception {
|
||||
public void testNormalizeRuntimeVariablesShouldEnforceHundredMbSingleFileLimit() throws Exception {
|
||||
WorkflowRunningParameterResolver resolver = newResolver();
|
||||
Map<String, Object> variables = new LinkedHashMap<>();
|
||||
variables.put("attachments", fileValue(
|
||||
"accepted.pdf",
|
||||
"/files/accepted.pdf",
|
||||
20L * 1024L * 1024L
|
||||
100L * 1024L * 1024L
|
||||
));
|
||||
|
||||
Map<String, Object> normalized = resolver.normalizeRuntimeVariables(
|
||||
@@ -226,13 +226,47 @@ public class WorkflowRunningParameterResolverTest {
|
||||
variables.put("attachments", fileValue(
|
||||
"oversized.pdf",
|
||||
"/files/oversized.pdf",
|
||||
20L * 1024L * 1024L + 1L
|
||||
100L * 1024L * 1024L + 1L
|
||||
));
|
||||
try {
|
||||
resolver.normalizeRuntimeVariables(workflowContentWithStartParameters(), variables);
|
||||
Assert.fail("expected BusinessException");
|
||||
} catch (BusinessException exception) {
|
||||
Assert.assertEquals("文件参数 attachments 中单个文件不能超过 20MB", exception.getMessage());
|
||||
Assert.assertEquals("文件参数 attachments 中单个文件不能超过 100MB", exception.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 文件参数总大小应允许 100MB 边界值,并拒绝超过边界的文件列表。
|
||||
*
|
||||
* @throws Exception 反射注入失败
|
||||
*/
|
||||
@Test
|
||||
public void testNormalizeRuntimeVariablesShouldEnforceHundredMbTotalFileLimit() throws Exception {
|
||||
WorkflowRunningParameterResolver resolver = newResolver();
|
||||
Map<String, Object> variables = new LinkedHashMap<>();
|
||||
variables.put("attachments", List.of(
|
||||
fileValue("first.pdf", "/files/first.pdf", 50L * 1024L * 1024L),
|
||||
fileValue("second.pdf", "/files/second.pdf", 50L * 1024L * 1024L)
|
||||
));
|
||||
|
||||
Map<String, Object> normalized = resolver.normalizeRuntimeVariables(
|
||||
workflowContentWithStartParameters(),
|
||||
variables
|
||||
);
|
||||
Assert.assertEquals(2, ((List<?>) normalized.get("attachments")).size());
|
||||
|
||||
variables.put("attachments", List.of(
|
||||
fileValue("first.pdf", "/files/first.pdf", 50L * 1024L * 1024L),
|
||||
fileValue("second.pdf", "/files/second.pdf", 50L * 1024L * 1024L + 1L)
|
||||
));
|
||||
try {
|
||||
resolver.normalizeRuntimeVariables(workflowContentWithStartParameters(), variables);
|
||||
Assert.fail("expected BusinessException");
|
||||
} catch (BusinessException exception) {
|
||||
Assert.assertEquals(
|
||||
"文件参数 attachments 的文件总大小不能超过 100MB",
|
||||
exception.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -264,10 +264,10 @@ public class DocNodeFileContentExtractorTest {
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证文档节点允许 20MB 边界值,并拒绝超过边界的文件。
|
||||
* 验证文档节点允许 100MB 边界值,并拒绝超过边界的文件。
|
||||
*/
|
||||
@Test
|
||||
public void shouldEnforceTwentyMbSingleFileLimit() {
|
||||
public void shouldEnforceHundredMbSingleFileLimit() {
|
||||
DocNodeFileContentExtractor extractor = new DocNodeFileContentExtractor(
|
||||
new RecordingDocumentParseBridgeService(),
|
||||
new FakeFileStorageService(),
|
||||
@@ -278,7 +278,7 @@ public class DocNodeFileContentExtractorTest {
|
||||
"/files/accepted.pdf",
|
||||
"application/pdf"
|
||||
);
|
||||
accepted.put("size", 20L * 1024L * 1024L);
|
||||
accepted.put("size", 100L * 1024L * 1024L);
|
||||
Assert.assertEquals(1, extractor.toDocumentSourceRefs(accepted).size());
|
||||
|
||||
Map<String, Object> oversized = buildFileValue(
|
||||
@@ -286,12 +286,72 @@ public class DocNodeFileContentExtractorTest {
|
||||
"/files/oversized.pdf",
|
||||
"application/pdf"
|
||||
);
|
||||
oversized.put("size", 20L * 1024L * 1024L + 1L);
|
||||
oversized.put("size", 100L * 1024L * 1024L + 1L);
|
||||
try {
|
||||
extractor.toDocumentSourceRefs(oversized);
|
||||
Assert.fail("expected BusinessException");
|
||||
} catch (BusinessException exception) {
|
||||
Assert.assertEquals("单个文件不能超过 20MB: oversized.pdf", exception.getMessage());
|
||||
Assert.assertEquals("单个文件不能超过 100MB: oversized.pdf", exception.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证文档节点允许总大小 100MB 边界值,并拒绝超过边界的文件列表。
|
||||
*/
|
||||
@Test
|
||||
public void shouldEnforceHundredMbTotalFileLimit() {
|
||||
DocNodeFileContentExtractor extractor = new DocNodeFileContentExtractor(
|
||||
new RecordingDocumentParseBridgeService(),
|
||||
new FakeFileStorageService(),
|
||||
new FakeReaderManager("plain text")
|
||||
);
|
||||
Map<String, Object> first = buildFileValue(
|
||||
"first.pdf",
|
||||
"/files/first.pdf",
|
||||
"application/pdf"
|
||||
);
|
||||
first.put("size", 50L * 1024L * 1024L);
|
||||
Map<String, Object> second = buildFileValue(
|
||||
"second.pdf",
|
||||
"/files/second.pdf",
|
||||
"application/pdf"
|
||||
);
|
||||
second.put("size", 50L * 1024L * 1024L);
|
||||
Assert.assertEquals(2, extractor.toDocumentSourceRefs(List.of(first, second)).size());
|
||||
|
||||
second.put("size", 50L * 1024L * 1024L + 1L);
|
||||
try {
|
||||
extractor.toDocumentSourceRefs(List.of(first, second));
|
||||
Assert.fail("expected BusinessException");
|
||||
} catch (BusinessException exception) {
|
||||
Assert.assertEquals("文件总大小不能超过 100MB", exception.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证文件元数据偏小时仍按实际读取量拒绝超过 100MB 的内容。
|
||||
*/
|
||||
@Test
|
||||
public void shouldEnforceActualStreamSizeLimit() {
|
||||
DocNodeFileContentExtractor extractor = new DocNodeFileContentExtractor(
|
||||
new RecordingDocumentParseBridgeService(),
|
||||
new OversizedFileStorageService(),
|
||||
new DrainingReaderManager()
|
||||
);
|
||||
Map<String, Object> input = buildFileValue(
|
||||
"oversized.txt",
|
||||
"/files/oversized.txt",
|
||||
"text/plain"
|
||||
);
|
||||
input.put("size", 1L);
|
||||
|
||||
try {
|
||||
extractor.extract(input);
|
||||
Assert.fail("expected BusinessException");
|
||||
} catch (BusinessException exception) {
|
||||
Assert.assertEquals(
|
||||
"文件实际读取大小超过 100MB: oversized.txt",
|
||||
exception.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -398,6 +458,74 @@ public class DocNodeFileContentExtractorTest {
|
||||
}
|
||||
}
|
||||
|
||||
private static class DrainingReaderManager extends ReaderManager {
|
||||
|
||||
@Override
|
||||
public ReadDocService getReader() {
|
||||
return (fileName, is) -> {
|
||||
byte[] buffer = new byte[8192];
|
||||
try {
|
||||
while (is.read(buffer) >= 0) {
|
||||
// 仅消费输入流,用于验证实际字节数保护,不保留大对象。
|
||||
}
|
||||
return "unreachable";
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
private static class OversizedFileStorageService implements FileStorageService {
|
||||
|
||||
@Override
|
||||
public String save(org.springframework.web.multipart.MultipartFile file) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void delete(String path) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public String save(File file, String prePath) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public InputStream readStream(String path) {
|
||||
return new InputStream() {
|
||||
private long remaining = 100L * 1024L * 1024L + 1L;
|
||||
|
||||
@Override
|
||||
public int read() {
|
||||
if (remaining <= 0L) {
|
||||
return -1;
|
||||
}
|
||||
remaining--;
|
||||
return 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read(byte[] buffer, int offset, int length) {
|
||||
if (remaining <= 0L) {
|
||||
return -1;
|
||||
}
|
||||
int count = (int) Math.min(remaining, length);
|
||||
java.util.Arrays.fill(buffer, offset, offset + count, (byte) 1);
|
||||
remaining -= count;
|
||||
return count;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getFileSize(String path) {
|
||||
// 模拟存储元数据不准确,实际流大小必须成为最终保护边界。
|
||||
return 1L;
|
||||
}
|
||||
}
|
||||
|
||||
private static class FailingFileStorageService implements FileStorageService {
|
||||
|
||||
@Override
|
||||
|
||||
Reference in New Issue
Block a user