fix: 将工作流文档解析上限统一为 100MiB
- 前后端统一单文件与总大小限制为 100MiB - 按实际读取字节数拦截超限文档并记录大小差异 - 补充存储、远程流与边界场景测试
This commit is contained in:
@@ -43,6 +43,7 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic
|
|||||||
|
|
||||||
private static final Logger LOG = LoggerFactory.getLogger(DocumentParseBridgeServiceImpl.class);
|
private static final Logger LOG = LoggerFactory.getLogger(DocumentParseBridgeServiceImpl.class);
|
||||||
private static final String DEFAULT_DOCUMENT_PARSE_SERVICE_BEAN_NAME = "documentParseService";
|
private static final String DEFAULT_DOCUMENT_PARSE_SERVICE_BEAN_NAME = "documentParseService";
|
||||||
|
private static final long WORKFLOW_TEXT_MAX_BYTES = 100L * 1024L * 1024L;
|
||||||
|
|
||||||
@Nullable
|
@Nullable
|
||||||
private final DocumentParseService defaultDocumentParseService;
|
private final DocumentParseService defaultDocumentParseService;
|
||||||
@@ -80,7 +81,7 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic
|
|||||||
@Override
|
@Override
|
||||||
public DocumentParsedResult parse(DocumentSourceRef source, DocumentParseScenario scenario) {
|
public DocumentParsedResult parse(DocumentSourceRef source, DocumentParseScenario scenario) {
|
||||||
try {
|
try {
|
||||||
LoadedDocumentSource loadedSource = prepareSupportedSource(source);
|
LoadedDocumentSource loadedSource = prepareSupportedSource(source, scenario);
|
||||||
LOG.info("桥接服务开始同步解析文档: fileName={}, contentType={}, scenario={}",
|
LOG.info("桥接服务开始同步解析文档: fileName={}, contentType={}, scenario={}",
|
||||||
loadedSource.getFileName(), loadedSource.getContentType(), scenario);
|
loadedSource.getFileName(), loadedSource.getContentType(), scenario);
|
||||||
DocumentParseService parseService = resolveService(loadedSource);
|
DocumentParseService parseService = resolveService(loadedSource);
|
||||||
@@ -106,7 +107,7 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic
|
|||||||
@Override
|
@Override
|
||||||
public DocumentParseTaskStatus submit(DocumentSourceRef source, DocumentParseScenario scenario) {
|
public DocumentParseTaskStatus submit(DocumentSourceRef source, DocumentParseScenario scenario) {
|
||||||
try {
|
try {
|
||||||
LoadedDocumentSource loadedSource = prepareSupportedSource(source);
|
LoadedDocumentSource loadedSource = prepareSupportedSource(source, scenario);
|
||||||
LOG.info("桥接服务开始提交异步解析任务: fileName={}, contentType={}, scenario={}",
|
LOG.info("桥接服务开始提交异步解析任务: fileName={}, contentType={}, scenario={}",
|
||||||
loadedSource.getFileName(), loadedSource.getContentType(), scenario);
|
loadedSource.getFileName(), loadedSource.getContentType(), scenario);
|
||||||
DocumentParseService parseService = resolveService(loadedSource);
|
DocumentParseService parseService = resolveService(loadedSource);
|
||||||
@@ -204,8 +205,12 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic
|
|||||||
return text == null ? 0 : text.length();
|
return text == null ? 0 : text.length();
|
||||||
}
|
}
|
||||||
|
|
||||||
private LoadedDocumentSource prepareSupportedSource(DocumentSourceRef source) {
|
private LoadedDocumentSource prepareSupportedSource(DocumentSourceRef source,
|
||||||
LoadedDocumentSource loadedSource = documentSourceLoader.load(source);
|
DocumentParseScenario scenario) {
|
||||||
|
long maxBytes = scenario == DocumentParseScenario.WORKFLOW_TEXT
|
||||||
|
? WORKFLOW_TEXT_MAX_BYTES
|
||||||
|
: 0L;
|
||||||
|
LoadedDocumentSource loadedSource = documentSourceLoader.load(source, maxBytes);
|
||||||
if (!isSupportedByBridge(loadedSource)) {
|
if (!isSupportedByBridge(loadedSource)) {
|
||||||
throw DocumentParseBridgeException.unsupportedSource("统一文档解析桥接当前仅支持 PDF、DOCX、PPTX、XLSX 文件");
|
throw DocumentParseBridgeException.unsupportedSource("统一文档解析桥接当前仅支持 PDF、DOCX、PPTX、XLSX 文件");
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
package tech.easyflow.ai.document.support;
|
package tech.easyflow.ai.document.support;
|
||||||
|
|
||||||
import org.springframework.beans.factory.annotation.Qualifier;
|
import org.springframework.beans.factory.annotation.Qualifier;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
import org.springframework.util.StringUtils;
|
import org.springframework.util.StringUtils;
|
||||||
import tech.easyflow.ai.document.exception.DocumentParseBridgeException;
|
import tech.easyflow.ai.document.exception.DocumentParseBridgeException;
|
||||||
@@ -23,6 +25,9 @@ import java.net.URLConnection;
|
|||||||
@Component
|
@Component
|
||||||
public class DocumentSourceLoader {
|
public class DocumentSourceLoader {
|
||||||
|
|
||||||
|
private static final Logger LOG =
|
||||||
|
LoggerFactory.getLogger(DocumentSourceLoader.class);
|
||||||
|
|
||||||
private final FileStorageService fileStorageService;
|
private final FileStorageService fileStorageService;
|
||||||
|
|
||||||
public DocumentSourceLoader(@Qualifier("default") FileStorageService fileStorageService) {
|
public DocumentSourceLoader(@Qualifier("default") FileStorageService fileStorageService) {
|
||||||
@@ -36,38 +41,52 @@ public class DocumentSourceLoader {
|
|||||||
* @return 内部已加载文档对象
|
* @return 内部已加载文档对象
|
||||||
*/
|
*/
|
||||||
public LoadedDocumentSource load(DocumentSourceRef sourceRef) {
|
public LoadedDocumentSource load(DocumentSourceRef sourceRef) {
|
||||||
|
return load(sourceRef, 0L);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 在实际字节数保护下加载文档源。
|
||||||
|
*
|
||||||
|
* @param sourceRef easyflow 文档源
|
||||||
|
* @param maxBytes 最大允许字节数;小于等于 0 时保持原有不限大小语义
|
||||||
|
* @return 内部已加载文档对象
|
||||||
|
*/
|
||||||
|
public LoadedDocumentSource load(DocumentSourceRef sourceRef, long maxBytes) {
|
||||||
if (sourceRef == null) {
|
if (sourceRef == null) {
|
||||||
throw DocumentParseBridgeException.unsupportedSource("文档源不能为空");
|
throw DocumentParseBridgeException.unsupportedSource("文档源不能为空");
|
||||||
}
|
}
|
||||||
if (hasContentBytes(sourceRef)) {
|
if (hasContentBytes(sourceRef)) {
|
||||||
|
long actualBytes = sourceRef.getContentBytes().length;
|
||||||
|
assertWithinLimit(actualBytes, maxBytes);
|
||||||
|
logSizeMismatch(sourceRef, actualBytes);
|
||||||
return buildLoadedSource(
|
return buildLoadedSource(
|
||||||
resolveFileName(sourceRef),
|
resolveFileName(sourceRef),
|
||||||
resolveContentType(sourceRef, resolveFileName(sourceRef)),
|
resolveContentType(sourceRef, resolveFileName(sourceRef)),
|
||||||
resolveSize(sourceRef, sourceRef.getContentBytes().length),
|
actualBytes,
|
||||||
sourceRef.getContentBytes()
|
sourceRef.getContentBytes()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
if (StringUtils.hasText(sourceRef.getFilePath())) {
|
if (StringUtils.hasText(sourceRef.getFilePath())) {
|
||||||
if (isRemoteUrl(sourceRef.getFilePath())) {
|
if (isRemoteUrl(sourceRef.getFilePath())) {
|
||||||
return loadFromRemoteValue(sourceRef, sourceRef.getFilePath());
|
return loadFromRemoteValue(sourceRef, sourceRef.getFilePath(), maxBytes);
|
||||||
}
|
}
|
||||||
return loadFromFilePath(sourceRef);
|
return loadFromFilePath(sourceRef, maxBytes);
|
||||||
}
|
}
|
||||||
if (StringUtils.hasText(sourceRef.getUrl())) {
|
if (StringUtils.hasText(sourceRef.getUrl())) {
|
||||||
return loadFromUrl(sourceRef);
|
return loadFromUrl(sourceRef, maxBytes);
|
||||||
}
|
}
|
||||||
throw DocumentParseBridgeException.unsupportedSource("文档源缺少 filePath、url 或 contentBytes");
|
throw DocumentParseBridgeException.unsupportedSource("文档源缺少 filePath、url 或 contentBytes");
|
||||||
}
|
}
|
||||||
|
|
||||||
private LoadedDocumentSource loadFromFilePath(DocumentSourceRef sourceRef) {
|
private LoadedDocumentSource loadFromFilePath(DocumentSourceRef sourceRef, long maxBytes) {
|
||||||
String fileName = resolveFileName(sourceRef);
|
String fileName = resolveFileName(sourceRef);
|
||||||
try (InputStream inputStream = fileStorageService.readStream(sourceRef.getFilePath())) {
|
try (InputStream inputStream = fileStorageService.readStream(sourceRef.getFilePath())) {
|
||||||
byte[] contentBytes = inputStream.readAllBytes();
|
byte[] contentBytes = DocumentInputStreamSupport.readBytes(inputStream, maxBytes);
|
||||||
long actualSize = sourceRef.getSize() != null ? sourceRef.getSize() : fileStorageService.getFileSize(sourceRef.getFilePath());
|
logSizeMismatch(sourceRef, contentBytes.length);
|
||||||
return buildLoadedSource(
|
return buildLoadedSource(
|
||||||
fileName,
|
fileName,
|
||||||
resolveContentType(sourceRef, fileName),
|
resolveContentType(sourceRef, fileName),
|
||||||
resolveSize(sourceRef, actualSize),
|
(long) contentBytes.length,
|
||||||
contentBytes
|
contentBytes
|
||||||
);
|
);
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
@@ -78,19 +97,21 @@ public class DocumentSourceLoader {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private LoadedDocumentSource loadFromUrl(DocumentSourceRef sourceRef) {
|
private LoadedDocumentSource loadFromUrl(DocumentSourceRef sourceRef, long maxBytes) {
|
||||||
return loadFromRemoteValue(sourceRef, sourceRef.getUrl());
|
return loadFromRemoteValue(sourceRef, sourceRef.getUrl(), maxBytes);
|
||||||
}
|
}
|
||||||
|
|
||||||
private LoadedDocumentSource loadFromRemoteValue(DocumentSourceRef sourceRef, String remoteUrl) {
|
private LoadedDocumentSource loadFromRemoteValue(DocumentSourceRef sourceRef,
|
||||||
|
String remoteUrl,
|
||||||
|
long maxBytes) {
|
||||||
String fileName = resolveFileName(sourceRef);
|
String fileName = resolveFileName(sourceRef);
|
||||||
try (InputStream inputStream =
|
try (InputStream inputStream = DocumentInputStreamSupport.openRemote(remoteUrl, maxBytes)) {
|
||||||
DocumentInputStreamSupport.openRemote(remoteUrl, 0L)) {
|
byte[] contentBytes = DocumentInputStreamSupport.readBytes(inputStream, maxBytes);
|
||||||
byte[] contentBytes = DocumentInputStreamSupport.readBytes(inputStream, 0L);
|
logSizeMismatch(sourceRef, contentBytes.length);
|
||||||
return buildLoadedSource(
|
return buildLoadedSource(
|
||||||
fileName,
|
fileName,
|
||||||
resolveContentType(sourceRef, fileName),
|
resolveContentType(sourceRef, fileName),
|
||||||
resolveSize(sourceRef, contentBytes.length),
|
(long) contentBytes.length,
|
||||||
contentBytes
|
contentBytes
|
||||||
);
|
);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
@@ -132,10 +153,6 @@ public class DocumentSourceLoader {
|
|||||||
return URLConnection.guessContentTypeFromName(fileName);
|
return URLConnection.guessContentTypeFromName(fileName);
|
||||||
}
|
}
|
||||||
|
|
||||||
private Long resolveSize(DocumentSourceRef sourceRef, long fallbackSize) {
|
|
||||||
return sourceRef.getSize() != null ? sourceRef.getSize() : fallbackSize;
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean hasContentBytes(DocumentSourceRef sourceRef) {
|
private boolean hasContentBytes(DocumentSourceRef sourceRef) {
|
||||||
return sourceRef.getContentBytes() != null && sourceRef.getContentBytes().length > 0;
|
return sourceRef.getContentBytes() != null && sourceRef.getContentBytes().length > 0;
|
||||||
}
|
}
|
||||||
@@ -143,4 +160,40 @@ public class DocumentSourceLoader {
|
|||||||
private boolean isRemoteUrl(String value) {
|
private boolean isRemoteUrl(String value) {
|
||||||
return value.startsWith("http://") || value.startsWith("https://");
|
return value.startsWith("http://") || value.startsWith("https://");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 记录上传声明大小与实际读取量不一致的情况。
|
||||||
|
*
|
||||||
|
* @param sourceRef 文档源
|
||||||
|
* @param actualBytes 实际读取字节数
|
||||||
|
*/
|
||||||
|
private void logSizeMismatch(DocumentSourceRef sourceRef, long actualBytes) {
|
||||||
|
Long declaredBytes = sourceRef.getSize();
|
||||||
|
if (declaredBytes == null || declaredBytes < 0L
|
||||||
|
|| declaredBytes == actualBytes) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
LOG.warn(
|
||||||
|
"文档声明大小与实际读取量不一致: fileName={}, "
|
||||||
|
+ "declaredBytes={}, actualBytes={}",
|
||||||
|
resolveFileName(sourceRef),
|
||||||
|
declaredBytes,
|
||||||
|
actualBytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 校验已确认的实际字节数。
|
||||||
|
*
|
||||||
|
* @param actualBytes 实际字节数
|
||||||
|
* @param maxBytes 最大允许字节数
|
||||||
|
*/
|
||||||
|
private void assertWithinLimit(long actualBytes, long maxBytes) {
|
||||||
|
if (maxBytes > 0L && actualBytes > maxBytes) {
|
||||||
|
throw DocumentParseBridgeException.sourceLoadFailed(
|
||||||
|
"文档实际大小超过限制: " + maxBytes + " bytes",
|
||||||
|
new DocumentInputStreamSupport.SizeLimitExceededException(
|
||||||
|
maxBytes,
|
||||||
|
actualBytes));
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -36,8 +36,8 @@ public class WorkflowRunningParameterResolver {
|
|||||||
private static final String DEFAULT_START_FORM_DESCRIPTION = "请先补充必要信息,再开始执行工作流。";
|
private static final String DEFAULT_START_FORM_DESCRIPTION = "请先补充必要信息,再开始执行工作流。";
|
||||||
private static final String DEFAULT_START_FORM_SUBMIT_TEXT = "开始";
|
private static final String DEFAULT_START_FORM_SUBMIT_TEXT = "开始";
|
||||||
private static final int FILE_MAX_COUNT = 10;
|
private static final int FILE_MAX_COUNT = 10;
|
||||||
private static final long FILE_MAX_SINGLE_SIZE = 20L * 1024 * 1024;
|
private static final long FILE_MAX_SINGLE_SIZE = 100L * 1024 * 1024;
|
||||||
private static final long FILE_MAX_TOTAL_SIZE = 50L * 1024 * 1024;
|
private static final long FILE_MAX_TOTAL_SIZE = 100L * 1024 * 1024;
|
||||||
private static final long IMAGE_MAX_SINGLE_SIZE = 10L * 1024 * 1024;
|
private static final long IMAGE_MAX_SINGLE_SIZE = 10L * 1024 * 1024;
|
||||||
|
|
||||||
@Resource
|
@Resource
|
||||||
@@ -524,7 +524,7 @@ public class WorkflowRunningParameterResolver {
|
|||||||
}
|
}
|
||||||
Long size = parseLong(fileMap.get("size"));
|
Long size = parseLong(fileMap.get("size"));
|
||||||
if (size != null && size > FILE_MAX_SINGLE_SIZE) {
|
if (size != null && size > FILE_MAX_SINGLE_SIZE) {
|
||||||
throw new BusinessException("文件参数 " + parameterName + " 中单个文件不能超过 20MB");
|
throw new BusinessException("文件参数 " + parameterName + " 中单个文件不能超过 100MB");
|
||||||
}
|
}
|
||||||
if (size != null && size > 0) {
|
if (size != null && size > 0) {
|
||||||
totalSize += size;
|
totalSize += size;
|
||||||
@@ -536,7 +536,7 @@ public class WorkflowRunningParameterResolver {
|
|||||||
throw new BusinessException("文件参数 " + parameterName + " 最多上传 10 个文件");
|
throw new BusinessException("文件参数 " + parameterName + " 最多上传 10 个文件");
|
||||||
}
|
}
|
||||||
if (totalSize > FILE_MAX_TOTAL_SIZE) {
|
if (totalSize > FILE_MAX_TOTAL_SIZE) {
|
||||||
throw new BusinessException("文件参数 " + parameterName + " 的文件总大小不能超过 50MB");
|
throw new BusinessException("文件参数 " + parameterName + " 的文件总大小不能超过 100MB");
|
||||||
}
|
}
|
||||||
return normalized;
|
return normalized;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,12 +1,16 @@
|
|||||||
package tech.easyflow.ai.node;
|
package tech.easyflow.ai.node;
|
||||||
|
|
||||||
import cn.hutool.http.HttpUtil;
|
import com.easyagents.flow.core.util.IoBulkhead;
|
||||||
import org.springframework.beans.factory.annotation.Qualifier;
|
import org.springframework.beans.factory.annotation.Qualifier;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
import tech.easyflow.ai.document.model.DocumentParseScenario;
|
import tech.easyflow.ai.document.model.DocumentParseScenario;
|
||||||
import tech.easyflow.ai.document.model.DocumentParsedResult;
|
import tech.easyflow.ai.document.model.DocumentParsedResult;
|
||||||
import tech.easyflow.ai.document.model.DocumentSourceRef;
|
import tech.easyflow.ai.document.model.DocumentSourceRef;
|
||||||
|
import tech.easyflow.ai.document.exception.DocumentParseBridgeException;
|
||||||
import tech.easyflow.ai.document.service.DocumentParseBridgeService;
|
import tech.easyflow.ai.document.service.DocumentParseBridgeService;
|
||||||
|
import tech.easyflow.ai.document.support.DocumentInputStreamSupport;
|
||||||
import tech.easyflow.ai.document.support.DocumentParseSourceType;
|
import tech.easyflow.ai.document.support.DocumentParseSourceType;
|
||||||
import tech.easyflow.common.filestorage.FileStorageService;
|
import tech.easyflow.common.filestorage.FileStorageService;
|
||||||
import tech.easyflow.common.util.StringUtil;
|
import tech.easyflow.common.util.StringUtil;
|
||||||
@@ -14,6 +18,9 @@ import tech.easyflow.common.web.exceptions.BusinessException;
|
|||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
|
import java.io.OutputStream;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
@@ -33,9 +40,11 @@ import java.util.Set;
|
|||||||
*/
|
*/
|
||||||
@Component
|
@Component
|
||||||
public class DocNodeFileContentExtractor {
|
public class DocNodeFileContentExtractor {
|
||||||
|
private static final Logger LOG =
|
||||||
|
LoggerFactory.getLogger(DocNodeFileContentExtractor.class);
|
||||||
private static final int FILE_MAX_COUNT = 10;
|
private static final int FILE_MAX_COUNT = 10;
|
||||||
private static final long FILE_MAX_SINGLE_SIZE = 20L * 1024 * 1024;
|
private static final long FILE_MAX_SINGLE_SIZE = 100L * 1024 * 1024;
|
||||||
private static final long FILE_MAX_TOTAL_SIZE = 50L * 1024 * 1024;
|
private static final long FILE_MAX_TOTAL_SIZE = 100L * 1024 * 1024;
|
||||||
|
|
||||||
private final DocumentParseBridgeService documentParseBridgeService;
|
private final DocumentParseBridgeService documentParseBridgeService;
|
||||||
private final FileStorageService fileStorageService;
|
private final FileStorageService fileStorageService;
|
||||||
@@ -145,7 +154,7 @@ public class DocNodeFileContentExtractor {
|
|||||||
}
|
}
|
||||||
Long size = sourceRef.getSize();
|
Long size = sourceRef.getSize();
|
||||||
if (size != null && size > FILE_MAX_SINGLE_SIZE) {
|
if (size != null && size > FILE_MAX_SINGLE_SIZE) {
|
||||||
throw new BusinessException("单个文件不能超过 20MB: " + sourceRef.getFileName());
|
throw new BusinessException("单个文件不能超过 100MB: " + sourceRef.getFileName());
|
||||||
}
|
}
|
||||||
if (size != null && size > 0) {
|
if (size != null && size > 0) {
|
||||||
totalSize += size;
|
totalSize += size;
|
||||||
@@ -156,7 +165,7 @@ public class DocNodeFileContentExtractor {
|
|||||||
throw new BusinessException("最多上传 10 个文件");
|
throw new BusinessException("最多上传 10 个文件");
|
||||||
}
|
}
|
||||||
if (totalSize > FILE_MAX_TOTAL_SIZE) {
|
if (totalSize > FILE_MAX_TOTAL_SIZE) {
|
||||||
throw new BusinessException("文件总大小不能超过 50MB");
|
throw new BusinessException("文件总大小不能超过 100MB");
|
||||||
}
|
}
|
||||||
if (sourceRefs.isEmpty()) {
|
if (sourceRefs.isEmpty()) {
|
||||||
throw new BusinessException("文件输入不能为空");
|
throw new BusinessException("文件输入不能为空");
|
||||||
@@ -207,7 +216,24 @@ public class DocNodeFileContentExtractor {
|
|||||||
* @return 桥接提取出的主文本
|
* @return 桥接提取出的主文本
|
||||||
*/
|
*/
|
||||||
private String extractBridgeContent(DocumentSourceRef sourceRef) {
|
private String extractBridgeContent(DocumentSourceRef sourceRef) {
|
||||||
DocumentParsedResult parsedResult = documentParseBridgeService.parse(sourceRef, DocumentParseScenario.WORKFLOW_TEXT);
|
DocumentParsedResult parsedResult;
|
||||||
|
try (IoBulkhead.Permit ignored =
|
||||||
|
IoBulkhead.documentParse().acquire(
|
||||||
|
"document:"
|
||||||
|
+ DocumentParseSourceType.resolve(
|
||||||
|
sourceRef.getFileName(),
|
||||||
|
sourceRef.getContentType()))) {
|
||||||
|
parsedResult = documentParseBridgeService.parse(
|
||||||
|
sourceRef,
|
||||||
|
DocumentParseScenario.WORKFLOW_TEXT);
|
||||||
|
} catch (DocumentParseBridgeException error) {
|
||||||
|
DocumentInputStreamSupport.SizeLimitExceededException sizeError =
|
||||||
|
findSizeLimitExceeded(error);
|
||||||
|
if (sizeError != null) {
|
||||||
|
throw actualSizeLimitException(sourceRef, sizeError);
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
String preferredText = parsedResult == null ? null : parsedResult.getPreferredText();
|
String preferredText = parsedResult == null ? null : parsedResult.getPreferredText();
|
||||||
if (StringUtil.hasText(preferredText)) {
|
if (StringUtil.hasText(preferredText)) {
|
||||||
return preferredText;
|
return preferredText;
|
||||||
@@ -222,29 +248,155 @@ public class DocNodeFileContentExtractor {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private String extractDefaultContent(DocumentSourceRef sourceRef) {
|
private String extractDefaultContent(DocumentSourceRef sourceRef) {
|
||||||
try (InputStream inputStream = openInputStream(sourceRef)) {
|
Path temporaryFile = null;
|
||||||
return readerManager.getReader().read(sourceRef.getFileName(), inputStream);
|
try {
|
||||||
|
temporaryFile = Files.createTempFile(
|
||||||
|
"easyflow-doc-node-", ".content");
|
||||||
|
copySourceToTemporaryFile(sourceRef, temporaryFile);
|
||||||
|
/*
|
||||||
|
* 源 HTTP/对象存储流在进入解析器前已经关闭,解析过程中不会嵌套占用
|
||||||
|
* 网络或存储 lane,避免慢解析耗尽上游连接许可。
|
||||||
|
*/
|
||||||
|
try (IoBulkhead.Permit ignored =
|
||||||
|
IoBulkhead.documentParse().acquire(
|
||||||
|
"document:default-reader");
|
||||||
|
InputStream inputStream =
|
||||||
|
Files.newInputStream(temporaryFile)) {
|
||||||
|
return readerManager.getReader().read(
|
||||||
|
sourceRef.getFileName(), inputStream);
|
||||||
|
}
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
|
DocumentInputStreamSupport.SizeLimitExceededException sizeError =
|
||||||
|
findSizeLimitExceeded(e);
|
||||||
|
if (sizeError != null) {
|
||||||
|
throw actualSizeLimitException(sourceRef, sizeError);
|
||||||
|
}
|
||||||
throw new RuntimeException("读取文件内容失败: " + sourceRef.getFilePath(), e);
|
throw new RuntimeException("读取文件内容失败: " + sourceRef.getFilePath(), e);
|
||||||
|
} catch (RuntimeException e) {
|
||||||
|
DocumentInputStreamSupport.SizeLimitExceededException sizeError =
|
||||||
|
findSizeLimitExceeded(e);
|
||||||
|
if (sizeError != null) {
|
||||||
|
throw actualSizeLimitException(sourceRef, sizeError);
|
||||||
|
}
|
||||||
|
throw e;
|
||||||
|
} finally {
|
||||||
|
if (temporaryFile != null) {
|
||||||
|
try {
|
||||||
|
Files.deleteIfExists(temporaryFile);
|
||||||
|
} catch (IOException cleanupError) {
|
||||||
|
// 临时文件由系统临时目录托管;清理失败只记录路径,不改变解析结果。
|
||||||
|
LOG.warn(
|
||||||
|
"清理文档解析临时文件失败,path={}",
|
||||||
|
temporaryFile,
|
||||||
|
cleanupError);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 将受限源流复制到临时文件并在返回前关闭上游连接。
|
||||||
|
*
|
||||||
|
* @param sourceRef 文档源
|
||||||
|
* @param target 临时文件
|
||||||
|
* @throws IOException 读取、限额或写入失败
|
||||||
|
*/
|
||||||
|
private void copySourceToTemporaryFile(
|
||||||
|
DocumentSourceRef sourceRef, Path target) throws IOException {
|
||||||
|
String filePath = sourceRef.getFilePath();
|
||||||
|
boolean localStorage = StringUtil.hasText(filePath)
|
||||||
|
&& !isRemoteUrl(filePath);
|
||||||
|
if (localStorage) {
|
||||||
|
try (IoBulkhead.Permit ignored =
|
||||||
|
IoBulkhead.storage().acquire("storage:document-read");
|
||||||
|
InputStream inputStream = openInputStream(sourceRef);
|
||||||
|
OutputStream outputStream = Files.newOutputStream(target)) {
|
||||||
|
copy(inputStream, outputStream);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try (InputStream inputStream = openInputStream(sourceRef);
|
||||||
|
OutputStream outputStream = Files.newOutputStream(target)) {
|
||||||
|
copy(inputStream, outputStream);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 使用固定缓冲区复制流。
|
||||||
|
*
|
||||||
|
* @param inputStream 输入
|
||||||
|
* @param outputStream 输出
|
||||||
|
* @throws IOException 复制失败
|
||||||
|
*/
|
||||||
|
private void copy(
|
||||||
|
InputStream inputStream, OutputStream outputStream)
|
||||||
|
throws IOException {
|
||||||
|
byte[] buffer = new byte[64 * 1024];
|
||||||
|
int read;
|
||||||
|
while ((read = inputStream.read(buffer)) != -1) {
|
||||||
|
outputStream.write(buffer, 0, read);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private InputStream openInputStream(DocumentSourceRef sourceRef) throws IOException {
|
private InputStream openInputStream(DocumentSourceRef sourceRef) throws IOException {
|
||||||
String filePath = sourceRef.getFilePath();
|
String filePath = sourceRef.getFilePath();
|
||||||
if (StringUtil.hasText(filePath) && isRemoteUrl(filePath)) {
|
if (StringUtil.hasText(filePath) && isRemoteUrl(filePath)) {
|
||||||
byte[] bytes = HttpUtil.downloadBytes(filePath);
|
return DocumentInputStreamSupport.openRemote(filePath, FILE_MAX_SINGLE_SIZE);
|
||||||
return new java.io.ByteArrayInputStream(bytes);
|
|
||||||
}
|
}
|
||||||
if (StringUtil.hasText(filePath)) {
|
if (StringUtil.hasText(filePath)) {
|
||||||
return fileStorageService.readStream(filePath);
|
return DocumentInputStreamSupport.limit(
|
||||||
|
fileStorageService.readStream(filePath),
|
||||||
|
FILE_MAX_SINGLE_SIZE);
|
||||||
}
|
}
|
||||||
if (StringUtil.hasText(sourceRef.getUrl())) {
|
if (StringUtil.hasText(sourceRef.getUrl())) {
|
||||||
byte[] bytes = HttpUtil.downloadBytes(sourceRef.getUrl());
|
return DocumentInputStreamSupport.openRemote(
|
||||||
return new java.io.ByteArrayInputStream(bytes);
|
sourceRef.getUrl(),
|
||||||
|
FILE_MAX_SINGLE_SIZE);
|
||||||
}
|
}
|
||||||
throw new IOException("文件输入缺少可读取路径");
|
throw new IOException("文件输入缺少可读取路径");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 从异常链中查找实际读取量超限异常。
|
||||||
|
*
|
||||||
|
* @param error 原始异常
|
||||||
|
* @return 超限异常;不存在时返回 null
|
||||||
|
*/
|
||||||
|
private DocumentInputStreamSupport.SizeLimitExceededException
|
||||||
|
findSizeLimitExceeded(Throwable error) {
|
||||||
|
Throwable current = error;
|
||||||
|
while (current != null) {
|
||||||
|
if (current instanceof
|
||||||
|
DocumentInputStreamSupport.SizeLimitExceededException
|
||||||
|
sizeError) {
|
||||||
|
return sizeError;
|
||||||
|
}
|
||||||
|
current = current.getCause();
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 记录实际读取量超限并生成面向工作流用户的业务异常。
|
||||||
|
*
|
||||||
|
* @param sourceRef 文档源
|
||||||
|
* @param sizeError 实际读取量超限异常
|
||||||
|
* @return 业务异常
|
||||||
|
*/
|
||||||
|
private BusinessException actualSizeLimitException(
|
||||||
|
DocumentSourceRef sourceRef,
|
||||||
|
DocumentInputStreamSupport.SizeLimitExceededException sizeError) {
|
||||||
|
LOG.warn(
|
||||||
|
"工作流文档实际读取量超过限制: fileName={}, "
|
||||||
|
+ "declaredBytes={}, actualBytes={}, maxBytes={}",
|
||||||
|
sourceRef.getFileName(),
|
||||||
|
sourceRef.getSize(),
|
||||||
|
sizeError.getActualBytes(),
|
||||||
|
sizeError.getMaxBytes());
|
||||||
|
return new BusinessException(
|
||||||
|
"文件实际读取大小超过 100MB: " + sourceRef.getFileName());
|
||||||
|
}
|
||||||
|
|
||||||
private boolean isRemoteUrl(String value) {
|
private boolean isRemoteUrl(String value) {
|
||||||
return value.startsWith("http://") || value.startsWith("https://");
|
return value.startsWith("http://") || value.startsWith("https://");
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -50,6 +50,31 @@ public class DocumentParseBridgeServiceImplTest {
|
|||||||
Assert.assertFalse(parseService.lastParseRequest.getReturnImages());
|
Assert.assertFalse(parseService.lastParseRequest.getReturnImages());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证工作流同步解析场景向文档加载器传递 100 MiB 上限。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void shouldApplyHundredMiBWorkflowTextLimit() {
|
||||||
|
FakePdfDocumentParseService parseService = new FakePdfDocumentParseService();
|
||||||
|
RecordingDocumentSourceLoader sourceLoader =
|
||||||
|
new RecordingDocumentSourceLoader();
|
||||||
|
DocumentParseBridgeServiceImpl bridgeService =
|
||||||
|
buildBridgeService(
|
||||||
|
parseService,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
parseService,
|
||||||
|
sourceLoader);
|
||||||
|
|
||||||
|
bridgeService.parse(
|
||||||
|
buildSource(),
|
||||||
|
DocumentParseScenario.WORKFLOW_TEXT);
|
||||||
|
|
||||||
|
Assert.assertEquals(
|
||||||
|
100L * 1024L * 1024L,
|
||||||
|
sourceLoader.maxBytes);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 验证异步提交、状态查询和结果查询链路可用。
|
* 验证异步提交、状态查询和结果查询链路可用。
|
||||||
*/
|
*/
|
||||||
@@ -171,12 +196,36 @@ public class DocumentParseBridgeServiceImplTest {
|
|||||||
PptxDocumentParseService pptxDocumentParseService,
|
PptxDocumentParseService pptxDocumentParseService,
|
||||||
XlsxDocumentParseService xlsxDocumentParseService,
|
XlsxDocumentParseService xlsxDocumentParseService,
|
||||||
DocumentParseService parseService) {
|
DocumentParseService parseService) {
|
||||||
|
return buildBridgeService(
|
||||||
|
pdfDocumentParseService,
|
||||||
|
pptxDocumentParseService,
|
||||||
|
xlsxDocumentParseService,
|
||||||
|
parseService,
|
||||||
|
new DocumentSourceLoader(new InMemoryFileStorageService()));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 使用指定文档加载器创建桥接服务。
|
||||||
|
*
|
||||||
|
* @param pdfDocumentParseService PDF 解析服务
|
||||||
|
* @param pptxDocumentParseService PPTX 解析服务
|
||||||
|
* @param xlsxDocumentParseService XLSX 解析服务
|
||||||
|
* @param parseService 默认解析服务
|
||||||
|
* @param sourceLoader 文档加载器
|
||||||
|
* @return 文档解析桥接服务
|
||||||
|
*/
|
||||||
|
private DocumentParseBridgeServiceImpl buildBridgeService(
|
||||||
|
PdfDocumentParseService pdfDocumentParseService,
|
||||||
|
PptxDocumentParseService pptxDocumentParseService,
|
||||||
|
XlsxDocumentParseService xlsxDocumentParseService,
|
||||||
|
DocumentParseService parseService,
|
||||||
|
DocumentSourceLoader sourceLoader) {
|
||||||
return new DocumentParseBridgeServiceImpl(
|
return new DocumentParseBridgeServiceImpl(
|
||||||
parseService,
|
parseService,
|
||||||
pdfDocumentParseService,
|
pdfDocumentParseService,
|
||||||
pptxDocumentParseService,
|
pptxDocumentParseService,
|
||||||
xlsxDocumentParseService,
|
xlsxDocumentParseService,
|
||||||
new DocumentSourceLoader(new InMemoryFileStorageService()),
|
sourceLoader,
|
||||||
new DocumentParseRequestFactory(),
|
new DocumentParseRequestFactory(),
|
||||||
new DocumentParseResultMapper()
|
new DocumentParseResultMapper()
|
||||||
);
|
);
|
||||||
@@ -215,6 +264,33 @@ public class DocumentParseBridgeServiceImplTest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 记录桥接服务传入的文档大小上限。
|
||||||
|
*/
|
||||||
|
private static class RecordingDocumentSourceLoader extends DocumentSourceLoader {
|
||||||
|
|
||||||
|
private long maxBytes;
|
||||||
|
|
||||||
|
private RecordingDocumentSourceLoader() {
|
||||||
|
super(new InMemoryFileStorageService());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 记录大小上限后执行真实的小文件加载。
|
||||||
|
*
|
||||||
|
* @param sourceRef 文档源
|
||||||
|
* @param maxBytes 最大允许字节数
|
||||||
|
* @return 已加载文档
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public tech.easyflow.ai.document.support.LoadedDocumentSource load(
|
||||||
|
DocumentSourceRef sourceRef,
|
||||||
|
long maxBytes) {
|
||||||
|
this.maxBytes = maxBytes;
|
||||||
|
return super.load(sourceRef, maxBytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private static class FakePdfDocumentParseService implements PdfDocumentParseService {
|
private static class FakePdfDocumentParseService implements PdfDocumentParseService {
|
||||||
|
|
||||||
private ParseRequest lastParseRequest;
|
private ParseRequest lastParseRequest;
|
||||||
|
|||||||
@@ -81,6 +81,64 @@ public class DocumentSourceLoaderTest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证远端响应声明值不可信时仍按实际读取字节数拒绝超限内容。
|
||||||
|
*
|
||||||
|
* @throws IOException 测试服务启动失败时抛出
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void shouldRejectRemoteContentThatExceedsActualByteLimit() throws IOException {
|
||||||
|
DocumentSourceLoader loader = new DocumentSourceLoader(new FailingFileStorageService());
|
||||||
|
HttpServer server = HttpServer.create(new InetSocketAddress(0), 0);
|
||||||
|
byte[] body = "123456789".getBytes(StandardCharsets.UTF_8);
|
||||||
|
server.createContext("/oversized.pdf", exchange -> {
|
||||||
|
exchange.sendResponseHeaders(200, 0);
|
||||||
|
exchange.getResponseBody().write(body);
|
||||||
|
exchange.close();
|
||||||
|
});
|
||||||
|
server.start();
|
||||||
|
try {
|
||||||
|
DocumentSourceRef sourceRef = new DocumentSourceRef();
|
||||||
|
sourceRef.setFileName("oversized.pdf");
|
||||||
|
sourceRef.setFilePath(
|
||||||
|
"http://127.0.0.1:" + server.getAddress().getPort() + "/oversized.pdf");
|
||||||
|
sourceRef.setSize(1L);
|
||||||
|
|
||||||
|
try {
|
||||||
|
loader.load(sourceRef, 8L);
|
||||||
|
Assert.fail("expected DocumentParseBridgeException");
|
||||||
|
} catch (DocumentParseBridgeException exception) {
|
||||||
|
Assert.assertEquals("source_load_failed", exception.getCode());
|
||||||
|
Assert.assertTrue(exception.getCause()
|
||||||
|
instanceof DocumentInputStreamSupport.SizeLimitExceededException);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
server.stop(0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证存储元数据虚高时按实际读取量接受小文件。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void shouldUseActualBytesWhenStorageMetadataIsIncorrect() {
|
||||||
|
byte[] content = new byte[58 * 1024];
|
||||||
|
DocumentSourceLoader loader = new DocumentSourceLoader(
|
||||||
|
new IncorrectSizeFileStorageService(content));
|
||||||
|
DocumentSourceRef sourceRef = DocumentSourceRef.ofPath(
|
||||||
|
"/attachment/document.docx");
|
||||||
|
sourceRef.setSize((long) content.length);
|
||||||
|
|
||||||
|
LoadedDocumentSource loadedSource = loader.load(
|
||||||
|
sourceRef,
|
||||||
|
20L * 1024L * 1024L);
|
||||||
|
|
||||||
|
Assert.assertEquals(
|
||||||
|
content.length,
|
||||||
|
loadedSource.getSize().longValue());
|
||||||
|
Assert.assertArrayEquals(content, loadedSource.getContentBytes());
|
||||||
|
}
|
||||||
|
|
||||||
private static class FakeFileStorageService implements FileStorageService {
|
private static class FakeFileStorageService implements FileStorageService {
|
||||||
|
|
||||||
private final byte[] content;
|
private final byte[] content;
|
||||||
@@ -109,6 +167,22 @@ public class DocumentSourceLoaderTest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 返回错误大小元数据的存储测试替身。
|
||||||
|
*/
|
||||||
|
private static class IncorrectSizeFileStorageService
|
||||||
|
extends FakeFileStorageService {
|
||||||
|
|
||||||
|
private IncorrectSizeFileStorageService(byte[] content) {
|
||||||
|
super(content);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public long getFileSize(String path) {
|
||||||
|
return 20L * 1024L * 1024L + 1L;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private static class FailingFileStorageService implements FileStorageService {
|
private static class FailingFileStorageService implements FileStorageService {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|||||||
@@ -203,18 +203,18 @@ public class WorkflowRunningParameterResolverTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 文件参数应允许 20MB 边界值,并拒绝超过边界的文件。
|
* 文件参数应允许 100MB 边界值,并拒绝超过边界的文件。
|
||||||
*
|
*
|
||||||
* @throws Exception 反射注入失败
|
* @throws Exception 反射注入失败
|
||||||
*/
|
*/
|
||||||
@Test
|
@Test
|
||||||
public void testNormalizeRuntimeVariablesShouldEnforceTwentyMbSingleFileLimit() throws Exception {
|
public void testNormalizeRuntimeVariablesShouldEnforceHundredMbSingleFileLimit() throws Exception {
|
||||||
WorkflowRunningParameterResolver resolver = newResolver();
|
WorkflowRunningParameterResolver resolver = newResolver();
|
||||||
Map<String, Object> variables = new LinkedHashMap<>();
|
Map<String, Object> variables = new LinkedHashMap<>();
|
||||||
variables.put("attachments", fileValue(
|
variables.put("attachments", fileValue(
|
||||||
"accepted.pdf",
|
"accepted.pdf",
|
||||||
"/files/accepted.pdf",
|
"/files/accepted.pdf",
|
||||||
20L * 1024L * 1024L
|
100L * 1024L * 1024L
|
||||||
));
|
));
|
||||||
|
|
||||||
Map<String, Object> normalized = resolver.normalizeRuntimeVariables(
|
Map<String, Object> normalized = resolver.normalizeRuntimeVariables(
|
||||||
@@ -226,13 +226,47 @@ public class WorkflowRunningParameterResolverTest {
|
|||||||
variables.put("attachments", fileValue(
|
variables.put("attachments", fileValue(
|
||||||
"oversized.pdf",
|
"oversized.pdf",
|
||||||
"/files/oversized.pdf",
|
"/files/oversized.pdf",
|
||||||
20L * 1024L * 1024L + 1L
|
100L * 1024L * 1024L + 1L
|
||||||
));
|
));
|
||||||
try {
|
try {
|
||||||
resolver.normalizeRuntimeVariables(workflowContentWithStartParameters(), variables);
|
resolver.normalizeRuntimeVariables(workflowContentWithStartParameters(), variables);
|
||||||
Assert.fail("expected BusinessException");
|
Assert.fail("expected BusinessException");
|
||||||
} catch (BusinessException exception) {
|
} catch (BusinessException exception) {
|
||||||
Assert.assertEquals("文件参数 attachments 中单个文件不能超过 20MB", exception.getMessage());
|
Assert.assertEquals("文件参数 attachments 中单个文件不能超过 100MB", exception.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 文件参数总大小应允许 100MB 边界值,并拒绝超过边界的文件列表。
|
||||||
|
*
|
||||||
|
* @throws Exception 反射注入失败
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testNormalizeRuntimeVariablesShouldEnforceHundredMbTotalFileLimit() throws Exception {
|
||||||
|
WorkflowRunningParameterResolver resolver = newResolver();
|
||||||
|
Map<String, Object> variables = new LinkedHashMap<>();
|
||||||
|
variables.put("attachments", List.of(
|
||||||
|
fileValue("first.pdf", "/files/first.pdf", 50L * 1024L * 1024L),
|
||||||
|
fileValue("second.pdf", "/files/second.pdf", 50L * 1024L * 1024L)
|
||||||
|
));
|
||||||
|
|
||||||
|
Map<String, Object> normalized = resolver.normalizeRuntimeVariables(
|
||||||
|
workflowContentWithStartParameters(),
|
||||||
|
variables
|
||||||
|
);
|
||||||
|
Assert.assertEquals(2, ((List<?>) normalized.get("attachments")).size());
|
||||||
|
|
||||||
|
variables.put("attachments", List.of(
|
||||||
|
fileValue("first.pdf", "/files/first.pdf", 50L * 1024L * 1024L),
|
||||||
|
fileValue("second.pdf", "/files/second.pdf", 50L * 1024L * 1024L + 1L)
|
||||||
|
));
|
||||||
|
try {
|
||||||
|
resolver.normalizeRuntimeVariables(workflowContentWithStartParameters(), variables);
|
||||||
|
Assert.fail("expected BusinessException");
|
||||||
|
} catch (BusinessException exception) {
|
||||||
|
Assert.assertEquals(
|
||||||
|
"文件参数 attachments 的文件总大小不能超过 100MB",
|
||||||
|
exception.getMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -264,10 +264,10 @@ public class DocNodeFileContentExtractorTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 验证文档节点允许 20MB 边界值,并拒绝超过边界的文件。
|
* 验证文档节点允许 100MB 边界值,并拒绝超过边界的文件。
|
||||||
*/
|
*/
|
||||||
@Test
|
@Test
|
||||||
public void shouldEnforceTwentyMbSingleFileLimit() {
|
public void shouldEnforceHundredMbSingleFileLimit() {
|
||||||
DocNodeFileContentExtractor extractor = new DocNodeFileContentExtractor(
|
DocNodeFileContentExtractor extractor = new DocNodeFileContentExtractor(
|
||||||
new RecordingDocumentParseBridgeService(),
|
new RecordingDocumentParseBridgeService(),
|
||||||
new FakeFileStorageService(),
|
new FakeFileStorageService(),
|
||||||
@@ -278,7 +278,7 @@ public class DocNodeFileContentExtractorTest {
|
|||||||
"/files/accepted.pdf",
|
"/files/accepted.pdf",
|
||||||
"application/pdf"
|
"application/pdf"
|
||||||
);
|
);
|
||||||
accepted.put("size", 20L * 1024L * 1024L);
|
accepted.put("size", 100L * 1024L * 1024L);
|
||||||
Assert.assertEquals(1, extractor.toDocumentSourceRefs(accepted).size());
|
Assert.assertEquals(1, extractor.toDocumentSourceRefs(accepted).size());
|
||||||
|
|
||||||
Map<String, Object> oversized = buildFileValue(
|
Map<String, Object> oversized = buildFileValue(
|
||||||
@@ -286,12 +286,72 @@ public class DocNodeFileContentExtractorTest {
|
|||||||
"/files/oversized.pdf",
|
"/files/oversized.pdf",
|
||||||
"application/pdf"
|
"application/pdf"
|
||||||
);
|
);
|
||||||
oversized.put("size", 20L * 1024L * 1024L + 1L);
|
oversized.put("size", 100L * 1024L * 1024L + 1L);
|
||||||
try {
|
try {
|
||||||
extractor.toDocumentSourceRefs(oversized);
|
extractor.toDocumentSourceRefs(oversized);
|
||||||
Assert.fail("expected BusinessException");
|
Assert.fail("expected BusinessException");
|
||||||
} catch (BusinessException exception) {
|
} catch (BusinessException exception) {
|
||||||
Assert.assertEquals("单个文件不能超过 20MB: oversized.pdf", exception.getMessage());
|
Assert.assertEquals("单个文件不能超过 100MB: oversized.pdf", exception.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证文档节点允许总大小 100MB 边界值,并拒绝超过边界的文件列表。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void shouldEnforceHundredMbTotalFileLimit() {
|
||||||
|
DocNodeFileContentExtractor extractor = new DocNodeFileContentExtractor(
|
||||||
|
new RecordingDocumentParseBridgeService(),
|
||||||
|
new FakeFileStorageService(),
|
||||||
|
new FakeReaderManager("plain text")
|
||||||
|
);
|
||||||
|
Map<String, Object> first = buildFileValue(
|
||||||
|
"first.pdf",
|
||||||
|
"/files/first.pdf",
|
||||||
|
"application/pdf"
|
||||||
|
);
|
||||||
|
first.put("size", 50L * 1024L * 1024L);
|
||||||
|
Map<String, Object> second = buildFileValue(
|
||||||
|
"second.pdf",
|
||||||
|
"/files/second.pdf",
|
||||||
|
"application/pdf"
|
||||||
|
);
|
||||||
|
second.put("size", 50L * 1024L * 1024L);
|
||||||
|
Assert.assertEquals(2, extractor.toDocumentSourceRefs(List.of(first, second)).size());
|
||||||
|
|
||||||
|
second.put("size", 50L * 1024L * 1024L + 1L);
|
||||||
|
try {
|
||||||
|
extractor.toDocumentSourceRefs(List.of(first, second));
|
||||||
|
Assert.fail("expected BusinessException");
|
||||||
|
} catch (BusinessException exception) {
|
||||||
|
Assert.assertEquals("文件总大小不能超过 100MB", exception.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证文件元数据偏小时仍按实际读取量拒绝超过 100MB 的内容。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void shouldEnforceActualStreamSizeLimit() {
|
||||||
|
DocNodeFileContentExtractor extractor = new DocNodeFileContentExtractor(
|
||||||
|
new RecordingDocumentParseBridgeService(),
|
||||||
|
new OversizedFileStorageService(),
|
||||||
|
new DrainingReaderManager()
|
||||||
|
);
|
||||||
|
Map<String, Object> input = buildFileValue(
|
||||||
|
"oversized.txt",
|
||||||
|
"/files/oversized.txt",
|
||||||
|
"text/plain"
|
||||||
|
);
|
||||||
|
input.put("size", 1L);
|
||||||
|
|
||||||
|
try {
|
||||||
|
extractor.extract(input);
|
||||||
|
Assert.fail("expected BusinessException");
|
||||||
|
} catch (BusinessException exception) {
|
||||||
|
Assert.assertEquals(
|
||||||
|
"文件实际读取大小超过 100MB: oversized.txt",
|
||||||
|
exception.getMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -398,6 +458,74 @@ public class DocNodeFileContentExtractorTest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static class DrainingReaderManager extends ReaderManager {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ReadDocService getReader() {
|
||||||
|
return (fileName, is) -> {
|
||||||
|
byte[] buffer = new byte[8192];
|
||||||
|
try {
|
||||||
|
while (is.read(buffer) >= 0) {
|
||||||
|
// 仅消费输入流,用于验证实际字节数保护,不保留大对象。
|
||||||
|
}
|
||||||
|
return "unreachable";
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static class OversizedFileStorageService implements FileStorageService {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String save(org.springframework.web.multipart.MultipartFile file) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void delete(String path) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String save(File file, String prePath) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public InputStream readStream(String path) {
|
||||||
|
return new InputStream() {
|
||||||
|
private long remaining = 100L * 1024L * 1024L + 1L;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int read() {
|
||||||
|
if (remaining <= 0L) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
remaining--;
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int read(byte[] buffer, int offset, int length) {
|
||||||
|
if (remaining <= 0L) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
int count = (int) Math.min(remaining, length);
|
||||||
|
java.util.Arrays.fill(buffer, offset, offset + count, (byte) 1);
|
||||||
|
remaining -= count;
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public long getFileSize(String path) {
|
||||||
|
// 模拟存储元数据不准确,实际流大小必须成为最终保护边界。
|
||||||
|
return 1L;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private static class FailingFileStorageService implements FileStorageService {
|
private static class FailingFileStorageService implements FileStorageService {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|||||||
@@ -127,20 +127,20 @@ describe('workflowFileValue', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('上传前校验单文件大小限制', () => {
|
it('上传前校验单文件大小限制', () => {
|
||||||
const acceptedFile = new File(
|
const acceptedFile = new File([], 'accepted.pdf');
|
||||||
[new Uint8Array(20 * 1024 * 1024)],
|
const oversizedFile = new File([], 'large.pdf');
|
||||||
'accepted.pdf',
|
Object.defineProperty(acceptedFile, 'size', {
|
||||||
);
|
value: 100 * 1024 * 1024,
|
||||||
const oversizedFile = new File(
|
});
|
||||||
[new Uint8Array(20 * 1024 * 1024 + 1)],
|
Object.defineProperty(oversizedFile, 'size', {
|
||||||
'large.pdf',
|
value: 100 * 1024 * 1024 + 1,
|
||||||
);
|
});
|
||||||
|
|
||||||
expect(() =>
|
expect(() =>
|
||||||
validateWorkflowFileSelection([], [acceptedFile]),
|
validateWorkflowFileSelection([], [acceptedFile]),
|
||||||
).not.toThrow();
|
).not.toThrow();
|
||||||
expect(() => validateWorkflowFileSelection([], [oversizedFile])).toThrow(
|
expect(() => validateWorkflowFileSelection([], [oversizedFile])).toThrow(
|
||||||
'单个文件不能超过 20.0 MB',
|
'单个文件不能超过 100.0 MB',
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -149,13 +149,13 @@ describe('workflowFileValue', () => {
|
|||||||
{
|
{
|
||||||
fileName: 'existing.pdf',
|
fileName: 'existing.pdf',
|
||||||
filePath: 'https://example.com/existing.pdf',
|
filePath: 'https://example.com/existing.pdf',
|
||||||
size: 49 * 1024 * 1024,
|
size: 99 * 1024 * 1024,
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
const incomingFile = new File([new Uint8Array(2 * 1024 * 1024)], 'new.pdf');
|
const incomingFile = new File([new Uint8Array(2 * 1024 * 1024)], 'new.pdf');
|
||||||
|
|
||||||
expect(() =>
|
expect(() =>
|
||||||
validateWorkflowFileSelection(currentFiles, [incomingFile]),
|
validateWorkflowFileSelection(currentFiles, [incomingFile]),
|
||||||
).toThrow('文件总大小不能超过 50.0 MB');
|
).toThrow('文件总大小不能超过 100.0 MB');
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -21,8 +21,8 @@ export interface WorkflowResourceLike {
|
|||||||
|
|
||||||
export const WORKFLOW_FILE_LIMITS = {
|
export const WORKFLOW_FILE_LIMITS = {
|
||||||
maxCount: 10,
|
maxCount: 10,
|
||||||
maxSingleSize: 20 * 1024 * 1024,
|
maxSingleSize: 100 * 1024 * 1024,
|
||||||
maxTotalSize: 50 * 1024 * 1024,
|
maxTotalSize: 100 * 1024 * 1024,
|
||||||
} as const;
|
} as const;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -21,8 +21,8 @@ export interface WorkflowResourceLike {
|
|||||||
|
|
||||||
export const WORKFLOW_FILE_LIMITS = {
|
export const WORKFLOW_FILE_LIMITS = {
|
||||||
maxCount: 10,
|
maxCount: 10,
|
||||||
maxSingleSize: 20 * 1024 * 1024,
|
maxSingleSize: 100 * 1024 * 1024,
|
||||||
maxTotalSize: 50 * 1024 * 1024,
|
maxTotalSize: 100 * 1024 * 1024,
|
||||||
} as const;
|
} as const;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
Reference in New Issue
Block a user