From 5c29ca94074572d7a92d1850bf50d46f8f6720ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Fri, 31 Jul 2026 14:52:06 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E5=B0=86=E5=B7=A5=E4=BD=9C=E6=B5=81?= =?UTF-8?q?=E6=96=87=E6=A1=A3=E8=A7=A3=E6=9E=90=E4=B8=8A=E9=99=90=E7=BB=9F?= =?UTF-8?q?=E4=B8=80=E4=B8=BA=20100MiB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 前后端统一单文件与总大小限制为 100MiB - 按实际读取字节数拦截超限文档并记录大小差异 - 补充存储、远程流与边界场景测试 --- .../impl/DocumentParseBridgeServiceImpl.java | 13 +- .../support/DocumentSourceLoader.java | 91 +++++++-- .../WorkflowRunningParameterResolver.java | 8 +- .../ai/node/DocNodeFileContentExtractor.java | 178 ++++++++++++++++-- .../DocumentParseBridgeServiceImplTest.java | 78 +++++++- .../support/DocumentSourceLoaderTest.java | 74 ++++++++ .../WorkflowRunningParameterResolverTest.java | 44 ++++- .../node/DocNodeFileContentExtractorTest.java | 138 +++++++++++++- .../__tests__/workflowFileValue.test.ts | 22 +-- .../workflow/components/workflowFileValue.ts | 4 +- .../workflow/components/workflowFileValue.ts | 4 +- 11 files changed, 588 insertions(+), 66 deletions(-) diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/document/service/impl/DocumentParseBridgeServiceImpl.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/document/service/impl/DocumentParseBridgeServiceImpl.java index d0dc9f09..02af4fc5 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/document/service/impl/DocumentParseBridgeServiceImpl.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/document/service/impl/DocumentParseBridgeServiceImpl.java @@ -43,6 +43,7 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic private static final Logger LOG = LoggerFactory.getLogger(DocumentParseBridgeServiceImpl.class); private static final String DEFAULT_DOCUMENT_PARSE_SERVICE_BEAN_NAME = "documentParseService"; + private static final long WORKFLOW_TEXT_MAX_BYTES = 100L * 1024L * 1024L; @Nullable private final DocumentParseService defaultDocumentParseService; @@ -80,7 +81,7 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic @Override public DocumentParsedResult parse(DocumentSourceRef source, DocumentParseScenario scenario) { try { - LoadedDocumentSource loadedSource = prepareSupportedSource(source); + LoadedDocumentSource loadedSource = prepareSupportedSource(source, scenario); LOG.info("桥接服务开始同步解析文档: fileName={}, contentType={}, scenario={}", loadedSource.getFileName(), loadedSource.getContentType(), scenario); DocumentParseService parseService = resolveService(loadedSource); @@ -106,7 +107,7 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic @Override public DocumentParseTaskStatus submit(DocumentSourceRef source, DocumentParseScenario scenario) { try { - LoadedDocumentSource loadedSource = prepareSupportedSource(source); + LoadedDocumentSource loadedSource = prepareSupportedSource(source, scenario); LOG.info("桥接服务开始提交异步解析任务: fileName={}, contentType={}, scenario={}", loadedSource.getFileName(), loadedSource.getContentType(), scenario); DocumentParseService parseService = resolveService(loadedSource); @@ -204,8 +205,12 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic return text == null ? 0 : text.length(); } - private LoadedDocumentSource prepareSupportedSource(DocumentSourceRef source) { - LoadedDocumentSource loadedSource = documentSourceLoader.load(source); + private LoadedDocumentSource prepareSupportedSource(DocumentSourceRef source, + DocumentParseScenario scenario) { + long maxBytes = scenario == DocumentParseScenario.WORKFLOW_TEXT + ? WORKFLOW_TEXT_MAX_BYTES + : 0L; + LoadedDocumentSource loadedSource = documentSourceLoader.load(source, maxBytes); if (!isSupportedByBridge(loadedSource)) { throw DocumentParseBridgeException.unsupportedSource("统一文档解析桥接当前仅支持 PDF、DOCX、PPTX、XLSX 文件"); } diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/document/support/DocumentSourceLoader.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/document/support/DocumentSourceLoader.java index 3cf8d0c6..50996301 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/document/support/DocumentSourceLoader.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/document/support/DocumentSourceLoader.java @@ -1,6 +1,8 @@ package tech.easyflow.ai.document.support; import org.springframework.beans.factory.annotation.Qualifier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; import tech.easyflow.ai.document.exception.DocumentParseBridgeException; @@ -23,6 +25,9 @@ import java.net.URLConnection; @Component public class DocumentSourceLoader { + private static final Logger LOG = + LoggerFactory.getLogger(DocumentSourceLoader.class); + private final FileStorageService fileStorageService; public DocumentSourceLoader(@Qualifier("default") FileStorageService fileStorageService) { @@ -36,38 +41,52 @@ public class DocumentSourceLoader { * @return 内部已加载文档对象 */ public LoadedDocumentSource load(DocumentSourceRef sourceRef) { + return load(sourceRef, 0L); + } + + /** + * 在实际字节数保护下加载文档源。 + * + * @param sourceRef easyflow 文档源 + * @param maxBytes 最大允许字节数；小于等于 0 时保持原有不限大小语义 + * @return 内部已加载文档对象 + */ + public LoadedDocumentSource load(DocumentSourceRef sourceRef, long maxBytes) { if (sourceRef == null) { throw DocumentParseBridgeException.unsupportedSource("文档源不能为空"); } if (hasContentBytes(sourceRef)) { + long actualBytes = sourceRef.getContentBytes().length; + assertWithinLimit(actualBytes, maxBytes); + logSizeMismatch(sourceRef, actualBytes); return buildLoadedSource( resolveFileName(sourceRef), resolveContentType(sourceRef, resolveFileName(sourceRef)), - resolveSize(sourceRef, sourceRef.getContentBytes().length), + actualBytes, sourceRef.getContentBytes() ); } if (StringUtils.hasText(sourceRef.getFilePath())) { if (isRemoteUrl(sourceRef.getFilePath())) { - return loadFromRemoteValue(sourceRef, sourceRef.getFilePath()); + return loadFromRemoteValue(sourceRef, sourceRef.getFilePath(), maxBytes); } - return loadFromFilePath(sourceRef); + return loadFromFilePath(sourceRef, maxBytes); } if (StringUtils.hasText(sourceRef.getUrl())) { - return loadFromUrl(sourceRef); + return loadFromUrl(sourceRef, maxBytes); } throw DocumentParseBridgeException.unsupportedSource("文档源缺少 filePath、url 或 contentBytes"); } - private LoadedDocumentSource loadFromFilePath(DocumentSourceRef sourceRef) { + private LoadedDocumentSource loadFromFilePath(DocumentSourceRef sourceRef, long maxBytes) { String fileName = resolveFileName(sourceRef); try (InputStream inputStream = fileStorageService.readStream(sourceRef.getFilePath())) { - byte[] contentBytes = inputStream.readAllBytes(); - long actualSize = sourceRef.getSize() != null ? sourceRef.getSize() : fileStorageService.getFileSize(sourceRef.getFilePath()); + byte[] contentBytes = DocumentInputStreamSupport.readBytes(inputStream, maxBytes); + logSizeMismatch(sourceRef, contentBytes.length); return buildLoadedSource( fileName, resolveContentType(sourceRef, fileName), - resolveSize(sourceRef, actualSize), + (long) contentBytes.length, contentBytes ); } catch (IOException e) { @@ -78,19 +97,21 @@ public class DocumentSourceLoader { } } - private LoadedDocumentSource loadFromUrl(DocumentSourceRef sourceRef) { - return loadFromRemoteValue(sourceRef, sourceRef.getUrl()); + private LoadedDocumentSource loadFromUrl(DocumentSourceRef sourceRef, long maxBytes) { + return loadFromRemoteValue(sourceRef, sourceRef.getUrl(), maxBytes); } - private LoadedDocumentSource loadFromRemoteValue(DocumentSourceRef sourceRef, String remoteUrl) { + private LoadedDocumentSource loadFromRemoteValue(DocumentSourceRef sourceRef, + String remoteUrl, + long maxBytes) { String fileName = resolveFileName(sourceRef); - try (InputStream inputStream = - DocumentInputStreamSupport.openRemote(remoteUrl, 0L)) { - byte[] contentBytes = DocumentInputStreamSupport.readBytes(inputStream, 0L); + try (InputStream inputStream = DocumentInputStreamSupport.openRemote(remoteUrl, maxBytes)) { + byte[] contentBytes = DocumentInputStreamSupport.readBytes(inputStream, maxBytes); + logSizeMismatch(sourceRef, contentBytes.length); return buildLoadedSource( fileName, resolveContentType(sourceRef, fileName), - resolveSize(sourceRef, contentBytes.length), + (long) contentBytes.length, contentBytes ); } catch (Exception e) { @@ -132,10 +153,6 @@ public class DocumentSourceLoader { return URLConnection.guessContentTypeFromName(fileName); } - private Long resolveSize(DocumentSourceRef sourceRef, long fallbackSize) { - return sourceRef.getSize() != null ? sourceRef.getSize() : fallbackSize; - } - private boolean hasContentBytes(DocumentSourceRef sourceRef) { return sourceRef.getContentBytes() != null && sourceRef.getContentBytes().length > 0; } @@ -143,4 +160,40 @@ public class DocumentSourceLoader { private boolean isRemoteUrl(String value) { return value.startsWith("http://") || value.startsWith("https://"); } + + /** + * 记录上传声明大小与实际读取量不一致的情况。 + * + * @param sourceRef 文档源 + * @param actualBytes 实际读取字节数 + */ + private void logSizeMismatch(DocumentSourceRef sourceRef, long actualBytes) { + Long declaredBytes = sourceRef.getSize(); + if (declaredBytes == null || declaredBytes < 0L + || declaredBytes == actualBytes) { + return; + } + LOG.warn( + "文档声明大小与实际读取量不一致: fileName={}, " + + "declaredBytes={}, actualBytes={}", + resolveFileName(sourceRef), + declaredBytes, + actualBytes); + } + + /** + * 校验已确认的实际字节数。 + * + * @param actualBytes 实际字节数 + * @param maxBytes 最大允许字节数 + */ + private void assertWithinLimit(long actualBytes, long maxBytes) { + if (maxBytes > 0L && actualBytes > maxBytes) { + throw DocumentParseBridgeException.sourceLoadFailed( + "文档实际大小超过限制: " + maxBytes + " bytes", + new DocumentInputStreamSupport.SizeLimitExceededException( + maxBytes, + actualBytes)); + } + } } diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/service/WorkflowRunningParameterResolver.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/service/WorkflowRunningParameterResolver.java index acfe9ba5..9d939fb6 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/service/WorkflowRunningParameterResolver.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/service/WorkflowRunningParameterResolver.java @@ -36,8 +36,8 @@ public class WorkflowRunningParameterResolver { private static final String DEFAULT_START_FORM_DESCRIPTION = "请先补充必要信息，再开始执行工作流。"; private static final String DEFAULT_START_FORM_SUBMIT_TEXT = "开始"; private static final int FILE_MAX_COUNT = 10; - private static final long FILE_MAX_SINGLE_SIZE = 20L * 1024 * 1024; - private static final long FILE_MAX_TOTAL_SIZE = 50L * 1024 * 1024; + private static final long FILE_MAX_SINGLE_SIZE = 100L * 1024 * 1024; + private static final long FILE_MAX_TOTAL_SIZE = 100L * 1024 * 1024; private static final long IMAGE_MAX_SINGLE_SIZE = 10L * 1024 * 1024; @Resource @@ -524,7 +524,7 @@ public class WorkflowRunningParameterResolver { } Long size = parseLong(fileMap.get("size")); if (size != null && size > FILE_MAX_SINGLE_SIZE) { - throw new BusinessException("文件参数 " + parameterName + " 中单个文件不能超过 20MB"); + throw new BusinessException("文件参数 " + parameterName + " 中单个文件不能超过 100MB"); } if (size != null && size > 0) { totalSize += size; @@ -536,7 +536,7 @@ public class WorkflowRunningParameterResolver { throw new BusinessException("文件参数 " + parameterName + " 最多上传 10 个文件"); } if (totalSize > FILE_MAX_TOTAL_SIZE) { - throw new BusinessException("文件参数 " + parameterName + " 的文件总大小不能超过 50MB"); + throw new BusinessException("文件参数 " + parameterName + " 的文件总大小不能超过 100MB"); } return normalized; } diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/DocNodeFileContentExtractor.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/DocNodeFileContentExtractor.java index adfb31d7..3458a916 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/DocNodeFileContentExtractor.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/DocNodeFileContentExtractor.java @@ -1,12 +1,16 @@ package tech.easyflow.ai.node; -import cn.hutool.http.HttpUtil; +import com.easyagents.flow.core.util.IoBulkhead; import org.springframework.beans.factory.annotation.Qualifier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import tech.easyflow.ai.document.model.DocumentParseScenario; import tech.easyflow.ai.document.model.DocumentParsedResult; import tech.easyflow.ai.document.model.DocumentSourceRef; +import tech.easyflow.ai.document.exception.DocumentParseBridgeException; import tech.easyflow.ai.document.service.DocumentParseBridgeService; +import tech.easyflow.ai.document.support.DocumentInputStreamSupport; import tech.easyflow.ai.document.support.DocumentParseSourceType; import tech.easyflow.common.filestorage.FileStorageService; import tech.easyflow.common.util.StringUtil; @@ -14,6 +18,9 @@ import tech.easyflow.common.web.exceptions.BusinessException; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.LinkedHashMap; @@ -33,9 +40,11 @@ import java.util.Set; */ @Component public class DocNodeFileContentExtractor { + private static final Logger LOG = + LoggerFactory.getLogger(DocNodeFileContentExtractor.class); private static final int FILE_MAX_COUNT = 10; - private static final long FILE_MAX_SINGLE_SIZE = 20L * 1024 * 1024; - private static final long FILE_MAX_TOTAL_SIZE = 50L * 1024 * 1024; + private static final long FILE_MAX_SINGLE_SIZE = 100L * 1024 * 1024; + private static final long FILE_MAX_TOTAL_SIZE = 100L * 1024 * 1024; private final DocumentParseBridgeService documentParseBridgeService; private final FileStorageService fileStorageService; @@ -145,7 +154,7 @@ public class DocNodeFileContentExtractor { } Long size = sourceRef.getSize(); if (size != null && size > FILE_MAX_SINGLE_SIZE) { - throw new BusinessException("单个文件不能超过 20MB: " + sourceRef.getFileName()); + throw new BusinessException("单个文件不能超过 100MB: " + sourceRef.getFileName()); } if (size != null && size > 0) { totalSize += size; @@ -156,7 +165,7 @@ public class DocNodeFileContentExtractor { throw new BusinessException("最多上传 10 个文件"); } if (totalSize > FILE_MAX_TOTAL_SIZE) { - throw new BusinessException("文件总大小不能超过 50MB"); + throw new BusinessException("文件总大小不能超过 100MB"); } if (sourceRefs.isEmpty()) { throw new BusinessException("文件输入不能为空"); @@ -207,7 +216,24 @@ public class DocNodeFileContentExtractor { * @return 桥接提取出的主文本 */ private String extractBridgeContent(DocumentSourceRef sourceRef) { - DocumentParsedResult parsedResult = documentParseBridgeService.parse(sourceRef, DocumentParseScenario.WORKFLOW_TEXT); + DocumentParsedResult parsedResult; + try (IoBulkhead.Permit ignored = + IoBulkhead.documentParse().acquire( + "document:" + + DocumentParseSourceType.resolve( + sourceRef.getFileName(), + sourceRef.getContentType()))) { + parsedResult = documentParseBridgeService.parse( + sourceRef, + DocumentParseScenario.WORKFLOW_TEXT); + } catch (DocumentParseBridgeException error) { + DocumentInputStreamSupport.SizeLimitExceededException sizeError = + findSizeLimitExceeded(error); + if (sizeError != null) { + throw actualSizeLimitException(sourceRef, sizeError); + } + throw error; + } String preferredText = parsedResult == null ? null : parsedResult.getPreferredText(); if (StringUtil.hasText(preferredText)) { return preferredText; @@ -222,29 +248,155 @@ public class DocNodeFileContentExtractor { } private String extractDefaultContent(DocumentSourceRef sourceRef) { - try (InputStream inputStream = openInputStream(sourceRef)) { - return readerManager.getReader().read(sourceRef.getFileName(), inputStream); + Path temporaryFile = null; + try { + temporaryFile = Files.createTempFile( + "easyflow-doc-node-", ".content"); + copySourceToTemporaryFile(sourceRef, temporaryFile); + /* + * 源 HTTP/对象存储流在进入解析器前已经关闭，解析过程中不会嵌套占用 + * 网络或存储 lane，避免慢解析耗尽上游连接许可。 + */ + try (IoBulkhead.Permit ignored = + IoBulkhead.documentParse().acquire( + "document:default-reader"); + InputStream inputStream = + Files.newInputStream(temporaryFile)) { + return readerManager.getReader().read( + sourceRef.getFileName(), inputStream); + } } catch (IOException e) { + DocumentInputStreamSupport.SizeLimitExceededException sizeError = + findSizeLimitExceeded(e); + if (sizeError != null) { + throw actualSizeLimitException(sourceRef, sizeError); + } throw new RuntimeException("读取文件内容失败: " + sourceRef.getFilePath(), e); + } catch (RuntimeException e) { + DocumentInputStreamSupport.SizeLimitExceededException sizeError = + findSizeLimitExceeded(e); + if (sizeError != null) { + throw actualSizeLimitException(sourceRef, sizeError); + } + throw e; + } finally { + if (temporaryFile != null) { + try { + Files.deleteIfExists(temporaryFile); + } catch (IOException cleanupError) { + // 临时文件由系统临时目录托管；清理失败只记录路径，不改变解析结果。 + LOG.warn( + "清理文档解析临时文件失败，path={}", + temporaryFile, + cleanupError); + } + } + } + } + + /** + * 将受限源流复制到临时文件并在返回前关闭上游连接。 + * + * @param sourceRef 文档源 + * @param target 临时文件 + * @throws IOException 读取、限额或写入失败 + */ + private void copySourceToTemporaryFile( + DocumentSourceRef sourceRef, Path target) throws IOException { + String filePath = sourceRef.getFilePath(); + boolean localStorage = StringUtil.hasText(filePath) + && !isRemoteUrl(filePath); + if (localStorage) { + try (IoBulkhead.Permit ignored = + IoBulkhead.storage().acquire("storage:document-read"); + InputStream inputStream = openInputStream(sourceRef); + OutputStream outputStream = Files.newOutputStream(target)) { + copy(inputStream, outputStream); + } + return; + } + try (InputStream inputStream = openInputStream(sourceRef); + OutputStream outputStream = Files.newOutputStream(target)) { + copy(inputStream, outputStream); + } + } + + /** + * 使用固定缓冲区复制流。 + * + * @param inputStream 输入 + * @param outputStream 输出 + * @throws IOException 复制失败 + */ + private void copy( + InputStream inputStream, OutputStream outputStream) + throws IOException { + byte[] buffer = new byte[64 * 1024]; + int read; + while ((read = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, read); } } private InputStream openInputStream(DocumentSourceRef sourceRef) throws IOException { String filePath = sourceRef.getFilePath(); if (StringUtil.hasText(filePath) && isRemoteUrl(filePath)) { - byte[] bytes = HttpUtil.downloadBytes(filePath); - return new java.io.ByteArrayInputStream(bytes); + return DocumentInputStreamSupport.openRemote(filePath, FILE_MAX_SINGLE_SIZE); } if (StringUtil.hasText(filePath)) { - return fileStorageService.readStream(filePath); + return DocumentInputStreamSupport.limit( + fileStorageService.readStream(filePath), + FILE_MAX_SINGLE_SIZE); } if (StringUtil.hasText(sourceRef.getUrl())) { - byte[] bytes = HttpUtil.downloadBytes(sourceRef.getUrl()); - return new java.io.ByteArrayInputStream(bytes); + return DocumentInputStreamSupport.openRemote( + sourceRef.getUrl(), + FILE_MAX_SINGLE_SIZE); } throw new IOException("文件输入缺少可读取路径"); } + /** + * 从异常链中查找实际读取量超限异常。 + * + * @param error 原始异常 + * @return 超限异常；不存在时返回 null + */ + private DocumentInputStreamSupport.SizeLimitExceededException + findSizeLimitExceeded(Throwable error) { + Throwable current = error; + while (current != null) { + if (current instanceof + DocumentInputStreamSupport.SizeLimitExceededException + sizeError) { + return sizeError; + } + current = current.getCause(); + } + return null; + } + + /** + * 记录实际读取量超限并生成面向工作流用户的业务异常。 + * + * @param sourceRef 文档源 + * @param sizeError 实际读取量超限异常 + * @return 业务异常 + */ + private BusinessException actualSizeLimitException( + DocumentSourceRef sourceRef, + DocumentInputStreamSupport.SizeLimitExceededException sizeError) { + LOG.warn( + "工作流文档实际读取量超过限制: fileName={}, " + + "declaredBytes={}, actualBytes={}, maxBytes={}", + sourceRef.getFileName(), + sourceRef.getSize(), + sizeError.getActualBytes(), + sizeError.getMaxBytes()); + return new BusinessException( + "文件实际读取大小超过 100MB: " + sourceRef.getFileName()); + } + private boolean isRemoteUrl(String value) { return value.startsWith("http://") || value.startsWith("https://"); } diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/document/service/impl/DocumentParseBridgeServiceImplTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/document/service/impl/DocumentParseBridgeServiceImplTest.java index 548785bf..5ad3fe4b 100644 --- a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/document/service/impl/DocumentParseBridgeServiceImplTest.java +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/document/service/impl/DocumentParseBridgeServiceImplTest.java @@ -50,6 +50,31 @@ public class DocumentParseBridgeServiceImplTest { Assert.assertFalse(parseService.lastParseRequest.getReturnImages()); } + /** + * 验证工作流同步解析场景向文档加载器传递 100 MiB 上限。 + */ + @Test + public void shouldApplyHundredMiBWorkflowTextLimit() { + FakePdfDocumentParseService parseService = new FakePdfDocumentParseService(); + RecordingDocumentSourceLoader sourceLoader = + new RecordingDocumentSourceLoader(); + DocumentParseBridgeServiceImpl bridgeService = + buildBridgeService( + parseService, + null, + null, + parseService, + sourceLoader); + + bridgeService.parse( + buildSource(), + DocumentParseScenario.WORKFLOW_TEXT); + + Assert.assertEquals( + 100L * 1024L * 1024L, + sourceLoader.maxBytes); + } + /** * 验证异步提交、状态查询和结果查询链路可用。 */ @@ -171,12 +196,36 @@ public class DocumentParseBridgeServiceImplTest { PptxDocumentParseService pptxDocumentParseService, XlsxDocumentParseService xlsxDocumentParseService, DocumentParseService parseService) { + return buildBridgeService( + pdfDocumentParseService, + pptxDocumentParseService, + xlsxDocumentParseService, + parseService, + new DocumentSourceLoader(new InMemoryFileStorageService())); + } + + /** + * 使用指定文档加载器创建桥接服务。 + * + * @param pdfDocumentParseService PDF 解析服务 + * @param pptxDocumentParseService PPTX 解析服务 + * @param xlsxDocumentParseService XLSX 解析服务 + * @param parseService 默认解析服务 + * @param sourceLoader 文档加载器 + * @return 文档解析桥接服务 + */ + private DocumentParseBridgeServiceImpl buildBridgeService( + PdfDocumentParseService pdfDocumentParseService, + PptxDocumentParseService pptxDocumentParseService, + XlsxDocumentParseService xlsxDocumentParseService, + DocumentParseService parseService, + DocumentSourceLoader sourceLoader) { return new DocumentParseBridgeServiceImpl( parseService, pdfDocumentParseService, pptxDocumentParseService, xlsxDocumentParseService, - new DocumentSourceLoader(new InMemoryFileStorageService()), + sourceLoader, new DocumentParseRequestFactory(), new DocumentParseResultMapper() ); @@ -215,6 +264,33 @@ public class DocumentParseBridgeServiceImplTest { } } + /** + * 记录桥接服务传入的文档大小上限。 + */ + private static class RecordingDocumentSourceLoader extends DocumentSourceLoader { + + private long maxBytes; + + private RecordingDocumentSourceLoader() { + super(new InMemoryFileStorageService()); + } + + /** + * 记录大小上限后执行真实的小文件加载。 + * + * @param sourceRef 文档源 + * @param maxBytes 最大允许字节数 + * @return 已加载文档 + */ + @Override + public tech.easyflow.ai.document.support.LoadedDocumentSource load( + DocumentSourceRef sourceRef, + long maxBytes) { + this.maxBytes = maxBytes; + return super.load(sourceRef, maxBytes); + } + } + private static class FakePdfDocumentParseService implements PdfDocumentParseService { private ParseRequest lastParseRequest; diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/document/support/DocumentSourceLoaderTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/document/support/DocumentSourceLoaderTest.java index 85983ca5..2a845ce7 100644 --- a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/document/support/DocumentSourceLoaderTest.java +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/document/support/DocumentSourceLoaderTest.java @@ -81,6 +81,64 @@ public class DocumentSourceLoaderTest { } } + /** + * 验证远端响应声明值不可信时仍按实际读取字节数拒绝超限内容。 + * + * @throws IOException 测试服务启动失败时抛出 + */ + @Test + public void shouldRejectRemoteContentThatExceedsActualByteLimit() throws IOException { + DocumentSourceLoader loader = new DocumentSourceLoader(new FailingFileStorageService()); + HttpServer server = HttpServer.create(new InetSocketAddress(0), 0); + byte[] body = "123456789".getBytes(StandardCharsets.UTF_8); + server.createContext("/oversized.pdf", exchange -> { + exchange.sendResponseHeaders(200, 0); + exchange.getResponseBody().write(body); + exchange.close(); + }); + server.start(); + try { + DocumentSourceRef sourceRef = new DocumentSourceRef(); + sourceRef.setFileName("oversized.pdf"); + sourceRef.setFilePath( + "http://127.0.0.1:" + server.getAddress().getPort() + "/oversized.pdf"); + sourceRef.setSize(1L); + + try { + loader.load(sourceRef, 8L); + Assert.fail("expected DocumentParseBridgeException"); + } catch (DocumentParseBridgeException exception) { + Assert.assertEquals("source_load_failed", exception.getCode()); + Assert.assertTrue(exception.getCause() + instanceof DocumentInputStreamSupport.SizeLimitExceededException); + } + } finally { + server.stop(0); + } + } + + /** + * 验证存储元数据虚高时按实际读取量接受小文件。 + */ + @Test + public void shouldUseActualBytesWhenStorageMetadataIsIncorrect() { + byte[] content = new byte[58 * 1024]; + DocumentSourceLoader loader = new DocumentSourceLoader( + new IncorrectSizeFileStorageService(content)); + DocumentSourceRef sourceRef = DocumentSourceRef.ofPath( + "/attachment/document.docx"); + sourceRef.setSize((long) content.length); + + LoadedDocumentSource loadedSource = loader.load( + sourceRef, + 20L * 1024L * 1024L); + + Assert.assertEquals( + content.length, + loadedSource.getSize().longValue()); + Assert.assertArrayEquals(content, loadedSource.getContentBytes()); + } + private static class FakeFileStorageService implements FileStorageService { private final byte[] content; @@ -109,6 +167,22 @@ public class DocumentSourceLoaderTest { } } + /** + * 返回错误大小元数据的存储测试替身。 + */ + private static class IncorrectSizeFileStorageService + extends FakeFileStorageService { + + private IncorrectSizeFileStorageService(byte[] content) { + super(content); + } + + @Override + public long getFileSize(String path) { + return 20L * 1024L * 1024L + 1L; + } + } + private static class FailingFileStorageService implements FileStorageService { @Override diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/service/WorkflowRunningParameterResolverTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/service/WorkflowRunningParameterResolverTest.java index febd7435..dbafc1c1 100644 --- a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/service/WorkflowRunningParameterResolverTest.java +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/service/WorkflowRunningParameterResolverTest.java @@ -203,18 +203,18 @@ public class WorkflowRunningParameterResolverTest { } /** - * 文件参数应允许 20MB 边界值，并拒绝超过边界的文件。 + * 文件参数应允许 100MB 边界值，并拒绝超过边界的文件。 * * @throws Exception 反射注入失败 */ @Test - public void testNormalizeRuntimeVariablesShouldEnforceTwentyMbSingleFileLimit() throws Exception { + public void testNormalizeRuntimeVariablesShouldEnforceHundredMbSingleFileLimit() throws Exception { WorkflowRunningParameterResolver resolver = newResolver(); Map variables = new LinkedHashMap<>(); variables.put("attachments", fileValue( "accepted.pdf", "/files/accepted.pdf", - 20L * 1024L * 1024L + 100L * 1024L * 1024L )); Map normalized = resolver.normalizeRuntimeVariables( @@ -226,13 +226,47 @@ public class WorkflowRunningParameterResolverTest { variables.put("attachments", fileValue( "oversized.pdf", "/files/oversized.pdf", - 20L * 1024L * 1024L + 1L + 100L * 1024L * 1024L + 1L )); try { resolver.normalizeRuntimeVariables(workflowContentWithStartParameters(), variables); Assert.fail("expected BusinessException"); } catch (BusinessException exception) { - Assert.assertEquals("文件参数 attachments 中单个文件不能超过 20MB", exception.getMessage()); + Assert.assertEquals("文件参数 attachments 中单个文件不能超过 100MB", exception.getMessage()); + } + } + + /** + * 文件参数总大小应允许 100MB 边界值，并拒绝超过边界的文件列表。 + * + * @throws Exception 反射注入失败 + */ + @Test + public void testNormalizeRuntimeVariablesShouldEnforceHundredMbTotalFileLimit() throws Exception { + WorkflowRunningParameterResolver resolver = newResolver(); + Map variables = new LinkedHashMap<>(); + variables.put("attachments", List.of( + fileValue("first.pdf", "/files/first.pdf", 50L * 1024L * 1024L), + fileValue("second.pdf", "/files/second.pdf", 50L * 1024L * 1024L) + )); + + Map normalized = resolver.normalizeRuntimeVariables( + workflowContentWithStartParameters(), + variables + ); + Assert.assertEquals(2, ((List) normalized.get("attachments")).size()); + + variables.put("attachments", List.of( + fileValue("first.pdf", "/files/first.pdf", 50L * 1024L * 1024L), + fileValue("second.pdf", "/files/second.pdf", 50L * 1024L * 1024L + 1L) + )); + try { + resolver.normalizeRuntimeVariables(workflowContentWithStartParameters(), variables); + Assert.fail("expected BusinessException"); + } catch (BusinessException exception) { + Assert.assertEquals( + "文件参数 attachments 的文件总大小不能超过 100MB", + exception.getMessage()); } } diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/node/DocNodeFileContentExtractorTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/node/DocNodeFileContentExtractorTest.java index 743237e8..e54ae430 100644 --- a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/node/DocNodeFileContentExtractorTest.java +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/node/DocNodeFileContentExtractorTest.java @@ -264,10 +264,10 @@ public class DocNodeFileContentExtractorTest { } /** - * 验证文档节点允许 20MB 边界值，并拒绝超过边界的文件。 + * 验证文档节点允许 100MB 边界值，并拒绝超过边界的文件。 */ @Test - public void shouldEnforceTwentyMbSingleFileLimit() { + public void shouldEnforceHundredMbSingleFileLimit() { DocNodeFileContentExtractor extractor = new DocNodeFileContentExtractor( new RecordingDocumentParseBridgeService(), new FakeFileStorageService(), @@ -278,7 +278,7 @@ public class DocNodeFileContentExtractorTest { "/files/accepted.pdf", "application/pdf" ); - accepted.put("size", 20L * 1024L * 1024L); + accepted.put("size", 100L * 1024L * 1024L); Assert.assertEquals(1, extractor.toDocumentSourceRefs(accepted).size()); Map oversized = buildFileValue( @@ -286,12 +286,72 @@ public class DocNodeFileContentExtractorTest { "/files/oversized.pdf", "application/pdf" ); - oversized.put("size", 20L * 1024L * 1024L + 1L); + oversized.put("size", 100L * 1024L * 1024L + 1L); try { extractor.toDocumentSourceRefs(oversized); Assert.fail("expected BusinessException"); } catch (BusinessException exception) { - Assert.assertEquals("单个文件不能超过 20MB: oversized.pdf", exception.getMessage()); + Assert.assertEquals("单个文件不能超过 100MB: oversized.pdf", exception.getMessage()); + } + } + + /** + * 验证文档节点允许总大小 100MB 边界值，并拒绝超过边界的文件列表。 + */ + @Test + public void shouldEnforceHundredMbTotalFileLimit() { + DocNodeFileContentExtractor extractor = new DocNodeFileContentExtractor( + new RecordingDocumentParseBridgeService(), + new FakeFileStorageService(), + new FakeReaderManager("plain text") + ); + Map first = buildFileValue( + "first.pdf", + "/files/first.pdf", + "application/pdf" + ); + first.put("size", 50L * 1024L * 1024L); + Map second = buildFileValue( + "second.pdf", + "/files/second.pdf", + "application/pdf" + ); + second.put("size", 50L * 1024L * 1024L); + Assert.assertEquals(2, extractor.toDocumentSourceRefs(List.of(first, second)).size()); + + second.put("size", 50L * 1024L * 1024L + 1L); + try { + extractor.toDocumentSourceRefs(List.of(first, second)); + Assert.fail("expected BusinessException"); + } catch (BusinessException exception) { + Assert.assertEquals("文件总大小不能超过 100MB", exception.getMessage()); + } + } + + /** + * 验证文件元数据偏小时仍按实际读取量拒绝超过 100MB 的内容。 + */ + @Test + public void shouldEnforceActualStreamSizeLimit() { + DocNodeFileContentExtractor extractor = new DocNodeFileContentExtractor( + new RecordingDocumentParseBridgeService(), + new OversizedFileStorageService(), + new DrainingReaderManager() + ); + Map input = buildFileValue( + "oversized.txt", + "/files/oversized.txt", + "text/plain" + ); + input.put("size", 1L); + + try { + extractor.extract(input); + Assert.fail("expected BusinessException"); + } catch (BusinessException exception) { + Assert.assertEquals( + "文件实际读取大小超过 100MB: oversized.txt", + exception.getMessage()); } } @@ -398,6 +458,74 @@ public class DocNodeFileContentExtractorTest { } } + private static class DrainingReaderManager extends ReaderManager { + + @Override + public ReadDocService getReader() { + return (fileName, is) -> { + byte[] buffer = new byte[8192]; + try { + while (is.read(buffer) >= 0) { + // 仅消费输入流，用于验证实际字节数保护，不保留大对象。 + } + return "unreachable"; + } catch (IOException e) { + throw new RuntimeException(e); + } + }; + } + } + + private static class OversizedFileStorageService implements FileStorageService { + + @Override + public String save(org.springframework.web.multipart.MultipartFile file) { + return null; + } + + @Override + public void delete(String path) { + } + + @Override + public String save(File file, String prePath) { + return null; + } + + @Override + public InputStream readStream(String path) { + return new InputStream() { + private long remaining = 100L * 1024L * 1024L + 1L; + + @Override + public int read() { + if (remaining <= 0L) { + return -1; + } + remaining--; + return 1; + } + + @Override + public int read(byte[] buffer, int offset, int length) { + if (remaining <= 0L) { + return -1; + } + int count = (int) Math.min(remaining, length); + java.util.Arrays.fill(buffer, offset, offset + count, (byte) 1); + remaining -= count; + return count; + } + }; + } + + @Override + public long getFileSize(String path) { + // 模拟存储元数据不准确，实际流大小必须成为最终保护边界。 + return 1L; + } + } + private static class FailingFileStorageService implements FileStorageService { @Override diff --git a/easyflow-ui-admin/app/src/views/ai/workflow/components/__tests__/workflowFileValue.test.ts b/easyflow-ui-admin/app/src/views/ai/workflow/components/__tests__/workflowFileValue.test.ts index c6e8ad46..8f80f9d5 100644 --- a/easyflow-ui-admin/app/src/views/ai/workflow/components/__tests__/workflowFileValue.test.ts +++ b/easyflow-ui-admin/app/src/views/ai/workflow/components/__tests__/workflowFileValue.test.ts @@ -127,20 +127,20 @@ describe('workflowFileValue', () => { }); it('上传前校验单文件大小限制', () => { - const acceptedFile = new File( - [new Uint8Array(20 * 1024 * 1024)], - 'accepted.pdf', - ); - const oversizedFile = new File( - [new Uint8Array(20 * 1024 * 1024 + 1)], - 'large.pdf', - ); + const acceptedFile = new File([], 'accepted.pdf'); + const oversizedFile = new File([], 'large.pdf'); + Object.defineProperty(acceptedFile, 'size', { + value: 100 * 1024 * 1024, + }); + Object.defineProperty(oversizedFile, 'size', { + value: 100 * 1024 * 1024 + 1, + }); expect(() => validateWorkflowFileSelection([], [acceptedFile]), ).not.toThrow(); expect(() => validateWorkflowFileSelection([], [oversizedFile])).toThrow( - '单个文件不能超过 20.0 MB', + '单个文件不能超过 100.0 MB', ); }); @@ -149,13 +149,13 @@ describe('workflowFileValue', () => { { fileName: 'existing.pdf', filePath: 'https://example.com/existing.pdf', - size: 49 * 1024 * 1024, + size: 99 * 1024 * 1024, }, ]; const incomingFile = new File([new Uint8Array(2 * 1024 * 1024)], 'new.pdf'); expect(() => validateWorkflowFileSelection(currentFiles, [incomingFile]), - ).toThrow('文件总大小不能超过 50.0 MB'); + ).toThrow('文件总大小不能超过 100.0 MB'); }); }); diff --git a/easyflow-ui-admin/app/src/views/ai/workflow/components/workflowFileValue.ts b/easyflow-ui-admin/app/src/views/ai/workflow/components/workflowFileValue.ts index b3f5c05b..17d8bc20 100644 --- a/easyflow-ui-admin/app/src/views/ai/workflow/components/workflowFileValue.ts +++ b/easyflow-ui-admin/app/src/views/ai/workflow/components/workflowFileValue.ts @@ -21,8 +21,8 @@ export interface WorkflowResourceLike { export const WORKFLOW_FILE_LIMITS = { maxCount: 10, - maxSingleSize: 20 * 1024 * 1024, - maxTotalSize: 50 * 1024 * 1024, + maxSingleSize: 100 * 1024 * 1024, + maxTotalSize: 100 * 1024 * 1024, } as const; /** diff --git a/easyflow-ui-usercenter/app/src/views/ai/workflow/components/workflowFileValue.ts b/easyflow-ui-usercenter/app/src/views/ai/workflow/components/workflowFileValue.ts index b3f5c05b..17d8bc20 100644 --- a/easyflow-ui-usercenter/app/src/views/ai/workflow/components/workflowFileValue.ts +++ b/easyflow-ui-usercenter/app/src/views/ai/workflow/components/workflowFileValue.ts @@ -21,8 +21,8 @@ export interface WorkflowResourceLike { export const WORKFLOW_FILE_LIMITS = { maxCount: 10, - maxSingleSize: 20 * 1024 * 1024, - maxTotalSize: 50 * 1024 * 1024, + maxSingleSize: 100 * 1024 * 1024, + maxTotalSize: 100 * 1024 * 1024, } as const; /**