fix: 将工作流文档解析上限统一为 100MiB

- 前后端统一单文件与总大小限制为 100MiB - 按实际读取字节数拦截超限文档并记录大小差异 - 补充存储、远程流与边界场景测试
2026-07-31 14:52:06 +08:00
parent 4a0efe8879
commit 5c29ca9407
11 changed files with 588 additions and 66 deletions
--- a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/document/service/impl/DocumentParseBridgeServiceImplTest.java
+++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/document/service/impl/DocumentParseBridgeServiceImplTest.java
@@ -50,6 +50,31 @@ public class DocumentParseBridgeServiceImplTest {
        Assert.assertFalse(parseService.lastParseRequest.getReturnImages());
    }

+    /**
+     * 验证工作流同步解析场景向文档加载器传递 100 MiB 上限。
+     */
+    @Test
+    public void shouldApplyHundredMiBWorkflowTextLimit() {
+        FakePdfDocumentParseService parseService = new FakePdfDocumentParseService();
+        RecordingDocumentSourceLoader sourceLoader =
+                new RecordingDocumentSourceLoader();
+        DocumentParseBridgeServiceImpl bridgeService =
+                buildBridgeService(
+                        parseService,
+                        null,
+                        null,
+                        parseService,
+                        sourceLoader);
+
+        bridgeService.parse(
+                buildSource(),
+                DocumentParseScenario.WORKFLOW_TEXT);
+
+        Assert.assertEquals(
+                100L * 1024L * 1024L,
+                sourceLoader.maxBytes);
+    }
+
    /**
     * 验证异步提交、状态查询和结果查询链路可用。
     */
@@ -171,12 +196,36 @@ public class DocumentParseBridgeServiceImplTest {
                                                              PptxDocumentParseService pptxDocumentParseService,
                                                              XlsxDocumentParseService xlsxDocumentParseService,
                                                              DocumentParseService parseService) {
+        return buildBridgeService(
+                pdfDocumentParseService,
+                pptxDocumentParseService,
+                xlsxDocumentParseService,
+                parseService,
+                new DocumentSourceLoader(new InMemoryFileStorageService()));
+    }
+
+    /**
+     * 使用指定文档加载器创建桥接服务。
+     *
+     * @param pdfDocumentParseService PDF 解析服务
+     * @param pptxDocumentParseService PPTX 解析服务
+     * @param xlsxDocumentParseService XLSX 解析服务
+     * @param parseService 默认解析服务
+     * @param sourceLoader 文档加载器
+     * @return 文档解析桥接服务
+     */
+    private DocumentParseBridgeServiceImpl buildBridgeService(
+            PdfDocumentParseService pdfDocumentParseService,
+            PptxDocumentParseService pptxDocumentParseService,
+            XlsxDocumentParseService xlsxDocumentParseService,
+            DocumentParseService parseService,
+            DocumentSourceLoader sourceLoader) {
        return new DocumentParseBridgeServiceImpl(
            parseService,
            pdfDocumentParseService,
            pptxDocumentParseService,
            xlsxDocumentParseService,
-            new DocumentSourceLoader(new InMemoryFileStorageService()),
+            sourceLoader,
            new DocumentParseRequestFactory(),
            new DocumentParseResultMapper()
        );
@@ -215,6 +264,33 @@ public class DocumentParseBridgeServiceImplTest {
        }
    }

+    /**
+     * 记录桥接服务传入的文档大小上限。
+     */
+    private static class RecordingDocumentSourceLoader extends DocumentSourceLoader {
+
+        private long maxBytes;
+
+        private RecordingDocumentSourceLoader() {
+            super(new InMemoryFileStorageService());
+        }
+
+        /**
+         * 记录大小上限后执行真实的小文件加载。
+         *
+         * @param sourceRef 文档源
+         * @param maxBytes 最大允许字节数
+         * @return 已加载文档
+         */
+        @Override
+        public tech.easyflow.ai.document.support.LoadedDocumentSource load(
+                DocumentSourceRef sourceRef,
+                long maxBytes) {
+            this.maxBytes = maxBytes;
+            return super.load(sourceRef, maxBytes);
+        }
+    }
+
    private static class FakePdfDocumentParseService implements PdfDocumentParseService {

        private ParseRequest lastParseRequest;
--- a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/document/support/DocumentSourceLoaderTest.java
+++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/document/support/DocumentSourceLoaderTest.java
@@ -81,6 +81,64 @@ public class DocumentSourceLoaderTest {
        }
    }

+    /**
+     * 验证远端响应声明值不可信时仍按实际读取字节数拒绝超限内容。
+     *
+     * @throws IOException 测试服务启动失败时抛出
+     */
+    @Test
+    public void shouldRejectRemoteContentThatExceedsActualByteLimit() throws IOException {
+        DocumentSourceLoader loader = new DocumentSourceLoader(new FailingFileStorageService());
+        HttpServer server = HttpServer.create(new InetSocketAddress(0), 0);
+        byte[] body = "123456789".getBytes(StandardCharsets.UTF_8);
+        server.createContext("/oversized.pdf", exchange -> {
+            exchange.sendResponseHeaders(200, 0);
+            exchange.getResponseBody().write(body);
+            exchange.close();
+        });
+        server.start();
+        try {
+            DocumentSourceRef sourceRef = new DocumentSourceRef();
+            sourceRef.setFileName("oversized.pdf");
+            sourceRef.setFilePath(
+                    "http://127.0.0.1:" + server.getAddress().getPort() + "/oversized.pdf");
+            sourceRef.setSize(1L);
+
+            try {
+                loader.load(sourceRef, 8L);
+                Assert.fail("expected DocumentParseBridgeException");
+            } catch (DocumentParseBridgeException exception) {
+                Assert.assertEquals("source_load_failed", exception.getCode());
+                Assert.assertTrue(exception.getCause()
+                        instanceof DocumentInputStreamSupport.SizeLimitExceededException);
+            }
+        } finally {
+            server.stop(0);
+        }
+    }
+
+    /**
+     * 验证存储元数据虚高时按实际读取量接受小文件。
+     */
+    @Test
+    public void shouldUseActualBytesWhenStorageMetadataIsIncorrect() {
+        byte[] content = new byte[58 * 1024];
+        DocumentSourceLoader loader = new DocumentSourceLoader(
+                new IncorrectSizeFileStorageService(content));
+        DocumentSourceRef sourceRef = DocumentSourceRef.ofPath(
+                "/attachment/document.docx");
+        sourceRef.setSize((long) content.length);
+
+        LoadedDocumentSource loadedSource = loader.load(
+                sourceRef,
+                20L * 1024L * 1024L);
+
+        Assert.assertEquals(
+                content.length,
+                loadedSource.getSize().longValue());
+        Assert.assertArrayEquals(content, loadedSource.getContentBytes());
+    }
+
    private static class FakeFileStorageService implements FileStorageService {

        private final byte[] content;
@@ -109,6 +167,22 @@ public class DocumentSourceLoaderTest {
        }
    }

+    /**
+     * 返回错误大小元数据的存储测试替身。
+     */
+    private static class IncorrectSizeFileStorageService
+            extends FakeFileStorageService {
+
+        private IncorrectSizeFileStorageService(byte[] content) {
+            super(content);
+        }
+
+        @Override
+        public long getFileSize(String path) {
+            return 20L * 1024L * 1024L + 1L;
+        }
+    }
+
    private static class FailingFileStorageService implements FileStorageService {

        @Override
--- a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/service/WorkflowRunningParameterResolverTest.java
+++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/service/WorkflowRunningParameterResolverTest.java
@@ -203,18 +203,18 @@ public class WorkflowRunningParameterResolverTest {
    }

    /**
-     * 文件参数应允许 20MB 边界值，并拒绝超过边界的文件。
+     * 文件参数应允许 100MB 边界值，并拒绝超过边界的文件。
     *
     * @throws Exception 反射注入失败
     */
    @Test
-    public void testNormalizeRuntimeVariablesShouldEnforceTwentyMbSingleFileLimit() throws Exception {
+    public void testNormalizeRuntimeVariablesShouldEnforceHundredMbSingleFileLimit() throws Exception {
        WorkflowRunningParameterResolver resolver = newResolver();
        Map<String, Object> variables = new LinkedHashMap<>();
        variables.put("attachments", fileValue(
                "accepted.pdf",
                "/files/accepted.pdf",
-                20L * 1024L * 1024L
+                100L * 1024L * 1024L
        ));

        Map<String, Object> normalized = resolver.normalizeRuntimeVariables(
@@ -226,13 +226,47 @@ public class WorkflowRunningParameterResolverTest {
        variables.put("attachments", fileValue(
                "oversized.pdf",
                "/files/oversized.pdf",
-                20L * 1024L * 1024L + 1L
+                100L * 1024L * 1024L + 1L
        ));
        try {
            resolver.normalizeRuntimeVariables(workflowContentWithStartParameters(), variables);
            Assert.fail("expected BusinessException");
        } catch (BusinessException exception) {
-            Assert.assertEquals("文件参数 attachments 中单个文件不能超过 20MB", exception.getMessage());
+            Assert.assertEquals("文件参数 attachments 中单个文件不能超过 100MB", exception.getMessage());
+        }
+    }
+
+    /**
+     * 文件参数总大小应允许 100MB 边界值，并拒绝超过边界的文件列表。
+     *
+     * @throws Exception 反射注入失败
+     */
+    @Test
+    public void testNormalizeRuntimeVariablesShouldEnforceHundredMbTotalFileLimit() throws Exception {
+        WorkflowRunningParameterResolver resolver = newResolver();
+        Map<String, Object> variables = new LinkedHashMap<>();
+        variables.put("attachments", List.of(
+                fileValue("first.pdf", "/files/first.pdf", 50L * 1024L * 1024L),
+                fileValue("second.pdf", "/files/second.pdf", 50L * 1024L * 1024L)
+        ));
+
+        Map<String, Object> normalized = resolver.normalizeRuntimeVariables(
+                workflowContentWithStartParameters(),
+                variables
+        );
+        Assert.assertEquals(2, ((List<?>) normalized.get("attachments")).size());
+
+        variables.put("attachments", List.of(
+                fileValue("first.pdf", "/files/first.pdf", 50L * 1024L * 1024L),
+                fileValue("second.pdf", "/files/second.pdf", 50L * 1024L * 1024L + 1L)
+        ));
+        try {
+            resolver.normalizeRuntimeVariables(workflowContentWithStartParameters(), variables);
+            Assert.fail("expected BusinessException");
+        } catch (BusinessException exception) {
+            Assert.assertEquals(
+                    "文件参数 attachments 的文件总大小不能超过 100MB",
+                    exception.getMessage());
        }
    }

--- a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/node/DocNodeFileContentExtractorTest.java
+++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/node/DocNodeFileContentExtractorTest.java
@@ -264,10 +264,10 @@ public class DocNodeFileContentExtractorTest {
    }

    /**
-     * 验证文档节点允许 20MB 边界值，并拒绝超过边界的文件。
+     * 验证文档节点允许 100MB 边界值，并拒绝超过边界的文件。
     */
    @Test
-    public void shouldEnforceTwentyMbSingleFileLimit() {
+    public void shouldEnforceHundredMbSingleFileLimit() {
        DocNodeFileContentExtractor extractor = new DocNodeFileContentExtractor(
            new RecordingDocumentParseBridgeService(),
            new FakeFileStorageService(),
@@ -278,7 +278,7 @@ public class DocNodeFileContentExtractorTest {
            "/files/accepted.pdf",
            "application/pdf"
        );
-        accepted.put("size", 20L * 1024L * 1024L);
+        accepted.put("size", 100L * 1024L * 1024L);
        Assert.assertEquals(1, extractor.toDocumentSourceRefs(accepted).size());

        Map<String, Object> oversized = buildFileValue(
@@ -286,12 +286,72 @@ public class DocNodeFileContentExtractorTest {
            "/files/oversized.pdf",
            "application/pdf"
        );
-        oversized.put("size", 20L * 1024L * 1024L + 1L);
+        oversized.put("size", 100L * 1024L * 1024L + 1L);
        try {
            extractor.toDocumentSourceRefs(oversized);
            Assert.fail("expected BusinessException");
        } catch (BusinessException exception) {
-            Assert.assertEquals("单个文件不能超过 20MB: oversized.pdf", exception.getMessage());
+            Assert.assertEquals("单个文件不能超过 100MB: oversized.pdf", exception.getMessage());
+        }
+    }
+
+    /**
+     * 验证文档节点允许总大小 100MB 边界值，并拒绝超过边界的文件列表。
+     */
+    @Test
+    public void shouldEnforceHundredMbTotalFileLimit() {
+        DocNodeFileContentExtractor extractor = new DocNodeFileContentExtractor(
+            new RecordingDocumentParseBridgeService(),
+            new FakeFileStorageService(),
+            new FakeReaderManager("plain text")
+        );
+        Map<String, Object> first = buildFileValue(
+            "first.pdf",
+            "/files/first.pdf",
+            "application/pdf"
+        );
+        first.put("size", 50L * 1024L * 1024L);
+        Map<String, Object> second = buildFileValue(
+            "second.pdf",
+            "/files/second.pdf",
+            "application/pdf"
+        );
+        second.put("size", 50L * 1024L * 1024L);
+        Assert.assertEquals(2, extractor.toDocumentSourceRefs(List.of(first, second)).size());
+
+        second.put("size", 50L * 1024L * 1024L + 1L);
+        try {
+            extractor.toDocumentSourceRefs(List.of(first, second));
+            Assert.fail("expected BusinessException");
+        } catch (BusinessException exception) {
+            Assert.assertEquals("文件总大小不能超过 100MB", exception.getMessage());
+        }
+    }
+
+    /**
+     * 验证文件元数据偏小时仍按实际读取量拒绝超过 100MB 的内容。
+     */
+    @Test
+    public void shouldEnforceActualStreamSizeLimit() {
+        DocNodeFileContentExtractor extractor = new DocNodeFileContentExtractor(
+            new RecordingDocumentParseBridgeService(),
+            new OversizedFileStorageService(),
+            new DrainingReaderManager()
+        );
+        Map<String, Object> input = buildFileValue(
+            "oversized.txt",
+            "/files/oversized.txt",
+            "text/plain"
+        );
+        input.put("size", 1L);
+
+        try {
+            extractor.extract(input);
+            Assert.fail("expected BusinessException");
+        } catch (BusinessException exception) {
+            Assert.assertEquals(
+                "文件实际读取大小超过 100MB: oversized.txt",
+                exception.getMessage());
        }
    }

@@ -398,6 +458,74 @@ public class DocNodeFileContentExtractorTest {
        }
    }

+    private static class DrainingReaderManager extends ReaderManager {
+
+        @Override
+        public ReadDocService getReader() {
+            return (fileName, is) -> {
+                byte[] buffer = new byte[8192];
+                try {
+                    while (is.read(buffer) >= 0) {
+                        // 仅消费输入流，用于验证实际字节数保护，不保留大对象。
+                    }
+                    return "unreachable";
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+            };
+        }
+    }
+
+    private static class OversizedFileStorageService implements FileStorageService {
+
+        @Override
+        public String save(org.springframework.web.multipart.MultipartFile file) {
+            return null;
+        }
+
+        @Override
+        public void delete(String path) {
+        }
+
+        @Override
+        public String save(File file, String prePath) {
+            return null;
+        }
+
+        @Override
+        public InputStream readStream(String path) {
+            return new InputStream() {
+                private long remaining = 100L * 1024L * 1024L + 1L;
+
+                @Override
+                public int read() {
+                    if (remaining <= 0L) {
+                        return -1;
+                    }
+                    remaining--;
+                    return 1;
+                }
+
+                @Override
+                public int read(byte[] buffer, int offset, int length) {
+                    if (remaining <= 0L) {
+                        return -1;
+                    }
+                    int count = (int) Math.min(remaining, length);
+                    java.util.Arrays.fill(buffer, offset, offset + count, (byte) 1);
+                    remaining -= count;
+                    return count;
+                }
+            };
+        }
+
+        @Override
+        public long getFileSize(String path) {
+            // 模拟存储元数据不准确，实际流大小必须成为最终保护边界。
+            return 1L;
+        }
+    }
+
    private static class FailingFileStorageService implements FileStorageService {

        @Override