From 1e6158be7724846b88f747697e12c848a85af794 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Fri, 17 Jul 2026 19:54:26 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=AE=8C=E5=96=84=E6=99=BA=E8=83=BD?= =?UTF-8?q?=E4=BD=93=E5=9B=BE=E7=89=87=E8=81=8A=E5=A4=A9=E4=B8=8E=E4=BC=9A?= =?UTF-8?q?=E8=AF=9D=E6=81=A2=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 增加私有图片上传、绑定、历史回显与生命周期清理 - 支持输入草稿恢复、图片交互和模型图片能力约束 - 修复旧脏会话幂等删除与前端会话恢复 --- docker-compose.middleware.yml | 4 +- .../controller/agent/AgentController.java | 124 +++++ .../service/agent/AgentSessionService.java | 55 ++- .../agent/AgentSessionServiceTest.java | 152 +++++++ .../easyflow/core/chat/protocol/ChatType.java | 1 + .../easyflow-module-agent/pom.xml | 8 + .../agent/config/AgentMediaProperties.java | 56 +++ .../agent/config/AgentModuleConfig.java | 4 +- .../agent/runtime/AgentChatRequest.java | 17 + .../agent/runtime/AgentDraftChatRequest.java | 20 + .../agent/runtime/AgentRunService.java | 120 ++++- .../agent/runtime/AgentRuntimeCompiler.java | 2 + .../runtime/composer/AgentComposerDraft.java | 59 +++ .../composer/AgentComposerDraftService.java | 330 ++++++++++++++ .../composer/AgentComposerSession.java | 10 + .../agent/runtime/media/AgentBoundMedia.java | 13 + .../agent/runtime/media/AgentImageData.java | 32 ++ .../runtime/media/AgentImageProcessor.java | 213 +++++++++ .../media/AgentMediaCleanupScheduler.java | 40 ++ .../media/AgentMediaObjectStorage.java | 233 ++++++++++ .../runtime/media/AgentMediaService.java | 428 ++++++++++++++++++ .../runtime/media/AgentMediaUploadRecord.java | 96 ++++ .../runtime/media/AgentMediaUploadView.java | 51 +++ .../media/RedisAgentMediaUploadStore.java | 189 ++++++++ .../agent/service/impl/AgentServiceImpl.java | 2 + .../AgentRunServiceDraftAndHitlTest.java | 57 ++- .../AgentComposerDraftServiceTest.java | 101 +++++ .../media/AgentImageProcessorTest.java | 141 ++++++ .../runtime/media/AgentMediaServiceTest.java | 103 +++++ .../java/tech/easyflow/ai/entity/Model.java | 3 + .../src/main/resources/application.yml | 15 + .../src/components/ai-chat/AiChatPanel.vue | 40 +- .../components/ai-chat/AiPromptInput.test.ts | 64 ++- .../src/components/ai-chat/AiPromptInput.vue | 181 +++++++- .../app/src/components/ai-chat/mediaApi.ts | 101 +++++ .../ai-chat/useAgentComposerDraft.test.ts | 184 ++++++++ .../ai-chat/useAgentComposerDraft.ts | 403 +++++++++++++++++ .../ai-chat/useChatImageUploads.test.ts | 62 +++ .../components/ai-chat/useChatImageUploads.ts | 201 ++++++++ .../app/src/locales/langs/en-US/llm.json | 8 +- .../app/src/locales/langs/zh-CN/llm.json | 4 +- easyflow-ui-admin/app/src/store/auth.ts | 2 + .../app/src/utils/agent-chat-cache.ts | 83 ++++ .../adapters/agentTimelineAdapter.ts | 43 +- .../agentChatRuntimeManager.test.ts | 98 ++++ .../ai/agent-chat/agentChatRuntimeManager.ts | 186 ++++++-- .../app/src/views/ai/agent-chat/api.ts | 7 +- .../app/src/views/ai/agent-chat/index.vue | 302 ++++++++++-- .../ai/agent-chat/sessionRecovery.test.ts | 36 ++ .../views/ai/agent-chat/sessionRecovery.ts | 36 ++ .../agents/components/AgentInspectorPanel.vue | 7 + .../ai/agents/components/AgentTryoutPanel.vue | 96 +++- .../composables/useAgentTryoutRawRounds.ts | 94 +++- .../composables/useAgentTryoutStream.ts | 58 ++- .../app/src/views/ai/model/AddModelModal.vue | 14 +- .../chat-timeline/ChatImageAttachments.vue | 393 ++++++++++++++++ .../components/chat-timeline/ChatTimeline.vue | 5 +- .../chat-timeline/ChatTimelineItem.vue | 11 +- .../__tests__/ChatImageAttachments.test.ts | 86 ++++ .../src/components/chat-timeline/builder.ts | 18 +- .../src/components/chat-timeline/index.ts | 3 + .../src/components/chat-timeline/types.ts | 17 + 62 files changed, 5333 insertions(+), 189 deletions(-) create mode 100644 easyflow-api/easyflow-api-admin/src/test/java/tech/easyflow/admin/service/agent/AgentSessionServiceTest.java create mode 100644 easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/config/AgentMediaProperties.java create mode 100644 easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/composer/AgentComposerDraft.java create mode 100644 easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/composer/AgentComposerDraftService.java create mode 100644 easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/composer/AgentComposerSession.java create mode 100644 easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentBoundMedia.java create mode 100644 easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentImageData.java create mode 100644 easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentImageProcessor.java create mode 100644 easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentMediaCleanupScheduler.java create mode 100644 easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentMediaObjectStorage.java create mode 100644 easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentMediaService.java create mode 100644 easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentMediaUploadRecord.java create mode 100644 easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentMediaUploadView.java create mode 100644 easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/RedisAgentMediaUploadStore.java create mode 100644 easyflow-modules/easyflow-module-agent/src/test/java/tech/easyflow/agent/runtime/composer/AgentComposerDraftServiceTest.java create mode 100644 easyflow-modules/easyflow-module-agent/src/test/java/tech/easyflow/agent/runtime/media/AgentImageProcessorTest.java create mode 100644 easyflow-modules/easyflow-module-agent/src/test/java/tech/easyflow/agent/runtime/media/AgentMediaServiceTest.java create mode 100644 easyflow-ui-admin/app/src/components/ai-chat/mediaApi.ts create mode 100644 easyflow-ui-admin/app/src/components/ai-chat/useAgentComposerDraft.test.ts create mode 100644 easyflow-ui-admin/app/src/components/ai-chat/useAgentComposerDraft.ts create mode 100644 easyflow-ui-admin/app/src/components/ai-chat/useChatImageUploads.test.ts create mode 100644 easyflow-ui-admin/app/src/components/ai-chat/useChatImageUploads.ts create mode 100644 easyflow-ui-admin/app/src/utils/agent-chat-cache.ts create mode 100644 easyflow-ui-admin/app/src/views/ai/agent-chat/agentChatRuntimeManager.test.ts create mode 100644 easyflow-ui-admin/app/src/views/ai/agent-chat/sessionRecovery.test.ts create mode 100644 easyflow-ui-admin/app/src/views/ai/agent-chat/sessionRecovery.ts create mode 100644 easyflow-ui-admin/packages/effects/common-ui/src/components/chat-timeline/ChatImageAttachments.vue create mode 100644 easyflow-ui-admin/packages/effects/common-ui/src/components/chat-timeline/__tests__/ChatImageAttachments.test.ts diff --git a/docker-compose.middleware.yml b/docker-compose.middleware.yml index 6750edbf..286e79d3 100644 --- a/docker-compose.middleware.yml +++ b/docker-compose.middleware.yml @@ -72,6 +72,8 @@ services: TZ: Asia/Shanghai MINIO_ROOT_USER: easyflowadmin MINIO_ROOT_PASSWORD: easyflowadmin123 + MINIO_API_STALE_UPLOADS_EXPIRY: 24h + MINIO_API_STALE_UPLOADS_CLEANUP_INTERVAL: 6h ports: - "9000:9000" - "9001:9001" @@ -88,7 +90,7 @@ services: MINIO_ROOT_USER: easyflowadmin MINIO_ROOT_PASSWORD: easyflowadmin123 MINIO_ENDPOINT: http://minio:9000 - MINIO_BUCKETS: easyflow,milvus + MINIO_BUCKETS: easyflow,milvus,easyflow-agent-media MINIO_PUBLIC_BUCKETS: easyflow MINIO_ALIAS: local volumes: diff --git a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/agent/AgentController.java b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/agent/AgentController.java index 4fb9efa1..f4460cb2 100644 --- a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/agent/AgentController.java +++ b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/agent/AgentController.java @@ -4,10 +4,15 @@ import cn.dev33.satoken.annotation.SaCheckPermission; import com.mybatisflex.core.paginate.Page; import com.mybatisflex.core.query.QueryWrapper; import jakarta.servlet.http.HttpServletRequest; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.multipart.MultipartFile; import org.springframework.web.context.request.RequestContextHolder; import org.springframework.web.context.request.ServletRequestAttributes; import org.springframework.web.servlet.mvc.method.annotation.SseEmitter; @@ -19,6 +24,12 @@ import tech.easyflow.agent.publish.AgentPublishAppService; import tech.easyflow.agent.runtime.AgentChatRequest; import tech.easyflow.agent.runtime.AgentDraftChatRequest; import tech.easyflow.agent.runtime.AgentRunService; +import tech.easyflow.agent.runtime.composer.AgentComposerDraft; +import tech.easyflow.agent.runtime.composer.AgentComposerDraftService; +import tech.easyflow.agent.runtime.composer.AgentComposerSession; +import tech.easyflow.agent.runtime.media.AgentMediaService; +import tech.easyflow.agent.runtime.media.AgentMediaUploadView; +import com.easyagents.agent.runtime.media.AgentMediaResource; import tech.easyflow.agent.service.AgentApprovalStateService; import tech.easyflow.agent.service.AgentKnowledgeBindingService; import tech.easyflow.agent.service.AgentService; @@ -28,6 +39,8 @@ import tech.easyflow.approval.entity.vo.ApprovalActionResult; import tech.easyflow.common.domain.Result; import tech.easyflow.common.web.controller.BaseCurdController; import tech.easyflow.common.web.jsonbody.JsonBody; +import tech.easyflow.common.satoken.util.SaTokenUtil; +import tech.easyflow.log.annotation.LogReporterDisabled; import tech.easyflow.system.entity.vo.RoleCategoryAccessSnapshot; import tech.easyflow.system.enums.CategoryResourceType; import tech.easyflow.system.enums.ResourceAction; @@ -66,6 +79,10 @@ public class AgentController extends BaseCurdController { private AgentApprovalStateService agentApprovalStateService; @Resource private AiResourceCreatorNameSupport aiResourceCreatorNameSupport; + @Resource + private AgentMediaService agentMediaService; + @Resource + private AgentComposerDraftService agentComposerDraftService; /** * 创建 Agent 控制器。 @@ -162,6 +179,113 @@ public class AgentController extends BaseCurdController { return agentRunService.chatDraft(request); } + /** + * 上传一张 Agent 聊天临时图片。 + * + * @param file 图片文件 + * @param mode 聊天模式 + * @param agentId Agent ID + * @param sessionId 会话 ID + * @return 上传结果 + */ + @PostMapping(value = "/media/upload", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + public Result uploadMedia(@RequestParam("file") MultipartFile file, + @RequestParam("mode") String mode, + @RequestParam("agentId") String agentId, + @RequestParam("sessionId") String sessionId) { + return Result.ok(agentMediaService.upload(file, mode, agentId, sessionId, SaTokenUtil.getLoginAccount())); + } + + /** + * 删除当前账号尚未发送的临时图片。 + * + * @param uploadId 上传 ID + * @return 操作结果 + */ + @PostMapping("/media/delete") + public Result deleteMedia(@JsonBody(value = "uploadId", required = true) String uploadId) { + agentMediaService.deleteUpload(uploadId, SaTokenUtil.getLoginAccount()); + return Result.ok(); + } + + /** + * 通过鉴权代理读取 Agent 私有聊天图片。 + * + * @param reference 稳定图片引用 + * @return 图片响应 + */ + @GetMapping("/media/content") + @LogReporterDisabled + public ResponseEntity mediaContent(@RequestParam("reference") String reference) { + AgentMediaResource resource = agentMediaService.load(reference, SaTokenUtil.getLoginAccount()); + return ResponseEntity.ok() + .header(HttpHeaders.CACHE_CONTROL, "private, no-store") + .header(HttpHeaders.CONTENT_DISPOSITION, "inline") + .contentType(MediaType.parseMediaType(resource.mimeType())) + .contentLength(resource.bytes().length) + .body(resource.bytes()); + } + + /** + * 为输入框预分配稳定会话 ID。 + * + * @param mode 聊天模式 + * @return 会话信息 + */ + @PostMapping("/composer/session") + public Result allocateComposerSession( + @JsonBody(value = "mode", required = true) String mode) { + return Result.ok(agentComposerDraftService.allocateSession(mode)); + } + + /** + * 保存 Agent 输入草稿。 + * + * @param draft 输入草稿 + * @return 保存后的草稿 + */ + @PostMapping("/composer/draft/persist") + public Result saveComposerDraft(@JsonBody AgentComposerDraft draft) { + return Result.ok(agentComposerDraftService.save(draft, SaTokenUtil.getLoginAccount())); + } + + /** + * 获取当前会话或最近未发送会话的输入草稿。 + * + * @param mode 聊天模式 + * @param agentId Agent ID + * @param sessionId 会话 ID，可为空 + * @return 输入草稿 + */ + @GetMapping("/composer/draft") + public Result getComposerDraft(@RequestParam("mode") String mode, + @RequestParam("agentId") String agentId, + @RequestParam(value = "sessionId", required = false) String sessionId) { + return Result.ok(agentComposerDraftService.get(mode, agentId, sessionId, SaTokenUtil.getLoginAccount()) + .orElse(null)); + } + + /** + * 删除已发送或主动清空的输入草稿。 + * + * @param mode 聊天模式 + * @param agentId Agent ID + * @param sessionId 会话 ID + * @param imageUploadIds 调用方仍持有的上传 ID + * @param deleteUploads 是否同时删除临时图片 + * @return 操作结果 + */ + @PostMapping("/composer/draft/delete") + public Result deleteComposerDraft(@JsonBody(value = "mode", required = true) String mode, + @JsonBody(value = "agentId", required = true) String agentId, + @JsonBody(value = "sessionId", required = true) String sessionId, + @JsonBody(value = "imageUploadIds") List imageUploadIds, + @JsonBody(value = "deleteUploads") Boolean deleteUploads) { + agentComposerDraftService.delete(mode, agentId, sessionId, imageUploadIds, + !Boolean.FALSE.equals(deleteUploads), SaTokenUtil.getLoginAccount()); + return Result.ok(); + } + /** * 清理 Agent 草稿试运行会话。 * diff --git a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/service/agent/AgentSessionService.java b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/service/agent/AgentSessionService.java index 4e461809..0096883d 100644 --- a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/service/agent/AgentSessionService.java +++ b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/service/agent/AgentSessionService.java @@ -6,6 +6,8 @@ import org.springframework.util.StringUtils; import tech.easyflow.admin.dto.chatworkspace.*; import tech.easyflow.agent.entity.Agent; import tech.easyflow.agent.runtime.AgentRuntimeStateCleanupService; +import tech.easyflow.agent.runtime.composer.AgentComposerDraftService; +import tech.easyflow.agent.runtime.media.AgentMediaService; import tech.easyflow.agent.service.AgentService; import tech.easyflow.ai.entity.DocumentCollection; import tech.easyflow.ai.enums.PublishStatus; @@ -39,6 +41,8 @@ public class AgentSessionService { private final DocumentCollectionService documentCollectionService; private final ResourceAccessService resourceAccessService; private final AgentRuntimeStateCleanupService agentRuntimeStateCleanupService; + private final AgentMediaService agentMediaService; + private final AgentComposerDraftService agentComposerDraftService; private final ChatJsonSupport chatJsonSupport; /** @@ -50,6 +54,8 @@ public class AgentSessionService { * @param documentCollectionService 知识库服务 * @param resourceAccessService 资源访问服务 * @param agentRuntimeStateCleanupService Agent 运行态清理服务 + * @param agentMediaService Agent 媒体服务 + * @param agentComposerDraftService Agent 输入草稿服务 * @param chatJsonSupport 聊天 JSON 工具 */ public AgentSessionService(ChatSessionQueryService chatSessionQueryService, @@ -58,6 +64,8 @@ public class AgentSessionService { DocumentCollectionService documentCollectionService, ResourceAccessService resourceAccessService, AgentRuntimeStateCleanupService agentRuntimeStateCleanupService, + AgentMediaService agentMediaService, + AgentComposerDraftService agentComposerDraftService, ChatJsonSupport chatJsonSupport) { this.chatSessionQueryService = chatSessionQueryService; this.chatSessionCommandService = chatSessionCommandService; @@ -65,6 +73,8 @@ public class AgentSessionService { this.documentCollectionService = documentCollectionService; this.resourceAccessService = resourceAccessService; this.agentRuntimeStateCleanupService = agentRuntimeStateCleanupService; + this.agentMediaService = agentMediaService; + this.agentComposerDraftService = agentComposerDraftService; this.chatJsonSupport = chatJsonSupport; } @@ -186,21 +196,58 @@ public class AgentSessionService { * @param sessionId 会话 ID */ public void deleteCurrentUserSession(LoginAccount account, BigInteger sessionId) { - requireUserAgentSession(account, sessionId); + ChatSessionSummary summary = chatSessionQueryService.getSessionSummary(sessionId); + if (summary == null || Integer.valueOf(1).equals(summary.getIsDeleted())) { + // 上一次删除可能已写入删除标记但媒体清理失败，重试时继续清理当前用户目录。 + deleteComposerDraft(summary, account, sessionId); + agentMediaService.deleteFormalSession(sessionId.toString(), account); + return; + } + requireUserAgentSession(account, summary); agentRuntimeStateCleanupService.clearChatSession(sessionId, account.getId()); chatSessionCommandService.deleteSession(sessionId, account.getId(), account.getId()); + deleteComposerDraft(summary, account, sessionId); + agentMediaService.deleteFormalSession(sessionId.toString(), account); + } + + /** + * 删除会话对应的未发送草稿和临时图片。 + * + * @param summary 会话摘要 + * @param account 当前登录账号 + * @param sessionId 会话 ID + */ + private void deleteComposerDraft(ChatSessionSummary summary, LoginAccount account, BigInteger sessionId) { + if (summary == null || summary.getAssistantId() == null) { + return; + } + agentComposerDraftService.delete(AgentMediaService.MODE_FORMAL, + summary.getAssistantId().toString(), sessionId.toString(), account); } private ChatSessionSummary requireUserAgentSession(LoginAccount account, BigInteger sessionId) { ChatSessionSummary summary = chatSessionQueryService.getSessionSummary(sessionId); - if (summary == null || Integer.valueOf(1).equals(summary.getIsDeleted()) - || !ASSISTANT_CODE.equals(summary.getAssistantCode())) { + if (summary == null || Integer.valueOf(1).equals(summary.getIsDeleted())) { + throw new BusinessException("Agent 会话不存在"); + } + requireUserAgentSession(account, summary); + return summary; + } + + /** + * 校验会话属于当前用户且类型为 Agent。 + * + * @param account 当前登录账号 + * @param summary 会话摘要 + * @throws BusinessException 会话类型不匹配或不属于当前用户时抛出 + */ + private void requireUserAgentSession(LoginAccount account, ChatSessionSummary summary) { + if (!ASSISTANT_CODE.equals(summary.getAssistantCode())) { throw new BusinessException("Agent 会话不存在"); } if (!Objects.equals(summary.getUserId(), account.getId())) { throw new BusinessException("无权访问该 Agent 会话"); } - return summary; } private Map resolveAgentAvailability(List sessions) { diff --git a/easyflow-api/easyflow-api-admin/src/test/java/tech/easyflow/admin/service/agent/AgentSessionServiceTest.java b/easyflow-api/easyflow-api-admin/src/test/java/tech/easyflow/admin/service/agent/AgentSessionServiceTest.java new file mode 100644 index 00000000..2c8fcbf5 --- /dev/null +++ b/easyflow-api/easyflow-api-admin/src/test/java/tech/easyflow/admin/service/agent/AgentSessionServiceTest.java @@ -0,0 +1,152 @@ +package tech.easyflow.admin.service.agent; + +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; +import tech.easyflow.agent.runtime.AgentRuntimeStateCleanupService; +import tech.easyflow.agent.runtime.composer.AgentComposerDraftService; +import tech.easyflow.agent.runtime.media.AgentMediaService; +import tech.easyflow.agent.service.AgentService; +import tech.easyflow.ai.service.DocumentCollectionService; +import tech.easyflow.chatlog.domain.dto.ChatSessionSummary; +import tech.easyflow.chatlog.service.ChatSessionCommandService; +import tech.easyflow.chatlog.service.ChatSessionQueryService; +import tech.easyflow.chatlog.support.ChatJsonSupport; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.common.web.exceptions.BusinessException; +import tech.easyflow.system.service.ResourceAccessService; + +import java.math.BigInteger; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * {@link AgentSessionService} 会话删除测试。 + */ +public class AgentSessionServiceTest { + + private static final BigInteger ACCOUNT_ID = BigInteger.valueOf(7); + private static final BigInteger SESSION_ID = BigInteger.valueOf(101); + + private ChatSessionQueryService chatSessionQueryService; + private ChatSessionCommandService chatSessionCommandService; + private AgentRuntimeStateCleanupService agentRuntimeStateCleanupService; + private AgentMediaService agentMediaService; + private AgentComposerDraftService agentComposerDraftService; + private AgentSessionService service; + private LoginAccount account; + + /** + * 初始化测试依赖。 + */ + @BeforeMethod + public void setUp() { + chatSessionQueryService = mock(ChatSessionQueryService.class); + chatSessionCommandService = mock(ChatSessionCommandService.class); + agentRuntimeStateCleanupService = mock(AgentRuntimeStateCleanupService.class); + agentMediaService = mock(AgentMediaService.class); + agentComposerDraftService = mock(AgentComposerDraftService.class); + service = new AgentSessionService( + chatSessionQueryService, + chatSessionCommandService, + mock(AgentService.class), + mock(DocumentCollectionService.class), + mock(ResourceAccessService.class), + agentRuntimeStateCleanupService, + agentMediaService, + agentComposerDraftService, + mock(ChatJsonSupport.class) + ); + account = new LoginAccount(); + account.setId(ACCOUNT_ID); + } + + /** + * 验证正常删除会清理运行态、写入删除命令并删除媒体目录。 + */ + @Test + public void shouldDeleteActiveOwnedAgentSession() { + when(chatSessionQueryService.getSessionSummary(SESSION_ID)) + .thenReturn(buildSession(ACCOUNT_ID, 0, "AGENT")); + + service.deleteCurrentUserSession(account, SESSION_ID); + + verify(agentRuntimeStateCleanupService).clearChatSession(SESSION_ID, ACCOUNT_ID); + verify(chatSessionCommandService).deleteSession(SESSION_ID, ACCOUNT_ID, ACCOUNT_ID); + verify(agentComposerDraftService).delete(AgentMediaService.MODE_FORMAL, "9", "101", account); + verify(agentMediaService).deleteFormalSession(SESSION_ID.toString(), account); + } + + /** + * 验证会话已删除时重复请求仍会重试媒体目录清理并成功返回。 + */ + @Test + public void shouldRetryMediaCleanupForDeletedSession() { + when(chatSessionQueryService.getSessionSummary(SESSION_ID)) + .thenReturn(buildSession(ACCOUNT_ID, 1, "AGENT")); + + service.deleteCurrentUserSession(account, SESSION_ID); + + verify(agentRuntimeStateCleanupService, never()).clearChatSession(Mockito.any(), Mockito.any()); + verify(chatSessionCommandService, never()).deleteSession(Mockito.any(), Mockito.any(), Mockito.any()); + verify(agentComposerDraftService).delete(AgentMediaService.MODE_FORMAL, "9", "101", account); + verify(agentMediaService).deleteFormalSession(SESSION_ID.toString(), account); + } + + /** + * 验证查询不到会话时删除保持幂等，并按当前用户目录重试媒体清理。 + */ + @Test + public void shouldRetryMediaCleanupWhenSessionIsMissing() { + when(chatSessionQueryService.getSessionSummary(SESSION_ID)).thenReturn(null); + + service.deleteCurrentUserSession(account, SESSION_ID); + + verify(agentRuntimeStateCleanupService, never()).clearChatSession(Mockito.any(), Mockito.any()); + verify(chatSessionCommandService, never()).deleteSession(Mockito.any(), Mockito.any(), Mockito.any()); + verify(agentComposerDraftService, never()).delete(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + verify(agentMediaService).deleteFormalSession(SESSION_ID.toString(), account); + } + + /** + * 验证活动会话属于其他用户时仍拒绝删除，且不执行任何清理。 + */ + @Test + public void shouldRejectActiveSessionOwnedByAnotherUser() { + when(chatSessionQueryService.getSessionSummary(SESSION_ID)) + .thenReturn(buildSession(BigInteger.valueOf(8), 0, "AGENT")); + + BusinessException exception = Assert.expectThrows( + BusinessException.class, + () -> service.deleteCurrentUserSession(account, SESSION_ID) + ); + + Assert.assertEquals(exception.getMessage(), "无权访问该 Agent 会话"); + verify(agentRuntimeStateCleanupService, never()).clearChatSession(Mockito.any(), Mockito.any()); + verify(chatSessionCommandService, never()).deleteSession(Mockito.any(), Mockito.any(), Mockito.any()); + verify(agentComposerDraftService, never()).delete(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + verify(agentMediaService, never()).deleteFormalSession(Mockito.any(), Mockito.any()); + } + + /** + * 构造会话摘要。 + * + * @param userId 用户 ID + * @param isDeleted 删除标记 + * @param assistantCode 助手类型 + * @return 会话摘要 + */ + private ChatSessionSummary buildSession(BigInteger userId, Integer isDeleted, String assistantCode) { + ChatSessionSummary summary = new ChatSessionSummary(); + summary.setId(SESSION_ID); + summary.setUserId(userId); + summary.setIsDeleted(isDeleted); + summary.setAssistantCode(assistantCode); + summary.setAssistantId(BigInteger.valueOf(9)); + return summary; + } +} diff --git a/easyflow-commons/easyflow-common-chat-protocol/src/main/java/tech/easyflow/core/chat/protocol/ChatType.java b/easyflow-commons/easyflow-common-chat-protocol/src/main/java/tech/easyflow/core/chat/protocol/ChatType.java index 7ee9f942..26f3d578 100644 --- a/easyflow-commons/easyflow-common-chat-protocol/src/main/java/tech/easyflow/core/chat/protocol/ChatType.java +++ b/easyflow-commons/easyflow-common-chat-protocol/src/main/java/tech/easyflow/core/chat/protocol/ChatType.java @@ -8,6 +8,7 @@ public enum ChatType { STATUS, CITATIONS, SESSION_CREATED, + INPUT_ACCEPTED, ERROR, FORM_REQUEST, FORM_CANCEL, diff --git a/easyflow-modules/easyflow-module-agent/pom.xml b/easyflow-modules/easyflow-module-agent/pom.xml index 03c56d77..cf43c5a5 100644 --- a/easyflow-modules/easyflow-module-agent/pom.xml +++ b/easyflow-modules/easyflow-module-agent/pom.xml @@ -49,6 +49,14 @@ tech.easyflow easyflow-common-satoken + + org.dromara.x-file-storage + x-file-storage-spring + + + io.minio + minio + com.mybatis-flex mybatis-flex-spring-boot3-starter diff --git a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/config/AgentMediaProperties.java b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/config/AgentMediaProperties.java new file mode 100644 index 00000000..ae862f1b --- /dev/null +++ b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/config/AgentMediaProperties.java @@ -0,0 +1,56 @@ +package tech.easyflow.agent.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +import java.time.Duration; + +/** + * Agent 图片上传、草稿和对象存储配置。 + */ +@ConfigurationProperties(prefix = "easyflow.agent.media") +public class AgentMediaProperties { + + /** x-file-storage 中的私有 Agent 媒体平台。 */ + private String platform = "minio-agent-media"; + /** 单次对话最多图片数。 */ + private int maxImageCount = 5; + /** 单张图片最大字节数。 */ + private long maxImageBytes = 10L * 1024L * 1024L; + /** 单张图片最大像素数。 */ + private long maxImagePixels = 40_000_000L; + /** 临时图片有效期。 */ + private Duration uploadTtl = Duration.ofHours(24); + /** 输入草稿有效期。 */ + private Duration composerDraftTtl = Duration.ofHours(24); + /** 过期对象清理周期。 */ + private Duration cleanupInterval = Duration.ofMinutes(10); + + /** @return 私有媒体平台名称 */ + public String getPlatform() { return platform; } + /** @param platform 私有媒体平台名称 */ + public void setPlatform(String platform) { this.platform = platform; } + /** @return 单次图片上限 */ + public int getMaxImageCount() { return maxImageCount; } + /** @param maxImageCount 单次图片上限 */ + public void setMaxImageCount(int maxImageCount) { this.maxImageCount = maxImageCount; } + /** @return 单图字节上限 */ + public long getMaxImageBytes() { return maxImageBytes; } + /** @param maxImageBytes 单图字节上限 */ + public void setMaxImageBytes(long maxImageBytes) { this.maxImageBytes = maxImageBytes; } + /** @return 单图像素上限 */ + public long getMaxImagePixels() { return maxImagePixels; } + /** @param maxImagePixels 单图像素上限 */ + public void setMaxImagePixels(long maxImagePixels) { this.maxImagePixels = maxImagePixels; } + /** @return 临时图片 TTL */ + public Duration getUploadTtl() { return uploadTtl; } + /** @param uploadTtl 临时图片 TTL */ + public void setUploadTtl(Duration uploadTtl) { this.uploadTtl = uploadTtl; } + /** @return 草稿 TTL */ + public Duration getComposerDraftTtl() { return composerDraftTtl; } + /** @param composerDraftTtl 草稿 TTL */ + public void setComposerDraftTtl(Duration composerDraftTtl) { this.composerDraftTtl = composerDraftTtl; } + /** @return 清理周期 */ + public Duration getCleanupInterval() { return cleanupInterval; } + /** @param cleanupInterval 清理周期 */ + public void setCleanupInterval(Duration cleanupInterval) { this.cleanupInterval = cleanupInterval; } +} diff --git a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/config/AgentModuleConfig.java b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/config/AgentModuleConfig.java index 0e5a3dce..b7d66c6e 100644 --- a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/config/AgentModuleConfig.java +++ b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/config/AgentModuleConfig.java @@ -4,6 +4,7 @@ import org.mybatis.spring.annotation.MapperScan; import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.ComponentScan; +import org.springframework.scheduling.annotation.EnableScheduling; /** * Agent 模块自动配置。 @@ -11,6 +12,7 @@ import org.springframework.context.annotation.ComponentScan; @AutoConfiguration @MapperScan("tech.easyflow.agent.mapper") @ComponentScan("tech.easyflow.agent") -@EnableConfigurationProperties(AgentRuntimeProperties.class) +@EnableScheduling +@EnableConfigurationProperties({AgentRuntimeProperties.class, AgentMediaProperties.class}) public class AgentModuleConfig { } diff --git a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentChatRequest.java b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentChatRequest.java index 7eb703d0..7ab213f8 100644 --- a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentChatRequest.java +++ b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentChatRequest.java @@ -12,6 +12,7 @@ public class AgentChatRequest { private BigInteger agentId; private BigInteger sessionId; private String prompt; + private List imageUploadIds = new ArrayList<>(); private List capabilities = new ArrayList<>(); /** @@ -56,6 +57,22 @@ public class AgentChatRequest { */ public void setPrompt(String prompt) { this.prompt = prompt; } + /** + * 获取本轮临时图片上传 ID。 + * + * @return 图片上传 ID + */ + public List getImageUploadIds() { return imageUploadIds; } + + /** + * 设置本轮临时图片上传 ID。 + * + * @param imageUploadIds 图片上传 ID + */ + public void setImageUploadIds(List imageUploadIds) { + this.imageUploadIds = imageUploadIds == null ? new ArrayList<>() : new ArrayList<>(imageUploadIds); + } + /** * 获取本次聊天启用的临时能力。 * diff --git a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentDraftChatRequest.java b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentDraftChatRequest.java index 4e740033..2684c21d 100644 --- a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentDraftChatRequest.java +++ b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentDraftChatRequest.java @@ -5,6 +5,7 @@ import tech.easyflow.agent.entity.AgentKnowledgeBinding; import tech.easyflow.agent.entity.AgentToolBinding; import java.util.List; +import java.util.ArrayList; /** * Agent 草稿态纯文本试用请求。 @@ -16,6 +17,7 @@ public class AgentDraftChatRequest { private List knowledgeBindings; private String sessionId; private String prompt; + private List imageUploadIds = new ArrayList<>(); /** * 获取 Agent 草稿快照。 @@ -106,4 +108,22 @@ public class AgentDraftChatRequest { public void setPrompt(String prompt) { this.prompt = prompt; } + + /** + * 获取本轮临时图片上传 ID。 + * + * @return 图片上传 ID + */ + public List getImageUploadIds() { + return imageUploadIds; + } + + /** + * 设置本轮临时图片上传 ID。 + * + * @param imageUploadIds 图片上传 ID + */ + public void setImageUploadIds(List imageUploadIds) { + this.imageUploadIds = imageUploadIds == null ? new ArrayList<>() : new ArrayList<>(imageUploadIds); + } } diff --git a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentRunService.java b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentRunService.java index 6018a02b..cdebbcb4 100644 --- a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentRunService.java +++ b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentRunService.java @@ -8,6 +8,8 @@ import com.easyagents.agent.runtime.event.AgentRuntimeEventType; import com.easyagents.agent.runtime.message.AgentKnowledgeReference; import com.easyagents.agent.runtime.message.AgentMessage; import com.easyagents.agent.runtime.message.AgentMessageRole; +import com.easyagents.agent.runtime.message.AgentMediaBlock; +import com.easyagents.agent.runtime.message.AgentTextBlock; import com.easyagents.agent.runtime.persistence.session.AgentSessionStore; import com.mybatisflex.core.keygen.impl.SnowFlakeIDKeyGenerator; import org.slf4j.Logger; @@ -27,6 +29,9 @@ import tech.easyflow.agent.runtime.event.AgentRunEventRecorder; import tech.easyflow.agent.runtime.hitl.AgentHitlPendingService; import tech.easyflow.agent.runtime.lock.AgentRunLock; import tech.easyflow.agent.runtime.session.EasyFlowAgentSessionStore; +import tech.easyflow.agent.runtime.media.AgentBoundMedia; +import tech.easyflow.agent.runtime.media.AgentMediaService; +import tech.easyflow.agent.runtime.media.AgentMediaUploadRecord; import tech.easyflow.agent.service.AgentService; import tech.easyflow.ai.entity.DocumentCollection; import tech.easyflow.ai.entity.Mcp; @@ -36,6 +41,7 @@ import tech.easyflow.ai.enums.PublishStatus; import tech.easyflow.ai.rag.KnowledgeRetrievalModes; import tech.easyflow.ai.service.DocumentCollectionService; import tech.easyflow.ai.service.McpService; +import tech.easyflow.ai.service.ModelService; import tech.easyflow.ai.service.PluginItemService; import tech.easyflow.ai.service.WorkflowService; import tech.easyflow.chatlog.domain.dto.ChatSessionSummary; @@ -56,6 +62,8 @@ import javax.annotation.Resource; import java.math.BigInteger; import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; +import org.springframework.http.HttpStatus; +import org.springframework.web.server.ResponseStatusException; /** * Agent 管理端运行服务。 @@ -107,6 +115,10 @@ public class AgentRunService { private McpService mcpService; @Resource private DocumentCollectionService documentCollectionService; + @Resource + private ModelService modelService; + @Resource + private AgentMediaService agentMediaService; /** * 启动 Agent 聊天。 @@ -134,6 +146,10 @@ public class AgentRunService { AgentChatCapabilityService.AgentChatCapabilityResolution capabilityResolution = agentChatCapabilityService.apply(agent, chatRequest.getCapabilities(), account); agent = capabilityResolution.agent(); + assertImageCapability(agent, chatRequest.getImageUploadIds()); + List mediaUploads = agentMediaService.requireUploads( + chatRequest.getImageUploadIds(), AgentMediaService.MODE_FORMAL, + chatRequest.getAgentId().toString(), sessionId.toString(), account); String requestId = UUID.randomUUID().toString(); String traceId = UUID.randomUUID().toString(); // 组建会话上下文必要信息 @@ -143,7 +159,7 @@ public class AgentRunService { } applyFormalSessionTitle(chatContext, chatRequest.getPrompt(), existingSession); // 执行对话 - return run(agent, chatRequest.getPrompt(), requestId, traceId, sessionId.toString(), + return run(agent, chatRequest.getPrompt(), mediaUploads, account, requestId, traceId, sessionId.toString(), ASSISTANT_CODE, chatContext, true, easyFlowAgentSessionStore); } @@ -163,16 +179,22 @@ public class AgentRunService { if (runtimeSessionId == null || runtimeSessionId.isBlank()) { runtimeSessionId = "agent-draft-" + new SnowFlakeIDKeyGenerator().nextId(); } + assertImageCapability(agent, draftRequest.getImageUploadIds()); + List mediaUploads = agentMediaService.requireUploads( + draftRequest.getImageUploadIds(), AgentMediaService.MODE_DRAFT, + agent.getId().toString(), runtimeSessionId, account); BigInteger chatSessionId = BigInteger.valueOf(new SnowFlakeIDKeyGenerator().nextId()); String requestId = UUID.randomUUID().toString(); String traceId = UUID.randomUUID().toString(); ChatRuntimeContext chatContext = buildChatRuntimeContext(agent, chatSessionId, draftRequest.getPrompt(), account, DRAFT_ASSISTANT_CODE); - return run(agent, draftRequest.getPrompt(), requestId, traceId, runtimeSessionId, + return run(agent, draftRequest.getPrompt(), mediaUploads, account, requestId, traceId, runtimeSessionId, DRAFT_ASSISTANT_CODE, chatContext, false, draftAgentSessionStore); } private SseEmitter run(Agent agent, String prompt, + List mediaUploads, + LoginAccount account, String requestId, String traceId, String runtimeSessionId, @@ -185,6 +207,7 @@ public class AgentRunService { AgentRunLock.Handle lockHandle = acquireRunLock(agent, runtimeSessionId); boolean submitted = false; try { + List boundMedia; if (persistChatlog) { // 持久化会话初始信息 chatRuntimeManager.prepareSession(chatContext); @@ -192,9 +215,23 @@ public class AgentRunService { chatRuntimeManager.recordFailure(chatContext, new BusinessException("客户端连接已断开，Agent 运行已取消")); return chatSseEmitter.getEmitter(); } - chatRuntimeManager.recordUserMessage(chatContext, buildUserRuntimeMessage(chatContext, prompt)); + BigInteger messageId = BigInteger.valueOf(new SnowFlakeIDKeyGenerator().nextId()); + boundMedia = agentMediaService.bindFormal(mediaUploads, chatContext.getSessionId().toString(), + messageId.toString(), account); + chatRuntimeManager.recordUserMessage(chatContext, + buildUserRuntimeMessage(chatContext, messageId, prompt, boundMedia)); + if (!sendInputAccepted(chatSseEmitter, chatContext.getSessionId(), messageId, boundMedia)) { + chatRuntimeManager.recordFailure(chatContext, new BusinessException("客户端连接已断开，Agent 运行已取消")); + return chatSseEmitter.getEmitter(); + } + } else { + boundMedia = agentMediaService.bindDraft(mediaUploads); + if (!sendInputAccepted(chatSseEmitter, null, null, boundMedia)) { + return chatSseEmitter.getEmitter(); + } } - threadPoolTaskExecutor.execute(() -> startRuntime(agent, prompt, requestId, traceId, runtimeSessionId, + AgentMessage userMessage = buildAgentMessage(prompt, boundMedia); + threadPoolTaskExecutor.execute(() -> startRuntime(agent, userMessage, account, requestId, traceId, runtimeSessionId, assistantCode, chatContext, chatSseEmitter, persistChatlog, runtimeSessionStore, lockHandle)); submitted = true; return chatSseEmitter.getEmitter(); @@ -342,7 +379,8 @@ public class AgentRunService { } private void startRuntime(Agent agent, - String prompt, + AgentMessage userMessage, + LoginAccount account, String requestId, String traceId, String runtimeSessionId, @@ -374,6 +412,7 @@ public class AgentRunService { request.setToolInvokers(bundle.getToolInvokers()); request.setKnowledgeRetrievers(bundle.getKnowledgeRetrievers()); request.setSessionStore(runtimeSessionStore); + request.setMediaResolver(agentMediaService.runtimeResolver(account)); request.getMetadata().put("assistantCode", assistantCode); runtime.init(request); // 注册会话运行时管理 @@ -409,7 +448,7 @@ public class AgentRunService { return; } agentRunRegistry.bindSubscription(requestId, - runtime.stream(AgentMessage.text(AgentMessageRole.USER, prompt)).subscribe( + runtime.stream(userMessage).subscribe( runContext.eventConsumer(), runContext.errorConsumer(), runContext.completionHandler() @@ -1012,8 +1051,8 @@ public class AgentRunService { * @return 最长 200 字符的会话标题 */ private String toSessionTitle(String prompt) { - if (prompt == null) { - return null; + if (prompt == null || prompt.isBlank()) { + return "图片对话"; } return prompt.length() > 200 ? prompt.substring(0, 200) : prompt; } @@ -1028,17 +1067,43 @@ public class AgentRunService { return context; } - private ChatRuntimeMessage buildUserRuntimeMessage(ChatRuntimeContext context, String prompt) { + private ChatRuntimeMessage buildUserRuntimeMessage(ChatRuntimeContext context, + BigInteger messageId, + String prompt, + List media) { ChatRuntimeMessage message = new ChatRuntimeMessage(); + message.setMessageId(messageId); message.setRole("user"); - message.setContentType("TEXT"); + message.setContentType(media == null || media.isEmpty() ? "TEXT" : "MULTIMODAL"); message.setContentText(prompt); + if (media != null && !media.isEmpty()) { + message.getContentPayload().put("images", media.stream().map(AgentBoundMedia::payload).toList()); + } message.setCreatedAt(new Date()); message.setSenderId(context.getUserId()); message.setSenderName(context.getUserName()); return message; } + private AgentMessage buildAgentMessage(String prompt, List media) { + AgentMessage message = new AgentMessage(); + message.setRole(AgentMessageRole.USER); + List blocks = new ArrayList<>(); + if (prompt != null && !prompt.isBlank()) { + blocks.add(new AgentTextBlock(prompt)); + } + if (media != null) { + for (AgentBoundMedia item : media) { + AgentMediaBlock image = new AgentMediaBlock("image"); + image.setReference(item.reference()); + image.setMimeType(item.mimeType()); + blocks.add(image); + } + } + message.setContentBlocks(blocks); + return message; + } + private ChatRuntimeMessage buildAssistantRuntimeMessage(ChatRuntimeContext context, String content) { return buildAssistantRuntimeMessage(context, content, new ChatAssistantAccumulator(), List.of()); } @@ -1092,11 +1157,29 @@ public class AgentRunService { Map.of("sessionId", sessionId.toString())); } + private boolean sendInputAccepted(ChatSseEmitter chatSseEmitter, + BigInteger sessionId, + BigInteger messageId, + List boundMedia) { + Map payload = new LinkedHashMap<>(); + if (sessionId != null) { + payload.put("sessionId", sessionId.toString()); + } + if (messageId != null) { + payload.put("messageId", messageId.toString()); + } + if (boundMedia != null && !boundMedia.isEmpty()) { + payload.put("images", boundMedia.stream().map(AgentBoundMedia::payload).toList()); + } + return sendEnvelope(chatSseEmitter, ChatDomain.SYSTEM, ChatType.INPUT_ACCEPTED, payload); + } + private void validateChatRequest(AgentChatRequest request) { if (request == null || request.getAgentId() == null) { throw new BusinessException("Agent ID 不能为空"); } - if (request.getPrompt() == null || request.getPrompt().isBlank()) { + if ((request.getPrompt() == null || request.getPrompt().isBlank()) + && (request.getImageUploadIds() == null || request.getImageUploadIds().isEmpty())) { throw new BusinessException("Agent 输入不能为空"); } } @@ -1108,11 +1191,24 @@ public class AgentRunService { if (request.getAgent().getModelId() == null) { throw new BusinessException("Agent 模型不能为空"); } - if (request.getPrompt() == null || request.getPrompt().isBlank()) { + if ((request.getPrompt() == null || request.getPrompt().isBlank()) + && (request.getImageUploadIds() == null || request.getImageUploadIds().isEmpty())) { throw new BusinessException("Agent 输入不能为空"); } } + private void assertImageCapability(Agent agent, List imageUploadIds) { + if (imageUploadIds == null || imageUploadIds.isEmpty()) { + return; + } + tech.easyflow.ai.entity.Model model = agent == null || agent.getModelId() == null + ? null + : modelService.getModelInstance(agent.getModelId()); + if (model == null || !Boolean.TRUE.equals(model.getSupportImage())) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "当前 Agent 模型未启用多模态图片能力"); + } + } + private LoginAccount requireCurrentLoginAccount() { try { return SaTokenUtil.getLoginAccount(); diff --git a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentRuntimeCompiler.java b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentRuntimeCompiler.java index 453bd4b1..227b4f46 100644 --- a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentRuntimeCompiler.java +++ b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentRuntimeCompiler.java @@ -112,6 +112,8 @@ public class AgentRuntimeCompiler { spec.setBaseUrl(stringValue(config, "baseUrl", model.getEndpoint())); spec.setEndpointPath(stringValue(config, "endpointPath", model.getRequestPath())); spec.setApiKey(stringValue(config, "apiKey", model.getApiKey())); + spec.setSupportImage(Boolean.TRUE.equals(model.getSupportImage())); + spec.setSupportImageBase64Only(Boolean.TRUE.equals(model.getSupportImageB64Only())); spec.getMetadata().put("modelId", model.getId()); return spec; } diff --git a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/composer/AgentComposerDraft.java b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/composer/AgentComposerDraft.java new file mode 100644 index 00000000..63bc2962 --- /dev/null +++ b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/composer/AgentComposerDraft.java @@ -0,0 +1,59 @@ +package tech.easyflow.agent.runtime.composer; + +import tech.easyflow.agent.runtime.media.AgentMediaUploadView; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +/** + * 可跨刷新恢复的 Agent 聊天输入草稿。 + */ +public class AgentComposerDraft { + + private String mode; + private String agentId; + private String sessionId; + private String text; + private List imageUploadIds = new ArrayList<>(); + private List images = new ArrayList<>(); + private long revision; + private Instant expiresAt; + + /** @return 聊天模式 */ + public String getMode() { return mode; } + /** @param mode 聊天模式 */ + public void setMode(String mode) { this.mode = mode; } + /** @return Agent ID */ + public String getAgentId() { return agentId; } + /** @param agentId Agent ID */ + public void setAgentId(String agentId) { this.agentId = agentId; } + /** @return 会话 ID */ + public String getSessionId() { return sessionId; } + /** @param sessionId 会话 ID */ + public void setSessionId(String sessionId) { this.sessionId = sessionId; } + /** @return 输入文本 */ + public String getText() { return text; } + /** @param text 输入文本 */ + public void setText(String text) { this.text = text; } + /** @return 图片上传 ID */ + public List getImageUploadIds() { return imageUploadIds; } + /** @param imageUploadIds 图片上传 ID */ + public void setImageUploadIds(List imageUploadIds) { + this.imageUploadIds = imageUploadIds == null ? new ArrayList<>() : new ArrayList<>(imageUploadIds); + } + /** @return 图片展示信息 */ + public List getImages() { return images; } + /** @param images 图片展示信息 */ + public void setImages(List images) { + this.images = images == null ? new ArrayList<>() : new ArrayList<>(images); + } + /** @return 草稿修订号 */ + public long getRevision() { return revision; } + /** @param revision 草稿修订号 */ + public void setRevision(long revision) { this.revision = revision; } + /** @return 过期时间 */ + public Instant getExpiresAt() { return expiresAt; } + /** @param expiresAt 过期时间 */ + public void setExpiresAt(Instant expiresAt) { this.expiresAt = expiresAt; } +} diff --git a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/composer/AgentComposerDraftService.java b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/composer/AgentComposerDraftService.java new file mode 100644 index 00000000..77abf227 --- /dev/null +++ b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/composer/AgentComposerDraftService.java @@ -0,0 +1,330 @@ +package tech.easyflow.agent.runtime.composer; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.mybatisflex.core.keygen.impl.SnowFlakeIDKeyGenerator; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.data.redis.core.script.DefaultRedisScript; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import org.springframework.web.server.ResponseStatusException; +import tech.easyflow.agent.config.AgentMediaProperties; +import tech.easyflow.agent.runtime.media.AgentMediaService; +import tech.easyflow.agent.runtime.media.AgentMediaUploadRecord; +import tech.easyflow.agent.runtime.media.AgentMediaUploadView; +import tech.easyflow.common.entity.LoginAccount; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Optional; + +/** + * 基于 Redis 的 Agent 输入草稿与当前未发送会话管理服务。 + */ +@Service +public class AgentComposerDraftService { + + private static final String DRAFT_PREFIX = "easyflow:agent:composer:draft:"; + private static final String REVISION_PREFIX = "easyflow:agent:composer:revision:"; + private static final String ACTIVE_PREFIX = "easyflow:agent:composer:active:"; + private static final DefaultRedisScript SAVE_SCRIPT = new DefaultRedisScript<>(""" + local current = redis.call('GET', KEYS[2]) + if current and tonumber(current) ~= tonumber(ARGV[1]) then + return -1 + end + redis.call('SET', KEYS[1], ARGV[2], 'EX', ARGV[4]) + redis.call('SET', KEYS[2], ARGV[3], 'EX', ARGV[4]) + redis.call('SET', KEYS[3], ARGV[5], 'EX', ARGV[4]) + return tonumber(ARGV[3]) + """, Long.class); + + private final StringRedisTemplate redisTemplate; + private final ObjectMapper objectMapper; + private final AgentMediaProperties properties; + private final AgentMediaService mediaService; + + /** + * 创建输入草稿服务。 + * + * @param redisTemplate Redis 模板 + * @param objectMapper JSON 映射器 + * @param properties Agent 媒体配置 + * @param mediaService Agent 媒体服务 + */ + public AgentComposerDraftService(StringRedisTemplate redisTemplate, + ObjectMapper objectMapper, + AgentMediaProperties properties, + AgentMediaService mediaService) { + this.redisTemplate = redisTemplate; + this.objectMapper = objectMapper; + this.properties = properties; + this.mediaService = mediaService; + } + + /** + * 为正式或草稿聊天预分配稳定会话 ID。 + * + * @param mode 聊天模式 + * @return 会话信息 + */ + public AgentComposerSession allocateSession(String mode) { + String safeMode = mode(mode); + long id = new SnowFlakeIDKeyGenerator().nextId(); + return new AgentComposerSession(safeMode, + AgentMediaService.MODE_DRAFT.equals(safeMode) ? "agent-draft-" + id : String.valueOf(id)); + } + + /** + * 保存输入草稿并对相关图片续期。 + * + * @param draft 草稿内容 + * @param account 当前账号 + * @return 保存后的草稿 + */ + public AgentComposerDraft save(AgentComposerDraft draft, LoginAccount account) { + if (draft == null) { + throw badRequest("聊天草稿不能为空"); + } + Identity identity = identity(account); + String safeMode = mode(draft.getMode()); + String agentId = text(draft.getAgentId(), "Agent ID 不能为空"); + String sessionId = text(draft.getSessionId(), "会话 ID 不能为空"); + validateSessionId(safeMode, sessionId); + List uploads = mediaService.requireUploads(draft.getImageUploadIds(), safeMode, + agentId, sessionId, account); + for (AgentMediaUploadRecord upload : uploads) { + mediaService.bindDraft(List.of(upload)); + } + draft.setMode(safeMode); + draft.setAgentId(agentId); + draft.setSessionId(sessionId); + draft.setText(draft.getText() == null ? "" : draft.getText()); + if (draft.getText().length() > 100_000) { + throw badRequest("输入内容过长"); + } + draft.setImages(uploads.stream().map(this::toView).toList()); + long expectedRevision = Math.max(0L, draft.getRevision()); + long nextRevision = expectedRevision + 1L; + draft.setRevision(nextRevision); + Duration ttl = draftTtl(); + draft.setExpiresAt(Instant.now().plus(ttl)); + String scope = scope(identity, safeMode, agentId, sessionId); + try { + Long result = redisTemplate.execute(SAVE_SCRIPT, + List.of(DRAFT_PREFIX + scope, REVISION_PREFIX + scope, activeKey(identity, safeMode, agentId)), + String.valueOf(expectedRevision), objectMapper.writeValueAsString(draft), + String.valueOf(nextRevision), String.valueOf(Math.max(1L, ttl.toSeconds())), sessionId); + if (result == null || result < 0L) { + throw new ResponseStatusException(HttpStatus.CONFLICT, "聊天草稿已在其他窗口更新，请刷新后重试"); + } + return draft; + } catch (ResponseStatusException error) { + throw error; + } catch (Exception error) { + throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "聊天草稿保存失败", error); + } + } + + /** + * 获取指定会话或最近未发送会话的草稿，读取不延长 TTL。 + * + * @param mode 聊天模式 + * @param agentId Agent ID + * @param sessionId 会话 ID，可为空 + * @param account 当前账号 + * @return 草稿 + */ + public Optional get(String mode, + String agentId, + String sessionId, + LoginAccount account) { + Identity identity = identity(account); + String safeMode = mode(mode); + String safeAgentId = text(agentId, "Agent ID 不能为空"); + String resolvedSessionId = sessionId; + if (!StringUtils.hasText(resolvedSessionId)) { + resolvedSessionId = redisTemplate.opsForValue().get(activeKey(identity, safeMode, safeAgentId)); + } + if (!StringUtils.hasText(resolvedSessionId)) { + return Optional.empty(); + } + validateSessionId(safeMode, resolvedSessionId); + String value = redisTemplate.opsForValue().get(DRAFT_PREFIX + + scope(identity, safeMode, safeAgentId, resolvedSessionId)); + if (!StringUtils.hasText(value)) { + return Optional.empty(); + } + try { + AgentComposerDraft draft = objectMapper.readValue(value, AgentComposerDraft.class); + List uploads = availableUploads(draft.getImageUploadIds(), safeMode, + safeAgentId, resolvedSessionId, account); + draft.setImageUploadIds(uploads.stream().map(AgentMediaUploadRecord::getUploadId).toList()); + draft.setImages(uploads.stream().map(this::toView).toList()); + return Optional.of(draft); + } catch (Exception error) { + if (error instanceof ResponseStatusException responseError) { + throw responseError; + } + throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "聊天草稿读取失败", error); + } + } + + /** + * 读取仍有效的草稿图片，跳过自然过期项并保留权限或归属异常。 + * + * @param uploadIds 图片上传 ID + * @param mode 聊天模式 + * @param agentId Agent ID + * @param sessionId 会话 ID + * @param account 当前账号 + * @return 仍有效且保持原顺序的图片 + */ + private List availableUploads(List uploadIds, + String mode, + String agentId, + String sessionId, + LoginAccount account) { + List available = new ArrayList<>(); + for (String uploadId : uploadIds == null ? List.of() : uploadIds) { + try { + available.addAll(mediaService.requireUploads(List.of(uploadId), mode, agentId, sessionId, account)); + } catch (ResponseStatusException error) { + if (error.getStatusCode() == HttpStatus.BAD_REQUEST + && "图片已过期，请重新上传".equals(error.getReason())) { + continue; + } + throw error; + } + } + return available; + } + + /** + * 删除已发送或主动清空的输入草稿。 + * + * @param mode 聊天模式 + * @param agentId Agent ID + * @param sessionId 会话 ID + * @param account 当前账号 + */ + public void delete(String mode, String agentId, String sessionId, LoginAccount account) { + delete(mode, agentId, sessionId, List.of(), true, account); + } + + /** + * 删除输入草稿，并按调用语义选择是否清理临时图片。 + * + * @param mode 聊天模式 + * @param agentId Agent ID + * @param sessionId 会话 ID + * @param requestedUploadIds 调用方当前持有的上传 ID + * @param deleteUploads 是否删除临时图片 + * @param account 当前账号 + */ + public void delete(String mode, + String agentId, + String sessionId, + List requestedUploadIds, + boolean deleteUploads, + LoginAccount account) { + Identity identity = identity(account); + String safeMode = mode(mode); + String safeAgentId = text(agentId, "Agent ID 不能为空"); + String safeSessionId = text(sessionId, "会话 ID 不能为空"); + validateSessionId(safeMode, safeSessionId); + String scope = scope(identity, safeMode, safeAgentId, safeSessionId); + String draftKey = DRAFT_PREFIX + scope; + if (deleteUploads) { + LinkedHashSet uploadIds = new LinkedHashSet<>(); + String serializedDraft = redisTemplate.opsForValue().get(draftKey); + if (StringUtils.hasText(serializedDraft)) { + try { + AgentComposerDraft storedDraft = objectMapper.readValue(serializedDraft, AgentComposerDraft.class); + if (storedDraft.getImageUploadIds() != null) { + uploadIds.addAll(storedDraft.getImageUploadIds()); + } + } catch (Exception error) { + throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "聊天草稿删除失败", error); + } + } + if (requestedUploadIds != null) { + uploadIds.addAll(requestedUploadIds); + } + mediaService.deleteUploadsForScope(new ArrayList<>(uploadIds), safeMode, + safeAgentId, safeSessionId, account); + } + redisTemplate.delete(List.of(draftKey, REVISION_PREFIX + scope)); + String activeKey = activeKey(identity, safeMode, safeAgentId); + String activeSession = redisTemplate.opsForValue().get(activeKey); + if (safeSessionId.equals(activeSession)) { + redisTemplate.delete(activeKey); + } + } + + private AgentMediaUploadView toView(AgentMediaUploadRecord record) { + AgentMediaUploadView view = new AgentMediaUploadView(); + view.setUploadId(record.getUploadId()); + view.setName(record.getOriginalName()); + view.setMimeType(record.getMimeType()); + view.setSize(record.getSize()); + view.setWidth(record.getWidth()); + view.setHeight(record.getHeight()); + view.setExpiresAt(record.getExpiresAt()); + view.setPreviewUrl("/api/v1/agent/media/content?reference=draft%3A" + record.getUploadId()); + return view; + } + + private void validateSessionId(String mode, String sessionId) { + boolean valid = AgentMediaService.MODE_DRAFT.equals(mode) + ? sessionId.matches("agent-draft-\\d+") + : sessionId.matches("\\d+"); + if (!valid) { + throw badRequest("会话 ID 无效"); + } + } + + private String scope(Identity identity, String mode, String agentId, String sessionId) { + return identity.tenantId + ":" + identity.userId + ":" + mode + ":" + agentId + ":" + sessionId; + } + + private String activeKey(Identity identity, String mode, String agentId) { + return ACTIVE_PREFIX + identity.tenantId + ":" + identity.userId + ":" + mode + ":" + agentId; + } + + private String mode(String value) { + String normalized = value == null ? "" : value.trim().toUpperCase(Locale.ROOT); + if (!AgentMediaService.MODE_FORMAL.equals(normalized) && !AgentMediaService.MODE_DRAFT.equals(normalized)) { + throw badRequest("聊天模式无效"); + } + return normalized; + } + + private String text(String value, String message) { + if (!StringUtils.hasText(value) || value.length() > 200) { + throw badRequest(message); + } + return value.trim(); + } + + private Duration draftTtl() { + Duration ttl = properties.getComposerDraftTtl(); + return ttl == null || ttl.isZero() || ttl.isNegative() ? Duration.ofHours(24) : ttl; + } + + private Identity identity(LoginAccount account) { + if (account == null || account.getId() == null || account.getTenantId() == null) { + throw new ResponseStatusException(HttpStatus.UNAUTHORIZED, "当前登录状态失效"); + } + return new Identity(account.getTenantId().toString(), account.getId().toString()); + } + + private ResponseStatusException badRequest(String message) { + return new ResponseStatusException(HttpStatus.BAD_REQUEST, message); + } + + private record Identity(String tenantId, String userId) { } +} diff --git a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/composer/AgentComposerSession.java b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/composer/AgentComposerSession.java new file mode 100644 index 00000000..78113ef5 --- /dev/null +++ b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/composer/AgentComposerSession.java @@ -0,0 +1,10 @@ +package tech.easyflow.agent.runtime.composer; + +/** + * 输入框使用的预分配会话标识。 + * + * @param mode 聊天模式 + * @param sessionId 会话 ID + */ +public record AgentComposerSession(String mode, String sessionId) { +} diff --git a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentBoundMedia.java b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentBoundMedia.java new file mode 100644 index 00000000..6d5f9838 --- /dev/null +++ b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentBoundMedia.java @@ -0,0 +1,13 @@ +package tech.easyflow.agent.runtime.media; + +import java.util.Map; + +/** + * 已绑定到聊天消息的图片引用与展示载荷。 + * + * @param reference 运行时稳定引用 + * @param mimeType MIME 类型 + * @param payload 聊天历史展示载荷 + */ +public record AgentBoundMedia(String reference, String mimeType, Map payload) { +} diff --git a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentImageData.java b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentImageData.java new file mode 100644 index 00000000..8630570d --- /dev/null +++ b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentImageData.java @@ -0,0 +1,32 @@ +package tech.easyflow.agent.runtime.media; + +import java.util.Arrays; + +/** + * 校验并规范化后的 Agent 图片。 + * + * @param bytes 图片字节 + * @param mimeType MIME 类型 + * @param extension 规范化扩展名 + * @param width 宽度 + * @param height 高度 + * @param sha256 SHA-256 摘要 + */ +public record AgentImageData(byte[] bytes, + String mimeType, + String extension, + int width, + int height, + String sha256) { + + /** + * 创建不可变图片数据。 + */ + public AgentImageData { + bytes = bytes == null ? new byte[0] : Arrays.copyOf(bytes, bytes.length); + } + + /** @return 图片字节副本 */ + @Override + public byte[] bytes() { return Arrays.copyOf(bytes, bytes.length); } +} diff --git a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentImageProcessor.java b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentImageProcessor.java new file mode 100644 index 00000000..e14fa108 --- /dev/null +++ b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentImageProcessor.java @@ -0,0 +1,213 @@ +package tech.easyflow.agent.runtime.media; + +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Component; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.server.ResponseStatusException; +import tech.easyflow.agent.config.AgentMediaProperties; + +import javax.imageio.ImageIO; +import javax.imageio.ImageReader; +import javax.imageio.stream.ImageInputStream; +import java.awt.image.BufferedImage; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.HexFormat; +import java.util.Iterator; + +/** + * Agent 聊天图片格式识别、尺寸校验与规范化处理器。 + */ +@Component +public class AgentImageProcessor { + + private final AgentMediaProperties properties; + + /** + * 创建图片处理器。 + * + * @param properties Agent 媒体配置 + */ + public AgentImageProcessor(AgentMediaProperties properties) { + this.properties = properties; + } + + /** + * 校验并规范化上传图片。 + * + * @param file 上传文件 + * @return 规范化图片 + */ + public AgentImageData process(MultipartFile file) { + if (file == null || file.isEmpty()) { + throw badRequest("请选择要上传的图片"); + } + if (file.getSize() > properties.getMaxImageBytes()) { + throw badRequest("单张图片不能超过 10 MiB"); + } + try { + byte[] source = file.getBytes(); + ImageFormat format = detectFormat(source); + Dimensions dimensions = format == ImageFormat.WEBP + ? webpDimensions(source) + : imageIoDimensions(source); + validateDimensions(dimensions); + byte[] normalized = source; + String mimeType = format.mimeType; + String extension = format.extension; + // 动图仅保留首帧，BMP 转 PNG，避免模型端格式兼容差异。 + if (format == ImageFormat.GIF || format == ImageFormat.BMP) { + BufferedImage image = ImageIO.read(new ByteArrayInputStream(source)); + if (image == null) { + throw badRequest("图片内容无法解析"); + } + ByteArrayOutputStream output = new ByteArrayOutputStream(); + if (!ImageIO.write(image, "png", output)) { + throw badRequest("图片格式转换失败"); + } + normalized = output.toByteArray(); + mimeType = "image/png"; + extension = "png"; + } + if (normalized.length > properties.getMaxImageBytes()) { + throw badRequest("处理后的图片不能超过 10 MiB"); + } + return new AgentImageData(normalized, mimeType, extension, + dimensions.width, dimensions.height, sha256(normalized)); + } catch (ResponseStatusException error) { + throw error; + } catch (Exception error) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "图片处理失败", error); + } + } + + private ImageFormat detectFormat(byte[] bytes) { + if (bytes.length >= 8 + && bytes[0] == (byte) 0x89 && bytes[1] == 0x50 && bytes[2] == 0x4e && bytes[3] == 0x47) { + return ImageFormat.PNG; + } + if (bytes.length >= 3 && bytes[0] == (byte) 0xff && bytes[1] == (byte) 0xd8 && bytes[2] == (byte) 0xff) { + return ImageFormat.JPEG; + } + if (bytes.length >= 6) { + String header = new String(bytes, 0, 6, StandardCharsets.US_ASCII); + if ("GIF87a".equals(header) || "GIF89a".equals(header)) { + return ImageFormat.GIF; + } + } + if (bytes.length >= 2 && bytes[0] == 'B' && bytes[1] == 'M') { + return ImageFormat.BMP; + } + if (bytes.length >= 12 + && "RIFF".equals(ascii(bytes, 0, 4)) + && "WEBP".equals(ascii(bytes, 8, 4))) { + return ImageFormat.WEBP; + } + throw badRequest("仅支持 PNG、JPG、JPEG、WebP、GIF、BMP 图片"); + } + + private Dimensions imageIoDimensions(byte[] bytes) throws Exception { + try (ImageInputStream input = ImageIO.createImageInputStream(new ByteArrayInputStream(bytes))) { + Iterator readers = ImageIO.getImageReaders(input); + if (!readers.hasNext()) { + throw badRequest("图片内容无法解析"); + } + ImageReader reader = readers.next(); + try { + reader.setInput(input, true, true); + return new Dimensions(reader.getWidth(0), reader.getHeight(0)); + } finally { + reader.dispose(); + } + } + } + + private Dimensions webpDimensions(byte[] bytes) { + int offset = 12; + while (offset + 8 <= bytes.length) { + String chunk = ascii(bytes, offset, 4); + int size = littleEndianInt(bytes, offset + 4); + int data = offset + 8; + if (size < 0 || data + size > bytes.length) { + break; + } + if ("VP8X".equals(chunk) && size >= 10) { + return new Dimensions(1 + littleEndian24(bytes, data + 4), + 1 + littleEndian24(bytes, data + 7)); + } + if ("VP8 ".equals(chunk) && size >= 10 + && bytes[data + 3] == (byte) 0x9d && bytes[data + 4] == 0x01 && bytes[data + 5] == 0x2a) { + return new Dimensions(littleEndian16(bytes, data + 6) & 0x3fff, + littleEndian16(bytes, data + 8) & 0x3fff); + } + if ("VP8L".equals(chunk) && size >= 5 && bytes[data] == 0x2f) { + int b1 = unsigned(bytes[data + 1]); + int b2 = unsigned(bytes[data + 2]); + int b3 = unsigned(bytes[data + 3]); + int b4 = unsigned(bytes[data + 4]); + int width = 1 + ((b1 | (b2 << 8)) & 0x3fff); + int height = 1 + (((b2 >> 6) | (b3 << 2) | (b4 << 10)) & 0x3fff); + return new Dimensions(width, height); + } + offset = data + size + (size & 1); + } + throw badRequest("WebP 图片内容无法解析"); + } + + private void validateDimensions(Dimensions dimensions) { + if (dimensions.width <= 0 || dimensions.height <= 0) { + throw badRequest("图片尺寸无效"); + } + long pixels = (long) dimensions.width * dimensions.height; + if (pixels > properties.getMaxImagePixels()) { + throw badRequest("图片像素过大，请压缩后重试"); + } + } + + private String sha256(byte[] bytes) throws Exception { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes)); + } + + private String ascii(byte[] bytes, int offset, int length) { + return new String(bytes, offset, length, StandardCharsets.US_ASCII); + } + + private int littleEndian16(byte[] bytes, int offset) { + return unsigned(bytes[offset]) | (unsigned(bytes[offset + 1]) << 8); + } + + private int littleEndian24(byte[] bytes, int offset) { + return unsigned(bytes[offset]) | (unsigned(bytes[offset + 1]) << 8) | (unsigned(bytes[offset + 2]) << 16); + } + + private int littleEndianInt(byte[] bytes, int offset) { + return unsigned(bytes[offset]) | (unsigned(bytes[offset + 1]) << 8) + | (unsigned(bytes[offset + 2]) << 16) | (unsigned(bytes[offset + 3]) << 24); + } + + private int unsigned(byte value) { return value & 0xff; } + + private ResponseStatusException badRequest(String message) { + return new ResponseStatusException(HttpStatus.BAD_REQUEST, message); + } + + private record Dimensions(int width, int height) { } + + private enum ImageFormat { + PNG("image/png", "png"), + JPEG("image/jpeg", "jpg"), + WEBP("image/webp", "webp"), + GIF("image/gif", "gif"), + BMP("image/bmp", "bmp"); + + private final String mimeType; + private final String extension; + + ImageFormat(String mimeType, String extension) { + this.mimeType = mimeType; + this.extension = extension; + } + } +} diff --git a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentMediaCleanupScheduler.java b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentMediaCleanupScheduler.java new file mode 100644 index 00000000..a15092d8 --- /dev/null +++ b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentMediaCleanupScheduler.java @@ -0,0 +1,40 @@ +package tech.easyflow.agent.runtime.media; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +/** + * 定期删除 Redis 凭据已过期的 MinIO 临时图片。 + */ +@Component +public class AgentMediaCleanupScheduler { + + private static final Logger LOG = LoggerFactory.getLogger(AgentMediaCleanupScheduler.class); + private final AgentMediaService mediaService; + + /** + * 创建清理任务。 + * + * @param mediaService Agent 媒体服务 + */ + public AgentMediaCleanupScheduler(AgentMediaService mediaService) { + this.mediaService = mediaService; + } + + /** + * 清理一批过期临时图片。 + */ + @Scheduled(fixedDelayString = "${easyflow.agent.media.cleanup-interval:10m}") + public void cleanup() { + try { + int cleaned = mediaService.cleanupExpired(200); + if (cleaned > 0) { + LOG.info("Cleaned {} expired Agent media objects", cleaned); + } + } catch (Exception error) { + LOG.error("Agent media cleanup failed", error); + } + } +} diff --git a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentMediaObjectStorage.java b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentMediaObjectStorage.java new file mode 100644 index 00000000..2dc17117 --- /dev/null +++ b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentMediaObjectStorage.java @@ -0,0 +1,233 @@ +package tech.easyflow.agent.runtime.media; + +import io.minio.*; +import io.minio.errors.ErrorResponseException; +import io.minio.messages.DeleteError; +import io.minio.messages.DeleteObject; +import io.minio.messages.Item; +import org.dromara.x.file.storage.core.FileStorageService; +import org.dromara.x.file.storage.core.platform.MinioFileStorage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.event.EventListener; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.web.server.ResponseStatusException; +import tech.easyflow.agent.config.AgentMediaProperties; + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * 基于独立私有 MinIO 桶的 Agent 媒体对象存储。 + */ +@Service +public class AgentMediaObjectStorage { + + private static final Logger LOG = LoggerFactory.getLogger(AgentMediaObjectStorage.class); + private final FileStorageService fileStorageService; + private final AgentMediaProperties properties; + private final AtomicBoolean bucketReady = new AtomicBoolean(false); + + /** + * 创建 Agent 媒体对象存储。 + * + * @param fileStorageService x-file-storage 服务 + * @param properties Agent 媒体配置 + */ + public AgentMediaObjectStorage(FileStorageService fileStorageService, AgentMediaProperties properties) { + this.fileStorageService = fileStorageService; + this.properties = properties; + } + + /** + * 应用启动后预创建私有桶；失败时保留完整日志，首次使用会再次尝试。 + */ + @EventListener(ApplicationReadyEvent.class) + public void initializeBucket() { + try { + ensureBucket(); + } catch (Exception error) { + LOG.error("Agent media bucket initialization failed", error); + } + } + + /** + * 写入对象。 + * + * @param objectKey 对象键 + * @param data 对象字节 + * @param mimeType MIME 类型 + */ + public void put(String objectKey, byte[] data, String mimeType) { + try { + MinioFileStorage storage = storage(); + ensureBucket(); + storage.getClient().putObject(PutObjectArgs.builder() + .bucket(storage.getBucketName()) + .object(fullKey(storage, objectKey)) + .stream(new java.io.ByteArrayInputStream(data), data.length, -1) + .contentType(mimeType) + .build()); + } catch (Exception error) { + throw storageError("图片上传到对象存储失败", error); + } + } + + /** + * 读取对象并限制最大内存占用。 + * + * @param objectKey 对象键 + * @param maxBytes 最大读取字节数 + * @return 对象字节 + */ + public byte[] get(String objectKey, long maxBytes) { + try { + MinioFileStorage storage = storage(); + ensureBucket(); + try (InputStream input = storage.getClient().getObject(GetObjectArgs.builder() + .bucket(storage.getBucketName()).object(fullKey(storage, objectKey)).build()); + ByteArrayOutputStream output = new ByteArrayOutputStream()) { + byte[] buffer = new byte[8192]; + long total = 0L; + int read; + while ((read = input.read(buffer)) >= 0) { + total += read; + if (total > maxBytes) { + throw new ResponseStatusException(HttpStatus.PAYLOAD_TOO_LARGE, "图片对象超过允许大小"); + } + output.write(buffer, 0, read); + } + return output.toByteArray(); + } + } catch (ResponseStatusException error) { + throw error; + } catch (ErrorResponseException error) { + String code = error.errorResponse() == null ? null : error.errorResponse().code(); + if ("NoSuchKey".equals(code) || "NoSuchObject".equals(code)) { + throw new ResponseStatusException(HttpStatus.NOT_FOUND, "图片不存在或已删除", error); + } + throw storageError("图片对象读取失败", error); + } catch (Exception error) { + throw storageError("图片对象读取失败", error); + } + } + + /** + * 在私有桶内复制对象。 + * + * @param sourceKey 源对象键 + * @param targetKey 目标对象键 + */ + public void copy(String sourceKey, String targetKey) { + try { + MinioFileStorage storage = storage(); + ensureBucket(); + storage.getClient().copyObject(CopyObjectArgs.builder() + .bucket(storage.getBucketName()) + .object(fullKey(storage, targetKey)) + .source(CopySource.builder().bucket(storage.getBucketName()) + .object(fullKey(storage, sourceKey)).build()) + .build()); + } catch (Exception error) { + throw storageError("图片对象归档失败", error); + } + } + + /** + * 删除对象，重复删除视为成功。 + * + * @param objectKey 对象键 + */ + public void delete(String objectKey) { + try { + MinioFileStorage storage = storage(); + ensureBucket(); + storage.getClient().removeObject(RemoveObjectArgs.builder() + .bucket(storage.getBucketName()).object(fullKey(storage, objectKey)).build()); + } catch (Exception error) { + throw storageError("图片对象删除失败", error); + } + } + + /** + * 删除指定前缀下的全部对象。 + * + * @param objectPrefix 对象键前缀 + */ + public void deletePrefix(String objectPrefix) { + try { + MinioFileStorage storage = storage(); + ensureBucket(); + String fullPrefix = fullKey(storage, objectPrefix); + List batch = new ArrayList<>(1000); + for (Result result : storage.getClient().listObjects(ListObjectsArgs.builder() + .bucket(storage.getBucketName()) + .prefix(fullPrefix) + .recursive(true) + .build())) { + batch.add(new DeleteObject(result.get().objectName())); + if (batch.size() == 1000) { + removeBatch(storage, batch); + batch.clear(); + } + } + removeBatch(storage, batch); + } catch (Exception error) { + throw storageError("图片对象目录删除失败", error); + } + } + + private synchronized void ensureBucket() throws Exception { + if (bucketReady.get()) { + return; + } + MinioFileStorage storage = storage(); + boolean exists = storage.getClient().bucketExists( + BucketExistsArgs.builder().bucket(storage.getBucketName()).build()); + if (!exists) { + storage.getClient().makeBucket(MakeBucketArgs.builder().bucket(storage.getBucketName()).build()); + } + bucketReady.set(true); + } + + private MinioFileStorage storage() { + MinioFileStorage storage = fileStorageService.getFileStorage(properties.getPlatform()); + if (storage == null) { + throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, + "Agent 私有媒体存储平台未配置"); + } + return storage; + } + + private void removeBatch(MinioFileStorage storage, List objects) throws Exception { + if (objects.isEmpty()) { + return; + } + Iterable> errors = storage.getClient().removeObjects(RemoveObjectsArgs.builder() + .bucket(storage.getBucketName()) + .objects(List.copyOf(objects)) + .build()); + for (Result result : errors) { + DeleteError error = result.get(); + throw new IllegalStateException("删除 MinIO 对象失败: " + error.objectName() + ", " + error.message()); + } + } + + private String fullKey(MinioFileStorage storage, String objectKey) { + String basePath = storage.getBasePath(); + if (basePath == null || basePath.isBlank()) { + return objectKey; + } + return basePath.replaceAll("/+$", "") + "/" + objectKey.replaceAll("^/+", ""); + } + + private ResponseStatusException storageError(String message, Exception error) { + LOG.error(message, error); + return new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, message, error); + } +} diff --git a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentMediaService.java b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentMediaService.java new file mode 100644 index 00000000..6da3ea4b --- /dev/null +++ b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentMediaService.java @@ -0,0 +1,428 @@ +package tech.easyflow.agent.runtime.media; + +import com.easyagents.agent.runtime.media.AgentMediaResolver; +import com.easyagents.agent.runtime.media.AgentMediaResource; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.server.ResponseStatusException; +import org.springframework.web.util.UriUtils; +import tech.easyflow.agent.config.AgentMediaProperties; +import tech.easyflow.common.entity.LoginAccount; + +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.*; + +/** + * Agent 图片上传、归档、鉴权读取和运行时解析服务。 + */ +@Service +public class AgentMediaService { + + public static final String MODE_FORMAL = "FORMAL"; + public static final String MODE_DRAFT = "DRAFT"; + + private final AgentImageProcessor imageProcessor; + private final AgentMediaObjectStorage objectStorage; + private final RedisAgentMediaUploadStore uploadStore; + private final AgentMediaProperties properties; + + /** + * 创建 Agent 媒体服务。 + * + * @param imageProcessor 图片处理器 + * @param objectStorage 私有对象存储 + * @param uploadStore 临时凭据存储 + * @param properties Agent 媒体配置 + */ + public AgentMediaService(AgentImageProcessor imageProcessor, + AgentMediaObjectStorage objectStorage, + RedisAgentMediaUploadStore uploadStore, + AgentMediaProperties properties) { + this.imageProcessor = imageProcessor; + this.objectStorage = objectStorage; + this.uploadStore = uploadStore; + this.properties = properties; + } + + /** + * 上传一张聊天临时图片。 + * + * @param file 图片文件 + * @param mode 聊天模式 + * @param agentId Agent ID + * @param sessionId 会话 ID + * @param account 当前账号 + * @return 上传结果 + */ + public AgentMediaUploadView upload(MultipartFile file, + String mode, + String agentId, + String sessionId, + LoginAccount account) { + String safeMode = requireMode(mode); + String safeAgentId = requireText(agentId, "Agent ID 不能为空"); + String safeSessionId = requireText(sessionId, "会话 ID 不能为空"); + Identity identity = identity(account); + AgentImageData image = imageProcessor.process(file); + String uploadId = UUID.randomUUID().toString().replace("-", ""); + String objectKey = "temp/%s/%s/%s.%s".formatted(identity.tenantId, identity.userId, + uploadId, image.extension()); + objectStorage.put(objectKey, image.bytes(), image.mimeType()); + AgentMediaUploadRecord record = new AgentMediaUploadRecord(); + record.setUploadId(uploadId); + record.setTenantId(identity.tenantId); + record.setUserId(identity.userId); + record.setMode(safeMode); + record.setAgentId(safeAgentId); + record.setSessionId(safeSessionId); + record.setObjectKey(objectKey); + record.setOriginalName(safeOriginalName(file.getOriginalFilename(), image.extension())); + record.setMimeType(image.mimeType()); + record.setExtension(image.extension()); + record.setSize(image.bytes().length); + record.setWidth(image.width()); + record.setHeight(image.height()); + record.setSha256(image.sha256()); + record.setCreatedAt(Instant.now()); + try { + uploadStore.create(record); + } catch (RuntimeException error) { + objectStorage.delete(objectKey); + throw error; + } + return toView(record); + } + + /** + * 删除当前账号尚未绑定的临时图片。 + * + * @param uploadId 上传 ID + * @param account 当前账号 + */ + public void deleteUpload(String uploadId, LoginAccount account) { + AgentMediaUploadRecord record = uploadStore.find(uploadId).orElse(null); + if (record == null) { + return; + } + assertOwner(record, identity(account)); + objectStorage.delete(record.getObjectKey()); + uploadStore.delete(record); + } + + /** + * 幂等删除指定草稿作用域内的临时图片。 + * + * @param uploadIds 上传 ID + * @param mode 聊天模式 + * @param agentId Agent ID + * @param sessionId 会话 ID + * @param account 当前账号 + */ + public void deleteUploadsForScope(List uploadIds, + String mode, + String agentId, + String sessionId, + LoginAccount account) { + Identity identity = identity(account); + String safeMode = requireMode(mode); + String safeAgentId = requireText(agentId, "Agent ID 不能为空"); + String safeSessionId = requireText(sessionId, "会话 ID 不能为空"); + for (String uploadId : uploadIds == null ? List.of() : uploadIds.stream() + .filter(StringUtils::hasText).map(String::trim).distinct().toList()) { + AgentMediaUploadRecord record = uploadStore.find(uploadId).orElse(null); + if (record == null) { + continue; + } + assertOwner(record, identity); + if (!safeMode.equals(record.getMode()) + || !safeAgentId.equals(record.getAgentId()) + || !safeSessionId.equals(record.getSessionId())) { + throw new ResponseStatusException(HttpStatus.FORBIDDEN, "图片不属于当前聊天会话"); + } + objectStorage.delete(record.getObjectKey()); + uploadStore.delete(record); + } + } + + /** + * 校验并取得本轮图片上传凭据。 + * + * @param uploadIds 上传 ID + * @param mode 聊天模式 + * @param agentId Agent ID + * @param sessionId 会话 ID + * @param account 当前账号 + * @return 有序上传凭据 + */ + public List requireUploads(List uploadIds, + String mode, + String agentId, + String sessionId, + LoginAccount account) { + List ids = uploadIds == null ? List.of() : uploadIds.stream() + .filter(StringUtils::hasText).map(String::trim).distinct().toList(); + if (ids.size() > properties.getMaxImageCount()) { + throw badRequest("每次最多发送 " + properties.getMaxImageCount() + " 张图片"); + } + if (ids.isEmpty()) { + return List.of(); + } + Identity identity = identity(account); + String safeMode = requireMode(mode); + String safeAgentId = requireText(agentId, "Agent ID 不能为空"); + String safeSessionId = requireText(sessionId, "会话 ID 不能为空"); + List records = new ArrayList<>(ids.size()); + for (String uploadId : ids) { + AgentMediaUploadRecord record = uploadStore.find(uploadId) + .orElseThrow(() -> badRequest("图片已过期，请重新上传")); + assertOwner(record, identity); + if (!safeMode.equals(record.getMode()) + || !safeAgentId.equals(record.getAgentId()) + || !safeSessionId.equals(record.getSessionId())) { + throw new ResponseStatusException(HttpStatus.FORBIDDEN, "图片不属于当前聊天会话"); + } + records.add(record); + } + return records; + } + + /** + * 将正式聊天图片从临时区归档到消息目录。 + * + * @param uploads 临时图片 + * @param sessionId 会话 ID + * @param messageId 消息 ID + * @param account 当前账号 + * @return 已绑定图片 + */ + public List bindFormal(List uploads, + String sessionId, + String messageId, + LoginAccount account) { + if (uploads == null || uploads.isEmpty()) { + return List.of(); + } + Identity identity = identity(account); + List result = new ArrayList<>(uploads.size()); + for (int index = 0; index < uploads.size(); index++) { + AgentMediaUploadRecord upload = uploads.get(index); + assertOwner(upload, identity); + String reference = upload.getFormalReference(); + if (StringUtils.hasText(reference)) { + FormalReference formal = parseFormalReference(reference); + if (!digits(sessionId).equals(formal.sessionId)) { + throw new ResponseStatusException(HttpStatus.CONFLICT, "图片已绑定到其他聊天会话"); + } + result.add(new AgentBoundMedia(reference, upload.getMimeType(), displayPayload(upload, reference))); + continue; + } + reference = formalReference(sessionId, messageId, index, upload.getExtension()); + String targetKey = formalObjectKey(identity, sessionId, messageId, index, upload.getExtension()); + objectStorage.copy(upload.getObjectKey(), targetKey); + uploadStore.markFormalBinding(upload, reference); + result.add(new AgentBoundMedia(reference, upload.getMimeType(), displayPayload(upload, reference))); + } + return result; + } + + /** + * 删除当前账号一个正式会话归档的全部图片。 + * + * @param sessionId 会话 ID + * @param account 当前账号 + */ + public void deleteFormalSession(String sessionId, LoginAccount account) { + Identity identity = identity(account); + objectStorage.deletePrefix("formal/%s/%s/%s/".formatted( + identity.tenantId, identity.userId, digits(sessionId))); + } + + /** + * 为草稿试运行保留临时图片引用并续期。 + * + * @param uploads 临时图片 + * @return 已绑定图片 + */ + public List bindDraft(List uploads) { + if (uploads == null || uploads.isEmpty()) { + return List.of(); + } + List result = new ArrayList<>(uploads.size()); + for (AgentMediaUploadRecord upload : uploads) { + uploadStore.renew(upload); + String reference = "draft:" + upload.getUploadId(); + result.add(new AgentBoundMedia(reference, upload.getMimeType(), displayPayload(upload, reference))); + } + return result; + } + + /** + * 读取当前用户可访问的图片引用。 + * + * @param reference 稳定图片引用 + * @param account 当前账号 + * @return 图片资源 + */ + public AgentMediaResource load(String reference, LoginAccount account) { + Identity identity = identity(account); + if (reference != null && reference.startsWith("draft:")) { + String uploadId = reference.substring("draft:".length()); + AgentMediaUploadRecord record = uploadStore.find(uploadId) + .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "图片已过期")); + assertOwner(record, identity); + return new AgentMediaResource(record.getMimeType(), + objectStorage.get(record.getObjectKey(), properties.getMaxImageBytes())); + } + FormalReference formal = parseFormalReference(reference); + String key = formalObjectKey(identity, formal.sessionId, formal.messageId, formal.index, formal.extension); + return new AgentMediaResource(mimeType(formal.extension), + objectStorage.get(key, properties.getMaxImageBytes())); + } + + /** + * 创建仅能访问当前账号媒体的 easy-agents 解析器。 + * + * @param account 当前账号 + * @return 运行时媒体解析器 + */ + public AgentMediaResolver runtimeResolver(LoginAccount account) { + return reference -> { + AgentMediaResource resource = load(reference, account); + if (reference != null && reference.startsWith("draft:")) { + uploadStore.find(reference.substring("draft:".length())).ifPresent(uploadStore::renew); + } + return resource; + }; + } + + /** + * 清理过期临时对象。 + * + * @param limit 单次清理上限 + * @return 清理数量 + */ + public int cleanupExpired(int limit) { + int cleaned = 0; + for (AgentMediaUploadRecord indexed : uploadStore.expired(limit)) { + Optional current = uploadStore.find(indexed.getUploadId()); + if (current.isPresent() && current.get().getExpiresAt() != null + && current.get().getExpiresAt().isAfter(Instant.now())) { + uploadStore.removeExpiryIndex(indexed); + continue; + } + objectStorage.delete(indexed.getObjectKey()); + current.ifPresent(uploadStore::delete); + uploadStore.removeExpiryIndex(indexed); + cleaned++; + } + return cleaned; + } + + private AgentMediaUploadView toView(AgentMediaUploadRecord record) { + AgentMediaUploadView view = new AgentMediaUploadView(); + view.setUploadId(record.getUploadId()); + view.setName(record.getOriginalName()); + view.setMimeType(record.getMimeType()); + view.setSize(record.getSize()); + view.setWidth(record.getWidth()); + view.setHeight(record.getHeight()); + view.setExpiresAt(record.getExpiresAt()); + view.setPreviewUrl("/api/v1/agent/media/content?reference=" + + UriUtils.encodeQueryParam("draft:" + record.getUploadId(), StandardCharsets.UTF_8)); + return view; + } + + private Map displayPayload(AgentMediaUploadRecord upload, String reference) { + Map payload = new LinkedHashMap<>(); + payload.put("imageRef", reference); + payload.put("name", upload.getOriginalName()); + payload.put("mimeType", upload.getMimeType()); + payload.put("size", upload.getSize()); + payload.put("width", upload.getWidth()); + payload.put("height", upload.getHeight()); + payload.put("previewUrl", "/api/v1/agent/media/content?reference=" + + UriUtils.encodeQueryParam(reference, StandardCharsets.UTF_8)); + return payload; + } + + private String formalReference(String sessionId, String messageId, int index, String extension) { + return "formal:" + sessionId + ":" + messageId + ":" + index + ":" + extension; + } + + private FormalReference parseFormalReference(String reference) { + if (reference == null || !reference.startsWith("formal:")) { + throw badRequest("图片引用无效"); + } + String[] parts = reference.split(":", -1); + if (parts.length != 5 || !parts[1].matches("\\d+") || !parts[2].matches("\\d+") + || !parts[3].matches("\\d+") || !Set.of("png", "jpg", "webp").contains(parts[4])) { + throw badRequest("图片引用无效"); + } + return new FormalReference(parts[1], parts[2], Integer.parseInt(parts[3]), parts[4]); + } + + private String formalObjectKey(Identity identity, String sessionId, String messageId, int index, String extension) { + return "formal/%s/%s/%s/%s/%d.%s".formatted(identity.tenantId, identity.userId, + digits(sessionId), digits(messageId), index, extension); + } + + private String digits(String value) { + if (value == null || !value.matches("\\d+")) { + throw badRequest("聊天消息标识无效"); + } + return value; + } + + private String mimeType(String extension) { + return switch (extension) { + case "jpg" -> "image/jpeg"; + case "webp" -> "image/webp"; + default -> "image/png"; + }; + } + + private void assertOwner(AgentMediaUploadRecord record, Identity identity) { + if (record == null || !identity.tenantId.equals(record.getTenantId()) + || !identity.userId.equals(record.getUserId())) { + throw new ResponseStatusException(HttpStatus.FORBIDDEN, "无权访问该图片"); + } + } + + private Identity identity(LoginAccount account) { + if (account == null || account.getId() == null || account.getTenantId() == null) { + throw new ResponseStatusException(HttpStatus.UNAUTHORIZED, "当前登录状态失效"); + } + return new Identity(account.getTenantId().toString(), account.getId().toString()); + } + + private String requireMode(String mode) { + String normalized = mode == null ? "" : mode.trim().toUpperCase(Locale.ROOT); + if (!MODE_FORMAL.equals(normalized) && !MODE_DRAFT.equals(normalized)) { + throw badRequest("聊天模式无效"); + } + return normalized; + } + + private String requireText(String value, String message) { + if (!StringUtils.hasText(value) || value.length() > 200) { + throw badRequest(message); + } + return value.trim(); + } + + private String safeOriginalName(String value, String extension) { + String name = StringUtils.hasText(value) ? value.replaceAll("[\\r\\n\\t]", " ").trim() : "image." + extension; + return name.length() > 200 ? name.substring(0, 200) : name; + } + + private ResponseStatusException badRequest(String message) { + return new ResponseStatusException(HttpStatus.BAD_REQUEST, message); + } + + private record Identity(String tenantId, String userId) { } + private record FormalReference(String sessionId, String messageId, int index, String extension) { } +} diff --git a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentMediaUploadRecord.java b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentMediaUploadRecord.java new file mode 100644 index 00000000..5535ddf3 --- /dev/null +++ b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentMediaUploadRecord.java @@ -0,0 +1,96 @@ +package tech.easyflow.agent.runtime.media; + +import java.time.Instant; + +/** + * Redis 中保存的 Agent 临时图片上传凭据。 + */ +public class AgentMediaUploadRecord { + + private String uploadId; + private String tenantId; + private String userId; + private String mode; + private String agentId; + private String sessionId; + private String objectKey; + private String originalName; + private String mimeType; + private String extension; + private long size; + private int width; + private int height; + private String sha256; + private String formalReference; + private Instant createdAt; + private Instant expiresAt; + + /** @return 上传 ID */ + public String getUploadId() { return uploadId; } + /** @param uploadId 上传 ID */ + public void setUploadId(String uploadId) { this.uploadId = uploadId; } + /** @return 租户 ID */ + public String getTenantId() { return tenantId; } + /** @param tenantId 租户 ID */ + public void setTenantId(String tenantId) { this.tenantId = tenantId; } + /** @return 用户 ID */ + public String getUserId() { return userId; } + /** @param userId 用户 ID */ + public void setUserId(String userId) { this.userId = userId; } + /** @return 聊天模式 */ + public String getMode() { return mode; } + /** @param mode 聊天模式 */ + public void setMode(String mode) { this.mode = mode; } + /** @return Agent ID */ + public String getAgentId() { return agentId; } + /** @param agentId Agent ID */ + public void setAgentId(String agentId) { this.agentId = agentId; } + /** @return 会话 ID */ + public String getSessionId() { return sessionId; } + /** @param sessionId 会话 ID */ + public void setSessionId(String sessionId) { this.sessionId = sessionId; } + /** @return 对象键 */ + public String getObjectKey() { return objectKey; } + /** @param objectKey 对象键 */ + public void setObjectKey(String objectKey) { this.objectKey = objectKey; } + /** @return 原文件名 */ + public String getOriginalName() { return originalName; } + /** @param originalName 原文件名 */ + public void setOriginalName(String originalName) { this.originalName = originalName; } + /** @return MIME 类型 */ + public String getMimeType() { return mimeType; } + /** @param mimeType MIME 类型 */ + public void setMimeType(String mimeType) { this.mimeType = mimeType; } + /** @return 扩展名 */ + public String getExtension() { return extension; } + /** @param extension 扩展名 */ + public void setExtension(String extension) { this.extension = extension; } + /** @return 文件大小 */ + public long getSize() { return size; } + /** @param size 文件大小 */ + public void setSize(long size) { this.size = size; } + /** @return 图片宽度 */ + public int getWidth() { return width; } + /** @param width 图片宽度 */ + public void setWidth(int width) { this.width = width; } + /** @return 图片高度 */ + public int getHeight() { return height; } + /** @param height 图片高度 */ + public void setHeight(int height) { this.height = height; } + /** @return SHA-256 */ + public String getSha256() { return sha256; } + /** @param sha256 SHA-256 */ + public void setSha256(String sha256) { this.sha256 = sha256; } + /** @return 已完成归档的正式图片引用 */ + public String getFormalReference() { return formalReference; } + /** @param formalReference 已完成归档的正式图片引用 */ + public void setFormalReference(String formalReference) { this.formalReference = formalReference; } + /** @return 创建时间 */ + public Instant getCreatedAt() { return createdAt; } + /** @param createdAt 创建时间 */ + public void setCreatedAt(Instant createdAt) { this.createdAt = createdAt; } + /** @return 过期时间 */ + public Instant getExpiresAt() { return expiresAt; } + /** @param expiresAt 过期时间 */ + public void setExpiresAt(Instant expiresAt) { this.expiresAt = expiresAt; } +} diff --git a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentMediaUploadView.java b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentMediaUploadView.java new file mode 100644 index 00000000..11ac8a03 --- /dev/null +++ b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/AgentMediaUploadView.java @@ -0,0 +1,51 @@ +package tech.easyflow.agent.runtime.media; + +import java.time.Instant; + +/** + * 返回给聊天输入框的临时图片信息。 + */ +public class AgentMediaUploadView { + + private String uploadId; + private String name; + private String mimeType; + private long size; + private int width; + private int height; + private String previewUrl; + private Instant expiresAt; + + /** @return 上传 ID */ + public String getUploadId() { return uploadId; } + /** @param uploadId 上传 ID */ + public void setUploadId(String uploadId) { this.uploadId = uploadId; } + /** @return 文件名 */ + public String getName() { return name; } + /** @param name 文件名 */ + public void setName(String name) { this.name = name; } + /** @return MIME 类型 */ + public String getMimeType() { return mimeType; } + /** @param mimeType MIME 类型 */ + public void setMimeType(String mimeType) { this.mimeType = mimeType; } + /** @return 文件大小 */ + public long getSize() { return size; } + /** @param size 文件大小 */ + public void setSize(long size) { this.size = size; } + /** @return 图片宽度 */ + public int getWidth() { return width; } + /** @param width 图片宽度 */ + public void setWidth(int width) { this.width = width; } + /** @return 图片高度 */ + public int getHeight() { return height; } + /** @param height 图片高度 */ + public void setHeight(int height) { this.height = height; } + /** @return 鉴权预览地址 */ + public String getPreviewUrl() { return previewUrl; } + /** @param previewUrl 鉴权预览地址 */ + public void setPreviewUrl(String previewUrl) { this.previewUrl = previewUrl; } + /** @return 过期时间 */ + public Instant getExpiresAt() { return expiresAt; } + /** @param expiresAt 过期时间 */ + public void setExpiresAt(Instant expiresAt) { this.expiresAt = expiresAt; } +} diff --git a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/RedisAgentMediaUploadStore.java b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/RedisAgentMediaUploadStore.java new file mode 100644 index 00000000..96be8ff2 --- /dev/null +++ b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/media/RedisAgentMediaUploadStore.java @@ -0,0 +1,189 @@ +package tech.easyflow.agent.runtime.media; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import tech.easyflow.agent.config.AgentMediaProperties; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +/** + * Agent 临时图片凭据与过期对象索引的 Redis 存储。 + */ +@Service +public class RedisAgentMediaUploadStore { + + private static final String KEY_PREFIX = "easyflow:agent:image-upload:"; + private static final String EXPIRY_INDEX = "easyflow:agent:image-upload:expiry"; + + private final StringRedisTemplate redisTemplate; + private final ObjectMapper objectMapper; + private final AgentMediaProperties properties; + + /** + * 创建临时图片凭据存储。 + * + * @param redisTemplate Redis 模板 + * @param objectMapper JSON 映射器 + * @param properties Agent 媒体配置 + */ + public RedisAgentMediaUploadStore(StringRedisTemplate redisTemplate, + ObjectMapper objectMapper, + AgentMediaProperties properties) { + this.redisTemplate = redisTemplate; + this.objectMapper = objectMapper; + this.properties = properties; + } + + /** + * 创建临时图片凭据和清理索引。 + * + * @param record 上传凭据 + */ + public void create(AgentMediaUploadRecord record) { + Duration ttl = uploadTtl(); + Instant expiresAt = Instant.now().plus(ttl); + record.setExpiresAt(expiresAt); + write(record, ttl); + redisTemplate.opsForZSet().add(EXPIRY_INDEX, cleanupMember(record), expiresAt.toEpochMilli()); + } + + /** + * 查找上传凭据，读取操作不延长 TTL。 + * + * @param uploadId 上传 ID + * @return 上传凭据 + */ + public Optional find(String uploadId) { + if (!StringUtils.hasText(uploadId)) { + return Optional.empty(); + } + String value = redisTemplate.opsForValue().get(key(uploadId)); + return StringUtils.hasText(value) ? Optional.of(read(value)) : Optional.empty(); + } + + /** + * 续期仍被草稿或试运行会话使用的图片。 + * + * @param record 上传凭据 + */ + public void renew(AgentMediaUploadRecord record) { + Duration ttl = uploadTtl(); + String previousMember = cleanupMember(record); + record.setExpiresAt(Instant.now().plus(ttl)); + write(record, ttl); + redisTemplate.opsForZSet().remove(EXPIRY_INDEX, previousMember); + redisTemplate.opsForZSet().add(EXPIRY_INDEX, cleanupMember(record), record.getExpiresAt().toEpochMilli()); + } + + /** + * 记录临时上传已归档到正式图片，且不延长原有 TTL。 + * + * @param record 上传凭据 + * @param formalReference 正式图片引用 + */ + public void markFormalBinding(AgentMediaUploadRecord record, String formalReference) { + if (record == null || !StringUtils.hasText(formalReference)) { + throw new IllegalArgumentException("正式图片绑定信息不能为空"); + } + String previousMember = cleanupMember(record); + Instant expiresAt = record.getExpiresAt(); + if (expiresAt == null) { + expiresAt = Instant.now().plus(uploadTtl()); + record.setExpiresAt(expiresAt); + } + Duration remaining = Duration.between(Instant.now(), expiresAt); + if (remaining.isZero() || remaining.isNegative()) { + throw new IllegalStateException("图片上传凭据已过期"); + } + record.setFormalReference(formalReference); + write(record, remaining); + redisTemplate.opsForZSet().remove(EXPIRY_INDEX, previousMember); + redisTemplate.opsForZSet().add(EXPIRY_INDEX, cleanupMember(record), + record.getExpiresAt().toEpochMilli()); + } + + /** + * 删除已归档的临时凭据和清理索引。 + * + * @param record 上传凭据 + */ + public void delete(AgentMediaUploadRecord record) { + if (record == null) { + return; + } + redisTemplate.delete(key(record.getUploadId())); + redisTemplate.opsForZSet().remove(EXPIRY_INDEX, cleanupMember(record)); + } + + /** + * 获取已到期的清理索引成员。 + * + * @param limit 最大数量 + * @return 到期上传凭据 + */ + public List expired(int limit) { + Set members = redisTemplate.opsForZSet() + .rangeByScore(EXPIRY_INDEX, 0, Instant.now().toEpochMilli(), 0, Math.max(1, limit)); + List records = new ArrayList<>(); + if (members == null) { + return records; + } + for (String member : members) { + try { + records.add(objectMapper.readValue(member, AgentMediaUploadRecord.class)); + } catch (Exception error) { + redisTemplate.opsForZSet().remove(EXPIRY_INDEX, member); + } + } + return records; + } + + /** + * 删除一个清理索引成员。 + * + * @param record 上传凭据 + */ + public void removeExpiryIndex(AgentMediaUploadRecord record) { + redisTemplate.opsForZSet().remove(EXPIRY_INDEX, cleanupMember(record)); + } + + private void write(AgentMediaUploadRecord record, Duration ttl) { + try { + redisTemplate.opsForValue().set(key(record.getUploadId()), objectMapper.writeValueAsString(record), + Math.max(1L, ttl.toSeconds()), TimeUnit.SECONDS); + } catch (Exception error) { + throw new IllegalStateException("写入 Agent 图片上传凭据失败", error); + } + } + + private AgentMediaUploadRecord read(String value) { + try { + return objectMapper.readValue(value, AgentMediaUploadRecord.class); + } catch (Exception error) { + throw new IllegalStateException("读取 Agent 图片上传凭据失败", error); + } + } + + private String cleanupMember(AgentMediaUploadRecord record) { + try { + return objectMapper.writeValueAsString(record); + } catch (Exception error) { + throw new IllegalStateException("写入 Agent 图片清理索引失败", error); + } + } + + private String key(String uploadId) { return KEY_PREFIX + uploadId; } + + private Duration uploadTtl() { + Duration ttl = properties.getUploadTtl(); + return ttl == null || ttl.isNegative() || ttl.isZero() ? Duration.ofHours(24) : ttl; + } +} diff --git a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/service/impl/AgentServiceImpl.java b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/service/impl/AgentServiceImpl.java index cb06502f..09ddc087 100644 --- a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/service/impl/AgentServiceImpl.java +++ b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/service/impl/AgentServiceImpl.java @@ -249,6 +249,8 @@ public class AgentServiceImpl extends ServiceImpl implements summary.put("title", model.getTitle()); summary.put("modelName", model.getModelName()); summary.put("providerType", model.getModelProvider() == null ? null : model.getModelProvider().getProviderType()); + summary.put("supportImage", Boolean.TRUE.equals(model.getSupportImage())); + summary.put("supportImageB64Only", Boolean.TRUE.equals(model.getSupportImageB64Only())); return summary; } diff --git a/easyflow-modules/easyflow-module-agent/src/test/java/tech/easyflow/agent/runtime/AgentRunServiceDraftAndHitlTest.java b/easyflow-modules/easyflow-module-agent/src/test/java/tech/easyflow/agent/runtime/AgentRunServiceDraftAndHitlTest.java index 6e6909b9..f259eb16 100644 --- a/easyflow-modules/easyflow-module-agent/src/test/java/tech/easyflow/agent/runtime/AgentRunServiceDraftAndHitlTest.java +++ b/easyflow-modules/easyflow-module-agent/src/test/java/tech/easyflow/agent/runtime/AgentRunServiceDraftAndHitlTest.java @@ -11,6 +11,7 @@ import com.easyagents.agent.runtime.persistence.session.AgentSessionStore; import com.easyagents.agent.runtime.persistence.session.memory.InMemoryAgentSessionStore; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import tech.easyflow.agent.entity.AgentHitlPending; import tech.easyflow.agent.entity.Agent; import tech.easyflow.agent.entity.AgentKnowledgeBinding; @@ -21,6 +22,8 @@ import tech.easyflow.agent.distributed.AgentRuntimeRouteRegistry; import tech.easyflow.agent.runtime.event.AgentRunEventRecorder; import tech.easyflow.agent.runtime.hitl.AgentHitlPendingService; import tech.easyflow.agent.runtime.lock.AgentRunLock; +import tech.easyflow.agent.runtime.media.AgentBoundMedia; +import tech.easyflow.agent.runtime.media.AgentMediaService; import tech.easyflow.chatlog.domain.dto.ChatSessionSummary; import tech.easyflow.common.entity.LoginAccount; import tech.easyflow.common.web.exceptions.BusinessException; @@ -399,6 +402,33 @@ public class AgentRunServiceDraftAndHitlTest { Assert.assertEquals(Boolean.TRUE, payload.get(0).get("faqCollection")); } + /** + * 验证输入接收事件会回传正式图片展示信息。 + * + * @throws Exception 反射调用失败时抛出 + */ + @Test + public void sendInputAcceptedShouldExposeBoundImages() throws Exception { + AgentRunService service = new AgentRunService(); + RecordingChatSseEmitter emitter = new RecordingChatSseEmitter(); + AgentBoundMedia image = new AgentBoundMedia("formal:101:201:0:png", "image/png", + Map.of("imageRef", "formal:101:201:0:png", + "previewUrl", "/api/v1/agent/media/content?reference=formal:101:201:0:png")); + + boolean sent = invoke(service, "sendInputAccepted", + new Class[]{ChatSseEmitter.class, BigInteger.class, BigInteger.class, List.class}, + emitter, BigInteger.valueOf(101), BigInteger.valueOf(201), List.of(image)); + + Assert.assertTrue(sent); + Assert.assertEquals(1, emitter.envelopes.size()); + Assert.assertEquals(ChatType.INPUT_ACCEPTED, emitter.envelopes.get(0).getType()); + @SuppressWarnings("unchecked") + Map payload = (Map) emitter.envelopes.get(0).getPayload(); + Assert.assertEquals("101", payload.get("sessionId")); + Assert.assertEquals("201", payload.get("messageId")); + Assert.assertEquals(List.of(image.payload()), payload.get("images")); + } + /** * 验证未保存草稿会生成临时 Agent ID，并把绑定指向该运行 ID。 * @@ -463,12 +493,16 @@ public class AgentRunServiceDraftAndHitlTest { Agent agent = new Agent(); agent.setId(BigInteger.valueOf(100)); ChatRuntimeContext context = chatContext(); + LoginAccount account = new LoginAccount(); + account.setId(BigInteger.ONE); + account.setTenantId(BigInteger.ONE); Exception thrown = Assert.assertThrows(Exception.class, () -> invoke(service, "run", - new Class[]{Agent.class, String.class, String.class, String.class, String.class, - String.class, ChatRuntimeContext.class, boolean.class, AgentSessionStore.class}, - agent, "你好", "request-lock", "trace-lock", "session-lock", "AGENT", context, true, - new InMemoryAgentSessionStore())); + new Class[]{Agent.class, String.class, List.class, LoginAccount.class, String.class, + String.class, String.class, String.class, ChatRuntimeContext.class, boolean.class, + AgentSessionStore.class}, + agent, "你好", List.of(), account, "request-lock", "trace-lock", "session-lock", "AGENT", + context, true, new InMemoryAgentSessionStore())); Assert.assertTrue(rootCause(thrown) instanceof BusinessException); Assert.assertEquals(0, chatRuntimeManager.prepareSessionCount); @@ -490,14 +524,21 @@ public class AgentRunServiceDraftAndHitlTest { setField(service, "agentRuntimeCompiler", compiler); setField(service, "agentRuntimeFactory", runtimeFactory); setField(service, "agentRunRegistry", new AgentRunRegistry()); + AgentMediaService mediaService = Mockito.mock(AgentMediaService.class); + Mockito.when(mediaService.runtimeResolver(Mockito.any())).thenReturn(reference -> null); + setField(service, "agentMediaService", mediaService); Agent agent = new Agent(); agent.setId(BigInteger.valueOf(100)); + LoginAccount account = new LoginAccount(); + account.setId(BigInteger.ONE); + account.setTenantId(BigInteger.ONE); invoke(service, "startRuntime", - new Class[]{Agent.class, String.class, String.class, String.class, String.class, String.class, - ChatRuntimeContext.class, ChatSseEmitter.class, boolean.class, AgentSessionStore.class, - AgentRunLock.Handle.class}, - agent, "你好", "request-draft", "trace-draft", "agent-draft-100", "AGENT_DRAFT", + new Class[]{Agent.class, AgentMessage.class, LoginAccount.class, String.class, String.class, + String.class, String.class, ChatRuntimeContext.class, ChatSseEmitter.class, boolean.class, + AgentSessionStore.class, AgentRunLock.Handle.class}, + agent, AgentMessage.text(AgentMessageRole.USER, "你好"), account, + "request-draft", "trace-draft", "agent-draft-100", "AGENT_DRAFT", chatContext(), new RecordingChatSseEmitter(), false, draftStore, null); Assert.assertSame(draftStore, runtime.initRequest.getSessionStore()); diff --git a/easyflow-modules/easyflow-module-agent/src/test/java/tech/easyflow/agent/runtime/composer/AgentComposerDraftServiceTest.java b/easyflow-modules/easyflow-module-agent/src/test/java/tech/easyflow/agent/runtime/composer/AgentComposerDraftServiceTest.java new file mode 100644 index 00000000..a1c6328e --- /dev/null +++ b/easyflow-modules/easyflow-module-agent/src/test/java/tech/easyflow/agent/runtime/composer/AgentComposerDraftServiceTest.java @@ -0,0 +1,101 @@ +package tech.easyflow.agent.runtime.composer; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.data.redis.core.ValueOperations; +import tech.easyflow.agent.config.AgentMediaProperties; +import tech.easyflow.agent.runtime.media.AgentMediaService; +import tech.easyflow.common.entity.LoginAccount; + +import java.math.BigInteger; +import java.util.List; + +/** + * {@link AgentComposerDraftService} 草稿清理测试。 + */ +public class AgentComposerDraftServiceTest { + + private static final String DRAFT_KEY = "easyflow:agent:composer:draft:2:7:FORMAL:9:101"; + private static final String ACTIVE_KEY = "easyflow:agent:composer:active:2:7:FORMAL:9"; + + private StringRedisTemplate redisTemplate; + private ValueOperations valueOperations; + private AgentMediaService mediaService; + private AgentComposerDraftService service; + private LoginAccount account; + + /** + * 初始化草稿服务测试依赖。 + */ + @Before + @SuppressWarnings("unchecked") + public void setUp() { + redisTemplate = Mockito.mock(StringRedisTemplate.class); + valueOperations = Mockito.mock(ValueOperations.class); + Mockito.when(redisTemplate.opsForValue()).thenReturn(valueOperations); + mediaService = Mockito.mock(AgentMediaService.class); + service = new AgentComposerDraftService(redisTemplate, new ObjectMapper(), + new AgentMediaProperties(), mediaService); + account = new LoginAccount(); + account.setTenantId(BigInteger.valueOf(2)); + account.setId(BigInteger.valueOf(7)); + } + + /** + * 验证删除草稿时合并服务端与调用方上传 ID 并清理临时图片。 + * + * @throws Exception JSON 编码失败时抛出 + */ + @Test + public void deleteShouldCleanupDraftUploadsBeforeRedisKeys() throws Exception { + AgentComposerDraft draft = new AgentComposerDraft(); + draft.setImageUploadIds(List.of("upload-1", "upload-2")); + Mockito.when(valueOperations.get(DRAFT_KEY)) + .thenReturn(new ObjectMapper().writeValueAsString(draft)); + Mockito.when(valueOperations.get(ACTIVE_KEY)).thenReturn("101"); + + service.delete("FORMAL", "9", "101", List.of("upload-2", "upload-3"), true, account); + + Mockito.verify(mediaService).deleteUploadsForScope( + List.of("upload-1", "upload-2", "upload-3"), "FORMAL", "9", "101", account); + Mockito.verify(redisTemplate).delete(List.of(DRAFT_KEY, + "easyflow:agent:composer:revision:2:7:FORMAL:9:101")); + Mockito.verify(redisTemplate).delete(ACTIVE_KEY); + } + + /** + * 验证关闭上传清理时只删除草稿缓存。 + */ + @Test + public void deleteShouldKeepUploadsWhenCleanupIsDisabled() { + Mockito.when(valueOperations.get(ACTIVE_KEY)).thenReturn("101"); + + service.delete("FORMAL", "9", "101", List.of("upload-1"), false, account); + + Mockito.verifyNoInteractions(mediaService); + Mockito.verify(redisTemplate).delete(List.of(DRAFT_KEY, + "easyflow:agent:composer:revision:2:7:FORMAL:9:101")); + } + + /** + * 验证旧版草稿未保存图片 ID 字段时仍能完成清理。 + * + * @throws Exception JSON 编码失败时抛出 + */ + @Test + public void deleteShouldHandleLegacyDraftWithoutImageUploadIds() throws Exception { + AgentComposerDraft draft = new AgentComposerDraft(); + Mockito.when(valueOperations.get(DRAFT_KEY)) + .thenReturn(new ObjectMapper().writeValueAsString(draft)); + + service.delete("FORMAL", "9", "101", List.of(), true, account); + + Mockito.verify(mediaService).deleteUploadsForScope( + List.of(), "FORMAL", "9", "101", account); + Mockito.verify(redisTemplate).delete(List.of(DRAFT_KEY, + "easyflow:agent:composer:revision:2:7:FORMAL:9:101")); + } +} diff --git a/easyflow-modules/easyflow-module-agent/src/test/java/tech/easyflow/agent/runtime/media/AgentImageProcessorTest.java b/easyflow-modules/easyflow-module-agent/src/test/java/tech/easyflow/agent/runtime/media/AgentImageProcessorTest.java new file mode 100644 index 00000000..eb151b20 --- /dev/null +++ b/easyflow-modules/easyflow-module-agent/src/test/java/tech/easyflow/agent/runtime/media/AgentImageProcessorTest.java @@ -0,0 +1,141 @@ +package tech.easyflow.agent.runtime.media; + +import org.junit.Assert; +import org.junit.Test; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.server.ResponseStatusException; +import tech.easyflow.agent.config.AgentMediaProperties; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; + +/** + * Agent 图片格式识别、规范化和资源限制测试。 + */ +public class AgentImageProcessorTest { + + /** + * 验证 PNG 会保留格式、尺寸和稳定摘要。 + * + * @throws Exception 图片生成失败时抛出 + */ + @Test + public void processShouldAcceptPngAndExposeMetadata() throws Exception { + AgentImageProcessor processor = new AgentImageProcessor(new AgentMediaProperties()); + + AgentImageData image = processor.process(file("sample.png", "image/png", imageBytes("png", 3, 2))); + + Assert.assertEquals("image/png", image.mimeType()); + Assert.assertEquals("png", image.extension()); + Assert.assertEquals(3, image.width()); + Assert.assertEquals(2, image.height()); + Assert.assertEquals(64, image.sha256().length()); + } + + /** + * 验证 GIF 首帧会转换为模型兼容的 PNG。 + * + * @throws Exception 图片生成失败时抛出 + */ + @Test + public void processShouldNormalizeGifToPng() throws Exception { + AgentImageProcessor processor = new AgentImageProcessor(new AgentMediaProperties()); + + AgentImageData image = processor.process(file("sample.gif", "image/gif", imageBytes("gif", 4, 3))); + + Assert.assertEquals("image/png", image.mimeType()); + Assert.assertEquals("png", image.extension()); + Assert.assertEquals(4, image.width()); + Assert.assertEquals(3, image.height()); + Assert.assertArrayEquals(new byte[]{(byte) 0x89, 0x50, 0x4e, 0x47}, + java.util.Arrays.copyOf(image.bytes(), 4)); + } + + /** + * 验证扩展名无法绕过真实图片内容校验。 + */ + @Test + public void processShouldRejectUnsupportedContent() { + AgentImageProcessor processor = new AgentImageProcessor(new AgentMediaProperties()); + + ResponseStatusException error = Assert.assertThrows(ResponseStatusException.class, + () -> processor.process(file("fake.png", "image/png", "not-an-image".getBytes()))); + + Assert.assertEquals(400, error.getStatusCode().value()); + } + + /** + * 验证像素上限会在图片解码前置阶段生效。 + * + * @throws Exception 图片生成失败时抛出 + */ + @Test + public void processShouldRejectImageOverPixelLimit() throws Exception { + AgentMediaProperties properties = new AgentMediaProperties(); + properties.setMaxImagePixels(3); + AgentImageProcessor processor = new AgentImageProcessor(properties); + + ResponseStatusException error = Assert.assertThrows(ResponseStatusException.class, + () -> processor.process(file("large.png", "image/png", imageBytes("png", 2, 2)))); + + Assert.assertEquals(400, error.getStatusCode().value()); + Assert.assertTrue(error.getReason().contains("像素过大")); + } + + private byte[] imageBytes(String format, int width, int height) throws IOException { + BufferedImage image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + Assert.assertTrue(ImageIO.write(image, format, output)); + return output.toByteArray(); + } + + private MultipartFile file(String name, String contentType, byte[] bytes) { + return new ByteArrayMultipartFile(name, contentType, bytes); + } + + /** + * 无磁盘依赖的测试 MultipartFile。 + */ + private static final class ByteArrayMultipartFile implements MultipartFile { + private final String name; + private final String contentType; + private final byte[] bytes; + + private ByteArrayMultipartFile(String name, String contentType, byte[] bytes) { + this.name = name; + this.contentType = contentType; + this.bytes = bytes; + } + + @Override + public String getName() { return "file"; } + + @Override + public String getOriginalFilename() { return name; } + + @Override + public String getContentType() { return contentType; } + + @Override + public boolean isEmpty() { return bytes.length == 0; } + + @Override + public long getSize() { return bytes.length; } + + @Override + public byte[] getBytes() { return bytes.clone(); } + + @Override + public InputStream getInputStream() { return new ByteArrayInputStream(bytes); } + + @Override + public void transferTo(File destination) throws IOException { + java.nio.file.Files.write(destination.toPath(), bytes); + } + } +} diff --git a/easyflow-modules/easyflow-module-agent/src/test/java/tech/easyflow/agent/runtime/media/AgentMediaServiceTest.java b/easyflow-modules/easyflow-module-agent/src/test/java/tech/easyflow/agent/runtime/media/AgentMediaServiceTest.java new file mode 100644 index 00000000..404bd88b --- /dev/null +++ b/easyflow-modules/easyflow-module-agent/src/test/java/tech/easyflow/agent/runtime/media/AgentMediaServiceTest.java @@ -0,0 +1,103 @@ +package tech.easyflow.agent.runtime.media; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.http.HttpStatus; +import org.springframework.web.server.ResponseStatusException; +import tech.easyflow.agent.config.AgentMediaProperties; +import tech.easyflow.common.entity.LoginAccount; + +import java.math.BigInteger; +import java.time.Instant; +import java.util.List; + +/** + * {@link AgentMediaService} 正式图片绑定测试。 + */ +public class AgentMediaServiceTest { + + private AgentMediaObjectStorage objectStorage; + private RedisAgentMediaUploadStore uploadStore; + private AgentMediaService service; + private LoginAccount account; + private AgentMediaUploadRecord upload; + + /** + * 初始化媒体服务测试依赖。 + */ + @Before + public void setUp() { + objectStorage = Mockito.mock(AgentMediaObjectStorage.class); + uploadStore = Mockito.mock(RedisAgentMediaUploadStore.class); + service = new AgentMediaService(Mockito.mock(AgentImageProcessor.class), objectStorage, + uploadStore, new AgentMediaProperties()); + account = new LoginAccount(); + account.setTenantId(BigInteger.valueOf(2)); + account.setId(BigInteger.valueOf(7)); + upload = buildUpload(); + Mockito.doAnswer(invocation -> { + upload.setFormalReference(invocation.getArgument(1)); + return null; + }).when(uploadStore).markFormalBinding(Mockito.same(upload), Mockito.anyString()); + } + + /** + * 验证重复绑定同一上传凭据时复用首次正式引用。 + */ + @Test + public void bindFormalShouldReuseExistingBinding() { + List first = service.bindFormal(List.of(upload), "101", "201", account); + List second = service.bindFormal(List.of(upload), "101", "202", account); + + Assert.assertEquals("formal:101:201:0:png", first.get(0).reference()); + Assert.assertEquals(first.get(0).reference(), second.get(0).reference()); + Assert.assertEquals("/api/v1/agent/media/content?reference=formal:101:201:0:png", + second.get(0).payload().get("previewUrl")); + Mockito.verify(objectStorage, Mockito.times(1)) + .copy("temp/2/7/upload-1.png", "formal/2/7/101/201/0.png"); + Mockito.verify(uploadStore, Mockito.times(1)) + .markFormalBinding(upload, "formal:101:201:0:png"); + Mockito.verify(objectStorage, Mockito.never()).delete(Mockito.anyString()); + Mockito.verify(uploadStore, Mockito.never()).delete(Mockito.any()); + } + + /** + * 验证已绑定图片不能被复用到其他正式会话。 + */ + @Test + public void bindFormalShouldRejectAnotherSession() { + upload.setFormalReference("formal:101:201:0:png"); + + ResponseStatusException error = Assert.assertThrows(ResponseStatusException.class, + () -> service.bindFormal(List.of(upload), "102", "202", account)); + + Assert.assertEquals(HttpStatus.CONFLICT, error.getStatusCode()); + Mockito.verify(objectStorage, Mockito.never()).copy(Mockito.anyString(), Mockito.anyString()); + } + + /** + * 构造一条有效的临时图片凭据。 + * + * @return 临时图片凭据 + */ + private AgentMediaUploadRecord buildUpload() { + AgentMediaUploadRecord record = new AgentMediaUploadRecord(); + record.setUploadId("upload-1"); + record.setTenantId("2"); + record.setUserId("7"); + record.setMode(AgentMediaService.MODE_FORMAL); + record.setAgentId("9"); + record.setSessionId("101"); + record.setObjectKey("temp/2/7/upload-1.png"); + record.setOriginalName("sample.png"); + record.setMimeType("image/png"); + record.setExtension("png"); + record.setSize(128L); + record.setWidth(32); + record.setHeight(32); + record.setExpiresAt(Instant.now().plusSeconds(3600)); + return record; + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/Model.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/Model.java index 43a451de..4f2f60f7 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/Model.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/Model.java @@ -67,6 +67,7 @@ public class Model extends ModelBase { } ollamaChatConfig.setModel(checkAndGetModelName()); ollamaChatConfig.setProvider(getModelProvider().getProviderName()); + ollamaChatConfig.setSupportImageBase64Only(getSupportImageB64Only()); return new OllamaChatModel(ollamaChatConfig); case "deepseek": DeepseekConfig deepseekConfig = new DeepseekConfig(); @@ -78,6 +79,7 @@ public class Model extends ModelBase { deepseekConfig.setSupportThinking(Boolean.TRUE); deepseekConfig.setThinkingProtocol("deepseek"); deepseekConfig.setNeedReasoningContentForToolMessage(Boolean.TRUE); + deepseekConfig.setSupportImageBase64Only(getSupportImageB64Only()); if (getSupportToolMessage() != null) { deepseekConfig.setSupportToolMessage(getSupportToolMessage()); } @@ -89,6 +91,7 @@ public class Model extends ModelBase { openAIChatConfig.setApiKey(checkAndGetApiKey()); openAIChatConfig.setModel(checkAndGetModelName()); openAIChatConfig.setRequestPath(checkAndGetRequestPath()); + openAIChatConfig.setSupportImageBase64Only(getSupportImageB64Only()); if (getSupportToolMessage() != null) { openAIChatConfig.setSupportToolMessage(getSupportToolMessage()); } diff --git a/easyflow-starter/easyflow-starter-all/src/main/resources/application.yml b/easyflow-starter/easyflow-starter-all/src/main/resources/application.yml index 2e7b0707..9a9e590c 100644 --- a/easyflow-starter/easyflow-starter-all/src/main/resources/application.yml +++ b/easyflow-starter/easyflow-starter-all/src/main/resources/application.yml @@ -156,6 +156,14 @@ easyflow: command-topic-prefix: easyflow:agent-runtime-command command-result-timeout: 5s command-result-ttl: 5m + media: + platform: minio-agent-media + max-image-count: 5 + max-image-bytes: 10485760 + max-image-pixels: 40000000 + upload-ttl: 24h + composer-draft-ttl: 24h + cleanup-interval: 10m login: # 放行接口路径 excludes: /api/v1/auth/**, /static/**, /userCenter/auth/**, /userCenter/public/** @@ -213,6 +221,13 @@ dromara: # minio 对象对外访问链接 domain: http://127.0.0.1:39000/easyflow/ base-path: attachment + - platform: minio-agent-media + enable-storage: true + access-key: easyflowadmin + secret-key: easyflowadmin123 + end-point: http://127.0.0.1:39000 + bucket-name: easyflow-agent-media + base-path: agent-chat # easy-agents 文档解析统一配置 easy-agents: diff --git a/easyflow-ui-admin/app/src/components/ai-chat/AiChatPanel.vue b/easyflow-ui-admin/app/src/components/ai-chat/AiChatPanel.vue index 4cbdff3a..7e7b7e9b 100644 --- a/easyflow-ui-admin/app/src/components/ai-chat/AiChatPanel.vue +++ b/easyflow-ui-admin/app/src/components/ai-chat/AiChatPanel.vue @@ -1,5 +1,6 @@