feat: 完善智能体图片聊天与会话恢复
- 增加私有图片上传、绑定、历史回显与生命周期清理 - 支持输入草稿恢复、图片交互和模型图片能力约束 - 修复旧脏会话幂等删除与前端会话恢复
This commit is contained in:
@@ -11,6 +11,7 @@ import com.easyagents.agent.runtime.persistence.session.AgentSessionStore;
|
||||
import com.easyagents.agent.runtime.persistence.session.memory.InMemoryAgentSessionStore;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.Mockito;
|
||||
import tech.easyflow.agent.entity.AgentHitlPending;
|
||||
import tech.easyflow.agent.entity.Agent;
|
||||
import tech.easyflow.agent.entity.AgentKnowledgeBinding;
|
||||
@@ -21,6 +22,8 @@ import tech.easyflow.agent.distributed.AgentRuntimeRouteRegistry;
|
||||
import tech.easyflow.agent.runtime.event.AgentRunEventRecorder;
|
||||
import tech.easyflow.agent.runtime.hitl.AgentHitlPendingService;
|
||||
import tech.easyflow.agent.runtime.lock.AgentRunLock;
|
||||
import tech.easyflow.agent.runtime.media.AgentBoundMedia;
|
||||
import tech.easyflow.agent.runtime.media.AgentMediaService;
|
||||
import tech.easyflow.chatlog.domain.dto.ChatSessionSummary;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
@@ -399,6 +402,33 @@ public class AgentRunServiceDraftAndHitlTest {
|
||||
Assert.assertEquals(Boolean.TRUE, payload.get(0).get("faqCollection"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证输入接收事件会回传正式图片展示信息。
|
||||
*
|
||||
* @throws Exception 反射调用失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void sendInputAcceptedShouldExposeBoundImages() throws Exception {
|
||||
AgentRunService service = new AgentRunService();
|
||||
RecordingChatSseEmitter emitter = new RecordingChatSseEmitter();
|
||||
AgentBoundMedia image = new AgentBoundMedia("formal:101:201:0:png", "image/png",
|
||||
Map.of("imageRef", "formal:101:201:0:png",
|
||||
"previewUrl", "/api/v1/agent/media/content?reference=formal:101:201:0:png"));
|
||||
|
||||
boolean sent = invoke(service, "sendInputAccepted",
|
||||
new Class<?>[]{ChatSseEmitter.class, BigInteger.class, BigInteger.class, List.class},
|
||||
emitter, BigInteger.valueOf(101), BigInteger.valueOf(201), List.of(image));
|
||||
|
||||
Assert.assertTrue(sent);
|
||||
Assert.assertEquals(1, emitter.envelopes.size());
|
||||
Assert.assertEquals(ChatType.INPUT_ACCEPTED, emitter.envelopes.get(0).getType());
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object> payload = (Map<String, Object>) emitter.envelopes.get(0).getPayload();
|
||||
Assert.assertEquals("101", payload.get("sessionId"));
|
||||
Assert.assertEquals("201", payload.get("messageId"));
|
||||
Assert.assertEquals(List.of(image.payload()), payload.get("images"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证未保存草稿会生成临时 Agent ID，并把绑定指向该运行 ID。
|
||||
*
|
||||
@@ -463,12 +493,16 @@ public class AgentRunServiceDraftAndHitlTest {
|
||||
Agent agent = new Agent();
|
||||
agent.setId(BigInteger.valueOf(100));
|
||||
ChatRuntimeContext context = chatContext();
|
||||
LoginAccount account = new LoginAccount();
|
||||
account.setId(BigInteger.ONE);
|
||||
account.setTenantId(BigInteger.ONE);
|
||||
|
||||
Exception thrown = Assert.assertThrows(Exception.class, () -> invoke(service, "run",
|
||||
new Class<?>[]{Agent.class, String.class, String.class, String.class, String.class,
|
||||
String.class, ChatRuntimeContext.class, boolean.class, AgentSessionStore.class},
|
||||
agent, "你好", "request-lock", "trace-lock", "session-lock", "AGENT", context, true,
|
||||
new InMemoryAgentSessionStore()));
|
||||
new Class<?>[]{Agent.class, String.class, List.class, LoginAccount.class, String.class,
|
||||
String.class, String.class, String.class, ChatRuntimeContext.class, boolean.class,
|
||||
AgentSessionStore.class},
|
||||
agent, "你好", List.of(), account, "request-lock", "trace-lock", "session-lock", "AGENT",
|
||||
context, true, new InMemoryAgentSessionStore()));
|
||||
|
||||
Assert.assertTrue(rootCause(thrown) instanceof BusinessException);
|
||||
Assert.assertEquals(0, chatRuntimeManager.prepareSessionCount);
|
||||
@@ -490,14 +524,21 @@ public class AgentRunServiceDraftAndHitlTest {
|
||||
setField(service, "agentRuntimeCompiler", compiler);
|
||||
setField(service, "agentRuntimeFactory", runtimeFactory);
|
||||
setField(service, "agentRunRegistry", new AgentRunRegistry());
|
||||
AgentMediaService mediaService = Mockito.mock(AgentMediaService.class);
|
||||
Mockito.when(mediaService.runtimeResolver(Mockito.any())).thenReturn(reference -> null);
|
||||
setField(service, "agentMediaService", mediaService);
|
||||
|
||||
Agent agent = new Agent();
|
||||
agent.setId(BigInteger.valueOf(100));
|
||||
LoginAccount account = new LoginAccount();
|
||||
account.setId(BigInteger.ONE);
|
||||
account.setTenantId(BigInteger.ONE);
|
||||
invoke(service, "startRuntime",
|
||||
new Class<?>[]{Agent.class, String.class, String.class, String.class, String.class, String.class,
|
||||
ChatRuntimeContext.class, ChatSseEmitter.class, boolean.class, AgentSessionStore.class,
|
||||
AgentRunLock.Handle.class},
|
||||
agent, "你好", "request-draft", "trace-draft", "agent-draft-100", "AGENT_DRAFT",
|
||||
new Class<?>[]{Agent.class, AgentMessage.class, LoginAccount.class, String.class, String.class,
|
||||
String.class, String.class, ChatRuntimeContext.class, ChatSseEmitter.class, boolean.class,
|
||||
AgentSessionStore.class, AgentRunLock.Handle.class},
|
||||
agent, AgentMessage.text(AgentMessageRole.USER, "你好"), account,
|
||||
"request-draft", "trace-draft", "agent-draft-100", "AGENT_DRAFT",
|
||||
chatContext(), new RecordingChatSseEmitter(), false, draftStore, null);
|
||||
|
||||
Assert.assertSame(draftStore, runtime.initRequest.getSessionStore());
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
package tech.easyflow.agent.runtime.composer;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.data.redis.core.ValueOperations;
|
||||
import tech.easyflow.agent.config.AgentMediaProperties;
|
||||
import tech.easyflow.agent.runtime.media.AgentMediaService;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* {@link AgentComposerDraftService} 草稿清理测试。
|
||||
*/
|
||||
public class AgentComposerDraftServiceTest {
|
||||
|
||||
private static final String DRAFT_KEY = "easyflow:agent:composer:draft:2:7:FORMAL:9:101";
|
||||
private static final String ACTIVE_KEY = "easyflow:agent:composer:active:2:7:FORMAL:9";
|
||||
|
||||
private StringRedisTemplate redisTemplate;
|
||||
private ValueOperations<String, String> valueOperations;
|
||||
private AgentMediaService mediaService;
|
||||
private AgentComposerDraftService service;
|
||||
private LoginAccount account;
|
||||
|
||||
/**
|
||||
* 初始化草稿服务测试依赖。
|
||||
*/
|
||||
@Before
|
||||
@SuppressWarnings("unchecked")
|
||||
public void setUp() {
|
||||
redisTemplate = Mockito.mock(StringRedisTemplate.class);
|
||||
valueOperations = Mockito.mock(ValueOperations.class);
|
||||
Mockito.when(redisTemplate.opsForValue()).thenReturn(valueOperations);
|
||||
mediaService = Mockito.mock(AgentMediaService.class);
|
||||
service = new AgentComposerDraftService(redisTemplate, new ObjectMapper(),
|
||||
new AgentMediaProperties(), mediaService);
|
||||
account = new LoginAccount();
|
||||
account.setTenantId(BigInteger.valueOf(2));
|
||||
account.setId(BigInteger.valueOf(7));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证删除草稿时合并服务端与调用方上传 ID 并清理临时图片。
|
||||
*
|
||||
* @throws Exception JSON 编码失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void deleteShouldCleanupDraftUploadsBeforeRedisKeys() throws Exception {
|
||||
AgentComposerDraft draft = new AgentComposerDraft();
|
||||
draft.setImageUploadIds(List.of("upload-1", "upload-2"));
|
||||
Mockito.when(valueOperations.get(DRAFT_KEY))
|
||||
.thenReturn(new ObjectMapper().writeValueAsString(draft));
|
||||
Mockito.when(valueOperations.get(ACTIVE_KEY)).thenReturn("101");
|
||||
|
||||
service.delete("FORMAL", "9", "101", List.of("upload-2", "upload-3"), true, account);
|
||||
|
||||
Mockito.verify(mediaService).deleteUploadsForScope(
|
||||
List.of("upload-1", "upload-2", "upload-3"), "FORMAL", "9", "101", account);
|
||||
Mockito.verify(redisTemplate).delete(List.of(DRAFT_KEY,
|
||||
"easyflow:agent:composer:revision:2:7:FORMAL:9:101"));
|
||||
Mockito.verify(redisTemplate).delete(ACTIVE_KEY);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证关闭上传清理时只删除草稿缓存。
|
||||
*/
|
||||
@Test
|
||||
public void deleteShouldKeepUploadsWhenCleanupIsDisabled() {
|
||||
Mockito.when(valueOperations.get(ACTIVE_KEY)).thenReturn("101");
|
||||
|
||||
service.delete("FORMAL", "9", "101", List.of("upload-1"), false, account);
|
||||
|
||||
Mockito.verifyNoInteractions(mediaService);
|
||||
Mockito.verify(redisTemplate).delete(List.of(DRAFT_KEY,
|
||||
"easyflow:agent:composer:revision:2:7:FORMAL:9:101"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证旧版草稿未保存图片 ID 字段时仍能完成清理。
|
||||
*
|
||||
* @throws Exception JSON 编码失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void deleteShouldHandleLegacyDraftWithoutImageUploadIds() throws Exception {
|
||||
AgentComposerDraft draft = new AgentComposerDraft();
|
||||
Mockito.when(valueOperations.get(DRAFT_KEY))
|
||||
.thenReturn(new ObjectMapper().writeValueAsString(draft));
|
||||
|
||||
service.delete("FORMAL", "9", "101", List.of(), true, account);
|
||||
|
||||
Mockito.verify(mediaService).deleteUploadsForScope(
|
||||
List.of(), "FORMAL", "9", "101", account);
|
||||
Mockito.verify(redisTemplate).delete(List.of(DRAFT_KEY,
|
||||
"easyflow:agent:composer:revision:2:7:FORMAL:9:101"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
package tech.easyflow.agent.runtime.media;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
import tech.easyflow.agent.config.AgentMediaProperties;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
/**
|
||||
* Agent 图片格式识别、规范化和资源限制测试。
|
||||
*/
|
||||
public class AgentImageProcessorTest {
|
||||
|
||||
/**
|
||||
* 验证 PNG 会保留格式、尺寸和稳定摘要。
|
||||
*
|
||||
* @throws Exception 图片生成失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void processShouldAcceptPngAndExposeMetadata() throws Exception {
|
||||
AgentImageProcessor processor = new AgentImageProcessor(new AgentMediaProperties());
|
||||
|
||||
AgentImageData image = processor.process(file("sample.png", "image/png", imageBytes("png", 3, 2)));
|
||||
|
||||
Assert.assertEquals("image/png", image.mimeType());
|
||||
Assert.assertEquals("png", image.extension());
|
||||
Assert.assertEquals(3, image.width());
|
||||
Assert.assertEquals(2, image.height());
|
||||
Assert.assertEquals(64, image.sha256().length());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 GIF 首帧会转换为模型兼容的 PNG。
|
||||
*
|
||||
* @throws Exception 图片生成失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void processShouldNormalizeGifToPng() throws Exception {
|
||||
AgentImageProcessor processor = new AgentImageProcessor(new AgentMediaProperties());
|
||||
|
||||
AgentImageData image = processor.process(file("sample.gif", "image/gif", imageBytes("gif", 4, 3)));
|
||||
|
||||
Assert.assertEquals("image/png", image.mimeType());
|
||||
Assert.assertEquals("png", image.extension());
|
||||
Assert.assertEquals(4, image.width());
|
||||
Assert.assertEquals(3, image.height());
|
||||
Assert.assertArrayEquals(new byte[]{(byte) 0x89, 0x50, 0x4e, 0x47},
|
||||
java.util.Arrays.copyOf(image.bytes(), 4));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证扩展名无法绕过真实图片内容校验。
|
||||
*/
|
||||
@Test
|
||||
public void processShouldRejectUnsupportedContent() {
|
||||
AgentImageProcessor processor = new AgentImageProcessor(new AgentMediaProperties());
|
||||
|
||||
ResponseStatusException error = Assert.assertThrows(ResponseStatusException.class,
|
||||
() -> processor.process(file("fake.png", "image/png", "not-an-image".getBytes())));
|
||||
|
||||
Assert.assertEquals(400, error.getStatusCode().value());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证像素上限会在图片解码前置阶段生效。
|
||||
*
|
||||
* @throws Exception 图片生成失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void processShouldRejectImageOverPixelLimit() throws Exception {
|
||||
AgentMediaProperties properties = new AgentMediaProperties();
|
||||
properties.setMaxImagePixels(3);
|
||||
AgentImageProcessor processor = new AgentImageProcessor(properties);
|
||||
|
||||
ResponseStatusException error = Assert.assertThrows(ResponseStatusException.class,
|
||||
() -> processor.process(file("large.png", "image/png", imageBytes("png", 2, 2))));
|
||||
|
||||
Assert.assertEquals(400, error.getStatusCode().value());
|
||||
Assert.assertTrue(error.getReason().contains("像素过大"));
|
||||
}
|
||||
|
||||
private byte[] imageBytes(String format, int width, int height) throws IOException {
|
||||
BufferedImage image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
|
||||
ByteArrayOutputStream output = new ByteArrayOutputStream();
|
||||
Assert.assertTrue(ImageIO.write(image, format, output));
|
||||
return output.toByteArray();
|
||||
}
|
||||
|
||||
private MultipartFile file(String name, String contentType, byte[] bytes) {
|
||||
return new ByteArrayMultipartFile(name, contentType, bytes);
|
||||
}
|
||||
|
||||
/**
|
||||
* 无磁盘依赖的测试 MultipartFile。
|
||||
*/
|
||||
private static final class ByteArrayMultipartFile implements MultipartFile {
|
||||
private final String name;
|
||||
private final String contentType;
|
||||
private final byte[] bytes;
|
||||
|
||||
private ByteArrayMultipartFile(String name, String contentType, byte[] bytes) {
|
||||
this.name = name;
|
||||
this.contentType = contentType;
|
||||
this.bytes = bytes;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() { return "file"; }
|
||||
|
||||
@Override
|
||||
public String getOriginalFilename() { return name; }
|
||||
|
||||
@Override
|
||||
public String getContentType() { return contentType; }
|
||||
|
||||
@Override
|
||||
public boolean isEmpty() { return bytes.length == 0; }
|
||||
|
||||
@Override
|
||||
public long getSize() { return bytes.length; }
|
||||
|
||||
@Override
|
||||
public byte[] getBytes() { return bytes.clone(); }
|
||||
|
||||
@Override
|
||||
public InputStream getInputStream() { return new ByteArrayInputStream(bytes); }
|
||||
|
||||
@Override
|
||||
public void transferTo(File destination) throws IOException {
|
||||
java.nio.file.Files.write(destination.toPath(), bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package tech.easyflow.agent.runtime.media;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
import tech.easyflow.agent.config.AgentMediaProperties;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* {@link AgentMediaService} 正式图片绑定测试。
|
||||
*/
|
||||
public class AgentMediaServiceTest {
|
||||
|
||||
private AgentMediaObjectStorage objectStorage;
|
||||
private RedisAgentMediaUploadStore uploadStore;
|
||||
private AgentMediaService service;
|
||||
private LoginAccount account;
|
||||
private AgentMediaUploadRecord upload;
|
||||
|
||||
/**
|
||||
* 初始化媒体服务测试依赖。
|
||||
*/
|
||||
@Before
|
||||
public void setUp() {
|
||||
objectStorage = Mockito.mock(AgentMediaObjectStorage.class);
|
||||
uploadStore = Mockito.mock(RedisAgentMediaUploadStore.class);
|
||||
service = new AgentMediaService(Mockito.mock(AgentImageProcessor.class), objectStorage,
|
||||
uploadStore, new AgentMediaProperties());
|
||||
account = new LoginAccount();
|
||||
account.setTenantId(BigInteger.valueOf(2));
|
||||
account.setId(BigInteger.valueOf(7));
|
||||
upload = buildUpload();
|
||||
Mockito.doAnswer(invocation -> {
|
||||
upload.setFormalReference(invocation.getArgument(1));
|
||||
return null;
|
||||
}).when(uploadStore).markFormalBinding(Mockito.same(upload), Mockito.anyString());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证重复绑定同一上传凭据时复用首次正式引用。
|
||||
*/
|
||||
@Test
|
||||
public void bindFormalShouldReuseExistingBinding() {
|
||||
List<AgentBoundMedia> first = service.bindFormal(List.of(upload), "101", "201", account);
|
||||
List<AgentBoundMedia> second = service.bindFormal(List.of(upload), "101", "202", account);
|
||||
|
||||
Assert.assertEquals("formal:101:201:0:png", first.get(0).reference());
|
||||
Assert.assertEquals(first.get(0).reference(), second.get(0).reference());
|
||||
Assert.assertEquals("/api/v1/agent/media/content?reference=formal:101:201:0:png",
|
||||
second.get(0).payload().get("previewUrl"));
|
||||
Mockito.verify(objectStorage, Mockito.times(1))
|
||||
.copy("temp/2/7/upload-1.png", "formal/2/7/101/201/0.png");
|
||||
Mockito.verify(uploadStore, Mockito.times(1))
|
||||
.markFormalBinding(upload, "formal:101:201:0:png");
|
||||
Mockito.verify(objectStorage, Mockito.never()).delete(Mockito.anyString());
|
||||
Mockito.verify(uploadStore, Mockito.never()).delete(Mockito.any());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证已绑定图片不能被复用到其他正式会话。
|
||||
*/
|
||||
@Test
|
||||
public void bindFormalShouldRejectAnotherSession() {
|
||||
upload.setFormalReference("formal:101:201:0:png");
|
||||
|
||||
ResponseStatusException error = Assert.assertThrows(ResponseStatusException.class,
|
||||
() -> service.bindFormal(List.of(upload), "102", "202", account));
|
||||
|
||||
Assert.assertEquals(HttpStatus.CONFLICT, error.getStatusCode());
|
||||
Mockito.verify(objectStorage, Mockito.never()).copy(Mockito.anyString(), Mockito.anyString());
|
||||
}
|
||||
|
||||
/**
|
||||
* 构造一条有效的临时图片凭据。
|
||||
*
|
||||
* @return 临时图片凭据
|
||||
*/
|
||||
private AgentMediaUploadRecord buildUpload() {
|
||||
AgentMediaUploadRecord record = new AgentMediaUploadRecord();
|
||||
record.setUploadId("upload-1");
|
||||
record.setTenantId("2");
|
||||
record.setUserId("7");
|
||||
record.setMode(AgentMediaService.MODE_FORMAL);
|
||||
record.setAgentId("9");
|
||||
record.setSessionId("101");
|
||||
record.setObjectKey("temp/2/7/upload-1.png");
|
||||
record.setOriginalName("sample.png");
|
||||
record.setMimeType("image/png");
|
||||
record.setExtension("png");
|
||||
record.setSize(128L);
|
||||
record.setWidth(32);
|
||||
record.setHeight(32);
|
||||
record.setExpiresAt(Instant.now().plusSeconds(3600));
|
||||
return record;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user