EasyFlow-Tech-Dev/Easy-Agents
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 打通智能体图片媒体运行链路

Browse Source 
- 增加稳定媒体引用解析与模型图片能力校验

- 修复图片 Data URI 与模型完整文本读取

- 补充媒体解析和消息兼容测试
This commit is contained in:
陈子默
2026-07-17 19:45:37 +08:00
parent 66da0c9039
commit f057900f7a
19 changed files with 629 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/AgentInitRequest.java
View File

@@ -1,6 +1,7 @@
package com.easyagents.agent.runtime;
import com.easyagents.agent.runtime.knowledge.AgentKnowledgeRetriever;
import com.easyagents.agent.runtime.media.AgentMediaResolver;
import com.easyagents.agent.runtime.persistence.conversation.AgentConversationRecorder;
import com.easyagents.agent.runtime.persistence.conversation.noop.NoopAgentConversationRecorder;
import com.easyagents.agent.runtime.persistence.session.AgentSessionStore;
@@ -55,6 +56,11 @@ public class AgentInitRequest {
*/
private Map<String, Object> metadata = new LinkedHashMap<>();
/**
* 媒体引用解析器，仅在模型调用前解析稳定引用。
*/
private AgentMediaResolver mediaResolver;
/**
* 获取会话ID。
*
@@ -200,4 +206,22 @@ public class AgentInitRequest {
public void setMetadata(Map<String, Object> metadata) {
this.metadata = metadata == null ? new LinkedHashMap<>() : metadata;
}
/**
* 获取媒体引用解析器。
*
* @return 媒体引用解析器
*/
public AgentMediaResolver getMediaResolver() {
return mediaResolver;
}
/**
* 设置媒体引用解析器。
*
* @param mediaResolver 媒体引用解析器
*/
public void setMediaResolver(AgentMediaResolver mediaResolver) {
this.mediaResolver = mediaResolver;
}
}

35
easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeMessageAdapter.java
View File

@@ -5,7 +5,9 @@ import io.agentscope.core.message.*;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Base64;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -15,6 +17,9 @@ import java.util.Map;
*/
public class AgentScopeMessageAdapter {
/** AgentScope URLSource 中用于持久化业务媒体引用的私有协议。 */
public static final String MEDIA_REFERENCE_SCHEME = "easyagents-media://";
/**
* 将运行时消息转换为 AgentScope 消息。
*
@@ -276,6 +281,11 @@ public class AgentScopeMessageAdapter {
.data(block.getData())
.build();
}
if (block.getReference() != null && !block.getReference().isBlank()) {
String encodedReference = Base64.getUrlEncoder().withoutPadding()
.encodeToString(block.getReference().getBytes(StandardCharsets.UTF_8));
return URLSource.builder().url(MEDIA_REFERENCE_SCHEME + encodedReference).build();
}
return URLSource.builder().url(block.getUrl()).build();
}
@@ -286,7 +296,30 @@ public class AgentScopeMessageAdapter {
return;
}
if (source instanceof URLSource urlSource) {
block.setUrl(urlSource.getUrl());
String url = urlSource.getUrl();
if (url != null && url.startsWith(MEDIA_REFERENCE_SCHEME)) {
block.setReference(decodeReference(url));
} else {
block.setUrl(url);
}
}
}
/**
* 从 AgentScope 私有 URLSource 解码业务媒体引用。
*
* @param url 私有协议 URL
* @return 原始业务媒体引用
*/
public static String decodeReference(String url) {
if (url == null || !url.startsWith(MEDIA_REFERENCE_SCHEME)) {
return null;
}
String encoded = url.substring(MEDIA_REFERENCE_SCHEME.length());
try {
return new String(Base64.getUrlDecoder().decode(encoded), StandardCharsets.UTF_8);
} catch (IllegalArgumentException error) {
throw new IllegalArgumentException("Invalid agent media reference.", error);
}
}

8
easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeReActRuntime.java
View File

@@ -3,6 +3,7 @@ package com.easyagents.agent.runtime.agentscope;
import com.easyagents.agent.runtime.*;
import com.easyagents.agent.runtime.event.*;
import com.easyagents.agent.runtime.event.interceptor.AutoContextInterceptor;
import com.easyagents.agent.runtime.event.interceptor.MediaReferenceInterceptor;
import com.easyagents.agent.runtime.event.interceptor.ToolHitlInterceptor;
import com.easyagents.agent.runtime.event.observer.AgentRuntimeErrorObserver;
import com.easyagents.agent.runtime.event.observer.ReasoningLifecycleObserver;
@@ -303,6 +304,12 @@ public class AgentScopeReActRuntime implements AgentRuntime {
if (userMessage.getContentBlocks() == null || userMessage.getContentBlocks().isEmpty()) {
throw new AgentRuntimeException("Agent user message content is required.");
}
boolean containsImage = userMessage.getContentBlocks().stream()
.anyMatch(block -> block instanceof AgentMediaBlock mediaBlock
&& "image".equalsIgnoreCase(mediaBlock.getMediaKind()));
if (containsImage && !initRequest.getAgentDefinition().getModelSpec().isSupportImage()) {
throw new AgentRuntimeException("The configured model does not support image input.");
}
}
/**
@@ -1087,6 +1094,7 @@ public class AgentScopeReActRuntime implements AgentRuntime {
if (memory instanceof AutoContextMemory) {
interceptors.add(new AutoContextInterceptor(eventBridge, memoryResult.getAutoContextConfig()));
}
interceptors.add(new MediaReferenceInterceptor(initRequest.getMediaResolver()));
List<AgentToolSpec> runtimeToolSpecs = mergeToolSpecs(definition.getToolSpecs(), toolkitBuildResult.mcpToolSpecs(),
toolkitBuildResult.operateToolSpecs());
interceptors.add(new ToolHitlInterceptor(eventBridge, approvalCoordinator,

159
easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/event/interceptor/MediaReferenceInterceptor.java Normal file
View File

@@ -0,0 +1,159 @@
package com.easyagents.agent.runtime.event.interceptor;
import com.easyagents.agent.runtime.AgentRuntimeException;
import com.easyagents.agent.runtime.agentscope.AgentScopeMessageAdapter;
import com.easyagents.agent.runtime.event.AgentRuntimeInterceptor;
import com.easyagents.agent.runtime.media.AgentMediaResolver;
import com.easyagents.agent.runtime.media.AgentMediaResource;
import io.agentscope.core.hook.HookEvent;
import io.agentscope.core.hook.PreReasoningEvent;
import io.agentscope.core.message.*;
import reactor.core.publisher.Mono;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
/**
* 在推理前将持久化的业务媒体引用解析为模型可用的 Base64 内容。
*
* <p>输入消息会被深拷贝，AgentScope memory/session 中仍保留小体积稳定引用。</p>
*/
public class MediaReferenceInterceptor implements AgentRuntimeInterceptor {
private final AgentMediaResolver mediaResolver;
/**
* 创建媒体引用干预器。
*
* @param mediaResolver 媒体引用解析器
*/
public MediaReferenceInterceptor(AgentMediaResolver mediaResolver) {
this.mediaResolver = mediaResolver;
}
/**
* 解析推理输入中的内部媒体引用。
*
* @param event AgentScope Hook 事件
* @param <T> Hook 事件类型
* @return 处理后的事件
*/
@Override
public <T extends HookEvent> Mono<T> intercept(T event) {
if (event instanceof PreReasoningEvent preReasoningEvent) {
preReasoningEvent.setInputMessages(resolveMessages(preReasoningEvent.getInputMessages()));
}
return Mono.just(event);
}
/**
* 在 AutoContext 完成消息重写后执行媒体解析。
*
* @return 执行优先级
*/
@Override
public int priority() {
return 10;
}
private List<Msg> resolveMessages(List<Msg> messages) {
if (messages == null || messages.isEmpty()) {
return List.of();
}
List<Msg> resolved = new ArrayList<>(messages.size());
for (Msg message : messages) {
List<ContentBlock> content = resolveBlocks(message.getContent());
resolved.add(Msg.builder()
.id(message.getId())
.name(message.getName())
.role(message.getRole())
.content(content)
.metadata(message.getMetadata())
.timestamp(message.getTimestamp())
.build());
}
return resolved;
}
private List<ContentBlock> resolveBlocks(List<ContentBlock> blocks) {
if (blocks == null || blocks.isEmpty()) {
return List.of();
}
List<ContentBlock> resolved = new ArrayList<>(blocks.size());
for (ContentBlock block : blocks) {
if (block instanceof ImageBlock imageBlock) {
resolved.add(resolveImage(imageBlock));
} else if (block instanceof AudioBlock audioBlock) {
resolved.add(resolveAudio(audioBlock));
} else if (block instanceof VideoBlock videoBlock) {
resolved.add(resolveVideo(videoBlock));
} else {
resolved.add(block);
}
}
return resolved;
}
private ImageBlock resolveImage(ImageBlock block) {
AgentMediaResource resource = resolve(block.getSource());
if (resource == null) {
return block;
}
return ImageBlock.builder()
.source(base64Source(resource))
.minPixels(block.getMinPixels())
.maxPixels(block.getMaxPixels())
.build();
}
private AudioBlock resolveAudio(AudioBlock block) {
AgentMediaResource resource = resolve(block.getSource());
return resource == null ? block : AudioBlock.builder().source(base64Source(resource)).build();
}
private VideoBlock resolveVideo(VideoBlock block) {
AgentMediaResource resource = resolve(block.getSource());
if (resource == null) {
return block;
}
return VideoBlock.builder()
.source(base64Source(resource))
.fps(block.getFps())
.maxFrames(block.getMaxFrames())
.minPixels(block.getMinPixels())
.maxPixels(block.getMaxPixels())
.totalPixels(block.getTotalPixels())
.build();
}
private AgentMediaResource resolve(Source source) {
if (!(source instanceof URLSource urlSource)
|| urlSource.getUrl() == null
|| !urlSource.getUrl().startsWith(AgentScopeMessageAdapter.MEDIA_REFERENCE_SCHEME)) {
return null;
}
if (mediaResolver == null) {
throw new AgentRuntimeException("Agent media resolver is required for media references.");
}
String reference;
try {
reference = AgentScopeMessageAdapter.decodeReference(urlSource.getUrl());
} catch (IllegalArgumentException error) {
throw new AgentRuntimeException("Agent media reference is invalid.", error);
}
AgentMediaResource resource = mediaResolver.resolve(reference);
if (resource == null || resource.bytes().length == 0 || resource.mimeType() == null
|| resource.mimeType().isBlank()) {
throw new AgentRuntimeException("Agent media resource is empty.");
}
return resource;
}
private Base64Source base64Source(AgentMediaResource resource) {
return Base64Source.builder()
.mediaType(resource.mimeType())
.data(Base64.getEncoder().encodeToString(resource.bytes()))
.build();
}
}

17
easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/media/AgentMediaResolver.java Normal file
View File

@@ -0,0 +1,17 @@
package com.easyagents.agent.runtime.media;
/**
* 在模型调用边界解析业务侧稳定媒体引用。
*/
@FunctionalInterface
public interface AgentMediaResolver {
/**
* 解析媒体引用。
*
* @param reference 业务侧稳定媒体引用
* @return 媒体资源
* @throws RuntimeException 引用无效、越权或资源读取失败时抛出
*/
AgentMediaResource resolve(String reference);
}

29
easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/media/AgentMediaResource.java Normal file
View File

@@ -0,0 +1,29 @@
package com.easyagents.agent.runtime.media;
import java.util.Arrays;
/**
* 模型调用前解析得到的媒体资源。
*
* @param mimeType MIME 类型
* @param bytes 媒体字节
*/
public record AgentMediaResource(String mimeType, byte[] bytes) {
/**
* 创建不可变媒体资源。
*/
public AgentMediaResource {
bytes = bytes == null ? new byte[0] : Arrays.copyOf(bytes, bytes.length);
}
/**
* 返回媒体字节副本。
*
* @return 媒体字节副本
*/
@Override
public byte[] bytes() {
return Arrays.copyOf(bytes, bytes.length);
}
}

20
easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/message/AgentMediaBlock.java
View File

@@ -9,6 +9,7 @@ import java.util.Map;
public class AgentMediaBlock extends AgentContentBlock {
private String mimeType;
private String reference;
private String url;
private String data;
private Integer minPixels;
@@ -64,6 +65,24 @@ public class AgentMediaBlock extends AgentContentBlock {
this.mimeType = mimeType;
}
/**
* 获取由业务侧解析的稳定媒体引用。
*
* @return 媒体引用
*/
public String getReference() {
return reference;
}
/**
* 设置由业务侧解析的稳定媒体引用。
*
* @param reference 媒体引用
*/
public void setReference(String reference) {
this.reference = reference;
}
/**
* 获取 URL。
*
@@ -209,6 +228,7 @@ public class AgentMediaBlock extends AgentContentBlock {
Map<String, Object> metadata = new LinkedHashMap<>(getMetadata());
metadata.put("mediaKind", mediaKind);
metadata.put("mimeType", mimeType);
metadata.put("reference", reference);
metadata.put("url", url);
metadata.put("data", data);
metadata.put("minPixels", minPixels);

15
easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentScopeStatefulRuntimeTest.java
View File

@@ -17,6 +17,7 @@ import com.easyagents.agent.runtime.memory.AgentMemoryPolicy;
import com.easyagents.agent.runtime.memory.AgentMemorySnapshot;
import com.easyagents.agent.runtime.message.AgentMessage;
import com.easyagents.agent.runtime.message.AgentMessageRole;
import com.easyagents.agent.runtime.message.AgentMediaBlock;
import com.easyagents.agent.runtime.model.AgentModelSpec;
import com.easyagents.agent.runtime.persistence.session.memory.InMemoryAgentSessionStore;
import com.easyagents.agent.runtime.skill.AgentSkillBoxSpec;
@@ -78,6 +79,20 @@ public class AgentScopeStatefulRuntimeTest {
runtime.stream(message);
}
@Test(expected = AgentRuntimeException.class)
public void shouldRejectImageWhenModelCapabilityIsDisabled() {
AgentScopeReActRuntime runtime = fakeRuntime();
AgentInitRequest request = initRequest();
runtime.init(request);
AgentMessage message = new AgentMessage();
message.setRole(AgentMessageRole.USER);
AgentMediaBlock image = new AgentMediaBlock("image");
image.setReference("draft:upload-1");
message.setContentBlocks(List.of(image));
runtime.stream(message);
}
@Test(expected = AgentRuntimeException.class)
public void shouldRejectDuplicateInit() {
AgentScopeReActRuntime runtime = fakeRuntime();

117
easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/event/interceptor/MediaReferenceInterceptorTest.java Normal file
View File

@@ -0,0 +1,117 @@
package com.easyagents.agent.runtime.event.interceptor;
import com.easyagents.agent.runtime.agentscope.AgentScopeMessageAdapter;
import com.easyagents.agent.runtime.media.AgentMediaResource;
import com.easyagents.agent.runtime.message.AgentMediaBlock;
import io.agentscope.core.ReActAgent;
import io.agentscope.core.hook.PreReasoningEvent;
import io.agentscope.core.message.Base64Source;
import io.agentscope.core.message.ImageBlock;
import io.agentscope.core.message.Msg;
import io.agentscope.core.message.MsgRole;
import io.agentscope.core.message.URLSource;
import io.agentscope.core.model.ChatResponse;
import io.agentscope.core.model.GenerateOptions;
import io.agentscope.core.model.Model;
import io.agentscope.core.model.ToolSchema;
import io.agentscope.core.tool.Toolkit;
import org.junit.Assert;
import org.junit.Test;
import reactor.core.publisher.Flux;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
/**
* 测试模型调用边界的媒体引用解析。
*/
public class MediaReferenceInterceptorTest {
/**
* 验证私有引用仅在模型调用边界转换为 Base64，原始记忆消息保持稳定引用。
*/
@Test
public void shouldResolveReferenceToBase64WithoutMutatingOriginalMessage() {
AgentScopeMessageAdapter adapter = new AgentScopeMessageAdapter();
AgentMediaBlock mediaBlock = new AgentMediaBlock("image");
mediaBlock.setReference("draft:upload-1");
mediaBlock.setMimeType("image/png");
Msg original = Msg.builder()
.id("message-1")
.name("user")
.role(MsgRole.USER)
.content(adapter.toContentBlock(mediaBlock))
.build();
AtomicReference<String> resolvedReference = new AtomicReference<>();
MediaReferenceInterceptor interceptor = new MediaReferenceInterceptor(reference -> {
resolvedReference.set(reference);
return new AgentMediaResource("image/png", "png-data".getBytes(StandardCharsets.UTF_8));
});
ReActAgent agent = ReActAgent.builder()
.name("media-test-agent")
.sysPrompt("system")
.model(new EmptyModel())
.toolkit(new Toolkit())
.build();
PreReasoningEvent event = new PreReasoningEvent(agent, "reasoning-1", null, List.of(original));
interceptor.intercept(event).block();
ImageBlock originalImage = original.getFirstContentBlock(ImageBlock.class);
ImageBlock resolvedImage = event.getInputMessages().get(0).getFirstContentBlock(ImageBlock.class);
Assert.assertTrue(originalImage.getSource() instanceof URLSource);
Assert.assertTrue(((URLSource) originalImage.getSource()).getUrl()
.startsWith(AgentScopeMessageAdapter.MEDIA_REFERENCE_SCHEME));
Assert.assertTrue(resolvedImage.getSource() instanceof Base64Source);
Assert.assertEquals("draft:upload-1", resolvedReference.get());
Assert.assertEquals("cG5nLWRhdGE=", ((Base64Source) resolvedImage.getSource()).getData());
}
/**
* 验证消息适配器序列化后仍能恢复不透明媒体引用。
*/
@Test
public void shouldRoundTripStableReferenceThroughMessageAdapter() {
AgentScopeMessageAdapter adapter = new AgentScopeMessageAdapter();
AgentMediaBlock mediaBlock = new AgentMediaBlock("image");
mediaBlock.setReference("formal:message-1:0:image/png");
AgentMediaBlock restored = (AgentMediaBlock) adapter.toAgentBlock(adapter.toContentBlock(mediaBlock));
Assert.assertEquals("formal:message-1:0:image/png", restored.getReference());
Assert.assertNull(restored.getData());
Assert.assertNull(restored.getUrl());
}
/**
* 仅用于构建真实 AgentScope 事件的空模型。
*/
private static final class EmptyModel implements Model {
/**
* 返回空响应流，当前测试不会实际调用模型。
*
* @param messages 输入消息
* @param toolSchemas 工具定义
* @param options 生成参数
* @return 空响应流
*/
@Override
public Flux<ChatResponse> stream(List<Msg> messages,
List<ToolSchema> toolSchemas,
GenerateOptions options) {
return Flux.empty();
}
/**
* 返回测试模型名称。
*
* @return 测试模型名称
*/
@Override
public String getModelName() {
return "media-test-model";
}
}
}