diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/AgentInitRequest.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/AgentInitRequest.java index ac2c7aa..1e647b8 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/AgentInitRequest.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/AgentInitRequest.java @@ -1,6 +1,7 @@ package com.easyagents.agent.runtime; import com.easyagents.agent.runtime.knowledge.AgentKnowledgeRetriever; +import com.easyagents.agent.runtime.media.AgentMediaResolver; import com.easyagents.agent.runtime.persistence.conversation.AgentConversationRecorder; import com.easyagents.agent.runtime.persistence.conversation.noop.NoopAgentConversationRecorder; import com.easyagents.agent.runtime.persistence.session.AgentSessionStore; @@ -55,6 +56,11 @@ public class AgentInitRequest { */ private Map metadata = new LinkedHashMap<>(); + /** + * 媒体引用解析器，仅在模型调用前解析稳定引用。 + */ + private AgentMediaResolver mediaResolver; + /** * 获取会话ID。 * @@ -200,4 +206,22 @@ public class AgentInitRequest { public void setMetadata(Map metadata) { this.metadata = metadata == null ? new LinkedHashMap<>() : metadata; } + + /** + * 获取媒体引用解析器。 + * + * @return 媒体引用解析器 + */ + public AgentMediaResolver getMediaResolver() { + return mediaResolver; + } + + /** + * 设置媒体引用解析器。 + * + * @param mediaResolver 媒体引用解析器 + */ + public void setMediaResolver(AgentMediaResolver mediaResolver) { + this.mediaResolver = mediaResolver; + } } diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeMessageAdapter.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeMessageAdapter.java index 71672eb..6b55439 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeMessageAdapter.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeMessageAdapter.java @@ -5,7 +5,9 @@ import io.agentscope.core.message.*; import java.time.Instant; import java.time.OffsetDateTime; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Base64; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -15,6 +17,9 @@ import java.util.Map; */ public class AgentScopeMessageAdapter { + /** AgentScope URLSource 中用于持久化业务媒体引用的私有协议。 */ + public static final String MEDIA_REFERENCE_SCHEME = "easyagents-media://"; + /** * 将运行时消息转换为 AgentScope 消息。 * @@ -276,6 +281,11 @@ public class AgentScopeMessageAdapter { .data(block.getData()) .build(); } + if (block.getReference() != null && !block.getReference().isBlank()) { + String encodedReference = Base64.getUrlEncoder().withoutPadding() + .encodeToString(block.getReference().getBytes(StandardCharsets.UTF_8)); + return URLSource.builder().url(MEDIA_REFERENCE_SCHEME + encodedReference).build(); + } return URLSource.builder().url(block.getUrl()).build(); } @@ -286,7 +296,30 @@ public class AgentScopeMessageAdapter { return; } if (source instanceof URLSource urlSource) { - block.setUrl(urlSource.getUrl()); + String url = urlSource.getUrl(); + if (url != null && url.startsWith(MEDIA_REFERENCE_SCHEME)) { + block.setReference(decodeReference(url)); + } else { + block.setUrl(url); + } + } + } + + /** + * 从 AgentScope 私有 URLSource 解码业务媒体引用。 + * + * @param url 私有协议 URL + * @return 原始业务媒体引用 + */ + public static String decodeReference(String url) { + if (url == null || !url.startsWith(MEDIA_REFERENCE_SCHEME)) { + return null; + } + String encoded = url.substring(MEDIA_REFERENCE_SCHEME.length()); + try { + return new String(Base64.getUrlDecoder().decode(encoded), StandardCharsets.UTF_8); + } catch (IllegalArgumentException error) { + throw new IllegalArgumentException("Invalid agent media reference.", error); } } diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeReActRuntime.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeReActRuntime.java index 27e073d..7530e33 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeReActRuntime.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeReActRuntime.java @@ -3,6 +3,7 @@ package com.easyagents.agent.runtime.agentscope; import com.easyagents.agent.runtime.*; import com.easyagents.agent.runtime.event.*; import com.easyagents.agent.runtime.event.interceptor.AutoContextInterceptor; +import com.easyagents.agent.runtime.event.interceptor.MediaReferenceInterceptor; import com.easyagents.agent.runtime.event.interceptor.ToolHitlInterceptor; import com.easyagents.agent.runtime.event.observer.AgentRuntimeErrorObserver; import com.easyagents.agent.runtime.event.observer.ReasoningLifecycleObserver; @@ -303,6 +304,12 @@ public class AgentScopeReActRuntime implements AgentRuntime { if (userMessage.getContentBlocks() == null || userMessage.getContentBlocks().isEmpty()) { throw new AgentRuntimeException("Agent user message content is required."); } + boolean containsImage = userMessage.getContentBlocks().stream() + .anyMatch(block -> block instanceof AgentMediaBlock mediaBlock + && "image".equalsIgnoreCase(mediaBlock.getMediaKind())); + if (containsImage && !initRequest.getAgentDefinition().getModelSpec().isSupportImage()) { + throw new AgentRuntimeException("The configured model does not support image input."); + } } /** @@ -1087,6 +1094,7 @@ public class AgentScopeReActRuntime implements AgentRuntime { if (memory instanceof AutoContextMemory) { interceptors.add(new AutoContextInterceptor(eventBridge, memoryResult.getAutoContextConfig())); } + interceptors.add(new MediaReferenceInterceptor(initRequest.getMediaResolver())); List runtimeToolSpecs = mergeToolSpecs(definition.getToolSpecs(), toolkitBuildResult.mcpToolSpecs(), toolkitBuildResult.operateToolSpecs()); interceptors.add(new ToolHitlInterceptor(eventBridge, approvalCoordinator, diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/event/interceptor/MediaReferenceInterceptor.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/event/interceptor/MediaReferenceInterceptor.java new file mode 100644 index 0000000..ec99472 --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/event/interceptor/MediaReferenceInterceptor.java @@ -0,0 +1,159 @@ +package com.easyagents.agent.runtime.event.interceptor; + +import com.easyagents.agent.runtime.AgentRuntimeException; +import com.easyagents.agent.runtime.agentscope.AgentScopeMessageAdapter; +import com.easyagents.agent.runtime.event.AgentRuntimeInterceptor; +import com.easyagents.agent.runtime.media.AgentMediaResolver; +import com.easyagents.agent.runtime.media.AgentMediaResource; +import io.agentscope.core.hook.HookEvent; +import io.agentscope.core.hook.PreReasoningEvent; +import io.agentscope.core.message.*; +import reactor.core.publisher.Mono; + +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; + +/** + * 在推理前将持久化的业务媒体引用解析为模型可用的 Base64 内容。 + * + *

输入消息会被深拷贝，AgentScope memory/session 中仍保留小体积稳定引用。

+ */ +public class MediaReferenceInterceptor implements AgentRuntimeInterceptor { + + private final AgentMediaResolver mediaResolver; + + /** + * 创建媒体引用干预器。 + * + * @param mediaResolver 媒体引用解析器 + */ + public MediaReferenceInterceptor(AgentMediaResolver mediaResolver) { + this.mediaResolver = mediaResolver; + } + + /** + * 解析推理输入中的内部媒体引用。 + * + * @param event AgentScope Hook 事件 + * @param Hook 事件类型 + * @return 处理后的事件 + */ + @Override + public Mono intercept(T event) { + if (event instanceof PreReasoningEvent preReasoningEvent) { + preReasoningEvent.setInputMessages(resolveMessages(preReasoningEvent.getInputMessages())); + } + return Mono.just(event); + } + + /** + * 在 AutoContext 完成消息重写后执行媒体解析。 + * + * @return 执行优先级 + */ + @Override + public int priority() { + return 10; + } + + private List resolveMessages(List messages) { + if (messages == null || messages.isEmpty()) { + return List.of(); + } + List resolved = new ArrayList<>(messages.size()); + for (Msg message : messages) { + List content = resolveBlocks(message.getContent()); + resolved.add(Msg.builder() + .id(message.getId()) + .name(message.getName()) + .role(message.getRole()) + .content(content) + .metadata(message.getMetadata()) + .timestamp(message.getTimestamp()) + .build()); + } + return resolved; + } + + private List resolveBlocks(List blocks) { + if (blocks == null || blocks.isEmpty()) { + return List.of(); + } + List resolved = new ArrayList<>(blocks.size()); + for (ContentBlock block : blocks) { + if (block instanceof ImageBlock imageBlock) { + resolved.add(resolveImage(imageBlock)); + } else if (block instanceof AudioBlock audioBlock) { + resolved.add(resolveAudio(audioBlock)); + } else if (block instanceof VideoBlock videoBlock) { + resolved.add(resolveVideo(videoBlock)); + } else { + resolved.add(block); + } + } + return resolved; + } + + private ImageBlock resolveImage(ImageBlock block) { + AgentMediaResource resource = resolve(block.getSource()); + if (resource == null) { + return block; + } + return ImageBlock.builder() + .source(base64Source(resource)) + .minPixels(block.getMinPixels()) + .maxPixels(block.getMaxPixels()) + .build(); + } + + private AudioBlock resolveAudio(AudioBlock block) { + AgentMediaResource resource = resolve(block.getSource()); + return resource == null ? block : AudioBlock.builder().source(base64Source(resource)).build(); + } + + private VideoBlock resolveVideo(VideoBlock block) { + AgentMediaResource resource = resolve(block.getSource()); + if (resource == null) { + return block; + } + return VideoBlock.builder() + .source(base64Source(resource)) + .fps(block.getFps()) + .maxFrames(block.getMaxFrames()) + .minPixels(block.getMinPixels()) + .maxPixels(block.getMaxPixels()) + .totalPixels(block.getTotalPixels()) + .build(); + } + + private AgentMediaResource resolve(Source source) { + if (!(source instanceof URLSource urlSource) + || urlSource.getUrl() == null + || !urlSource.getUrl().startsWith(AgentScopeMessageAdapter.MEDIA_REFERENCE_SCHEME)) { + return null; + } + if (mediaResolver == null) { + throw new AgentRuntimeException("Agent media resolver is required for media references."); + } + String reference; + try { + reference = AgentScopeMessageAdapter.decodeReference(urlSource.getUrl()); + } catch (IllegalArgumentException error) { + throw new AgentRuntimeException("Agent media reference is invalid.", error); + } + AgentMediaResource resource = mediaResolver.resolve(reference); + if (resource == null || resource.bytes().length == 0 || resource.mimeType() == null + || resource.mimeType().isBlank()) { + throw new AgentRuntimeException("Agent media resource is empty."); + } + return resource; + } + + private Base64Source base64Source(AgentMediaResource resource) { + return Base64Source.builder() + .mediaType(resource.mimeType()) + .data(Base64.getEncoder().encodeToString(resource.bytes())) + .build(); + } +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/media/AgentMediaResolver.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/media/AgentMediaResolver.java new file mode 100644 index 0000000..0b6d424 --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/media/AgentMediaResolver.java @@ -0,0 +1,17 @@ +package com.easyagents.agent.runtime.media; + +/** + * 在模型调用边界解析业务侧稳定媒体引用。 + */ +@FunctionalInterface +public interface AgentMediaResolver { + + /** + * 解析媒体引用。 + * + * @param reference 业务侧稳定媒体引用 + * @return 媒体资源 + * @throws RuntimeException 引用无效、越权或资源读取失败时抛出 + */ + AgentMediaResource resolve(String reference); +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/media/AgentMediaResource.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/media/AgentMediaResource.java new file mode 100644 index 0000000..f229a40 --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/media/AgentMediaResource.java @@ -0,0 +1,29 @@ +package com.easyagents.agent.runtime.media; + +import java.util.Arrays; + +/** + * 模型调用前解析得到的媒体资源。 + * + * @param mimeType MIME 类型 + * @param bytes 媒体字节 + */ +public record AgentMediaResource(String mimeType, byte[] bytes) { + + /** + * 创建不可变媒体资源。 + */ + public AgentMediaResource { + bytes = bytes == null ? new byte[0] : Arrays.copyOf(bytes, bytes.length); + } + + /** + * 返回媒体字节副本。 + * + * @return 媒体字节副本 + */ + @Override + public byte[] bytes() { + return Arrays.copyOf(bytes, bytes.length); + } +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/message/AgentMediaBlock.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/message/AgentMediaBlock.java index 8835e51..3ee561a 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/message/AgentMediaBlock.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/message/AgentMediaBlock.java @@ -9,6 +9,7 @@ import java.util.Map; public class AgentMediaBlock extends AgentContentBlock { private String mimeType; + private String reference; private String url; private String data; private Integer minPixels; @@ -64,6 +65,24 @@ public class AgentMediaBlock extends AgentContentBlock { this.mimeType = mimeType; } + /** + * 获取由业务侧解析的稳定媒体引用。 + * + * @return 媒体引用 + */ + public String getReference() { + return reference; + } + + /** + * 设置由业务侧解析的稳定媒体引用。 + * + * @param reference 媒体引用 + */ + public void setReference(String reference) { + this.reference = reference; + } + /** * 获取 URL。 * @@ -209,6 +228,7 @@ public class AgentMediaBlock extends AgentContentBlock { Map metadata = new LinkedHashMap<>(getMetadata()); metadata.put("mediaKind", mediaKind); metadata.put("mimeType", mimeType); + metadata.put("reference", reference); metadata.put("url", url); metadata.put("data", data); metadata.put("minPixels", minPixels); diff --git a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentScopeStatefulRuntimeTest.java b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentScopeStatefulRuntimeTest.java index ba15b66..50028ae 100644 --- a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentScopeStatefulRuntimeTest.java +++ b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentScopeStatefulRuntimeTest.java @@ -17,6 +17,7 @@ import com.easyagents.agent.runtime.memory.AgentMemoryPolicy; import com.easyagents.agent.runtime.memory.AgentMemorySnapshot; import com.easyagents.agent.runtime.message.AgentMessage; import com.easyagents.agent.runtime.message.AgentMessageRole; +import com.easyagents.agent.runtime.message.AgentMediaBlock; import com.easyagents.agent.runtime.model.AgentModelSpec; import com.easyagents.agent.runtime.persistence.session.memory.InMemoryAgentSessionStore; import com.easyagents.agent.runtime.skill.AgentSkillBoxSpec; @@ -78,6 +79,20 @@ public class AgentScopeStatefulRuntimeTest { runtime.stream(message); } + @Test(expected = AgentRuntimeException.class) + public void shouldRejectImageWhenModelCapabilityIsDisabled() { + AgentScopeReActRuntime runtime = fakeRuntime(); + AgentInitRequest request = initRequest(); + runtime.init(request); + AgentMessage message = new AgentMessage(); + message.setRole(AgentMessageRole.USER); + AgentMediaBlock image = new AgentMediaBlock("image"); + image.setReference("draft:upload-1"); + message.setContentBlocks(List.of(image)); + + runtime.stream(message); + } + @Test(expected = AgentRuntimeException.class) public void shouldRejectDuplicateInit() { AgentScopeReActRuntime runtime = fakeRuntime(); diff --git a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/event/interceptor/MediaReferenceInterceptorTest.java b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/event/interceptor/MediaReferenceInterceptorTest.java new file mode 100644 index 0000000..6fa1cf2 --- /dev/null +++ b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/event/interceptor/MediaReferenceInterceptorTest.java @@ -0,0 +1,117 @@ +package com.easyagents.agent.runtime.event.interceptor; + +import com.easyagents.agent.runtime.agentscope.AgentScopeMessageAdapter; +import com.easyagents.agent.runtime.media.AgentMediaResource; +import com.easyagents.agent.runtime.message.AgentMediaBlock; +import io.agentscope.core.ReActAgent; +import io.agentscope.core.hook.PreReasoningEvent; +import io.agentscope.core.message.Base64Source; +import io.agentscope.core.message.ImageBlock; +import io.agentscope.core.message.Msg; +import io.agentscope.core.message.MsgRole; +import io.agentscope.core.message.URLSource; +import io.agentscope.core.model.ChatResponse; +import io.agentscope.core.model.GenerateOptions; +import io.agentscope.core.model.Model; +import io.agentscope.core.model.ToolSchema; +import io.agentscope.core.tool.Toolkit; +import org.junit.Assert; +import org.junit.Test; +import reactor.core.publisher.Flux; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +/** + * 测试模型调用边界的媒体引用解析。 + */ +public class MediaReferenceInterceptorTest { + + /** + * 验证私有引用仅在模型调用边界转换为 Base64，原始记忆消息保持稳定引用。 + */ + @Test + public void shouldResolveReferenceToBase64WithoutMutatingOriginalMessage() { + AgentScopeMessageAdapter adapter = new AgentScopeMessageAdapter(); + AgentMediaBlock mediaBlock = new AgentMediaBlock("image"); + mediaBlock.setReference("draft:upload-1"); + mediaBlock.setMimeType("image/png"); + Msg original = Msg.builder() + .id("message-1") + .name("user") + .role(MsgRole.USER) + .content(adapter.toContentBlock(mediaBlock)) + .build(); + AtomicReference resolvedReference = new AtomicReference<>(); + MediaReferenceInterceptor interceptor = new MediaReferenceInterceptor(reference -> { + resolvedReference.set(reference); + return new AgentMediaResource("image/png", "png-data".getBytes(StandardCharsets.UTF_8)); + }); + ReActAgent agent = ReActAgent.builder() + .name("media-test-agent") + .sysPrompt("system") + .model(new EmptyModel()) + .toolkit(new Toolkit()) + .build(); + PreReasoningEvent event = new PreReasoningEvent(agent, "reasoning-1", null, List.of(original)); + + interceptor.intercept(event).block(); + + ImageBlock originalImage = original.getFirstContentBlock(ImageBlock.class); + ImageBlock resolvedImage = event.getInputMessages().get(0).getFirstContentBlock(ImageBlock.class); + Assert.assertTrue(originalImage.getSource() instanceof URLSource); + Assert.assertTrue(((URLSource) originalImage.getSource()).getUrl() + .startsWith(AgentScopeMessageAdapter.MEDIA_REFERENCE_SCHEME)); + Assert.assertTrue(resolvedImage.getSource() instanceof Base64Source); + Assert.assertEquals("draft:upload-1", resolvedReference.get()); + Assert.assertEquals("cG5nLWRhdGE=", ((Base64Source) resolvedImage.getSource()).getData()); + } + + /** + * 验证消息适配器序列化后仍能恢复不透明媒体引用。 + */ + @Test + public void shouldRoundTripStableReferenceThroughMessageAdapter() { + AgentScopeMessageAdapter adapter = new AgentScopeMessageAdapter(); + AgentMediaBlock mediaBlock = new AgentMediaBlock("image"); + mediaBlock.setReference("formal:message-1:0:image/png"); + + AgentMediaBlock restored = (AgentMediaBlock) adapter.toAgentBlock(adapter.toContentBlock(mediaBlock)); + + Assert.assertEquals("formal:message-1:0:image/png", restored.getReference()); + Assert.assertNull(restored.getData()); + Assert.assertNull(restored.getUrl()); + } + + /** + * 仅用于构建真实 AgentScope 事件的空模型。 + */ + private static final class EmptyModel implements Model { + + /** + * 返回空响应流，当前测试不会实际调用模型。 + * + * @param messages 输入消息 + * @param toolSchemas 工具定义 + * @param options 生成参数 + * @return 空响应流 + */ + @Override + public Flux stream(List messages, + List toolSchemas, + GenerateOptions options) { + return Flux.empty(); + } + + /** + * 返回测试模型名称。 + * + * @return 测试模型名称 + */ + @Override + public String getModelName() { + return "media-test-model"; + } + } +} diff --git a/easy-agents-core/src/main/java/com/easyagents/core/agent/react/ReActAgent.java b/easy-agents-core/src/main/java/com/easyagents/core/agent/react/ReActAgent.java index 2e89959..bfcbcdb 100644 --- a/easy-agents-core/src/main/java/com/easyagents/core/agent/react/ReActAgent.java +++ b/easy-agents-core/src/main/java/com/easyagents/core/agent/react/ReActAgent.java @@ -233,7 +233,7 @@ public class ReActAgent implements IAgent { AiMessageResponse response = chatModel.chat(memoryPrompt, chatOptions); notifyOnChatResponse(response); - String content = response.getMessage().getContent(); + String content = response.getMessage().getTextContent(); AiMessage message = new AiMessage(content); // 请求用户输入 diff --git a/easy-agents-core/src/main/java/com/easyagents/core/message/AiMessage.java b/easy-agents-core/src/main/java/com/easyagents/core/message/AiMessage.java index 7daa0b7..d986eb2 100644 --- a/easy-agents-core/src/main/java/com/easyagents/core/message/AiMessage.java +++ b/easy-agents-core/src/main/java/com/easyagents/core/message/AiMessage.java @@ -19,6 +19,9 @@ import com.easyagents.core.util.StringUtil; import java.util.*; +/** + * 表示模型生成的文本、推理内容和工具调用消息。 + */ public class AiMessage extends AbstractTextMessage { private Integer index; @@ -192,8 +195,13 @@ public class AiMessage extends AbstractTextMessage { this.localTotalTokens = localTotalTokens; } + /** + * 获取当前消息的完整回复文本。 + * + * @return 已聚合的完整文本；未聚合时返回当前文本 + */ public String getFullContent() { - return fullContent; + return fullContent != null ? fullContent : content; } public void setFullContent(String fullContent) { @@ -226,7 +234,7 @@ public class AiMessage extends AbstractTextMessage { @Override public String getTextContent() { - return fullContent; + return getFullContent(); } /** @@ -283,8 +291,13 @@ public class AiMessage extends AbstractTextMessage { this.toolCalls = toolCalls; } + /** + * 获取当前消息的完整推理文本。 + * + * @return 已聚合的完整推理文本；未聚合时返回当前推理文本 + */ public String getFullReasoningContent() { - return fullReasoningContent; + return fullReasoningContent != null ? fullReasoningContent : reasoningContent; } public void setFullReasoningContent(String fullReasoningContent) { diff --git a/easy-agents-core/src/main/java/com/easyagents/core/model/chat/ChatModel.java b/easy-agents-core/src/main/java/com/easyagents/core/model/chat/ChatModel.java index c2a0c2d..b9e9225 100644 --- a/easy-agents-core/src/main/java/com/easyagents/core/model/chat/ChatModel.java +++ b/easy-agents-core/src/main/java/com/easyagents/core/model/chat/ChatModel.java @@ -33,7 +33,7 @@ public interface ChatModel { if (response != null && response.isError()) { throw new ModelException(response.getErrorMessage()); } - return response != null && response.getMessage() != null ? response.getMessage().getContent() : null; + return response != null && response.getMessage() != null ? response.getMessage().getTextContent() : null; } default AiMessageResponse chat(Prompt prompt) { diff --git a/easy-agents-core/src/main/java/com/easyagents/core/model/chat/ChatObservabilityInterceptor.java b/easy-agents-core/src/main/java/com/easyagents/core/model/chat/ChatObservabilityInterceptor.java index d7274fa..7f6ac9c 100644 --- a/easy-agents-core/src/main/java/com/easyagents/core/model/chat/ChatObservabilityInterceptor.java +++ b/easy-agents-core/src/main/java/com/easyagents/core/model/chat/ChatObservabilityInterceptor.java @@ -177,7 +177,7 @@ public class ChatObservabilityInterceptor implements ChatInterceptor { private void enrichSpan(Span span, AiMessage msg) { if (msg != null) { span.setAttribute("llm.total_tokens", msg.getEffectiveTotalTokens()); - String content = msg.getContent(); + String content = msg.getTextContent(); if (content != null) { span.setAttribute("llm.response", content.substring(0, Math.min(content.length(), MAX_RESPONSE_LENGTH_FOR_SPAN))); diff --git a/easy-agents-core/src/main/java/com/easyagents/core/parser/impl/DefaultAiMessageParser.java b/easy-agents-core/src/main/java/com/easyagents/core/parser/impl/DefaultAiMessageParser.java index fd31027..578f6bd 100644 --- a/easy-agents-core/src/main/java/com/easyagents/core/parser/impl/DefaultAiMessageParser.java +++ b/easy-agents-core/src/main/java/com/easyagents/core/parser/impl/DefaultAiMessageParser.java @@ -32,6 +32,9 @@ import java.util.List; import java.util.Map; +/** + * 解析 OpenAI-compatible 同步响应和流式增量消息。 + */ public class DefaultAiMessageParser implements AiMessageParser { private JSONPath contentPath; @@ -171,11 +174,16 @@ public class DefaultAiMessageParser implements AiMessageParser { } } else { if (this.contentPath != null) { - aiMessage.setContent((String) this.contentPath.eval(rootJson)); + String content = (String) this.contentPath.eval(rootJson); + // 非流式响应已经是完整结果，同时填充当前内容和完整内容。 + aiMessage.setContent(content); + aiMessage.setFullContent(content); } if (this.reasoningContentPath != null) { - aiMessage.setReasoningContent((String) this.reasoningContentPath.eval(rootJson)); + String reasoningContent = (String) this.reasoningContentPath.eval(rootJson); + aiMessage.setReasoningContent(reasoningContent); + aiMessage.setFullReasoningContent(reasoningContent); } if (this.toolCallsJsonPath != null) { toolCallsJsonArray = (JSONArray) this.toolCallsJsonPath.eval(rootJson); diff --git a/easy-agents-core/src/main/java/com/easyagents/core/util/ImageUtil.java b/easy-agents-core/src/main/java/com/easyagents/core/util/ImageUtil.java index 97a6819..f3fa7a6 100644 --- a/easy-agents-core/src/main/java/com/easyagents/core/util/ImageUtil.java +++ b/easy-agents-core/src/main/java/com/easyagents/core/util/ImageUtil.java @@ -50,7 +50,7 @@ public class ImageUtil { } /** - * 将图片 URL 转换为 Data URI 格式的字符串（例如：image/jpeg;base64,...） + * 将图片 URL 转换为 Data URI 格式的字符串（例如：data:image/jpeg;base64,...） * * @throws IllegalArgumentException 如果 URL 无效或无法获取内容 */ @@ -108,6 +108,6 @@ public class ImageUtil { public static String imageBytesToDataUri(byte[] data, String mimeType) { String base64 = Base64.getEncoder().encodeToString(data); - return mimeType + ";base64," + base64; + return "data:" + mimeType + ";base64," + base64; } } diff --git a/easy-agents-core/src/test/java/com/easyagents/core/message/AiMessageTest.java b/easy-agents-core/src/test/java/com/easyagents/core/message/AiMessageTest.java new file mode 100644 index 0000000..e626a6c --- /dev/null +++ b/easy-agents-core/src/test/java/com/easyagents/core/message/AiMessageTest.java @@ -0,0 +1,51 @@ +package com.easyagents.core.message; + +import com.easyagents.core.util.LocalTokenCounter; +import org.junit.Assert; +import org.junit.Test; + +/** + * AI 消息文本语义测试。 + */ +public class AiMessageTest { + + /** + * 验证仅设置当前内容时，完整文本接口仍能返回模型回复。 + */ + @Test + public void currentContentShouldBeAvailableAsFullText() { + AiMessage message = new AiMessage(); + message.setContent("2026"); + + Assert.assertEquals("2026", message.getContent()); + Assert.assertEquals("2026", message.getFullContent()); + Assert.assertEquals("2026", message.getTextContent()); + Assert.assertTrue(LocalTokenCounter.countCompletionTokens(message) + > LocalTokenCounter.countCompletionTokens(new AiMessage())); + } + + /** + * 验证流式消息保留增量内容，同时完整文本接口返回聚合内容。 + */ + @Test + public void aggregatedContentShouldNotReplaceStreamingDelta() { + AiMessage message = new AiMessage(); + message.setContent("26"); + message.setFullContent("2026"); + + Assert.assertEquals("26", message.getContent()); + Assert.assertEquals("2026", message.getFullContent()); + Assert.assertEquals("2026", message.getTextContent()); + } + + /** + * 验证仅设置当前推理内容时，完整推理接口能够正确回退。 + */ + @Test + public void reasoningContentShouldBeAvailableAsFullReasoningText() { + AiMessage message = new AiMessage(); + message.setReasoningContent("识别图片中的数字"); + + Assert.assertEquals("识别图片中的数字", message.getFullReasoningContent()); + } +} diff --git a/easy-agents-core/src/test/java/com/easyagents/core/test/parser/DefaultAiMessageParserTest.java b/easy-agents-core/src/test/java/com/easyagents/core/test/parser/DefaultAiMessageParserTest.java new file mode 100644 index 0000000..52ed9bb --- /dev/null +++ b/easy-agents-core/src/test/java/com/easyagents/core/test/parser/DefaultAiMessageParserTest.java @@ -0,0 +1,87 @@ +package com.easyagents.core.test.parser; + +import com.alibaba.fastjson2.JSON; +import com.easyagents.core.message.AiMessage; +import com.easyagents.core.model.chat.ChatContext; +import com.easyagents.core.model.chat.ChatOptions; +import com.easyagents.core.parser.impl.DefaultAiMessageParser; +import org.junit.Assert; +import org.junit.Test; + +/** + * OpenAI-compatible AI 消息解析测试。 + */ +public class DefaultAiMessageParserTest { + + /** + * 验证非流式响应同时填充当前内容和完整内容。 + */ + @Test + public void nonStreamingResponseShouldPopulateCompleteContent() { + String response = """ + { + "choices": [{ + "message": { + "content": "2026", + "reasoning_content": "读取图片中的数字" + }, + "index": 0, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 221, + "completion_tokens": 51, + "total_tokens": 272 + } + } + """; + + AiMessage message = DefaultAiMessageParser.getOpenAIMessageParser() + .parse(JSON.parseObject(response), context(false)); + + Assert.assertEquals("2026", message.getContent()); + Assert.assertEquals("2026", message.getFullContent()); + Assert.assertEquals("2026", message.getTextContent()); + Assert.assertEquals("读取图片中的数字", message.getReasoningContent()); + Assert.assertEquals("读取图片中的数字", message.getFullReasoningContent()); + Assert.assertEquals(Integer.valueOf(272), message.getTotalTokens()); + } + + /** + * 验证流式解析继续暴露原始增量内容。 + */ + @Test + public void streamingResponseShouldKeepDeltaContent() { + String response = """ + { + "choices": [{ + "delta": { + "content": "20", + "reasoning_content": "读取" + }, + "index": 0 + }] + } + """; + + AiMessage message = DefaultAiMessageParser.getOpenAIMessageParser() + .parse(JSON.parseObject(response), context(true)); + + Assert.assertEquals("20", message.getContent()); + Assert.assertEquals("读取", message.getReasoningContent()); + } + + /** + * 创建指定流式模式的解析上下文。 + * + * @param streaming 是否解析流式增量 + * @return 解析上下文 + */ + private ChatContext context(boolean streaming) { + ChatOptions options = new ChatOptions(); + options.setStreaming(streaming); + ChatContext context = new ChatContext(); + context.setOptions(options); + return context; + } +} diff --git a/easy-agents-core/src/test/java/com/easyagents/core/test/util/ImageUtilTest.java b/easy-agents-core/src/test/java/com/easyagents/core/test/util/ImageUtilTest.java new file mode 100644 index 0000000..7895a92 --- /dev/null +++ b/easy-agents-core/src/test/java/com/easyagents/core/test/util/ImageUtilTest.java @@ -0,0 +1,36 @@ +package com.easyagents.core.test.util; + +import com.easyagents.core.message.UserMessage; +import com.easyagents.core.util.ImageUtil; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +/** + * 图片 Data URI 转换测试。 + */ +public class ImageUtilTest { + + /** + * 验证图片字节会生成符合标准的完整 Data URI。 + */ + @Test + public void shouldAddDataSchemeWhenEncodingImageBytes() { + String dataUri = ImageUtil.imageBytesToDataUri(new byte[]{1, 2, 3}, "image/png"); + + Assert.assertEquals("data:image/png;base64,AQID", dataUri); + } + + /** + * 验证用户消息添加图片字节时保留完整 Data URI。 + */ + @Test + public void shouldStoreCompleteDataUriInUserMessage() { + UserMessage message = new UserMessage(); + + message.addImageBytes(new byte[]{1, 2, 3}, "image/png"); + + Assert.assertEquals(List.of("data:image/png;base64,AQID"), message.getImageUrls()); + } +} diff --git a/easy-agents-support/src/main/java/com/easyagents/flow/support/provider/EasyAgentsLlm.java b/easy-agents-support/src/main/java/com/easyagents/flow/support/provider/EasyAgentsLlm.java index 881ab7c..9ab2422 100644 --- a/easy-agents-support/src/main/java/com/easyagents/flow/support/provider/EasyAgentsLlm.java +++ b/easy-agents-support/src/main/java/com/easyagents/flow/support/provider/EasyAgentsLlm.java @@ -61,7 +61,7 @@ public class EasyAgentsLlm implements Llm { AiMessage aiMessage = response.getMessage(); if (aiMessage != null) { - return aiMessage.getContent(); + return aiMessage.getTextContent(); } throw new RuntimeException("EasyAgentsLlm can not get aiMessage!");