EasyFlow-Tech-Dev/Easy-Agents
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 打通智能体图片媒体运行链路

Browse Source 
- 增加稳定媒体引用解析与模型图片能力校验

- 修复图片 Data URI 与模型完整文本读取

- 补充媒体解析和消息兼容测试
This commit is contained in:
陈子默
2026-07-17 19:45:37 +08:00
parent 66da0c9039
commit f057900f7a
19 changed files with 629 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
easy-agents-core/src/main/java/com/easyagents/core/agent/react/ReActAgent.java
View File

@@ -233,7 +233,7 @@ public class ReActAgent implements IAgent {
AiMessageResponse response = chatModel.chat(memoryPrompt, chatOptions);
notifyOnChatResponse(response);
String content = response.getMessage().getContent();
String content = response.getMessage().getTextContent();
AiMessage message = new AiMessage(content);
// 请求用户输入

19
easy-agents-core/src/main/java/com/easyagents/core/message/AiMessage.java
View File

@@ -19,6 +19,9 @@ import com.easyagents.core.util.StringUtil;
import java.util.*;
/**
* 表示模型生成的文本、推理内容和工具调用消息。
*/
public class AiMessage extends AbstractTextMessage<AiMessage> {
private Integer index;
@@ -192,8 +195,13 @@ public class AiMessage extends AbstractTextMessage<AiMessage> {
this.localTotalTokens = localTotalTokens;
}
/**
* 获取当前消息的完整回复文本。
*
* @return 已聚合的完整文本；未聚合时返回当前文本
*/
public String getFullContent() {
return fullContent;
return fullContent != null ? fullContent : content;
}
public void setFullContent(String fullContent) {
@@ -226,7 +234,7 @@ public class AiMessage extends AbstractTextMessage<AiMessage> {
@Override
public String getTextContent() {
return fullContent;
return getFullContent();
}
/**
@@ -283,8 +291,13 @@ public class AiMessage extends AbstractTextMessage<AiMessage> {
this.toolCalls = toolCalls;
}
/**
* 获取当前消息的完整推理文本。
*
* @return 已聚合的完整推理文本；未聚合时返回当前推理文本
*/
public String getFullReasoningContent() {
return fullReasoningContent;
return fullReasoningContent != null ? fullReasoningContent : reasoningContent;
}
public void setFullReasoningContent(String fullReasoningContent) {

2
easy-agents-core/src/main/java/com/easyagents/core/model/chat/ChatModel.java
View File

@@ -33,7 +33,7 @@ public interface ChatModel {
if (response != null && response.isError()) {
throw new ModelException(response.getErrorMessage());
}
return response != null && response.getMessage() != null ? response.getMessage().getContent() : null;
return response != null && response.getMessage() != null ? response.getMessage().getTextContent() : null;
}
default AiMessageResponse chat(Prompt prompt) {

2
easy-agents-core/src/main/java/com/easyagents/core/model/chat/ChatObservabilityInterceptor.java
View File

@@ -177,7 +177,7 @@ public class ChatObservabilityInterceptor implements ChatInterceptor {
private void enrichSpan(Span span, AiMessage msg) {
if (msg != null) {
span.setAttribute("llm.total_tokens", msg.getEffectiveTotalTokens());
String content = msg.getContent();
String content = msg.getTextContent();
if (content != null) {
span.setAttribute("llm.response",
content.substring(0, Math.min(content.length(), MAX_RESPONSE_LENGTH_FOR_SPAN)));

12
easy-agents-core/src/main/java/com/easyagents/core/parser/impl/DefaultAiMessageParser.java
View File

@@ -32,6 +32,9 @@ import java.util.List;
import java.util.Map;
/**
* 解析 OpenAI-compatible 同步响应和流式增量消息。
*/
public class DefaultAiMessageParser implements AiMessageParser<JSONObject> {
private JSONPath contentPath;
@@ -171,11 +174,16 @@ public class DefaultAiMessageParser implements AiMessageParser<JSONObject> {
}
} else {
if (this.contentPath != null) {
aiMessage.setContent((String) this.contentPath.eval(rootJson));
String content = (String) this.contentPath.eval(rootJson);
// 非流式响应已经是完整结果，同时填充当前内容和完整内容。
aiMessage.setContent(content);
aiMessage.setFullContent(content);
}
if (this.reasoningContentPath != null) {
aiMessage.setReasoningContent((String) this.reasoningContentPath.eval(rootJson));
String reasoningContent = (String) this.reasoningContentPath.eval(rootJson);
aiMessage.setReasoningContent(reasoningContent);
aiMessage.setFullReasoningContent(reasoningContent);
}
if (this.toolCallsJsonPath != null) {
toolCallsJsonArray = (JSONArray) this.toolCallsJsonPath.eval(rootJson);

4
easy-agents-core/src/main/java/com/easyagents/core/util/ImageUtil.java
View File

@@ -50,7 +50,7 @@ public class ImageUtil {
}
/**
* 将图片 URL 转换为 Data URI 格式的字符串（例如：image/jpeg;base64,...）
* 将图片 URL 转换为 Data URI 格式的字符串（例如：data:image/jpeg;base64,...）
*
* @throws IllegalArgumentException 如果 URL 无效或无法获取内容
*/
@@ -108,6 +108,6 @@ public class ImageUtil {
public static String imageBytesToDataUri(byte[] data, String mimeType) {
String base64 = Base64.getEncoder().encodeToString(data);
return mimeType + ";base64," + base64;
return "data:" + mimeType + ";base64," + base64;
}
}

51
easy-agents-core/src/test/java/com/easyagents/core/message/AiMessageTest.java Normal file
View File

@@ -0,0 +1,51 @@
package com.easyagents.core.message;
import com.easyagents.core.util.LocalTokenCounter;
import org.junit.Assert;
import org.junit.Test;
/**
* AI 消息文本语义测试。
*/
public class AiMessageTest {
/**
* 验证仅设置当前内容时，完整文本接口仍能返回模型回复。
*/
@Test
public void currentContentShouldBeAvailableAsFullText() {
AiMessage message = new AiMessage();
message.setContent("2026");
Assert.assertEquals("2026", message.getContent());
Assert.assertEquals("2026", message.getFullContent());
Assert.assertEquals("2026", message.getTextContent());
Assert.assertTrue(LocalTokenCounter.countCompletionTokens(message)
> LocalTokenCounter.countCompletionTokens(new AiMessage()));
}
/**
* 验证流式消息保留增量内容，同时完整文本接口返回聚合内容。
*/
@Test
public void aggregatedContentShouldNotReplaceStreamingDelta() {
AiMessage message = new AiMessage();
message.setContent("26");
message.setFullContent("2026");
Assert.assertEquals("26", message.getContent());
Assert.assertEquals("2026", message.getFullContent());
Assert.assertEquals("2026", message.getTextContent());
}
/**
* 验证仅设置当前推理内容时，完整推理接口能够正确回退。
*/
@Test
public void reasoningContentShouldBeAvailableAsFullReasoningText() {
AiMessage message = new AiMessage();
message.setReasoningContent("识别图片中的数字");
Assert.assertEquals("识别图片中的数字", message.getFullReasoningContent());
}
}

87
easy-agents-core/src/test/java/com/easyagents/core/test/parser/DefaultAiMessageParserTest.java Normal file
View File

@@ -0,0 +1,87 @@
package com.easyagents.core.test.parser;
import com.alibaba.fastjson2.JSON;
import com.easyagents.core.message.AiMessage;
import com.easyagents.core.model.chat.ChatContext;
import com.easyagents.core.model.chat.ChatOptions;
import com.easyagents.core.parser.impl.DefaultAiMessageParser;
import org.junit.Assert;
import org.junit.Test;
/**
* OpenAI-compatible AI 消息解析测试。
*/
public class DefaultAiMessageParserTest {
/**
* 验证非流式响应同时填充当前内容和完整内容。
*/
@Test
public void nonStreamingResponseShouldPopulateCompleteContent() {
String response = """
{
"choices": [{
"message": {
"content": "2026",
"reasoning_content": "读取图片中的数字"
},
"index": 0,
"finish_reason": "stop"
}],
"usage": {
"prompt_tokens": 221,
"completion_tokens": 51,
"total_tokens": 272
}
}
""";
AiMessage message = DefaultAiMessageParser.getOpenAIMessageParser()
.parse(JSON.parseObject(response), context(false));
Assert.assertEquals("2026", message.getContent());
Assert.assertEquals("2026", message.getFullContent());
Assert.assertEquals("2026", message.getTextContent());
Assert.assertEquals("读取图片中的数字", message.getReasoningContent());
Assert.assertEquals("读取图片中的数字", message.getFullReasoningContent());
Assert.assertEquals(Integer.valueOf(272), message.getTotalTokens());
}
/**
* 验证流式解析继续暴露原始增量内容。
*/
@Test
public void streamingResponseShouldKeepDeltaContent() {
String response = """
{
"choices": [{
"delta": {
"content": "20",
"reasoning_content": "读取"
},
"index": 0
}]
}
""";
AiMessage message = DefaultAiMessageParser.getOpenAIMessageParser()
.parse(JSON.parseObject(response), context(true));
Assert.assertEquals("20", message.getContent());
Assert.assertEquals("读取", message.getReasoningContent());
}
/**
* 创建指定流式模式的解析上下文。
*
* @param streaming 是否解析流式增量
* @return 解析上下文
*/
private ChatContext context(boolean streaming) {
ChatOptions options = new ChatOptions();
options.setStreaming(streaming);
ChatContext context = new ChatContext();
context.setOptions(options);
return context;
}
}

36
easy-agents-core/src/test/java/com/easyagents/core/test/util/ImageUtilTest.java Normal file
View File

@@ -0,0 +1,36 @@
package com.easyagents.core.test.util;
import com.easyagents.core.message.UserMessage;
import com.easyagents.core.util.ImageUtil;
import org.junit.Assert;
import org.junit.Test;
import java.util.List;
/**
* 图片 Data URI 转换测试。
*/
public class ImageUtilTest {
/**
* 验证图片字节会生成符合标准的完整 Data URI。
*/
@Test
public void shouldAddDataSchemeWhenEncodingImageBytes() {
String dataUri = ImageUtil.imageBytesToDataUri(new byte[]{1, 2, 3}, "image/png");
Assert.assertEquals("data:image/png;base64,AQID", dataUri);
}
/**
* 验证用户消息添加图片字节时保留完整 Data URI。
*/
@Test
public void shouldStoreCompleteDataUriInUserMessage() {
UserMessage message = new UserMessage();
message.addImageBytes(new byte[]{1, 2, 3}, "image/png");
Assert.assertEquals(List.of("data:image/png;base64,AQID"), message.getImageUrls());
}
}