EasyFlow-Tech-Dev/EasyFlow
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 扩展模型消息内容块数组配置

Browse Source 
- 升级高级设置为消息级格式并兼容旧 system 配置

- 增加 VLM 图片首轮与文本追问的多轮连接验证
This commit is contained in:
陈子默
2026-07-28 12:24:42 +08:00
parent dc35ddc3e4
commit f392c896f9
5 changed files with 227 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentModelSpecMapper.java
View File

@@ -1,9 +1,9 @@
package tech.easyflow.agent.runtime;
import com.easyagents.agent.runtime.model.AgentHttpVersionPolicy;
import com.easyagents.agent.runtime.model.AgentMessageContentFormat;
import com.easyagents.agent.runtime.model.AgentModelProviderType;
import com.easyagents.agent.runtime.model.AgentModelSpec;
import com.easyagents.agent.runtime.model.AgentSystemContentFormat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import tech.easyflow.ai.entity.Model;
@@ -61,7 +61,7 @@ public final class AgentModelSpecMapper {
spec.setSupportImage(Boolean.TRUE.equals(model.getSupportImage()));
spec.setSupportImageBase64Only(Boolean.TRUE.equals(model.getSupportImageB64Only()));
spec.setHttpVersionPolicy(parseHttpVersionPolicy(model));
spec.setSystemContentFormat(parseSystemContentFormat(model));
spec.setMessageContentFormat(parseMessageContentFormat(model));
spec.getMetadata().put("modelId", model.getId());
if (providerType != null && !providerType.isBlank()) {
spec.getMetadata().put("sourceProviderType", providerType);
@@ -93,25 +93,30 @@ public final class AgentModelSpecMapper {
}
/**
* 解析模型配置中的 Agent system content 格式。
* 解析模型配置中的 Agent 消息 content 格式。
*
* @param model 模型配置
* @return system content 格式，缺失或非法时返回 STRING
* @return 消息 content 格式，缺失或非法时返回 STANDARD
*/
private static AgentSystemContentFormat parseSystemContentFormat(Model model) {
Object rawFormat = model.getOptions() == null
? null
: model.getOptions().get("agentSystemContentFormat");
private static AgentMessageContentFormat parseMessageContentFormat(Model model) {
Map<String, Object> options = model.getOptions();
Object rawFormat = options == null ? null : options.get("agentMessageContentFormat");
if ((rawFormat == null || String.valueOf(rawFormat).isBlank()) && options != null) {
rawFormat = options.get("agentSystemContentFormat");
}
if (rawFormat == null || String.valueOf(rawFormat).isBlank()) {
return AgentSystemContentFormat.STRING;
return AgentMessageContentFormat.STANDARD;
}
String normalizedFormat = String.valueOf(rawFormat).trim().toUpperCase(Locale.ROOT);
if ("STRING".equals(normalizedFormat)) {
return AgentMessageContentFormat.STANDARD;
}
try {
return AgentSystemContentFormat.valueOf(normalizedFormat);
return AgentMessageContentFormat.valueOf(normalizedFormat);
} catch (IllegalArgumentException exception) {
LOG.warn("Invalid Agent system content format '{}' for model {}, fallback to STRING",
LOG.warn("Invalid Agent message content format '{}' for model {}, fallback to STANDARD",
rawFormat, model.getId());
return AgentSystemContentFormat.STRING;
return AgentMessageContentFormat.STANDARD;
}
}

89
easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentScopeChatModelConnectivityVerifier.java
View File

@@ -9,6 +9,7 @@ import com.easyagents.agent.runtime.message.AgentMessage;
import com.easyagents.agent.runtime.message.AgentMessageRole;
import com.easyagents.agent.runtime.message.AgentTextBlock;
import com.easyagents.agent.runtime.model.AgentGenerationOptions;
import com.easyagents.agent.runtime.model.AgentMessageContentFormat;
import com.easyagents.agent.runtime.model.AgentModelFactory;
import com.easyagents.agent.runtime.model.AgentModelProviderType;
import com.easyagents.agent.runtime.model.AgentModelSpec;
@@ -108,10 +109,11 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
}
/**
* 使用一次非流式 Chat 请求优先同时验证连接、视觉与工具调用能力。
* 使用非流式 Chat 请求验证连接、视觉与工具调用能力。
*
* <p>当兼容接口明确拒绝工具参数时，追加一次不带工具的连接兜底请求，
* 避免将可用的普通对话模型误判为连接失败。</p>
* 避免将可用的普通对话模型误判为连接失败。严格内容块数组模式会先
* 追加一次多轮上下文验证。</p>
*
* @param model 已补齐供应商默认配置的模型
* @return 连接和工具能力验证结果
@@ -125,6 +127,9 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
String nonce = nonceSupplier.get();
try {
if (modelSpec.getMessageContentFormat() == AgentMessageContentFormat.TEXT_PARTS) {
verifyMessageContentCompatibility(modelSpec);
}
boolean supportTool = verifyProbeRequest(modelSpec, nonce);
return ChatModelVerificationResult.passed(effectiveHttpVersion, supportTool);
} catch (Exception exception) {
@@ -198,6 +203,24 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
validateTextResponse(modelSpec.isSupportImage(), aggregateText(responses));
}
/**
* 验证严格内容块数组模式可以携带完整多轮上下文。
*
* <p>支持图片时将图片放在首轮用户消息中，再追加助手历史和纯文本追问，
* 覆盖 VLM 首轮成功后继续对话的请求格式。</p>
*
* @param modelSpec 运行时模型声明
* @throws BusinessException 响应为空时抛出
*/
private void verifyMessageContentCompatibility(AgentModelSpec modelSpec) {
List<ChatResponse> responses = request(
modelSpec,
buildMessageContentCompatibilityHistory(modelSpec.isSupportImage()),
List.of(),
null);
validateTextResponse(false, aggregateText(responses));
}
/**
* 使用统一低成本参数执行一次非流式模型请求。
*
@@ -229,6 +252,44 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
List<ToolSchema> tools,
ToolChoice toolChoice,
int maxTokens) {
List<Msg> messages = List.of(
messageAdapter.toMsg(AgentMessage.text(
AgentMessageRole.SYSTEM, VERIFICATION_SYSTEM_PROMPT)),
messageAdapter.toMsg(verificationMessage));
return request(modelSpec, messages, tools, toolChoice, maxTokens);
}
/**
* 使用统一低成本参数执行一次指定上下文的非流式模型请求。
*
* @param modelSpec 运行时模型声明
* @param messages 完整上下文消息
* @param tools 工具 Schema
* @param toolChoice 工具选择策略
* @return 模型响应片段
*/
private List<ChatResponse> request(AgentModelSpec modelSpec,
List<Msg> messages,
List<ToolSchema> tools,
ToolChoice toolChoice) {
return request(modelSpec, messages, tools, toolChoice, MAX_TOKENS);
}
/**
* 使用指定输出预算执行一次指定上下文的非流式模型请求。
*
* @param modelSpec 运行时模型声明
* @param messages 完整上下文消息
* @param tools 工具 Schema
* @param toolChoice 工具选择策略
* @param maxTokens 最大输出 Token 数
* @return 模型响应片段
*/
private List<ChatResponse> request(AgentModelSpec modelSpec,
List<Msg> messages,
List<ToolSchema> tools,
ToolChoice toolChoice,
int maxTokens) {
AgentGenerationOptions generationOptions = new AgentGenerationOptions();
generationOptions.setStream(false);
generationOptions.setThinkingEnabled(false);
@@ -236,10 +297,6 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
generationOptions.setMaxTokens(maxTokens);
io.agentscope.core.model.Model agentScopeModel = modelFactory.create(modelSpec, generationOptions);
List<Msg> messages = List.of(
messageAdapter.toMsg(AgentMessage.text(
AgentMessageRole.SYSTEM, VERIFICATION_SYSTEM_PROMPT)),
messageAdapter.toMsg(verificationMessage));
GenerateOptions.Builder requestBuilder = GenerateOptions.builder()
.stream(false)
.maxTokens(maxTokens)
@@ -257,6 +314,26 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
.block(phaseTimeout.plusSeconds(1));
}
/**
* 构造包含首轮用户消息、助手历史和文本追问的验证上下文。
*
* @param supportImage 是否在首轮用户消息中附加验证图片
* @return AgentScope 多轮消息
*/
private List<Msg> buildMessageContentCompatibilityHistory(boolean supportImage) {
AgentMessage firstUserMessage = buildVerificationMessage(
supportImage ? "请阅读图片，等待下一条消息。" : "这是第一轮消息。",
supportImage);
return List.of(
messageAdapter.toMsg(AgentMessage.text(
AgentMessageRole.SYSTEM, VERIFICATION_SYSTEM_PROMPT)),
messageAdapter.toMsg(firstUserMessage),
messageAdapter.toMsg(AgentMessage.text(
AgentMessageRole.ASSISTANT, "已收到。")),
messageAdapter.toMsg(AgentMessage.text(
AgentMessageRole.USER, "请直接回复“你好”，不要补充其他内容。")));
}
/**
* 为支持该扩展字段的 OpenAI-compatible 服务显式关闭思考。
*

63
easyflow-modules/easyflow-module-agent/src/test/java/tech/easyflow/agent/runtime/AgentRuntimeCompilerModelConfigTest.java
View File

@@ -3,8 +3,8 @@ package tech.easyflow.agent.runtime;
import com.easyagents.agent.runtime.memory.AgentMemoryPolicy;
import com.easyagents.agent.runtime.model.AgentGenerationOptions;
import com.easyagents.agent.runtime.model.AgentHttpVersionPolicy;
import com.easyagents.agent.runtime.model.AgentMessageContentFormat;
import com.easyagents.agent.runtime.model.AgentModelSpec;
import com.easyagents.agent.runtime.model.AgentSystemContentFormat;
import org.junit.Assert;
import org.junit.Test;
import tech.easyflow.agent.entity.Agent;
@@ -68,33 +68,80 @@ public class AgentRuntimeCompilerModelConfigTest {
}
/**
* 验证模型 options 中的 system content 格式会编译到中立模型声明。
* 验证模型 options 中的消息 content 格式会编译到中立模型声明。
*
* @throws Exception 反射调用失败时抛出
*/
@Test
public void modelSystemContentFormatShouldCompileFromOptions() throws Exception {
public void modelMessageContentFormatShouldCompileFromOptions() throws Exception {
Model model = model(Map.of("agentMessageContentFormat", "TEXT_PARTS"));
AgentRuntimeCompiler compiler = compiler(model);
AgentModelSpec spec = invokeModelSpec(compiler);
Assert.assertEquals(AgentMessageContentFormat.TEXT_PARTS, spec.getMessageContentFormat());
}
/**
* 验证旧 system content 数组配置会迁移为消息级数组格式。
*
* @throws Exception 反射调用失败时抛出
*/
@Test
public void legacyTextPartsFormatShouldMigrateToMessageContentFormat() throws Exception {
Model model = model(Map.of("agentSystemContentFormat", "TEXT_PARTS"));
AgentRuntimeCompiler compiler = compiler(model);
AgentModelSpec spec = invokeModelSpec(compiler);
Assert.assertEquals(AgentSystemContentFormat.TEXT_PARTS, spec.getSystemContentFormat());
Assert.assertEquals(AgentMessageContentFormat.TEXT_PARTS, spec.getMessageContentFormat());
}
/**
* 验证未知 system content 格式安全回退到字符串。
* 验证旧字符串配置会迁移为标准格式。
*
* @throws Exception 反射调用失败时抛出
*/
@Test
public void unknownSystemContentFormatShouldFallbackToString() throws Exception {
Model model = model(Map.of("agentSystemContentFormat", "PARTS"));
public void legacyStringFormatShouldMigrateToStandard() throws Exception {
Model model = model(Map.of("agentSystemContentFormat", "STRING"));
AgentRuntimeCompiler compiler = compiler(model);
AgentModelSpec spec = invokeModelSpec(compiler);
Assert.assertEquals(AgentSystemContentFormat.STRING, spec.getSystemContentFormat());
Assert.assertEquals(AgentMessageContentFormat.STANDARD, spec.getMessageContentFormat());
}
/**
* 验证新旧配置并存时优先使用消息级配置。
*
* @throws Exception 反射调用失败时抛出
*/
@Test
public void messageContentFormatShouldTakePrecedenceOverLegacyFormat() throws Exception {
Model model = model(Map.of(
"agentMessageContentFormat", "STANDARD",
"agentSystemContentFormat", "TEXT_PARTS"));
AgentRuntimeCompiler compiler = compiler(model);
AgentModelSpec spec = invokeModelSpec(compiler);
Assert.assertEquals(AgentMessageContentFormat.STANDARD, spec.getMessageContentFormat());
}
/**
* 验证未知消息 content 格式安全回退到标准格式。
*
* @throws Exception 反射调用失败时抛出
*/
@Test
public void unknownMessageContentFormatShouldFallbackToStandard() throws Exception {
Model model = model(Map.of("agentMessageContentFormat", "PARTS"));
AgentRuntimeCompiler compiler = compiler(model);
AgentModelSpec spec = invokeModelSpec(compiler);
Assert.assertEquals(AgentMessageContentFormat.STANDARD, spec.getMessageContentFormat());
}
/**

48
easyflow-modules/easyflow-module-agent/src/test/java/tech/easyflow/agent/runtime/AgentScopeChatModelConnectivityVerifierTest.java
View File

@@ -88,7 +88,7 @@ public class AgentScopeChatModelConnectivityVerifierTest {
Flux.just(toolResponse(TEST_NONCE, null)));
ChatModelVerificationResult result = verifier(factory).verify(
model(false, "self-hosted"));
model(false, Map.of(), "self-hosted"));
Assert.assertEquals(ModelVerificationStatus.PASSED, result.getStatus());
Map<String, Object> bodyParams = factory.getFactoryAdditionalBodyParams().get(0);
@@ -173,6 +173,33 @@ public class AgentScopeChatModelConnectivityVerifierTest {
Assert.assertEquals((byte) 0x50, imageBytes[1]);
}
/**
* 验证严格内容块模式会先发送包含图片首轮和文本追问的完整上下文。
*/
@Test
public void shouldVerifyStrictMessageContentFormatWithVlmHistory() {
RecordingModelFactory factory = new RecordingModelFactory(
Flux.just(textResponse("你好")),
Flux.just(toolResponse(TEST_NONCE, VlmVerificationImage.VERIFICATION_CODE)));
ChatModelVerificationResult result = verifier(factory).verify(model(
true,
Map.of("agentMessageContentFormat", "TEXT_PARTS")));
Assert.assertEquals(ModelVerificationStatus.PASSED, result.getStatus());
Assert.assertEquals(List.of(0, 1), factory.getToolCounts());
List<Msg> history = factory.getMessageBatches().get(0);
Assert.assertEquals(List.of(
MsgRole.SYSTEM,
MsgRole.USER,
MsgRole.ASSISTANT,
MsgRole.USER),
history.stream().map(Msg::getRole).toList());
Assert.assertTrue(history.get(1).getContent().stream().anyMatch(ImageBlock.class::isInstance));
Assert.assertEquals("请直接回复“你好”，不要补充其他内容。",
history.get(3).getTextContent());
}
/**
* 创建待验证模型。
*
@@ -180,17 +207,31 @@ public class AgentScopeChatModelConnectivityVerifierTest {
* @return 测试模型
*/
private Model model(boolean supportImage) {
return model(supportImage, "gpustack");
return model(supportImage, Map.of());
}
/**
* 创建携带指定 options 的待验证模型。
*
* @param supportImage 是否支持图片
* @param options 模型扩展配置
* @return 测试模型
*/
private Model model(boolean supportImage, Map<String, Object> options) {
return model(supportImage, options, "gpustack");
}
/**
* 创建携带指定供应商类型的待验证模型。
*
* @param supportImage 是否支持图片
* @param options 模型扩展配置
* @param providerType 供应商类型
* @return 测试模型
*/
private Model model(boolean supportImage, String providerType) {
private Model model(boolean supportImage,
Map<String, Object> options,
String providerType) {
Model model = new Model();
model.setId(BigInteger.TEN);
model.setModelName("test-model");
@@ -198,6 +239,7 @@ public class AgentScopeChatModelConnectivityVerifierTest {
model.setRequestPath("/v1/chat/completions");
model.setApiKey("test-key");
model.setSupportImage(supportImage);
model.setOptions(options);
ModelProvider provider = new ModelProvider();
provider.setProviderType(providerType);
model.setModelProvider(provider);