EasyFlow-Tech-Dev/EasyFlow
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 完善模型能力识别与验证

Browse Source 
- 自动识别模型类型、视觉、推理和工具能力并保留手动覆盖

- 使用 AgentScope 工具与视觉探测并统一管理端配置反馈
This commit is contained in:
陈子默
2026-07-27 19:40:23 +08:00
parent 0dc5c3ca55
commit 567fd12706
21 changed files with 1059 additions and 378 deletions
Download Patch File Download Diff File Expand all files Collapse all files

233
easyflow-modules/easyflow-module-agent/src/test/java/tech/easyflow/agent/runtime/AgentScopeChatModelConnectivityVerifierTest.java
View File

@@ -9,8 +9,10 @@ import io.agentscope.core.message.ImageBlock;
import io.agentscope.core.message.Msg;
import io.agentscope.core.message.MsgRole;
import io.agentscope.core.message.TextBlock;
import io.agentscope.core.message.ToolUseBlock;
import io.agentscope.core.model.ChatResponse;
import io.agentscope.core.model.GenerateOptions;
import io.agentscope.core.model.ToolChoice;
import io.agentscope.core.model.ToolSchema;
import org.junit.Assert;
import org.junit.Test;
@@ -27,101 +29,102 @@ import java.time.Duration;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
import java.util.Map;
/**
* AgentScope 双阶段模型连通性验证测试。
* AgentScope 单次优先模型连接与工具能力验证测试。
*/
public class AgentScopeChatModelConnectivityVerifierTest {
/**
* 验证非流式与流式阶段按顺序执行并返回通过状态。
*/
@Test
public void shouldPassWhenBothPhasesReturnText() {
RecordingModelFactory factory = new RecordingModelFactory(
Flux.just(response("你好")),
Flux.just(response("你"), response("好")));
AgentScopeChatModelConnectivityVerifier verifier = verifier(factory);
ChatModelVerificationResult result = verifier.verify(model(false));
Assert.assertEquals(ModelVerificationStatus.PASSED, result.getStatus());
Assert.assertEquals("HTTP_1_1", result.getEffectiveHttpVersion());
Assert.assertEquals(List.of(false, true), factory.getFactoryStreams());
Assert.assertEquals(List.of(false, true), factory.getRequestStreams());
Assert.assertEquals(List.of(MsgRole.SYSTEM, MsgRole.USER), factory.getMessageRoles().get(0));
Assert.assertEquals(List.of(false, false), factory.getEnableThinkingValues());
Assert.assertEquals(List.of(false, false), factory.getChatTemplateThinkingValues());
}
private static final String TEST_NONCE = "probe-nonce";
/**
* 验证基础连接通过而流式阶段失败时返回部分通过结果。
* 验证一次请求正确返回工具调用时同时确认连接和工具能力。
*/
@Test
public void shouldReturnPartialWhenStreamingPhaseFails() {
public void shouldPassConnectionAndToolProbeInOneRequest() {
RecordingModelFactory factory = new RecordingModelFactory(
Flux.just(response("你好")),
Flux.error(new IllegalStateException("stream failed")));
Flux.just(toolResponse(TEST_NONCE, null)));
ChatModelVerificationResult result = verifier(factory).verify(model(false));
Assert.assertEquals(ModelVerificationStatus.PARTIAL, result.getStatus());
Assert.assertEquals(ModelVerificationStatus.PASSED, result.getStatus());
Assert.assertEquals(ModelVerificationStatus.PASSED, result.getNonStreaming());
Assert.assertEquals(ModelVerificationStatus.FAILED, result.getStreaming());
Assert.assertTrue(result.getMessage().contains("流式响应不可用"));
Assert.assertEquals(ModelVerificationStatus.SKIPPED, result.getStreaming());
Assert.assertEquals(Boolean.TRUE, result.getSupportTool());
Assert.assertEquals("验证通过", result.getMessage());
Assert.assertEquals(List.of(false), factory.getFactoryStreams());
Assert.assertEquals(List.of(false), factory.getRequestStreams());
Assert.assertEquals(List.of(1), factory.getToolCounts());
Assert.assertTrue(factory.getToolChoices().get(0) instanceof ToolChoice.Specific);
Assert.assertEquals(MsgRole.SYSTEM, factory.getMessageBatches().get(0).get(0).getRole());
Assert.assertEquals(MsgRole.USER, factory.getMessageBatches().get(0).get(1).getRole());
}
/**
* 验证基础连接失败时立即终止且返回业务失败。
* 验证模型返回普通文本时连接通过但工具能力保持关闭。
*/
@Test
public void shouldStopWhenNonStreamingPhaseFails() {
public void shouldPassConnectionAndMarkToolUnsupportedWhenTextReturned() {
RecordingModelFactory factory = new RecordingModelFactory(
Flux.just(textResponse("你好")));
ChatModelVerificationResult result = verifier(factory).verify(model(false));
Assert.assertEquals(ModelVerificationStatus.PASSED, result.getStatus());
Assert.assertEquals(Boolean.FALSE, result.getSupportTool());
Assert.assertEquals(List.of(1), factory.getToolCounts());
}
/**
* 验证接口明确拒绝工具参数时仅追加一次普通连接兜底。
*/
@Test
public void shouldFallbackToPlainRequestWhenToolChoiceIsRejected() {
RecordingModelFactory factory = new RecordingModelFactory(
Flux.error(new IllegalArgumentException("tool_choice is unsupported")),
Flux.just(textResponse("你好")));
ChatModelVerificationResult result = verifier(factory).verify(model(false));
Assert.assertEquals(ModelVerificationStatus.PASSED, result.getStatus());
Assert.assertEquals(Boolean.FALSE, result.getSupportTool());
Assert.assertEquals(List.of(1, 0), factory.getToolCounts());
Assert.assertTrue(factory.getToolChoices().get(0) instanceof ToolChoice.Specific);
Assert.assertNull(factory.getToolChoices().get(1));
}
/**
* 验证普通连接异常不会被工具兼容兜底掩盖。
*/
@Test
public void shouldFailWhenConnectivityRequestFails() {
RecordingModelFactory factory = new RecordingModelFactory(
Flux.error(new IllegalStateException("connection failed")));
try {
verifier(factory).verify(model(false));
Assert.fail("Expected base connectivity verification failure");
Assert.fail("Expected connectivity verification failure");
} catch (BusinessException exception) {
Assert.assertTrue(exception.getMessage().contains("基础连接验证失败"));
Assert.assertTrue(exception.getMessage().contains("连接验证失败"));
Assert.assertFalse(exception.getMessage().contains("connection failed"));
}
Assert.assertEquals(List.of(false), factory.getFactoryStreams());
Assert.assertEquals(List.of(1), factory.getToolCounts());
}
/**
* 验证流式阶段超时被归类为部分可用且不暴露底层异常。
* 验证 VLM 在同一次工具调用中返回图片验证码。
*/
@Test
public void shouldReturnPartialWhenStreamingPhaseTimesOut() {
public void shouldVerifyVlmAndToolCallInOneRequest() {
RecordingModelFactory factory = new RecordingModelFactory(
Flux.just(response("你好")),
Flux.never());
AgentScopeChatModelConnectivityVerifier verifier = new AgentScopeChatModelConnectivityVerifier(
factory,
new AgentScopeMessageAdapter(),
Duration.ofMillis(20));
ChatModelVerificationResult result = verifier.verify(model(false));
Assert.assertEquals(ModelVerificationStatus.PARTIAL, result.getStatus());
Assert.assertEquals("连接成功，流式响应不可用，可关闭智能体的模型流式响应。",
result.getMessage());
}
/**
* 验证 VLM 使用 Base64 图片，并能聚合流式文本增量。
*/
@Test
public void shouldVerifyVlmImageAndAggregateStreamingChunks() {
RecordingModelFactory factory = new RecordingModelFactory(
Flux.just(response("图片中的内容是“" + VlmVerificationImage.VERIFICATION_CODE + "”。")),
Flux.just(response("识别结果：58"), response("39")));
Flux.just(toolResponse(TEST_NONCE, VlmVerificationImage.VERIFICATION_CODE)));
ChatModelVerificationResult result = verifier(factory).verify(model(true));
Assert.assertEquals(ModelVerificationStatus.PASSED, result.getStatus());
Msg message = factory.getMessages().get(0);
Assert.assertEquals(Boolean.TRUE, result.getSupportTool());
Msg message = factory.getMessageBatches().get(0).get(1);
ImageBlock image = message.getContent().stream()
.filter(ImageBlock.class::isInstance)
.map(ImageBlock.class::cast)
@@ -154,7 +157,7 @@ public class AgentScopeChatModelConnectivityVerifierTest {
}
/**
* 创建使用测试工厂的验证器。
* 创建使用固定随机值的验证器。
*
* @param factory 记录型模型工厂
* @return 验证器
@@ -163,54 +166,73 @@ public class AgentScopeChatModelConnectivityVerifierTest {
return new AgentScopeChatModelConnectivityVerifier(
factory,
new AgentScopeMessageAdapter(),
Duration.ofSeconds(2));
Duration.ofSeconds(2),
() -> TEST_NONCE);
}
/**
* 创建单个文本响应片段。
* 创建文本响应。
*
* @param text 文本内容
* @return AgentScope 响应
*/
private ChatResponse response(String text) {
private ChatResponse textResponse(String text) {
return ChatResponse.builder()
.content(List.of(TextBlock.builder().text(text).build()))
.build();
}
/**
* 按阶段返回预设响应并记录调用参数的模型工厂。
* 创建探测工具调用响应。
*
* @param nonce 随机校验值
* @param imageCode 图片验证码
* @return AgentScope 响应
*/
private ChatResponse toolResponse(String nonce, String imageCode) {
Map<String, Object> input = imageCode == null
? Map.of("nonce", nonce)
: Map.of("nonce", nonce, "imageCode", imageCode);
return ChatResponse.builder()
.content(List.of(ToolUseBlock.builder()
.id("call-probe")
.name("easyflow_capability_probe")
.input(input)
.build()))
.build();
}
/**
* 按调用顺序返回预设响应并记录真实请求参数的模型工厂。
*/
private static final class RecordingModelFactory
implements AgentModelFactory<io.agentscope.core.model.Model> {
/** 每个阶段的预设响应。 */
private final List<Flux<ChatResponse>> phaseResponses;
/** 每次调用的预设响应。 */
private final List<Flux<ChatResponse>> responses;
/** 模型工厂收到的流式参数。 */
private final List<Boolean> factoryStreams = new ArrayList<>();
/** 模型请求收到的流式参数。 */
private final List<Boolean> requestStreams = new ArrayList<>();
/** 模型请求收到的消息。 */
private final List<Msg> messages = new ArrayList<>();
/** 各阶段请求消息的角色顺序。 */
private final List<List<MsgRole>> messageRoles = new ArrayList<>();
/** OpenAI-compatible 请求中的思考开关。 */
private final List<Object> enableThinkingValues = new ArrayList<>();
/** GPUStack 模板参数中的思考开关。 */
private final List<Object> chatTemplateThinkingValues = new ArrayList<>();
/** 模型请求收到的消息批次。 */
private final List<List<Msg>> messageBatches = new ArrayList<>();
/** 每次请求携带的工具数量。 */
private final List<Integer> toolCounts = new ArrayList<>();
/** 每次请求使用的工具选择策略。 */
private final List<ToolChoice> toolChoices = new ArrayList<>();
/**
* 创建记录型模型工厂。
*
* @param phaseResponses 每个阶段的预设响应
* @param responses 每次调用的预设响应
*/
@SafeVarargs
private RecordingModelFactory(Flux<ChatResponse>... phaseResponses) {
this.phaseResponses = List.of(phaseResponses);
private RecordingModelFactory(Flux<ChatResponse>... responses) {
this.responses = List.of(responses);
}
/**
* 创建当前验证阶段的测试模型。
* 创建当前验证请求使用的模型。
*
* @param modelSpec 模型声明
* @param generationOptions 生成参数
@@ -220,19 +242,12 @@ public class AgentScopeChatModelConnectivityVerifierTest {
public io.agentscope.core.model.Model create(
AgentModelSpec modelSpec,
AgentGenerationOptions generationOptions) {
int phaseIndex = factoryStreams.size();
int requestIndex = factoryStreams.size();
factoryStreams.add(Boolean.TRUE.equals(generationOptions.getStream()));
enableThinkingValues.add(
generationOptions.getAdditionalBodyParams().get("enable_thinking"));
Object templateOptions = generationOptions.getAdditionalBodyParams()
.get("chat_template_kwargs");
chatTemplateThinkingValues.add(templateOptions instanceof java.util.Map<?, ?> map
? map.get("enable_thinking")
: null);
Flux<ChatResponse> responses = phaseResponses.get(phaseIndex);
Flux<ChatResponse> response = responses.get(requestIndex);
return new io.agentscope.core.model.Model() {
/**
* 返回预设响应并记录真实请求参数。
* 返回预设响应并记录请求参数。
*
* @param inputMessages 模型消息
* @param tools 工具声明
@@ -245,9 +260,10 @@ public class AgentScopeChatModelConnectivityVerifierTest {
List<ToolSchema> tools,
GenerateOptions options) {
requestStreams.add(Boolean.TRUE.equals(options.getStream()));
messages.add(inputMessages.get(inputMessages.size() - 1));
messageRoles.add(inputMessages.stream().map(Msg::getRole).toList());
return responses;
messageBatches.add(List.copyOf(inputMessages));
toolCounts.add(tools.size());
toolChoices.add(options.getToolChoice());
return response;
}
/**
@@ -263,7 +279,7 @@ public class AgentScopeChatModelConnectivityVerifierTest {
}
/**
* 获取工厂流式参数记录。
* 获取模型工厂流式参数。
*
* @return 流式参数列表
*/
@@ -272,7 +288,7 @@ public class AgentScopeChatModelConnectivityVerifierTest {
}
/**
* 获取请求流式参数记录。
* 获取请求流式参数。
*
* @return 流式参数列表
*/
@@ -281,39 +297,30 @@ public class AgentScopeChatModelConnectivityVerifierTest {
}
/**
* 获取请求消息记录。
* 获取每次请求的验证消息批次。
*
* @return 消息列表
* @return 验证消息批次
*/
private List<Msg> getMessages() {
return messages;
private List<List<Msg>> getMessageBatches() {
return messageBatches;
}
/**
* 获取各阶段请求消息的角色顺序。
* 获取工具数量。
*
* @return 消息角色顺序
* @return 工具数量列表
*/
private List<List<MsgRole>> getMessageRoles() {
return messageRoles;
private List<Integer> getToolCounts() {
return toolCounts;
}
/**
* 获取 OpenAI-compatible 请求中的思考开关。
* 获取工具选择策略。
*
* @return 各阶段思考开关
* @return 工具选择策略列表
*/
private List<Object> getEnableThinkingValues() {
return enableThinkingValues;
}
/**
* 获取 GPUStack 模板参数中的思考开关。
*
* @return 各阶段模板思考开关
*/
private List<Object> getChatTemplateThinkingValues() {
return chatTemplateThinkingValues;
private List<ToolChoice> getToolChoices() {
return toolChoices;
}
}
}