fix: 修复推理模型连接验证截断
- 响应达到输出上限时使用 512 Token 单次重试 - 为自部署 vLLM/SGLang 传递关闭思考参数 - 补充截断重试与请求参数回归测试
This commit is contained in:
@@ -51,8 +51,10 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
|
|||||||
private static final String PROBE_NONCE_FIELD = "nonce";
|
private static final String PROBE_NONCE_FIELD = "nonce";
|
||||||
private static final String PROBE_IMAGE_FIELD = "imageCode";
|
private static final String PROBE_IMAGE_FIELD = "imageCode";
|
||||||
private static final String VERIFICATION_SYSTEM_PROMPT = "You are a model connectivity verification assistant.";
|
private static final String VERIFICATION_SYSTEM_PROMPT = "You are a model connectivity verification assistant.";
|
||||||
/** 为未识别关闭思考扩展参数的推理模型保留足够的小额输出预算。 */
|
/** 首次能力探测使用的低成本输出预算。 */
|
||||||
private static final int MAX_TOKENS = 128;
|
private static final int MAX_TOKENS = 128;
|
||||||
|
/** 首次响应被输出上限截断时使用的单次重试预算。 */
|
||||||
|
private static final int TRUNCATION_RETRY_MAX_TOKENS = 512;
|
||||||
private static final Duration DEFAULT_PHASE_TIMEOUT = Duration.ofSeconds(60);
|
private static final Duration DEFAULT_PHASE_TIMEOUT = Duration.ofSeconds(60);
|
||||||
|
|
||||||
/** AgentScope 模型工厂。 */
|
/** AgentScope 模型工厂。 */
|
||||||
@@ -158,11 +160,23 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
|
|||||||
*/
|
*/
|
||||||
private boolean verifyProbeRequest(AgentModelSpec modelSpec, String nonce) {
|
private boolean verifyProbeRequest(AgentModelSpec modelSpec, String nonce) {
|
||||||
AgentMessage message = buildProbeMessage(modelSpec.isSupportImage(), nonce);
|
AgentMessage message = buildProbeMessage(modelSpec.isSupportImage(), nonce);
|
||||||
|
List<ToolSchema> tools = List.of(buildProbeToolSchema(modelSpec.isSupportImage()));
|
||||||
|
ToolChoice toolChoice = new ToolChoice.Specific(PROBE_TOOL_NAME);
|
||||||
List<ChatResponse> responses = request(
|
List<ChatResponse> responses = request(
|
||||||
modelSpec,
|
modelSpec,
|
||||||
message,
|
message,
|
||||||
List.of(buildProbeToolSchema(modelSpec.isSupportImage())),
|
tools,
|
||||||
new ToolChoice.Specific(PROBE_TOOL_NAME));
|
toolChoice);
|
||||||
|
if (hasLengthLimitedResponse(responses)) {
|
||||||
|
LOG.info("Model probe response reached token limit, retry with larger budget, modelName={}, maxTokens={}",
|
||||||
|
modelSpec.getModelName(), TRUNCATION_RETRY_MAX_TOKENS);
|
||||||
|
responses = request(
|
||||||
|
modelSpec,
|
||||||
|
message,
|
||||||
|
tools,
|
||||||
|
toolChoice,
|
||||||
|
TRUNCATION_RETRY_MAX_TOKENS);
|
||||||
|
}
|
||||||
boolean validToolCall = hasValidProbeToolCall(
|
boolean validToolCall = hasValidProbeToolCall(
|
||||||
responses, nonce, modelSpec.isSupportImage());
|
responses, nonce, modelSpec.isSupportImage());
|
||||||
validateProbeResponse(modelSpec.isSupportImage(), responses, validToolCall);
|
validateProbeResponse(modelSpec.isSupportImage(), responses, validToolCall);
|
||||||
@@ -197,11 +211,29 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
|
|||||||
AgentMessage verificationMessage,
|
AgentMessage verificationMessage,
|
||||||
List<ToolSchema> tools,
|
List<ToolSchema> tools,
|
||||||
ToolChoice toolChoice) {
|
ToolChoice toolChoice) {
|
||||||
|
return request(modelSpec, verificationMessage, tools, toolChoice, MAX_TOKENS);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 使用指定输出预算执行一次非流式模型请求。
|
||||||
|
*
|
||||||
|
* @param modelSpec 运行时模型声明
|
||||||
|
* @param verificationMessage 验证消息
|
||||||
|
* @param tools 工具 Schema
|
||||||
|
* @param toolChoice 工具选择策略
|
||||||
|
* @param maxTokens 最大输出 Token 数
|
||||||
|
* @return 模型响应片段
|
||||||
|
*/
|
||||||
|
private List<ChatResponse> request(AgentModelSpec modelSpec,
|
||||||
|
AgentMessage verificationMessage,
|
||||||
|
List<ToolSchema> tools,
|
||||||
|
ToolChoice toolChoice,
|
||||||
|
int maxTokens) {
|
||||||
AgentGenerationOptions generationOptions = new AgentGenerationOptions();
|
AgentGenerationOptions generationOptions = new AgentGenerationOptions();
|
||||||
generationOptions.setStream(false);
|
generationOptions.setStream(false);
|
||||||
generationOptions.setThinkingEnabled(false);
|
generationOptions.setThinkingEnabled(false);
|
||||||
disableOpenAiCompatibleThinking(modelSpec, generationOptions);
|
disableOpenAiCompatibleThinking(modelSpec, generationOptions);
|
||||||
generationOptions.setMaxTokens(MAX_TOKENS);
|
generationOptions.setMaxTokens(maxTokens);
|
||||||
io.agentscope.core.model.Model agentScopeModel = modelFactory.create(modelSpec, generationOptions);
|
io.agentscope.core.model.Model agentScopeModel = modelFactory.create(modelSpec, generationOptions);
|
||||||
|
|
||||||
List<Msg> messages = List.of(
|
List<Msg> messages = List.of(
|
||||||
@@ -210,7 +242,7 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
|
|||||||
messageAdapter.toMsg(verificationMessage));
|
messageAdapter.toMsg(verificationMessage));
|
||||||
GenerateOptions.Builder requestBuilder = GenerateOptions.builder()
|
GenerateOptions.Builder requestBuilder = GenerateOptions.builder()
|
||||||
.stream(false)
|
.stream(false)
|
||||||
.maxTokens(MAX_TOKENS)
|
.maxTokens(maxTokens)
|
||||||
.executionConfig(ExecutionConfig.builder()
|
.executionConfig(ExecutionConfig.builder()
|
||||||
.timeout(phaseTimeout)
|
.timeout(phaseTimeout)
|
||||||
.maxAttempts(1)
|
.maxAttempts(1)
|
||||||
@@ -245,15 +277,34 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
|
|||||||
generationOptions.getAdditionalBodyParams().put("enable_thinking", false);
|
generationOptions.getAdditionalBodyParams().put("enable_thinking", false);
|
||||||
}
|
}
|
||||||
Object sourceProviderType = modelSpec.getMetadata().get("sourceProviderType");
|
Object sourceProviderType = modelSpec.getMetadata().get("sourceProviderType");
|
||||||
if (sourceProviderType != null
|
if (usesChatTemplateThinkingControl(sourceProviderType)) {
|
||||||
&& "gpustack".equalsIgnoreCase(String.valueOf(sourceProviderType))) {
|
// vLLM 与 SGLang 通过聊天模板参数控制 Qwen 思考模式。
|
||||||
// GPUStack 托管的 vLLM 模型通过 chat_template_kwargs 控制 Qwen 思考模式。
|
|
||||||
generationOptions.getAdditionalBodyParams().put(
|
generationOptions.getAdditionalBodyParams().put(
|
||||||
"chat_template_kwargs",
|
"chat_template_kwargs",
|
||||||
Map.of("enable_thinking", false));
|
Map.of("enable_thinking", false));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 判断供应商是否通过聊天模板参数控制思考模式。
|
||||||
|
*
|
||||||
|
* @param sourceProviderType 模型配置中的原始供应商类型
|
||||||
|
* @return GPUStack 或自部署 vLLM/SGLang 入口返回 true
|
||||||
|
*/
|
||||||
|
private boolean usesChatTemplateThinkingControl(Object sourceProviderType) {
|
||||||
|
if (sourceProviderType == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
String normalizedProviderType = String.valueOf(sourceProviderType)
|
||||||
|
.trim()
|
||||||
|
.toLowerCase(Locale.ROOT)
|
||||||
|
.replace('_', '-');
|
||||||
|
return "gpustack".equals(normalizedProviderType)
|
||||||
|
|| "self-hosted".equals(normalizedProviderType)
|
||||||
|
|| "vllm".equals(normalizedProviderType)
|
||||||
|
|| "sglang".equals(normalizedProviderType);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 创建同时验证连接、视觉和工具调用的消息。
|
* 创建同时验证连接、视觉和工具调用的消息。
|
||||||
*
|
*
|
||||||
@@ -371,6 +422,24 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 判断响应是否因为达到输出 Token 上限而结束。
|
||||||
|
*
|
||||||
|
* @param responses AgentScope 响应片段
|
||||||
|
* @return 任一响应的结束原因为 length 时返回 true
|
||||||
|
*/
|
||||||
|
private boolean hasLengthLimitedResponse(List<ChatResponse> responses) {
|
||||||
|
if (responses == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
for (ChatResponse response : responses) {
|
||||||
|
if (response != null && "length".equalsIgnoreCase(response.getFinishReason())) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 校验探测响应是否足以证明连接和视觉能力。
|
* 校验探测响应是否足以证明连接和视觉能力。
|
||||||
*
|
*
|
||||||
|
|||||||
@@ -61,6 +61,43 @@ public class AgentScopeChatModelConnectivityVerifierTest {
|
|||||||
Assert.assertEquals(MsgRole.USER, factory.getMessageBatches().get(0).get(1).getRole());
|
Assert.assertEquals(MsgRole.USER, factory.getMessageBatches().get(0).get(1).getRole());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证能力探测达到输出上限时使用更大预算重试一次。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void shouldRetryProbeWithLargerBudgetWhenResponseIsLengthLimited() {
|
||||||
|
RecordingModelFactory factory = new RecordingModelFactory(
|
||||||
|
Flux.just(lengthLimitedResponse()),
|
||||||
|
Flux.just(toolResponse(TEST_NONCE, null)));
|
||||||
|
|
||||||
|
ChatModelVerificationResult result = verifier(factory).verify(model(false));
|
||||||
|
|
||||||
|
Assert.assertEquals(ModelVerificationStatus.PASSED, result.getStatus());
|
||||||
|
Assert.assertEquals(Boolean.TRUE, result.getSupportTool());
|
||||||
|
Assert.assertEquals(List.of(1, 1), factory.getToolCounts());
|
||||||
|
Assert.assertEquals(List.of(128, 512), factory.getFactoryMaxTokens());
|
||||||
|
Assert.assertEquals(List.of(128, 512), factory.getRequestMaxTokens());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证自部署 vLLM/SGLang 入口通过聊天模板参数关闭思考。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void shouldDisableThinkingThroughChatTemplateKwargsForSelfHostedEndpoint() {
|
||||||
|
RecordingModelFactory factory = new RecordingModelFactory(
|
||||||
|
Flux.just(toolResponse(TEST_NONCE, null)));
|
||||||
|
|
||||||
|
ChatModelVerificationResult result = verifier(factory).verify(
|
||||||
|
model(false, "self-hosted"));
|
||||||
|
|
||||||
|
Assert.assertEquals(ModelVerificationStatus.PASSED, result.getStatus());
|
||||||
|
Map<String, Object> bodyParams = factory.getFactoryAdditionalBodyParams().get(0);
|
||||||
|
Assert.assertEquals(Boolean.FALSE, bodyParams.get("enable_thinking"));
|
||||||
|
Assert.assertEquals(
|
||||||
|
Map.of("enable_thinking", false),
|
||||||
|
bodyParams.get("chat_template_kwargs"));
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 验证模型返回普通文本时连接通过但工具能力保持关闭。
|
* 验证模型返回普通文本时连接通过但工具能力保持关闭。
|
||||||
*/
|
*/
|
||||||
@@ -143,6 +180,17 @@ public class AgentScopeChatModelConnectivityVerifierTest {
|
|||||||
* @return 测试模型
|
* @return 测试模型
|
||||||
*/
|
*/
|
||||||
private Model model(boolean supportImage) {
|
private Model model(boolean supportImage) {
|
||||||
|
return model(supportImage, "gpustack");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 创建携带指定供应商类型的待验证模型。
|
||||||
|
*
|
||||||
|
* @param supportImage 是否支持图片
|
||||||
|
* @param providerType 供应商类型
|
||||||
|
* @return 测试模型
|
||||||
|
*/
|
||||||
|
private Model model(boolean supportImage, String providerType) {
|
||||||
Model model = new Model();
|
Model model = new Model();
|
||||||
model.setId(BigInteger.TEN);
|
model.setId(BigInteger.TEN);
|
||||||
model.setModelName("test-model");
|
model.setModelName("test-model");
|
||||||
@@ -151,7 +199,7 @@ public class AgentScopeChatModelConnectivityVerifierTest {
|
|||||||
model.setApiKey("test-key");
|
model.setApiKey("test-key");
|
||||||
model.setSupportImage(supportImage);
|
model.setSupportImage(supportImage);
|
||||||
ModelProvider provider = new ModelProvider();
|
ModelProvider provider = new ModelProvider();
|
||||||
provider.setProviderType("gpustack");
|
provider.setProviderType(providerType);
|
||||||
model.setModelProvider(provider);
|
model.setModelProvider(provider);
|
||||||
return model;
|
return model;
|
||||||
}
|
}
|
||||||
@@ -182,6 +230,18 @@ public class AgentScopeChatModelConnectivityVerifierTest {
|
|||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 创建达到输出上限的响应。
|
||||||
|
*
|
||||||
|
* @return 输出被截断的 AgentScope 响应
|
||||||
|
*/
|
||||||
|
private ChatResponse lengthLimitedResponse() {
|
||||||
|
return ChatResponse.builder()
|
||||||
|
.content(List.of(TextBlock.builder().text("incomplete").build()))
|
||||||
|
.finishReason("length")
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 创建探测工具调用响应。
|
* 创建探测工具调用响应。
|
||||||
*
|
*
|
||||||
@@ -212,8 +272,14 @@ public class AgentScopeChatModelConnectivityVerifierTest {
|
|||||||
private final List<Flux<ChatResponse>> responses;
|
private final List<Flux<ChatResponse>> responses;
|
||||||
/** 模型工厂收到的流式参数。 */
|
/** 模型工厂收到的流式参数。 */
|
||||||
private final List<Boolean> factoryStreams = new ArrayList<>();
|
private final List<Boolean> factoryStreams = new ArrayList<>();
|
||||||
|
/** 模型工厂收到的最大输出 Token 数。 */
|
||||||
|
private final List<Integer> factoryMaxTokens = new ArrayList<>();
|
||||||
|
/** 模型工厂收到的额外请求体参数。 */
|
||||||
|
private final List<Map<String, Object>> factoryAdditionalBodyParams = new ArrayList<>();
|
||||||
/** 模型请求收到的流式参数。 */
|
/** 模型请求收到的流式参数。 */
|
||||||
private final List<Boolean> requestStreams = new ArrayList<>();
|
private final List<Boolean> requestStreams = new ArrayList<>();
|
||||||
|
/** 模型请求收到的最大输出 Token 数。 */
|
||||||
|
private final List<Integer> requestMaxTokens = new ArrayList<>();
|
||||||
/** 模型请求收到的消息批次。 */
|
/** 模型请求收到的消息批次。 */
|
||||||
private final List<List<Msg>> messageBatches = new ArrayList<>();
|
private final List<List<Msg>> messageBatches = new ArrayList<>();
|
||||||
/** 每次请求携带的工具数量。 */
|
/** 每次请求携带的工具数量。 */
|
||||||
@@ -244,6 +310,9 @@ public class AgentScopeChatModelConnectivityVerifierTest {
|
|||||||
AgentGenerationOptions generationOptions) {
|
AgentGenerationOptions generationOptions) {
|
||||||
int requestIndex = factoryStreams.size();
|
int requestIndex = factoryStreams.size();
|
||||||
factoryStreams.add(Boolean.TRUE.equals(generationOptions.getStream()));
|
factoryStreams.add(Boolean.TRUE.equals(generationOptions.getStream()));
|
||||||
|
factoryMaxTokens.add(generationOptions.getMaxTokens());
|
||||||
|
factoryAdditionalBodyParams.add(Map.copyOf(
|
||||||
|
generationOptions.getAdditionalBodyParams()));
|
||||||
Flux<ChatResponse> response = responses.get(requestIndex);
|
Flux<ChatResponse> response = responses.get(requestIndex);
|
||||||
return new io.agentscope.core.model.Model() {
|
return new io.agentscope.core.model.Model() {
|
||||||
/**
|
/**
|
||||||
@@ -260,6 +329,7 @@ public class AgentScopeChatModelConnectivityVerifierTest {
|
|||||||
List<ToolSchema> tools,
|
List<ToolSchema> tools,
|
||||||
GenerateOptions options) {
|
GenerateOptions options) {
|
||||||
requestStreams.add(Boolean.TRUE.equals(options.getStream()));
|
requestStreams.add(Boolean.TRUE.equals(options.getStream()));
|
||||||
|
requestMaxTokens.add(options.getMaxTokens());
|
||||||
messageBatches.add(List.copyOf(inputMessages));
|
messageBatches.add(List.copyOf(inputMessages));
|
||||||
toolCounts.add(tools.size());
|
toolCounts.add(tools.size());
|
||||||
toolChoices.add(options.getToolChoice());
|
toolChoices.add(options.getToolChoice());
|
||||||
@@ -287,6 +357,24 @@ public class AgentScopeChatModelConnectivityVerifierTest {
|
|||||||
return factoryStreams;
|
return factoryStreams;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取模型工厂收到的最大输出 Token 数。
|
||||||
|
*
|
||||||
|
* @return 最大输出 Token 数列表
|
||||||
|
*/
|
||||||
|
private List<Integer> getFactoryMaxTokens() {
|
||||||
|
return factoryMaxTokens;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取模型工厂收到的额外请求体参数。
|
||||||
|
*
|
||||||
|
* @return 额外请求体参数列表
|
||||||
|
*/
|
||||||
|
private List<Map<String, Object>> getFactoryAdditionalBodyParams() {
|
||||||
|
return factoryAdditionalBodyParams;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 获取请求流式参数。
|
* 获取请求流式参数。
|
||||||
*
|
*
|
||||||
@@ -296,6 +384,15 @@ public class AgentScopeChatModelConnectivityVerifierTest {
|
|||||||
return requestStreams;
|
return requestStreams;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取模型请求收到的最大输出 Token 数。
|
||||||
|
*
|
||||||
|
* @return 最大输出 Token 数列表
|
||||||
|
*/
|
||||||
|
private List<Integer> getRequestMaxTokens() {
|
||||||
|
return requestMaxTokens;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 获取每次请求的验证消息批次。
|
* 获取每次请求的验证消息批次。
|
||||||
*
|
*
|
||||||
|
|||||||
Reference in New Issue
Block a user