fix: 修复推理模型连接验证截断
- 响应达到输出上限时使用 512 Token 单次重试 - 为自部署 vLLM/SGLang 传递关闭思考参数 - 补充截断重试与请求参数回归测试
This commit is contained in:
@@ -51,8 +51,10 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
|
||||
private static final String PROBE_NONCE_FIELD = "nonce";
|
||||
private static final String PROBE_IMAGE_FIELD = "imageCode";
|
||||
private static final String VERIFICATION_SYSTEM_PROMPT = "You are a model connectivity verification assistant.";
|
||||
/** 为未识别关闭思考扩展参数的推理模型保留足够的小额输出预算。 */
|
||||
/** 首次能力探测使用的低成本输出预算。 */
|
||||
private static final int MAX_TOKENS = 128;
|
||||
/** 首次响应被输出上限截断时使用的单次重试预算。 */
|
||||
private static final int TRUNCATION_RETRY_MAX_TOKENS = 512;
|
||||
private static final Duration DEFAULT_PHASE_TIMEOUT = Duration.ofSeconds(60);
|
||||
|
||||
/** AgentScope 模型工厂。 */
|
||||
@@ -158,11 +160,23 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
|
||||
*/
|
||||
private boolean verifyProbeRequest(AgentModelSpec modelSpec, String nonce) {
|
||||
AgentMessage message = buildProbeMessage(modelSpec.isSupportImage(), nonce);
|
||||
List<ToolSchema> tools = List.of(buildProbeToolSchema(modelSpec.isSupportImage()));
|
||||
ToolChoice toolChoice = new ToolChoice.Specific(PROBE_TOOL_NAME);
|
||||
List<ChatResponse> responses = request(
|
||||
modelSpec,
|
||||
message,
|
||||
List.of(buildProbeToolSchema(modelSpec.isSupportImage())),
|
||||
new ToolChoice.Specific(PROBE_TOOL_NAME));
|
||||
tools,
|
||||
toolChoice);
|
||||
if (hasLengthLimitedResponse(responses)) {
|
||||
LOG.info("Model probe response reached token limit, retry with larger budget, modelName={}, maxTokens={}",
|
||||
modelSpec.getModelName(), TRUNCATION_RETRY_MAX_TOKENS);
|
||||
responses = request(
|
||||
modelSpec,
|
||||
message,
|
||||
tools,
|
||||
toolChoice,
|
||||
TRUNCATION_RETRY_MAX_TOKENS);
|
||||
}
|
||||
boolean validToolCall = hasValidProbeToolCall(
|
||||
responses, nonce, modelSpec.isSupportImage());
|
||||
validateProbeResponse(modelSpec.isSupportImage(), responses, validToolCall);
|
||||
@@ -197,11 +211,29 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
|
||||
AgentMessage verificationMessage,
|
||||
List<ToolSchema> tools,
|
||||
ToolChoice toolChoice) {
|
||||
return request(modelSpec, verificationMessage, tools, toolChoice, MAX_TOKENS);
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用指定输出预算执行一次非流式模型请求。
|
||||
*
|
||||
* @param modelSpec 运行时模型声明
|
||||
* @param verificationMessage 验证消息
|
||||
* @param tools 工具 Schema
|
||||
* @param toolChoice 工具选择策略
|
||||
* @param maxTokens 最大输出 Token 数
|
||||
* @return 模型响应片段
|
||||
*/
|
||||
private List<ChatResponse> request(AgentModelSpec modelSpec,
|
||||
AgentMessage verificationMessage,
|
||||
List<ToolSchema> tools,
|
||||
ToolChoice toolChoice,
|
||||
int maxTokens) {
|
||||
AgentGenerationOptions generationOptions = new AgentGenerationOptions();
|
||||
generationOptions.setStream(false);
|
||||
generationOptions.setThinkingEnabled(false);
|
||||
disableOpenAiCompatibleThinking(modelSpec, generationOptions);
|
||||
generationOptions.setMaxTokens(MAX_TOKENS);
|
||||
generationOptions.setMaxTokens(maxTokens);
|
||||
io.agentscope.core.model.Model agentScopeModel = modelFactory.create(modelSpec, generationOptions);
|
||||
|
||||
List<Msg> messages = List.of(
|
||||
@@ -210,7 +242,7 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
|
||||
messageAdapter.toMsg(verificationMessage));
|
||||
GenerateOptions.Builder requestBuilder = GenerateOptions.builder()
|
||||
.stream(false)
|
||||
.maxTokens(MAX_TOKENS)
|
||||
.maxTokens(maxTokens)
|
||||
.executionConfig(ExecutionConfig.builder()
|
||||
.timeout(phaseTimeout)
|
||||
.maxAttempts(1)
|
||||
@@ -245,15 +277,34 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
|
||||
generationOptions.getAdditionalBodyParams().put("enable_thinking", false);
|
||||
}
|
||||
Object sourceProviderType = modelSpec.getMetadata().get("sourceProviderType");
|
||||
if (sourceProviderType != null
|
||||
&& "gpustack".equalsIgnoreCase(String.valueOf(sourceProviderType))) {
|
||||
// GPUStack 托管的 vLLM 模型通过 chat_template_kwargs 控制 Qwen 思考模式。
|
||||
if (usesChatTemplateThinkingControl(sourceProviderType)) {
|
||||
// vLLM 与 SGLang 通过聊天模板参数控制 Qwen 思考模式。
|
||||
generationOptions.getAdditionalBodyParams().put(
|
||||
"chat_template_kwargs",
|
||||
Map.of("enable_thinking", false));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断供应商是否通过聊天模板参数控制思考模式。
|
||||
*
|
||||
* @param sourceProviderType 模型配置中的原始供应商类型
|
||||
* @return GPUStack 或自部署 vLLM/SGLang 入口返回 true
|
||||
*/
|
||||
private boolean usesChatTemplateThinkingControl(Object sourceProviderType) {
|
||||
if (sourceProviderType == null) {
|
||||
return false;
|
||||
}
|
||||
String normalizedProviderType = String.valueOf(sourceProviderType)
|
||||
.trim()
|
||||
.toLowerCase(Locale.ROOT)
|
||||
.replace('_', '-');
|
||||
return "gpustack".equals(normalizedProviderType)
|
||||
|| "self-hosted".equals(normalizedProviderType)
|
||||
|| "vllm".equals(normalizedProviderType)
|
||||
|| "sglang".equals(normalizedProviderType);
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建同时验证连接、视觉和工具调用的消息。
|
||||
*
|
||||
@@ -371,6 +422,24 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断响应是否因为达到输出 Token 上限而结束。
|
||||
*
|
||||
* @param responses AgentScope 响应片段
|
||||
* @return 任一响应的结束原因为 length 时返回 true
|
||||
*/
|
||||
private boolean hasLengthLimitedResponse(List<ChatResponse> responses) {
|
||||
if (responses == null) {
|
||||
return false;
|
||||
}
|
||||
for (ChatResponse response : responses) {
|
||||
if (response != null && "length".equalsIgnoreCase(response.getFinishReason())) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验探测响应是否足以证明连接和视觉能力。
|
||||
*
|
||||
|
||||
@@ -61,6 +61,43 @@ public class AgentScopeChatModelConnectivityVerifierTest {
|
||||
Assert.assertEquals(MsgRole.USER, factory.getMessageBatches().get(0).get(1).getRole());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证能力探测达到输出上限时使用更大预算重试一次。
|
||||
*/
|
||||
@Test
|
||||
public void shouldRetryProbeWithLargerBudgetWhenResponseIsLengthLimited() {
|
||||
RecordingModelFactory factory = new RecordingModelFactory(
|
||||
Flux.just(lengthLimitedResponse()),
|
||||
Flux.just(toolResponse(TEST_NONCE, null)));
|
||||
|
||||
ChatModelVerificationResult result = verifier(factory).verify(model(false));
|
||||
|
||||
Assert.assertEquals(ModelVerificationStatus.PASSED, result.getStatus());
|
||||
Assert.assertEquals(Boolean.TRUE, result.getSupportTool());
|
||||
Assert.assertEquals(List.of(1, 1), factory.getToolCounts());
|
||||
Assert.assertEquals(List.of(128, 512), factory.getFactoryMaxTokens());
|
||||
Assert.assertEquals(List.of(128, 512), factory.getRequestMaxTokens());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证自部署 vLLM/SGLang 入口通过聊天模板参数关闭思考。
|
||||
*/
|
||||
@Test
|
||||
public void shouldDisableThinkingThroughChatTemplateKwargsForSelfHostedEndpoint() {
|
||||
RecordingModelFactory factory = new RecordingModelFactory(
|
||||
Flux.just(toolResponse(TEST_NONCE, null)));
|
||||
|
||||
ChatModelVerificationResult result = verifier(factory).verify(
|
||||
model(false, "self-hosted"));
|
||||
|
||||
Assert.assertEquals(ModelVerificationStatus.PASSED, result.getStatus());
|
||||
Map<String, Object> bodyParams = factory.getFactoryAdditionalBodyParams().get(0);
|
||||
Assert.assertEquals(Boolean.FALSE, bodyParams.get("enable_thinking"));
|
||||
Assert.assertEquals(
|
||||
Map.of("enable_thinking", false),
|
||||
bodyParams.get("chat_template_kwargs"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证模型返回普通文本时连接通过但工具能力保持关闭。
|
||||
*/
|
||||
@@ -143,6 +180,17 @@ public class AgentScopeChatModelConnectivityVerifierTest {
|
||||
* @return 测试模型
|
||||
*/
|
||||
private Model model(boolean supportImage) {
|
||||
return model(supportImage, "gpustack");
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建携带指定供应商类型的待验证模型。
|
||||
*
|
||||
* @param supportImage 是否支持图片
|
||||
* @param providerType 供应商类型
|
||||
* @return 测试模型
|
||||
*/
|
||||
private Model model(boolean supportImage, String providerType) {
|
||||
Model model = new Model();
|
||||
model.setId(BigInteger.TEN);
|
||||
model.setModelName("test-model");
|
||||
@@ -151,7 +199,7 @@ public class AgentScopeChatModelConnectivityVerifierTest {
|
||||
model.setApiKey("test-key");
|
||||
model.setSupportImage(supportImage);
|
||||
ModelProvider provider = new ModelProvider();
|
||||
provider.setProviderType("gpustack");
|
||||
provider.setProviderType(providerType);
|
||||
model.setModelProvider(provider);
|
||||
return model;
|
||||
}
|
||||
@@ -182,6 +230,18 @@ public class AgentScopeChatModelConnectivityVerifierTest {
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建达到输出上限的响应。
|
||||
*
|
||||
* @return 输出被截断的 AgentScope 响应
|
||||
*/
|
||||
private ChatResponse lengthLimitedResponse() {
|
||||
return ChatResponse.builder()
|
||||
.content(List.of(TextBlock.builder().text("incomplete").build()))
|
||||
.finishReason("length")
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建探测工具调用响应。
|
||||
*
|
||||
@@ -212,8 +272,14 @@ public class AgentScopeChatModelConnectivityVerifierTest {
|
||||
private final List<Flux<ChatResponse>> responses;
|
||||
/** 模型工厂收到的流式参数。 */
|
||||
private final List<Boolean> factoryStreams = new ArrayList<>();
|
||||
/** 模型工厂收到的最大输出 Token 数。 */
|
||||
private final List<Integer> factoryMaxTokens = new ArrayList<>();
|
||||
/** 模型工厂收到的额外请求体参数。 */
|
||||
private final List<Map<String, Object>> factoryAdditionalBodyParams = new ArrayList<>();
|
||||
/** 模型请求收到的流式参数。 */
|
||||
private final List<Boolean> requestStreams = new ArrayList<>();
|
||||
/** 模型请求收到的最大输出 Token 数。 */
|
||||
private final List<Integer> requestMaxTokens = new ArrayList<>();
|
||||
/** 模型请求收到的消息批次。 */
|
||||
private final List<List<Msg>> messageBatches = new ArrayList<>();
|
||||
/** 每次请求携带的工具数量。 */
|
||||
@@ -244,6 +310,9 @@ public class AgentScopeChatModelConnectivityVerifierTest {
|
||||
AgentGenerationOptions generationOptions) {
|
||||
int requestIndex = factoryStreams.size();
|
||||
factoryStreams.add(Boolean.TRUE.equals(generationOptions.getStream()));
|
||||
factoryMaxTokens.add(generationOptions.getMaxTokens());
|
||||
factoryAdditionalBodyParams.add(Map.copyOf(
|
||||
generationOptions.getAdditionalBodyParams()));
|
||||
Flux<ChatResponse> response = responses.get(requestIndex);
|
||||
return new io.agentscope.core.model.Model() {
|
||||
/**
|
||||
@@ -260,6 +329,7 @@ public class AgentScopeChatModelConnectivityVerifierTest {
|
||||
List<ToolSchema> tools,
|
||||
GenerateOptions options) {
|
||||
requestStreams.add(Boolean.TRUE.equals(options.getStream()));
|
||||
requestMaxTokens.add(options.getMaxTokens());
|
||||
messageBatches.add(List.copyOf(inputMessages));
|
||||
toolCounts.add(tools.size());
|
||||
toolChoices.add(options.getToolChoice());
|
||||
@@ -287,6 +357,24 @@ public class AgentScopeChatModelConnectivityVerifierTest {
|
||||
return factoryStreams;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取模型工厂收到的最大输出 Token 数。
|
||||
*
|
||||
* @return 最大输出 Token 数列表
|
||||
*/
|
||||
private List<Integer> getFactoryMaxTokens() {
|
||||
return factoryMaxTokens;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取模型工厂收到的额外请求体参数。
|
||||
*
|
||||
* @return 额外请求体参数列表
|
||||
*/
|
||||
private List<Map<String, Object>> getFactoryAdditionalBodyParams() {
|
||||
return factoryAdditionalBodyParams;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取请求流式参数。
|
||||
*
|
||||
@@ -296,6 +384,15 @@ public class AgentScopeChatModelConnectivityVerifierTest {
|
||||
return requestStreams;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取模型请求收到的最大输出 Token 数。
|
||||
*
|
||||
* @return 最大输出 Token 数列表
|
||||
*/
|
||||
private List<Integer> getRequestMaxTokens() {
|
||||
return requestMaxTokens;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取每次请求的验证消息批次。
|
||||
*
|
||||
|
||||
Reference in New Issue
Block a user