fix: 修复推理模型连接验证截断
- 响应达到输出上限时使用 512 Token 单次重试 - 为自部署 vLLM/SGLang 传递关闭思考参数 - 补充截断重试与请求参数回归测试
This commit is contained in:
@@ -51,8 +51,10 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
|
||||
private static final String PROBE_NONCE_FIELD = "nonce";
|
||||
private static final String PROBE_IMAGE_FIELD = "imageCode";
|
||||
private static final String VERIFICATION_SYSTEM_PROMPT = "You are a model connectivity verification assistant.";
|
||||
/** 为未识别关闭思考扩展参数的推理模型保留足够的小额输出预算。 */
|
||||
/** 首次能力探测使用的低成本输出预算。 */
|
||||
private static final int MAX_TOKENS = 128;
|
||||
/** 首次响应被输出上限截断时使用的单次重试预算。 */
|
||||
private static final int TRUNCATION_RETRY_MAX_TOKENS = 512;
|
||||
private static final Duration DEFAULT_PHASE_TIMEOUT = Duration.ofSeconds(60);
|
||||
|
||||
/** AgentScope 模型工厂。 */
|
||||
@@ -158,11 +160,23 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
|
||||
*/
|
||||
private boolean verifyProbeRequest(AgentModelSpec modelSpec, String nonce) {
|
||||
AgentMessage message = buildProbeMessage(modelSpec.isSupportImage(), nonce);
|
||||
List<ToolSchema> tools = List.of(buildProbeToolSchema(modelSpec.isSupportImage()));
|
||||
ToolChoice toolChoice = new ToolChoice.Specific(PROBE_TOOL_NAME);
|
||||
List<ChatResponse> responses = request(
|
||||
modelSpec,
|
||||
message,
|
||||
List.of(buildProbeToolSchema(modelSpec.isSupportImage())),
|
||||
new ToolChoice.Specific(PROBE_TOOL_NAME));
|
||||
tools,
|
||||
toolChoice);
|
||||
if (hasLengthLimitedResponse(responses)) {
|
||||
LOG.info("Model probe response reached token limit, retry with larger budget, modelName={}, maxTokens={}",
|
||||
modelSpec.getModelName(), TRUNCATION_RETRY_MAX_TOKENS);
|
||||
responses = request(
|
||||
modelSpec,
|
||||
message,
|
||||
tools,
|
||||
toolChoice,
|
||||
TRUNCATION_RETRY_MAX_TOKENS);
|
||||
}
|
||||
boolean validToolCall = hasValidProbeToolCall(
|
||||
responses, nonce, modelSpec.isSupportImage());
|
||||
validateProbeResponse(modelSpec.isSupportImage(), responses, validToolCall);
|
||||
@@ -197,11 +211,29 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
|
||||
AgentMessage verificationMessage,
|
||||
List<ToolSchema> tools,
|
||||
ToolChoice toolChoice) {
|
||||
return request(modelSpec, verificationMessage, tools, toolChoice, MAX_TOKENS);
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用指定输出预算执行一次非流式模型请求。
|
||||
*
|
||||
* @param modelSpec 运行时模型声明
|
||||
* @param verificationMessage 验证消息
|
||||
* @param tools 工具 Schema
|
||||
* @param toolChoice 工具选择策略
|
||||
* @param maxTokens 最大输出 Token 数
|
||||
* @return 模型响应片段
|
||||
*/
|
||||
private List<ChatResponse> request(AgentModelSpec modelSpec,
|
||||
AgentMessage verificationMessage,
|
||||
List<ToolSchema> tools,
|
||||
ToolChoice toolChoice,
|
||||
int maxTokens) {
|
||||
AgentGenerationOptions generationOptions = new AgentGenerationOptions();
|
||||
generationOptions.setStream(false);
|
||||
generationOptions.setThinkingEnabled(false);
|
||||
disableOpenAiCompatibleThinking(modelSpec, generationOptions);
|
||||
generationOptions.setMaxTokens(MAX_TOKENS);
|
||||
generationOptions.setMaxTokens(maxTokens);
|
||||
io.agentscope.core.model.Model agentScopeModel = modelFactory.create(modelSpec, generationOptions);
|
||||
|
||||
List<Msg> messages = List.of(
|
||||
@@ -210,7 +242,7 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
|
||||
messageAdapter.toMsg(verificationMessage));
|
||||
GenerateOptions.Builder requestBuilder = GenerateOptions.builder()
|
||||
.stream(false)
|
||||
.maxTokens(MAX_TOKENS)
|
||||
.maxTokens(maxTokens)
|
||||
.executionConfig(ExecutionConfig.builder()
|
||||
.timeout(phaseTimeout)
|
||||
.maxAttempts(1)
|
||||
@@ -245,15 +277,34 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
|
||||
generationOptions.getAdditionalBodyParams().put("enable_thinking", false);
|
||||
}
|
||||
Object sourceProviderType = modelSpec.getMetadata().get("sourceProviderType");
|
||||
if (sourceProviderType != null
|
||||
&& "gpustack".equalsIgnoreCase(String.valueOf(sourceProviderType))) {
|
||||
// GPUStack 托管的 vLLM 模型通过 chat_template_kwargs 控制 Qwen 思考模式。
|
||||
if (usesChatTemplateThinkingControl(sourceProviderType)) {
|
||||
// vLLM 与 SGLang 通过聊天模板参数控制 Qwen 思考模式。
|
||||
generationOptions.getAdditionalBodyParams().put(
|
||||
"chat_template_kwargs",
|
||||
Map.of("enable_thinking", false));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断供应商是否通过聊天模板参数控制思考模式。
|
||||
*
|
||||
* @param sourceProviderType 模型配置中的原始供应商类型
|
||||
* @return GPUStack 或自部署 vLLM/SGLang 入口返回 true
|
||||
*/
|
||||
private boolean usesChatTemplateThinkingControl(Object sourceProviderType) {
|
||||
if (sourceProviderType == null) {
|
||||
return false;
|
||||
}
|
||||
String normalizedProviderType = String.valueOf(sourceProviderType)
|
||||
.trim()
|
||||
.toLowerCase(Locale.ROOT)
|
||||
.replace('_', '-');
|
||||
return "gpustack".equals(normalizedProviderType)
|
||||
|| "self-hosted".equals(normalizedProviderType)
|
||||
|| "vllm".equals(normalizedProviderType)
|
||||
|| "sglang".equals(normalizedProviderType);
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建同时验证连接、视觉和工具调用的消息。
|
||||
*
|
||||
@@ -371,6 +422,24 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断响应是否因为达到输出 Token 上限而结束。
|
||||
*
|
||||
* @param responses AgentScope 响应片段
|
||||
* @return 任一响应的结束原因为 length 时返回 true
|
||||
*/
|
||||
private boolean hasLengthLimitedResponse(List<ChatResponse> responses) {
|
||||
if (responses == null) {
|
||||
return false;
|
||||
}
|
||||
for (ChatResponse response : responses) {
|
||||
if (response != null && "length".equalsIgnoreCase(response.getFinishReason())) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验探测响应是否足以证明连接和视觉能力。
|
||||
*
|
||||
|
||||
Reference in New Issue
Block a user