feat: 完善模型能力识别与验证

- 自动识别模型类型、视觉、推理和工具能力并保留手动覆盖 - 使用 AgentScope 工具与视觉探测并统一管理端配置反馈
2026-07-27 19:40:23 +08:00
parent 0dc5c3ca55
commit 567fd12706
21 changed files with 1059 additions and 378 deletions
--- a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentScopeChatModelConnectivityVerifier.java
+++ b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentScopeChatModelConnectivityVerifier.java
@@ -15,9 +15,12 @@ import com.easyagents.agent.runtime.model.AgentModelSpec;
 import io.agentscope.core.message.ContentBlock;
 import io.agentscope.core.message.Msg;
 import io.agentscope.core.message.TextBlock;
+import io.agentscope.core.message.ToolUseBlock;
 import io.agentscope.core.model.ChatResponse;
 import io.agentscope.core.model.ExecutionConfig;
 import io.agentscope.core.model.GenerateOptions;
+import io.agentscope.core.model.ToolChoice;
+import io.agentscope.core.model.ToolSchema;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
@@ -30,8 +33,12 @@ import tech.easyflow.common.web.exceptions.BusinessException;
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Base64;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
+import java.util.UUID;
+import java.util.function.Supplier;

 /**
 * 使用 AgentScope 真实运行链路验证 Chat Model 与 VLM 连通性。
@@ -40,11 +47,13 @@ import java.util.Map;
 public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnectivityVerifier {

    private static final Logger LOG = LoggerFactory.getLogger(AgentScopeChatModelConnectivityVerifier.class);
+    private static final String PROBE_TOOL_NAME = "easyflow_capability_probe";
+    private static final String PROBE_NONCE_FIELD = "nonce";
+    private static final String PROBE_IMAGE_FIELD = "imageCode";
+    private static final String VERIFICATION_SYSTEM_PROMPT = "You are a model connectivity verification assistant.";
    /** 为未识别关闭思考扩展参数的推理模型保留足够的小额输出预算。 */
-    private static final int MAX_TOKENS = 256;
+    private static final int MAX_TOKENS = 128;
    private static final Duration DEFAULT_PHASE_TIMEOUT = Duration.ofSeconds(60);
-    private static final String VERIFICATION_SYSTEM_PROMPT =
-            "You are verifying model connectivity. Follow the user's instruction exactly.";

    /** AgentScope 模型工厂。 */
    private final AgentModelFactory<io.agentscope.core.model.Model> modelFactory;
@@ -52,12 +61,15 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
    private final AgentScopeMessageAdapter messageAdapter;
    /** 单阶段最大等待时间。 */
    private final Duration phaseTimeout;
+    /** 为每次工具能力探测生成随机校验值。 */
+    private final Supplier<String> nonceSupplier;

    /**
     * 使用生产运行时组件创建验证器。
     */
    public AgentScopeChatModelConnectivityVerifier() {
-        this(new AgentScopeModelFactory(), new AgentScopeMessageAdapter(), DEFAULT_PHASE_TIMEOUT);
+        this(new AgentScopeModelFactory(), new AgentScopeMessageAdapter(), DEFAULT_PHASE_TIMEOUT,
+                () -> UUID.randomUUID().toString());
    }

    /**
@@ -71,60 +83,122 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
            AgentModelFactory<io.agentscope.core.model.Model> modelFactory,
            AgentScopeMessageAdapter messageAdapter,
            Duration phaseTimeout) {
-        this.modelFactory = modelFactory;
-        this.messageAdapter = messageAdapter;
-        this.phaseTimeout = phaseTimeout;
+        this(modelFactory, messageAdapter, phaseTimeout, () -> UUID.randomUUID().toString());
    }

    /**
-     * 依次验证非流式基础连接与流式响应能力。
+     * 使用可控随机值创建验证器，供能力探测测试使用。
+     *
+     * @param modelFactory AgentScope 模型工厂
+     * @param messageAdapter 消息适配器
+     * @param phaseTimeout 单阶段最大等待时间
+     * @param nonceSupplier 探测随机值生成器
+     */
+    AgentScopeChatModelConnectivityVerifier(
+            AgentModelFactory<io.agentscope.core.model.Model> modelFactory,
+            AgentScopeMessageAdapter messageAdapter,
+            Duration phaseTimeout,
+            Supplier<String> nonceSupplier) {
+        this.modelFactory = modelFactory;
+        this.messageAdapter = messageAdapter;
+        this.phaseTimeout = phaseTimeout;
+        this.nonceSupplier = nonceSupplier;
+    }
+
+    /**
+     * 使用一次非流式 Chat 请求优先同时验证连接、视觉与工具调用能力。
+     *
+     * <p>当兼容接口明确拒绝工具参数时，追加一次不带工具的连接兜底请求，
+     * 避免将可用的普通对话模型误判为连接失败。</p>
     *
     * @param model 已补齐供应商默认配置的模型
-     * @return 双阶段验证结果
-     * @throws BusinessException 非流式基础连接失败时抛出
+     * @return 连接和工具能力验证结果
+     * @throws BusinessException 连接或视觉验证失败时抛出
     */
    @Override
    public ChatModelVerificationResult verify(Model model) {
        AgentModelSpec modelSpec = AgentModelSpecMapper.fromModel(model);
        String effectiveHttpVersion = AgentHttpTransportProvider.resolveEffectivePolicy(
                modelSpec.getHttpVersionPolicy(), modelSpec.getBaseUrl()).name();
-        AgentMessage verificationMessage = buildVerificationMessage(modelSpec.isSupportImage());
+        String nonce = nonceSupplier.get();

        try {
-            verifyPhase(modelSpec, verificationMessage, false);
-        } catch (BusinessException exception) {
-            LOG.error("AgentScope model base connectivity verification failed, modelId={}, httpPolicy={}",
-                    model.getId(), effectiveHttpVersion, exception);
-            throw exception;
+            boolean supportTool = verifyProbeRequest(modelSpec, nonce);
+            return ChatModelVerificationResult.passed(effectiveHttpVersion, supportTool);
        } catch (Exception exception) {
-            LOG.error("AgentScope model base connectivity verification failed, modelId={}, httpPolicy={}",
+            if (isToolCapabilityRejection(exception)) {
+                LOG.info("Model endpoint rejected tool probe, fallback to plain connectivity, modelId={}",
+                        model.getId());
+                try {
+                    verifyPlainRequest(modelSpec);
+                    return ChatModelVerificationResult.passed(effectiveHttpVersion, false);
+                } catch (Exception fallbackException) {
+                    LOG.error("AgentScope model fallback connectivity verification failed, modelId={}, httpPolicy={}",
+                            model.getId(), effectiveHttpVersion, fallbackException);
+                    throw new BusinessException(
+                            400, 1, "模型连接验证失败，请查看后端日志", fallbackException);
+                }
+            }
+            LOG.error("AgentScope model connectivity verification failed, modelId={}, httpPolicy={}",
                    model.getId(), effectiveHttpVersion, exception);
-            throw new BusinessException(400, 1, "模型基础连接验证失败，请查看后端日志", exception);
-        }
-
-        try {
-            verifyPhase(modelSpec, verificationMessage, true);
-            return ChatModelVerificationResult.passed(effectiveHttpVersion);
-        } catch (Exception exception) {
-            LOG.warn("AgentScope model streaming verification failed, modelId={}, httpPolicy={}",
-                    model.getId(), effectiveHttpVersion, exception);
-            return ChatModelVerificationResult.streamingUnavailable(effectiveHttpVersion);
+            if (exception instanceof BusinessException businessException) {
+                throw businessException;
+            }
+            throw new BusinessException(400, 1, "模型连接验证失败，请查看后端日志", exception);
        }
    }

    /**
-     * 执行一次指定流式模式的模型请求并校验响应。
+     * 执行携带无副作用工具 Schema 的能力探测请求。
+     *
+     * @param modelSpec 运行时模型声明
+     * @param nonce 本次探测随机值
+     * @return 模型是否正确返回指定工具调用
+     * @throws BusinessException 响应为空或视觉识别失败时抛出
+     */
+    private boolean verifyProbeRequest(AgentModelSpec modelSpec, String nonce) {
+        AgentMessage message = buildProbeMessage(modelSpec.isSupportImage(), nonce);
+        List<ChatResponse> responses = request(
+                modelSpec,
+                message,
+                List.of(buildProbeToolSchema(modelSpec.isSupportImage())),
+                new ToolChoice.Specific(PROBE_TOOL_NAME));
+        boolean validToolCall = hasValidProbeToolCall(
+                responses, nonce, modelSpec.isSupportImage());
+        validateProbeResponse(modelSpec.isSupportImage(), responses, validToolCall);
+        return validToolCall;
+    }
+
+    /**
+     * 执行不带工具的兼容性兜底请求。
+     *
+     * @param modelSpec 运行时模型声明
+     * @throws BusinessException 响应为空或视觉识别失败时抛出
+     */
+    private void verifyPlainRequest(AgentModelSpec modelSpec) {
+        List<ChatResponse> responses = request(
+                modelSpec,
+                buildPlainVerificationMessage(modelSpec.isSupportImage()),
+                List.of(),
+                null);
+        validateTextResponse(modelSpec.isSupportImage(), aggregateText(responses));
+    }
+
+    /**
+     * 使用统一低成本参数执行一次非流式模型请求。
     *
     * @param modelSpec 运行时模型声明
     * @param verificationMessage 验证消息
-     * @param stream 是否启用流式响应
-     * @throws BusinessException 响应为空或 VLM 图片识别错误时抛出
+     * @param tools 工具 Schema
+     * @param toolChoice 工具选择策略
+     * @return 模型响应片段
     */
-    private void verifyPhase(AgentModelSpec modelSpec,
-                             AgentMessage verificationMessage,
-                             boolean stream) {
+    private List<ChatResponse> request(AgentModelSpec modelSpec,
+                                       AgentMessage verificationMessage,
+                                       List<ToolSchema> tools,
+                                       ToolChoice toolChoice) {
        AgentGenerationOptions generationOptions = new AgentGenerationOptions();
-        generationOptions.setStream(stream);
+        generationOptions.setStream(false);
        generationOptions.setThinkingEnabled(false);
        disableOpenAiCompatibleThinking(modelSpec, generationOptions);
        generationOptions.setMaxTokens(MAX_TOKENS);
@@ -132,24 +206,23 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect

        List<Msg> messages = List.of(
                messageAdapter.toMsg(AgentMessage.text(
-                        AgentMessageRole.SYSTEM,
-                        VERIFICATION_SYSTEM_PROMPT)),
+                        AgentMessageRole.SYSTEM, VERIFICATION_SYSTEM_PROMPT)),
                messageAdapter.toMsg(verificationMessage));
-        GenerateOptions requestOptions = GenerateOptions.builder()
-                .stream(stream)
+        GenerateOptions.Builder requestBuilder = GenerateOptions.builder()
+                .stream(false)
                .maxTokens(MAX_TOKENS)
                .executionConfig(ExecutionConfig.builder()
                        .timeout(phaseTimeout)
                        .maxAttempts(1)
-                        .build())
-                .build();
-        List<ChatResponse> responses = agentScopeModel
-                .stream(messages, List.of(), requestOptions)
+                        .build());
+        if (toolChoice != null) {
+            requestBuilder.toolChoice(toolChoice);
+        }
+        return agentScopeModel
+                .stream(messages, tools, requestBuilder.build())
                .timeout(phaseTimeout)
                .collectList()
                .block(phaseTimeout.plusSeconds(1));
-        String responseText = aggregateText(responses);
-        validateResponse(modelSpec.isSupportImage(), responseText);
    }

    /**
@@ -182,19 +255,47 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
    }

    /**
-     * 创建文字模型或 VLM 的最小验证消息。
+     * 创建同时验证连接、视觉和工具调用的消息。
     *
     * @param supportImage 是否验证图片理解能力
-     * @return 验证消息
+     * @param nonce 本次探测随机值
+     * @return 工具能力探测消息
     */
-    private AgentMessage buildVerificationMessage(boolean supportImage) {
+    private AgentMessage buildProbeMessage(boolean supportImage, String nonce) {
+        String prompt = supportImage
+                ? "请调用 " + PROBE_TOOL_NAME + " 工具，将 nonce 设置为“" + nonce
+                + "”，并将图片中的验证码设置为 imageCode。不要直接回答。"
+                : "请调用 " + PROBE_TOOL_NAME + " 工具，并将 nonce 设置为“" + nonce
+                + "”。不要直接回答。";
+        return buildVerificationMessage(prompt, supportImage);
+    }
+
+    /**
+     * 创建不带工具的兼容性兜底消息。
+     *
+     * @param supportImage 是否验证图片理解能力
+     * @return 普通连接验证消息
+     */
+    private AgentMessage buildPlainVerificationMessage(boolean supportImage) {
+        String prompt = supportImage
+                ? "请直接输出图片中的内容，不要补充其他内容。"
+                : "请直接回复“你好”，不要补充其他内容。";
+        return buildVerificationMessage(prompt, supportImage);
+    }
+
+    /**
+     * 创建文字或 VLM 验证消息。
+     *
+     * @param prompt 验证提示词
+     * @param supportImage 是否附加验证图片
+     * @return AgentScope 消息
+     */
+    private AgentMessage buildVerificationMessage(String prompt, boolean supportImage) {
        if (!supportImage) {
-            return AgentMessage.text(
-                    AgentMessageRole.USER,
-                    "请直接回复“你好”，不要补充其他内容。");
+            return AgentMessage.text(AgentMessageRole.USER, prompt);
        }
        List<AgentContentBlock> blocks = new ArrayList<>();
-        blocks.add(new AgentTextBlock("请直接输出图片中的内容，不要补充其他内容。"));
+        blocks.add(new AgentTextBlock(prompt));
        AgentMediaBlock image = new AgentMediaBlock("image");
        image.setMimeType("image/png");
        image.setData(Base64.getEncoder().encodeToString(VlmVerificationImage.pngBytes()));
@@ -205,6 +306,111 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
        return message;
    }

+    /**
+     * 创建只声明参数、不绑定执行逻辑的探测工具 Schema。
+     *
+     * @param supportImage 是否包含图片验证码参数
+     * @return 探测工具 Schema
+     */
+    private ToolSchema buildProbeToolSchema(boolean supportImage) {
+        Map<String, Object> properties = new LinkedHashMap<>();
+        properties.put(PROBE_NONCE_FIELD, Map.of(
+                "type", "string",
+                "description", "原样返回用户提供的 nonce"));
+        List<String> required = new ArrayList<>();
+        required.add(PROBE_NONCE_FIELD);
+        if (supportImage) {
+            properties.put(PROBE_IMAGE_FIELD, Map.of(
+                    "type", "string",
+                    "description", "图片中的验证码"));
+            required.add(PROBE_IMAGE_FIELD);
+        }
+        return ToolSchema.builder()
+                .name(PROBE_TOOL_NAME)
+                .description("验证模型是否能生成结构化工具调用")
+                .parameters(Map.of(
+                        "type", "object",
+                        "properties", properties,
+                        "required", required))
+                .strict(false)
+                .build();
+    }
+
+    /**
+     * 判断响应是否包含参数正确的探测工具调用。
+     *
+     * @param responses AgentScope 响应片段
+     * @param nonce 本次探测随机值
+     * @param supportImage 是否同时验证图片
+     * @return 工具名和参数均正确返回 true
+     */
+    private boolean hasValidProbeToolCall(List<ChatResponse> responses,
+                                          String nonce,
+                                          boolean supportImage) {
+        if (responses == null) {
+            return false;
+        }
+        for (ChatResponse response : responses) {
+            if (response == null || response.getContent() == null) {
+                continue;
+            }
+            for (ContentBlock block : response.getContent()) {
+                if (!(block instanceof ToolUseBlock toolUse)
+                        || !PROBE_TOOL_NAME.equals(toolUse.getName())
+                        || toolUse.getInput() == null
+                        || !nonce.equals(String.valueOf(toolUse.getInput().get(PROBE_NONCE_FIELD)))) {
+                    continue;
+                }
+                if (!supportImage || VlmVerificationImage.VERIFICATION_CODE.equals(
+                        normalizeVerificationText(String.valueOf(
+                                toolUse.getInput().get(PROBE_IMAGE_FIELD))))) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+    /**
+     * 校验探测响应是否足以证明连接和视觉能力。
+     *
+     * @param supportImage 是否验证图片理解能力
+     * @param responses AgentScope 响应片段
+     * @param validToolCall 是否包含正确工具调用
+     * @throws BusinessException 响应为空或图片识别错误时抛出
+     */
+    private void validateProbeResponse(boolean supportImage,
+                                       List<ChatResponse> responses,
+                                       boolean validToolCall) {
+        if (validToolCall) {
+            return;
+        }
+        String responseText = aggregateText(responses);
+        if (responseText != null && !responseText.isBlank()) {
+            validateTextResponse(supportImage, responseText);
+            return;
+        }
+        if (hasAnyContent(responses) && !supportImage) {
+            return;
+        }
+        throw new BusinessException("模型未返回有效内容");
+    }
+
+    /**
+     * 判断模型是否返回任意内容块。
+     *
+     * @param responses AgentScope 响应片段
+     * @return 存在内容块返回 true
+     */
+    private boolean hasAnyContent(List<ChatResponse> responses) {
+        if (responses == null) {
+            return false;
+        }
+        return responses.stream()
+                .filter(response -> response != null && response.getContent() != null)
+                .anyMatch(response -> !response.getContent().isEmpty());
+    }
+
    /**
     * 聚合流式响应中的全部文本增量。
     *
@@ -236,7 +442,7 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
     * @param responseText 聚合后的响应文本
     * @throws BusinessException 响应为空或图片识别结果不匹配时抛出
     */
-    private void validateResponse(boolean supportImage, String responseText) {
+    private void validateTextResponse(boolean supportImage, String responseText) {
        if (responseText == null || responseText.isBlank()) {
            throw new BusinessException("模型未返回有效内容");
        }
@@ -278,4 +484,34 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect
                ? normalized
                : normalized.substring(0, 160) + "...";
    }
+
+    /**
+     * 判断异常是否明确来自工具或工具选择参数不兼容。
+     *
+     * @param exception 模型调用异常
+     * @return 工具参数被拒绝返回 true
+     */
+    private boolean isToolCapabilityRejection(Throwable exception) {
+        Throwable current = exception;
+        while (current != null) {
+            String message = current.getMessage();
+            if (message != null) {
+                String normalized = message.toLowerCase(Locale.ROOT);
+                if (normalized.contains("tool_choice")
+                        || normalized.contains("tool choice")
+                        || normalized.contains("tool_call")
+                        || normalized.contains("tool call")
+                        || normalized.contains("tools parameter")
+                        || normalized.contains("function calling")
+                        || (normalized.contains("tools")
+                                && (normalized.contains("unsupported")
+                                || normalized.contains("not support")
+                                || normalized.contains("invalid")))) {
+                    return true;
+                }
+            }
+            current = current.getCause();
+        }
+        return false;
+    }
 }