EasyFlow-Tech-Dev/EasyFlow
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 增强智能体模型调用兼容能力

Browse Source 
- 增加模型流式开关和 HTTP 传输策略配置

- 使用 AgentScope 执行基础连接、流式与 VLM 双阶段验证

- 固定多模态校验图片并统一验证状态展示
This commit is contained in:
陈子默
2026-07-17 19:57:06 +08:00
parent ba21f861f4
commit 791649c7d5
22 changed files with 1492 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

122
easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentModelSpecMapper.java Normal file
View File

@@ -0,0 +1,122 @@
package tech.easyflow.agent.runtime;
import com.easyagents.agent.runtime.model.AgentHttpVersionPolicy;
import com.easyagents.agent.runtime.model.AgentModelProviderType;
import com.easyagents.agent.runtime.model.AgentModelSpec;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import tech.easyflow.ai.entity.Model;
import java.util.Locale;
import java.util.Map;
/**
* 将 EasyFlow 模型配置映射为智能体运行时模型声明。
*/
public final class AgentModelSpecMapper {
private static final Logger LOG = LoggerFactory.getLogger(AgentModelSpecMapper.class);
/**
* 禁止实例化工具类。
*/
private AgentModelSpecMapper() {
}
/**
* 按模型持久化配置创建运行时模型声明。
*
* @param model 已补齐供应商默认配置的模型
* @return 运行时模型声明
* @throws IllegalArgumentException 模型为空时抛出
*/
public static AgentModelSpec fromModel(Model model) {
return fromModel(model, Map.of());
}
/**
* 按模型配置和 Agent 快照覆盖项创建运行时模型声明。
*
* @param model 已补齐供应商默认配置的模型
* @param overrides Agent 发布快照中的模型覆盖项
* @return 运行时模型声明
* @throws IllegalArgumentException 模型为空时抛出
*/
public static AgentModelSpec fromModel(Model model, Map<String, Object> overrides) {
if (model == null) {
throw new IllegalArgumentException("模型配置不能为空");
}
Map<String, Object> safeOverrides = overrides == null ? Map.of() : overrides;
AgentModelSpec spec = new AgentModelSpec();
String providerType = stringValue(
safeOverrides,
"providerType",
model.getModelProvider() == null ? null : model.getModelProvider().getProviderType());
spec.setProviderType(parseProviderType(providerType));
spec.setModelName(stringValue(safeOverrides, "modelName", model.getModelName()));
spec.setBaseUrl(stringValue(safeOverrides, "baseUrl", model.getEndpoint()));
spec.setEndpointPath(stringValue(safeOverrides, "endpointPath", model.getRequestPath()));
spec.setApiKey(stringValue(safeOverrides, "apiKey", model.getApiKey()));
spec.setSupportImage(Boolean.TRUE.equals(model.getSupportImage()));
spec.setSupportImageBase64Only(Boolean.TRUE.equals(model.getSupportImageB64Only()));
spec.setHttpVersionPolicy(parseHttpVersionPolicy(model));
spec.getMetadata().put("modelId", model.getId());
if (providerType != null && !providerType.isBlank()) {
spec.getMetadata().put("sourceProviderType", providerType);
}
return spec;
}
/**
* 解析模型配置中的 Agent HTTP 版本策略。
*
* @param model 模型配置
* @return HTTP 版本策略，缺失或非法时返回 AUTO
*/
private static AgentHttpVersionPolicy parseHttpVersionPolicy(Model model) {
Object rawPolicy = model.getOptions() == null
? null
: model.getOptions().get("agentHttpVersionPolicy");
if (rawPolicy == null || String.valueOf(rawPolicy).isBlank()) {
return AgentHttpVersionPolicy.AUTO;
}
String normalizedPolicy = String.valueOf(rawPolicy).trim().toUpperCase(Locale.ROOT);
try {
return AgentHttpVersionPolicy.valueOf(normalizedPolicy);
} catch (IllegalArgumentException exception) {
LOG.warn("Invalid Agent HTTP version policy '{}' for model {}, fallback to AUTO",
rawPolicy, model.getId());
return AgentHttpVersionPolicy.AUTO;
}
}
/**
* 解析 AgentScope 支持的模型供应商类型。
*
* @param providerType 供应商类型
* @return 运行时供应商类型，未知值按 OpenAI-compatible 处理
*/
private static AgentModelProviderType parseProviderType(String providerType) {
if (providerType == null || providerType.isBlank()) {
return AgentModelProviderType.OPENAI_COMPATIBLE;
}
try {
return AgentModelProviderType.valueOf(providerType.trim().toUpperCase(Locale.ROOT));
} catch (IllegalArgumentException ignored) {
return AgentModelProviderType.OPENAI_COMPATIBLE;
}
}
/**
* 读取字符串覆盖项。
*
* @param values 配置映射
* @param key 字段名
* @param defaultValue 默认值
* @return 覆盖值或默认值
*/
private static String stringValue(Map<String, Object> values, String key, String defaultValue) {
Object value = values.get(key);
return value == null ? defaultValue : String.valueOf(value);
}
}

25
easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentRuntimeCompiler.java
View File

@@ -13,7 +13,6 @@ import com.easyagents.agent.runtime.memory.AgentMemoryType;
import com.easyagents.agent.runtime.mcp.McpSpec;
import com.easyagents.agent.runtime.mcp.McpTransportType;
import com.easyagents.agent.runtime.model.AgentGenerationOptions;
import com.easyagents.agent.runtime.model.AgentModelProviderType;
import com.easyagents.agent.runtime.model.AgentModelSpec;
import com.easyagents.agent.runtime.tool.AgentToolCategory;
import com.easyagents.agent.runtime.tool.AgentToolResult;
@@ -104,18 +103,7 @@ public class AgentRuntimeCompiler {
if (model == null) {
throw new BusinessException("Agent 模型不存在");
}
Map<String, Object> config = agent.getModelConfigJson();
AgentModelSpec spec = new AgentModelSpec();
String providerType = stringValue(config, "providerType", model.getModelProvider() == null ? null : model.getModelProvider().getProviderType());
spec.setProviderType(parseProviderType(providerType));
spec.setModelName(stringValue(config, "modelName", model.getModelName()));
spec.setBaseUrl(stringValue(config, "baseUrl", model.getEndpoint()));
spec.setEndpointPath(stringValue(config, "endpointPath", model.getRequestPath()));
spec.setApiKey(stringValue(config, "apiKey", model.getApiKey()));
spec.setSupportImage(Boolean.TRUE.equals(model.getSupportImage()));
spec.setSupportImageBase64Only(Boolean.TRUE.equals(model.getSupportImageB64Only()));
spec.getMetadata().put("modelId", model.getId());
return spec;
return AgentModelSpecMapper.fromModel(model, agent.getModelConfigJson());
}
private AgentGenerationOptions buildGenerationOptions(Map<String, Object> config) {
@@ -756,17 +744,6 @@ public class AgentRuntimeCompiler {
return description == null ? "" : description;
}
private AgentModelProviderType parseProviderType(String providerType) {
if (providerType == null || providerType.isBlank()) {
return AgentModelProviderType.OPENAI_COMPATIBLE;
}
try {
return AgentModelProviderType.valueOf(providerType.trim().toUpperCase());
} catch (IllegalArgumentException ignored) {
return AgentModelProviderType.OPENAI_COMPATIBLE;
}
}
private AgentMemoryType memoryTypeValue(Map<String, Object> map, String key) {
String value = stringValue(map, key, AgentMemoryType.AUTO_CONTEXT.name());
try {

275
easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentScopeChatModelConnectivityVerifier.java Normal file
View File

@@ -0,0 +1,275 @@
package tech.easyflow.agent.runtime;
import com.easyagents.agent.runtime.agentscope.AgentHttpTransportProvider;
import com.easyagents.agent.runtime.agentscope.AgentScopeMessageAdapter;
import com.easyagents.agent.runtime.agentscope.AgentScopeModelFactory;
import com.easyagents.agent.runtime.message.AgentContentBlock;
import com.easyagents.agent.runtime.message.AgentMediaBlock;
import com.easyagents.agent.runtime.message.AgentMessage;
import com.easyagents.agent.runtime.message.AgentMessageRole;
import com.easyagents.agent.runtime.message.AgentTextBlock;
import com.easyagents.agent.runtime.model.AgentGenerationOptions;
import com.easyagents.agent.runtime.model.AgentModelFactory;
import com.easyagents.agent.runtime.model.AgentModelProviderType;
import com.easyagents.agent.runtime.model.AgentModelSpec;
import io.agentscope.core.message.ContentBlock;
import io.agentscope.core.message.Msg;
import io.agentscope.core.message.TextBlock;
import io.agentscope.core.model.ChatResponse;
import io.agentscope.core.model.ExecutionConfig;
import io.agentscope.core.model.GenerateOptions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import tech.easyflow.ai.entity.Model;
import tech.easyflow.ai.service.support.VlmVerificationImage;
import tech.easyflow.ai.service.verification.ChatModelConnectivityVerifier;
import tech.easyflow.ai.service.verification.ChatModelVerificationResult;
import tech.easyflow.common.web.exceptions.BusinessException;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
import java.util.Map;
/**
* 使用 AgentScope 真实运行链路验证 Chat Model 与 VLM 连通性。
*/
@Component
public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnectivityVerifier {
private static final Logger LOG = LoggerFactory.getLogger(AgentScopeChatModelConnectivityVerifier.class);
/** 为未识别关闭思考扩展参数的推理模型保留足够的小额输出预算。 */
private static final int MAX_TOKENS = 256;
private static final Duration DEFAULT_PHASE_TIMEOUT = Duration.ofSeconds(60);
/** AgentScope 模型工厂。 */
private final AgentModelFactory<io.agentscope.core.model.Model> modelFactory;
/** EasyAgents 与 AgentScope 的消息适配器。 */
private final AgentScopeMessageAdapter messageAdapter;
/** 单阶段最大等待时间。 */
private final Duration phaseTimeout;
/**
* 使用生产运行时组件创建验证器。
*/
public AgentScopeChatModelConnectivityVerifier() {
this(new AgentScopeModelFactory(), new AgentScopeMessageAdapter(), DEFAULT_PHASE_TIMEOUT);
}
/**
* 使用指定依赖创建验证器，供测试和受控运行环境使用。
*
* @param modelFactory AgentScope 模型工厂
* @param messageAdapter 消息适配器
* @param phaseTimeout 单阶段最大等待时间
*/
AgentScopeChatModelConnectivityVerifier(
AgentModelFactory<io.agentscope.core.model.Model> modelFactory,
AgentScopeMessageAdapter messageAdapter,
Duration phaseTimeout) {
this.modelFactory = modelFactory;
this.messageAdapter = messageAdapter;
this.phaseTimeout = phaseTimeout;
}
/**
* 依次验证非流式基础连接与流式响应能力。
*
* @param model 已补齐供应商默认配置的模型
* @return 双阶段验证结果
* @throws BusinessException 非流式基础连接失败时抛出
*/
@Override
public ChatModelVerificationResult verify(Model model) {
AgentModelSpec modelSpec = AgentModelSpecMapper.fromModel(model);
String effectiveHttpVersion = AgentHttpTransportProvider.resolveEffectivePolicy(
modelSpec.getHttpVersionPolicy(), modelSpec.getBaseUrl()).name();
AgentMessage verificationMessage = buildVerificationMessage(modelSpec.isSupportImage());
try {
verifyPhase(modelSpec, verificationMessage, false);
} catch (BusinessException exception) {
LOG.error("AgentScope model base connectivity verification failed, modelId={}, httpPolicy={}",
model.getId(), effectiveHttpVersion, exception);
throw exception;
} catch (Exception exception) {
LOG.error("AgentScope model base connectivity verification failed, modelId={}, httpPolicy={}",
model.getId(), effectiveHttpVersion, exception);
throw new BusinessException(400, 1, "模型基础连接验证失败，请查看后端日志", exception);
}
try {
verifyPhase(modelSpec, verificationMessage, true);
return ChatModelVerificationResult.passed(effectiveHttpVersion);
} catch (Exception exception) {
LOG.warn("AgentScope model streaming verification failed, modelId={}, httpPolicy={}",
model.getId(), effectiveHttpVersion, exception);
return ChatModelVerificationResult.streamingUnavailable(effectiveHttpVersion);
}
}
/**
* 执行一次指定流式模式的模型请求并校验响应。
*
* @param modelSpec 运行时模型声明
* @param verificationMessage 验证消息
* @param stream 是否启用流式响应
* @throws BusinessException 响应为空或 VLM 图片识别错误时抛出
*/
private void verifyPhase(AgentModelSpec modelSpec,
AgentMessage verificationMessage,
boolean stream) {
AgentGenerationOptions generationOptions = new AgentGenerationOptions();
generationOptions.setStream(stream);
generationOptions.setThinkingEnabled(false);
disableOpenAiCompatibleThinking(modelSpec, generationOptions);
generationOptions.setMaxTokens(MAX_TOKENS);
io.agentscope.core.model.Model agentScopeModel = modelFactory.create(modelSpec, generationOptions);
Msg message = messageAdapter.toMsg(verificationMessage);
GenerateOptions requestOptions = GenerateOptions.builder()
.stream(stream)
.maxTokens(MAX_TOKENS)
.executionConfig(ExecutionConfig.builder()
.timeout(phaseTimeout)
.maxAttempts(1)
.build())
.build();
List<ChatResponse> responses = agentScopeModel
.stream(List.of(message), List.of(), requestOptions)
.timeout(phaseTimeout)
.collectList()
.block(phaseTimeout.plusSeconds(1));
String responseText = aggregateText(responses);
validateResponse(modelSpec.isSupportImage(), responseText);
}
/**
* 为支持该扩展字段的 OpenAI-compatible 服务显式关闭思考。
*
* <p>AgentScope 的通用 OpenAI Formatter 不会读取中立的
* {@code thinkingEnabled} 字段，需通过扩展请求体传递；原生 OpenAI、
* DeepSeek 等服务不接收该非标准字段，因此仅对已知兼容入口设置。</p>
*
* @param modelSpec 运行时模型声明
* @param generationOptions 验证生成参数
*/
private void disableOpenAiCompatibleThinking(
AgentModelSpec modelSpec,
AgentGenerationOptions generationOptions) {
AgentModelProviderType providerType = modelSpec.getProviderType();
if (providerType == AgentModelProviderType.OPENAI_COMPATIBLE
|| providerType == AgentModelProviderType.CUSTOM
|| providerType == AgentModelProviderType.SILICONFLOW) {
generationOptions.getAdditionalBodyParams().put("enable_thinking", false);
}
Object sourceProviderType = modelSpec.getMetadata().get("sourceProviderType");
if (sourceProviderType != null
&& "gpustack".equalsIgnoreCase(String.valueOf(sourceProviderType))) {
// GPUStack 托管的 vLLM 模型通过 chat_template_kwargs 控制 Qwen 思考模式。
generationOptions.getAdditionalBodyParams().put(
"chat_template_kwargs",
Map.of("enable_thinking", false));
}
}
/**
* 创建文字模型或 VLM 的最小验证消息。
*
* @param supportImage 是否验证图片理解能力
* @return 验证消息
*/
private AgentMessage buildVerificationMessage(boolean supportImage) {
if (!supportImage) {
return AgentMessage.text(
AgentMessageRole.USER,
"请直接回复“你好”，不要补充其他内容。");
}
List<AgentContentBlock> blocks = new ArrayList<>();
blocks.add(new AgentTextBlock("请直接输出图片中的内容，不要补充其他内容。"));
AgentMediaBlock image = new AgentMediaBlock("image");
image.setMimeType("image/png");
image.setData(Base64.getEncoder().encodeToString(VlmVerificationImage.pngBytes()));
blocks.add(image);
AgentMessage message = new AgentMessage();
message.setRole(AgentMessageRole.USER);
message.setContentBlocks(blocks);
return message;
}
/**
* 聚合流式响应中的全部文本增量。
*
* @param responses AgentScope 响应片段
* @return 聚合后的文本
*/
private String aggregateText(List<ChatResponse> responses) {
StringBuilder result = new StringBuilder();
if (responses == null) {
return result.toString();
}
for (ChatResponse response : responses) {
if (response == null || response.getContent() == null) {
continue;
}
for (ContentBlock block : response.getContent()) {
if (block instanceof TextBlock textBlock && textBlock.getText() != null) {
result.append(textBlock.getText());
}
}
}
return result.toString();
}
/**
* 校验模型返回内容。
*
* @param supportImage 是否执行 VLM 图片识别校验
* @param responseText 聚合后的响应文本
* @throws BusinessException 响应为空或图片识别结果不匹配时抛出
*/
private void validateResponse(boolean supportImage, String responseText) {
if (responseText == null || responseText.isBlank()) {
throw new BusinessException("模型未返回有效内容");
}
if (supportImage
&& !normalizeVerificationText(responseText)
.contains(VlmVerificationImage.VERIFICATION_CODE)) {
LOG.warn("VLM verification response did not contain expected code, responseSummary={}",
responseSummary(responseText));
throw new BusinessException("多模态校验未通过，模型未正确识别验证图片");
}
}
/**
* 归一化 VLM 对固定验证码的常见排版输出。
*
* <p>模型可能将连续数字输出为 {@code 5 8 3 9} 或 {@code 5,8,3,9}，
* 这不影响图片识别结论，因此移除非字母数字字符后再校验。</p>
*
* @param responseText 模型验证回复
* @return 仅保留字母与数字的响应文本
*/
private String normalizeVerificationText(String responseText) {
return responseText == null
? ""
: responseText.replaceAll("[^\\p{L}\\p{N}]", "");
}
/**
* 生成固定验证回复的安全日志摘要。
*
* @param responseText 模型验证回复
* @return 移除控制字符且最长 160 字符的摘要
*/
private String responseSummary(String responseText) {
String normalized = responseText == null
? ""
: responseText.replaceAll("[\\p{Cntrl}]", " ").trim();
return normalized.length() <= 160
? normalized
: normalized.substring(0, 160) + "...";
}
}