From f392c896f959b2bfa6455dcf1db7f15be1985250 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Tue, 28 Jul 2026 12:24:42 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E6=89=A9=E5=B1=95=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E6=B6=88=E6=81=AF=E5=86=85=E5=AE=B9=E5=9D=97=E6=95=B0=E7=BB=84?= =?UTF-8?q?=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 升级高级设置为消息级格式并兼容旧 system 配置 - 增加 VLM 图片首轮与文本追问的多轮连接验证 --- .../agent/runtime/AgentModelSpecMapper.java | 29 +++--- ...entScopeChatModelConnectivityVerifier.java | 89 +++++++++++++++++-- .../AgentRuntimeCompilerModelConfigTest.java | 63 +++++++++++-- ...copeChatModelConnectivityVerifierTest.java | 48 +++++++++- .../app/src/views/ai/model/AddModelModal.vue | 43 +++++---- 5 files changed, 227 insertions(+), 45 deletions(-) diff --git a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentModelSpecMapper.java b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentModelSpecMapper.java index 3578f141..028f7530 100644 --- a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentModelSpecMapper.java +++ b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentModelSpecMapper.java @@ -1,9 +1,9 @@ package tech.easyflow.agent.runtime; import com.easyagents.agent.runtime.model.AgentHttpVersionPolicy; +import com.easyagents.agent.runtime.model.AgentMessageContentFormat; import com.easyagents.agent.runtime.model.AgentModelProviderType; import com.easyagents.agent.runtime.model.AgentModelSpec; -import com.easyagents.agent.runtime.model.AgentSystemContentFormat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import tech.easyflow.ai.entity.Model; @@ -61,7 +61,7 @@ public final class AgentModelSpecMapper { spec.setSupportImage(Boolean.TRUE.equals(model.getSupportImage())); spec.setSupportImageBase64Only(Boolean.TRUE.equals(model.getSupportImageB64Only())); spec.setHttpVersionPolicy(parseHttpVersionPolicy(model)); - spec.setSystemContentFormat(parseSystemContentFormat(model)); + spec.setMessageContentFormat(parseMessageContentFormat(model)); spec.getMetadata().put("modelId", model.getId()); if (providerType != null && !providerType.isBlank()) { spec.getMetadata().put("sourceProviderType", providerType); @@ -93,25 +93,30 @@ public final class AgentModelSpecMapper { } /** - * 解析模型配置中的 Agent system content 格式。 + * 解析模型配置中的 Agent 消息 content 格式。 * * @param model 模型配置 - * @return system content 格式，缺失或非法时返回 STRING + * @return 消息 content 格式，缺失或非法时返回 STANDARD */ - private static AgentSystemContentFormat parseSystemContentFormat(Model model) { - Object rawFormat = model.getOptions() == null - ? null - : model.getOptions().get("agentSystemContentFormat"); + private static AgentMessageContentFormat parseMessageContentFormat(Model model) { + Map options = model.getOptions(); + Object rawFormat = options == null ? null : options.get("agentMessageContentFormat"); + if ((rawFormat == null || String.valueOf(rawFormat).isBlank()) && options != null) { + rawFormat = options.get("agentSystemContentFormat"); + } if (rawFormat == null || String.valueOf(rawFormat).isBlank()) { - return AgentSystemContentFormat.STRING; + return AgentMessageContentFormat.STANDARD; } String normalizedFormat = String.valueOf(rawFormat).trim().toUpperCase(Locale.ROOT); + if ("STRING".equals(normalizedFormat)) { + return AgentMessageContentFormat.STANDARD; + } try { - return AgentSystemContentFormat.valueOf(normalizedFormat); + return AgentMessageContentFormat.valueOf(normalizedFormat); } catch (IllegalArgumentException exception) { - LOG.warn("Invalid Agent system content format '{}' for model {}, fallback to STRING", + LOG.warn("Invalid Agent message content format '{}' for model {}, fallback to STANDARD", rawFormat, model.getId()); - return AgentSystemContentFormat.STRING; + return AgentMessageContentFormat.STANDARD; } } diff --git a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentScopeChatModelConnectivityVerifier.java b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentScopeChatModelConnectivityVerifier.java index 47c1aca7..0716ed28 100644 --- a/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentScopeChatModelConnectivityVerifier.java +++ b/easyflow-modules/easyflow-module-agent/src/main/java/tech/easyflow/agent/runtime/AgentScopeChatModelConnectivityVerifier.java @@ -9,6 +9,7 @@ import com.easyagents.agent.runtime.message.AgentMessage; import com.easyagents.agent.runtime.message.AgentMessageRole; import com.easyagents.agent.runtime.message.AgentTextBlock; import com.easyagents.agent.runtime.model.AgentGenerationOptions; +import com.easyagents.agent.runtime.model.AgentMessageContentFormat; import com.easyagents.agent.runtime.model.AgentModelFactory; import com.easyagents.agent.runtime.model.AgentModelProviderType; import com.easyagents.agent.runtime.model.AgentModelSpec; @@ -108,10 +109,11 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect } /** - * 使用一次非流式 Chat 请求优先同时验证连接、视觉与工具调用能力。 + * 使用非流式 Chat 请求验证连接、视觉与工具调用能力。 * *

当兼容接口明确拒绝工具参数时，追加一次不带工具的连接兜底请求， - * 避免将可用的普通对话模型误判为连接失败。

+ * 避免将可用的普通对话模型误判为连接失败。严格内容块数组模式会先 + * 追加一次多轮上下文验证。

* * @param model 已补齐供应商默认配置的模型 * @return 连接和工具能力验证结果 @@ -125,6 +127,9 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect String nonce = nonceSupplier.get(); try { + if (modelSpec.getMessageContentFormat() == AgentMessageContentFormat.TEXT_PARTS) { + verifyMessageContentCompatibility(modelSpec); + } boolean supportTool = verifyProbeRequest(modelSpec, nonce); return ChatModelVerificationResult.passed(effectiveHttpVersion, supportTool); } catch (Exception exception) { @@ -198,6 +203,24 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect validateTextResponse(modelSpec.isSupportImage(), aggregateText(responses)); } + /** + * 验证严格内容块数组模式可以携带完整多轮上下文。 + * + *

支持图片时将图片放在首轮用户消息中，再追加助手历史和纯文本追问， + * 覆盖 VLM 首轮成功后继续对话的请求格式。

+ * + * @param modelSpec 运行时模型声明 + * @throws BusinessException 响应为空时抛出 + */ + private void verifyMessageContentCompatibility(AgentModelSpec modelSpec) { + List responses = request( + modelSpec, + buildMessageContentCompatibilityHistory(modelSpec.isSupportImage()), + List.of(), + null); + validateTextResponse(false, aggregateText(responses)); + } + /** * 使用统一低成本参数执行一次非流式模型请求。 * @@ -229,6 +252,44 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect List tools, ToolChoice toolChoice, int maxTokens) { + List messages = List.of( + messageAdapter.toMsg(AgentMessage.text( + AgentMessageRole.SYSTEM, VERIFICATION_SYSTEM_PROMPT)), + messageAdapter.toMsg(verificationMessage)); + return request(modelSpec, messages, tools, toolChoice, maxTokens); + } + + /** + * 使用统一低成本参数执行一次指定上下文的非流式模型请求。 + * + * @param modelSpec 运行时模型声明 + * @param messages 完整上下文消息 + * @param tools 工具 Schema + * @param toolChoice 工具选择策略 + * @return 模型响应片段 + */ + private List request(AgentModelSpec modelSpec, + List messages, + List tools, + ToolChoice toolChoice) { + return request(modelSpec, messages, tools, toolChoice, MAX_TOKENS); + } + + /** + * 使用指定输出预算执行一次指定上下文的非流式模型请求。 + * + * @param modelSpec 运行时模型声明 + * @param messages 完整上下文消息 + * @param tools 工具 Schema + * @param toolChoice 工具选择策略 + * @param maxTokens 最大输出 Token 数 + * @return 模型响应片段 + */ + private List request(AgentModelSpec modelSpec, + List messages, + List tools, + ToolChoice toolChoice, + int maxTokens) { AgentGenerationOptions generationOptions = new AgentGenerationOptions(); generationOptions.setStream(false); generationOptions.setThinkingEnabled(false); @@ -236,10 +297,6 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect generationOptions.setMaxTokens(maxTokens); io.agentscope.core.model.Model agentScopeModel = modelFactory.create(modelSpec, generationOptions); - List messages = List.of( - messageAdapter.toMsg(AgentMessage.text( - AgentMessageRole.SYSTEM, VERIFICATION_SYSTEM_PROMPT)), - messageAdapter.toMsg(verificationMessage)); GenerateOptions.Builder requestBuilder = GenerateOptions.builder() .stream(false) .maxTokens(maxTokens) @@ -257,6 +314,26 @@ public class AgentScopeChatModelConnectivityVerifier implements ChatModelConnect .block(phaseTimeout.plusSeconds(1)); } + /** + * 构造包含首轮用户消息、助手历史和文本追问的验证上下文。 + * + * @param supportImage 是否在首轮用户消息中附加验证图片 + * @return AgentScope 多轮消息 + */ + private List buildMessageContentCompatibilityHistory(boolean supportImage) { + AgentMessage firstUserMessage = buildVerificationMessage( + supportImage ? "请阅读图片，等待下一条消息。" : "这是第一轮消息。", + supportImage); + return List.of( + messageAdapter.toMsg(AgentMessage.text( + AgentMessageRole.SYSTEM, VERIFICATION_SYSTEM_PROMPT)), + messageAdapter.toMsg(firstUserMessage), + messageAdapter.toMsg(AgentMessage.text( + AgentMessageRole.ASSISTANT, "已收到。")), + messageAdapter.toMsg(AgentMessage.text( + AgentMessageRole.USER, "请直接回复“你好”，不要补充其他内容。"))); + } + /** * 为支持该扩展字段的 OpenAI-compatible 服务显式关闭思考。 * diff --git a/easyflow-modules/easyflow-module-agent/src/test/java/tech/easyflow/agent/runtime/AgentRuntimeCompilerModelConfigTest.java b/easyflow-modules/easyflow-module-agent/src/test/java/tech/easyflow/agent/runtime/AgentRuntimeCompilerModelConfigTest.java index 34d3a925..7d916cd5 100644 --- a/easyflow-modules/easyflow-module-agent/src/test/java/tech/easyflow/agent/runtime/AgentRuntimeCompilerModelConfigTest.java +++ b/easyflow-modules/easyflow-module-agent/src/test/java/tech/easyflow/agent/runtime/AgentRuntimeCompilerModelConfigTest.java @@ -3,8 +3,8 @@ package tech.easyflow.agent.runtime; import com.easyagents.agent.runtime.memory.AgentMemoryPolicy; import com.easyagents.agent.runtime.model.AgentGenerationOptions; import com.easyagents.agent.runtime.model.AgentHttpVersionPolicy; +import com.easyagents.agent.runtime.model.AgentMessageContentFormat; import com.easyagents.agent.runtime.model.AgentModelSpec; -import com.easyagents.agent.runtime.model.AgentSystemContentFormat; import org.junit.Assert; import org.junit.Test; import tech.easyflow.agent.entity.Agent; @@ -68,33 +68,80 @@ public class AgentRuntimeCompilerModelConfigTest { } /** - * 验证模型 options 中的 system content 格式会编译到中立模型声明。 + * 验证模型 options 中的消息 content 格式会编译到中立模型声明。 * * @throws Exception 反射调用失败时抛出 */ @Test - public void modelSystemContentFormatShouldCompileFromOptions() throws Exception { + public void modelMessageContentFormatShouldCompileFromOptions() throws Exception { + Model model = model(Map.of("agentMessageContentFormat", "TEXT_PARTS")); + AgentRuntimeCompiler compiler = compiler(model); + + AgentModelSpec spec = invokeModelSpec(compiler); + + Assert.assertEquals(AgentMessageContentFormat.TEXT_PARTS, spec.getMessageContentFormat()); + } + + /** + * 验证旧 system content 数组配置会迁移为消息级数组格式。 + * + * @throws Exception 反射调用失败时抛出 + */ + @Test + public void legacyTextPartsFormatShouldMigrateToMessageContentFormat() throws Exception { Model model = model(Map.of("agentSystemContentFormat", "TEXT_PARTS")); AgentRuntimeCompiler compiler = compiler(model); AgentModelSpec spec = invokeModelSpec(compiler); - Assert.assertEquals(AgentSystemContentFormat.TEXT_PARTS, spec.getSystemContentFormat()); + Assert.assertEquals(AgentMessageContentFormat.TEXT_PARTS, spec.getMessageContentFormat()); } /** - * 验证未知 system content 格式安全回退到字符串。 + * 验证旧字符串配置会迁移为标准格式。 * * @throws Exception 反射调用失败时抛出 */ @Test - public void unknownSystemContentFormatShouldFallbackToString() throws Exception { - Model model = model(Map.of("agentSystemContentFormat", "PARTS")); + public void legacyStringFormatShouldMigrateToStandard() throws Exception { + Model model = model(Map.of("agentSystemContentFormat", "STRING")); AgentRuntimeCompiler compiler = compiler(model); AgentModelSpec spec = invokeModelSpec(compiler); - Assert.assertEquals(AgentSystemContentFormat.STRING, spec.getSystemContentFormat()); + Assert.assertEquals(AgentMessageContentFormat.STANDARD, spec.getMessageContentFormat()); + } + + /** + * 验证新旧配置并存时优先使用消息级配置。 + * + * @throws Exception 反射调用失败时抛出 + */ + @Test + public void messageContentFormatShouldTakePrecedenceOverLegacyFormat() throws Exception { + Model model = model(Map.of( + "agentMessageContentFormat", "STANDARD", + "agentSystemContentFormat", "TEXT_PARTS")); + AgentRuntimeCompiler compiler = compiler(model); + + AgentModelSpec spec = invokeModelSpec(compiler); + + Assert.assertEquals(AgentMessageContentFormat.STANDARD, spec.getMessageContentFormat()); + } + + /** + * 验证未知消息 content 格式安全回退到标准格式。 + * + * @throws Exception 反射调用失败时抛出 + */ + @Test + public void unknownMessageContentFormatShouldFallbackToStandard() throws Exception { + Model model = model(Map.of("agentMessageContentFormat", "PARTS")); + AgentRuntimeCompiler compiler = compiler(model); + + AgentModelSpec spec = invokeModelSpec(compiler); + + Assert.assertEquals(AgentMessageContentFormat.STANDARD, spec.getMessageContentFormat()); } /** diff --git a/easyflow-modules/easyflow-module-agent/src/test/java/tech/easyflow/agent/runtime/AgentScopeChatModelConnectivityVerifierTest.java b/easyflow-modules/easyflow-module-agent/src/test/java/tech/easyflow/agent/runtime/AgentScopeChatModelConnectivityVerifierTest.java index b81a5cb9..878f5a38 100644 --- a/easyflow-modules/easyflow-module-agent/src/test/java/tech/easyflow/agent/runtime/AgentScopeChatModelConnectivityVerifierTest.java +++ b/easyflow-modules/easyflow-module-agent/src/test/java/tech/easyflow/agent/runtime/AgentScopeChatModelConnectivityVerifierTest.java @@ -88,7 +88,7 @@ public class AgentScopeChatModelConnectivityVerifierTest { Flux.just(toolResponse(TEST_NONCE, null))); ChatModelVerificationResult result = verifier(factory).verify( - model(false, "self-hosted")); + model(false, Map.of(), "self-hosted")); Assert.assertEquals(ModelVerificationStatus.PASSED, result.getStatus()); Map bodyParams = factory.getFactoryAdditionalBodyParams().get(0); @@ -173,6 +173,33 @@ public class AgentScopeChatModelConnectivityVerifierTest { Assert.assertEquals((byte) 0x50, imageBytes[1]); } + /** + * 验证严格内容块模式会先发送包含图片首轮和文本追问的完整上下文。 + */ + @Test + public void shouldVerifyStrictMessageContentFormatWithVlmHistory() { + RecordingModelFactory factory = new RecordingModelFactory( + Flux.just(textResponse("你好")), + Flux.just(toolResponse(TEST_NONCE, VlmVerificationImage.VERIFICATION_CODE))); + + ChatModelVerificationResult result = verifier(factory).verify(model( + true, + Map.of("agentMessageContentFormat", "TEXT_PARTS"))); + + Assert.assertEquals(ModelVerificationStatus.PASSED, result.getStatus()); + Assert.assertEquals(List.of(0, 1), factory.getToolCounts()); + List history = factory.getMessageBatches().get(0); + Assert.assertEquals(List.of( + MsgRole.SYSTEM, + MsgRole.USER, + MsgRole.ASSISTANT, + MsgRole.USER), + history.stream().map(Msg::getRole).toList()); + Assert.assertTrue(history.get(1).getContent().stream().anyMatch(ImageBlock.class::isInstance)); + Assert.assertEquals("请直接回复“你好”，不要补充其他内容。", + history.get(3).getTextContent()); + } + /** * 创建待验证模型。 * @@ -180,17 +207,31 @@ public class AgentScopeChatModelConnectivityVerifierTest { * @return 测试模型 */ private Model model(boolean supportImage) { - return model(supportImage, "gpustack"); + return model(supportImage, Map.of()); + } + + /** + * 创建携带指定 options 的待验证模型。 + * + * @param supportImage 是否支持图片 + * @param options 模型扩展配置 + * @return 测试模型 + */ + private Model model(boolean supportImage, Map options) { + return model(supportImage, options, "gpustack"); } /** * 创建携带指定供应商类型的待验证模型。 * * @param supportImage 是否支持图片 + * @param options 模型扩展配置 * @param providerType 供应商类型 * @return 测试模型 */ - private Model model(boolean supportImage, String providerType) { + private Model model(boolean supportImage, + Map options, + String providerType) { Model model = new Model(); model.setId(BigInteger.TEN); model.setModelName("test-model"); @@ -198,6 +239,7 @@ public class AgentScopeChatModelConnectivityVerifierTest { model.setRequestPath("/v1/chat/completions"); model.setApiKey("test-key"); model.setSupportImage(supportImage); + model.setOptions(options); ModelProvider provider = new ModelProvider(); provider.setProviderType(providerType); model.setModelProvider(provider); diff --git a/easyflow-ui-admin/app/src/views/ai/model/AddModelModal.vue b/easyflow-ui-admin/app/src/views/ai/model/AddModelModal.vue index ba8caaef..e5c16b11 100644 --- a/easyflow-ui-admin/app/src/views/ai/model/AddModelModal.vue +++ b/easyflow-ui-admin/app/src/views/ai/model/AddModelModal.vue @@ -34,11 +34,11 @@ import { import { resetModelAbility } from '#/views/ai/model/modelUtils/model-ability-utils'; type AgentHttpVersionPolicy = 'AUTO' | 'HTTP_1_1' | 'HTTP_2_PREFERRED'; -type AgentSystemContentFormat = 'STRING' | 'TEXT_PARTS'; +type AgentMessageContentFormat = 'STANDARD' | 'TEXT_PARTS'; interface ModelOptions { agentHttpVersionPolicy: AgentHttpVersionPolicy; - agentSystemContentFormat: AgentSystemContentFormat; + agentMessageContentFormat: AgentMessageContentFormat; [key: string]: unknown; } @@ -116,7 +116,7 @@ const formData = reactive({ supportToolMessage: null, options: { agentHttpVersionPolicy: 'AUTO', - agentSystemContentFormat: 'STRING', + agentMessageContentFormat: 'STANDARD', }, }); @@ -126,8 +126,8 @@ const agentHttpVersionOptions = [ { label: 'HTTP/2 优先', value: 'HTTP_2_PREFERRED' }, ] as const; -const agentSystemContentFormatOptions = [ - { label: '字符串（默认）', value: 'STRING' }, +const agentMessageContentFormatOptions = [ + { label: '标准格式（默认）', value: 'STANDARD' }, { label: '内容块数组', value: 'TEXT_PARTS' }, ] as const; @@ -140,10 +140,10 @@ const normalizeAgentHttpVersionPolicy = ( return 'AUTO'; }; -const normalizeAgentSystemContentFormat = ( +const normalizeAgentMessageContentFormat = ( value?: unknown, -): AgentSystemContentFormat => { - return value === 'TEXT_PARTS' ? 'TEXT_PARTS' : 'STRING'; +): AgentMessageContentFormat => { + return value === 'TEXT_PARTS' ? 'TEXT_PARTS' : 'STANDARD'; }; const normalizeModelOptions = (options?: unknown): ModelOptions => { @@ -151,14 +151,25 @@ const normalizeModelOptions = (options?: unknown): ModelOptions => { options && typeof options === 'object' && !Array.isArray(options) ? (options as Record) : {}; + const { + agentSystemContentFormat: legacyMessageContentFormat, + ...currentOptions + } = source; + const currentMessageContentFormat = source.agentMessageContentFormat; + const messageContentFormat = + currentMessageContentFormat === null || + currentMessageContentFormat === undefined || + (typeof currentMessageContentFormat === 'string' && + currentMessageContentFormat.trim() === '') + ? legacyMessageContentFormat + : currentMessageContentFormat; return { - ...source, + ...currentOptions, agentHttpVersionPolicy: normalizeAgentHttpVersionPolicy( source.agentHttpVersionPolicy, ), - agentSystemContentFormat: normalizeAgentSystemContentFormat( - source.agentSystemContentFormat, - ), + agentMessageContentFormat: + normalizeAgentMessageContentFormat(messageContentFormat), }; }; @@ -592,13 +603,13 @@ const save = async () => { /> - +