feat: 增强多实例分布式部署兼容
- 增加定时任务分布式锁并覆盖 chatlog、文档导入和 Agent HITL 过期扫描 - 增强 Redis MQ 多实例 consumer 标识、pending reclaim 和单条处理能力 - 增加文档导入状态 Redis 广播和 Agent HITL 跨节点路由确认
This commit is contained in:
@@ -37,6 +37,10 @@
|
||||
<groupId>tech.easyflow</groupId>
|
||||
<artifactId>easyflow-common-cache</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>tech.easyflow</groupId>
|
||||
<artifactId>easyflow-common-mq</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>tech.easyflow</groupId>
|
||||
<artifactId>easyflow-common-web</artifactId>
|
||||
@@ -63,5 +67,11 @@
|
||||
<version>${junit.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-core</artifactId>
|
||||
<version>5.12.0</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
package tech.easyflow.agent.config;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.UUID;
|
||||
|
||||
/**
|
||||
* Agent 运行态生产化配置。
|
||||
@@ -15,6 +17,36 @@ public class AgentRuntimeProperties {
|
||||
*/
|
||||
private Duration sessionCacheTtl = Duration.ofHours(24);
|
||||
|
||||
/**
|
||||
* 当前 Agent 运行实例 ID。
|
||||
*/
|
||||
private String instanceId = defaultInstanceId();
|
||||
|
||||
/**
|
||||
* Agent 运行路由 TTL。
|
||||
*/
|
||||
private Duration routeTtl = Duration.ofHours(24);
|
||||
|
||||
/**
|
||||
* Agent 运行命令 topic 前缀。
|
||||
*/
|
||||
private String commandTopicPrefix = "easyflow:agent-runtime-command";
|
||||
|
||||
/**
|
||||
* Agent 运行命令结果等待超时时间。
|
||||
*/
|
||||
private Duration commandResultTimeout = Duration.ofSeconds(5);
|
||||
|
||||
/**
|
||||
* Agent 运行命令结果缓存 TTL。
|
||||
*/
|
||||
private Duration commandResultTtl = Duration.ofMinutes(5);
|
||||
|
||||
/**
|
||||
* 当前进程启动代 ID。
|
||||
*/
|
||||
private final String bootId = UUID.randomUUID().toString();
|
||||
|
||||
/**
|
||||
* HITL pending 默认过期时间。
|
||||
*/
|
||||
@@ -53,6 +85,107 @@ public class AgentRuntimeProperties {
|
||||
this.sessionCacheTtl = sessionCacheTtl == null ? Duration.ofHours(24) : sessionCacheTtl;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取当前 Agent 运行实例 ID。
|
||||
*
|
||||
* @return 实例 ID
|
||||
*/
|
||||
public String getInstanceId() {
|
||||
return instanceId;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置当前 Agent 运行实例 ID。
|
||||
*
|
||||
* @param instanceId 实例 ID
|
||||
*/
|
||||
public void setInstanceId(String instanceId) {
|
||||
this.instanceId = StringUtils.hasText(instanceId) ? instanceId.trim() : defaultInstanceId();
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取 Agent 运行路由 TTL。
|
||||
*
|
||||
* @return 路由 TTL
|
||||
*/
|
||||
public Duration getRouteTtl() {
|
||||
return routeTtl;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置 Agent 运行路由 TTL。
|
||||
*
|
||||
* @param routeTtl 路由 TTL
|
||||
*/
|
||||
public void setRouteTtl(Duration routeTtl) {
|
||||
this.routeTtl = routeTtl == null ? Duration.ofHours(24) : routeTtl;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取 Agent 运行命令 topic 前缀。
|
||||
*
|
||||
* @return 命令 topic 前缀
|
||||
*/
|
||||
public String getCommandTopicPrefix() {
|
||||
return commandTopicPrefix;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置 Agent 运行命令 topic 前缀。
|
||||
*
|
||||
* @param commandTopicPrefix 命令 topic 前缀
|
||||
*/
|
||||
public void setCommandTopicPrefix(String commandTopicPrefix) {
|
||||
this.commandTopicPrefix = StringUtils.hasText(commandTopicPrefix)
|
||||
? commandTopicPrefix.trim()
|
||||
: "easyflow:agent-runtime-command";
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取 Agent 运行命令结果等待超时时间。
|
||||
*
|
||||
* @return 等待超时时间
|
||||
*/
|
||||
public Duration getCommandResultTimeout() {
|
||||
return commandResultTimeout;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置 Agent 运行命令结果等待超时时间。
|
||||
*
|
||||
* @param commandResultTimeout 等待超时时间
|
||||
*/
|
||||
public void setCommandResultTimeout(Duration commandResultTimeout) {
|
||||
this.commandResultTimeout = commandResultTimeout == null ? Duration.ofSeconds(5) : commandResultTimeout;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取 Agent 运行命令结果缓存 TTL。
|
||||
*
|
||||
* @return 结果缓存 TTL
|
||||
*/
|
||||
public Duration getCommandResultTtl() {
|
||||
return commandResultTtl;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置 Agent 运行命令结果缓存 TTL。
|
||||
*
|
||||
* @param commandResultTtl 结果缓存 TTL
|
||||
*/
|
||||
public void setCommandResultTtl(Duration commandResultTtl) {
|
||||
this.commandResultTtl = commandResultTtl == null ? Duration.ofMinutes(5) : commandResultTtl;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取当前进程启动代 ID。
|
||||
*
|
||||
* @return 启动代 ID
|
||||
*/
|
||||
public String getBootId() {
|
||||
return bootId;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取 HITL pending 默认过期时间。
|
||||
*
|
||||
@@ -124,4 +257,16 @@ public class AgentRuntimeProperties {
|
||||
public void setLockRenewInterval(Duration lockRenewInterval) {
|
||||
this.lockRenewInterval = lockRenewInterval == null ? Duration.ofMinutes(1) : lockRenewInterval;
|
||||
}
|
||||
|
||||
private static String defaultInstanceId() {
|
||||
String envInstanceId = System.getenv("EASYFLOW_INSTANCE_ID");
|
||||
if (StringUtils.hasText(envInstanceId)) {
|
||||
return envInstanceId.trim();
|
||||
}
|
||||
String hostName = System.getenv("HOSTNAME");
|
||||
if (StringUtils.hasText(hostName)) {
|
||||
return hostName.trim();
|
||||
}
|
||||
return UUID.randomUUID().toString();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
package tech.easyflow.agent.distributed;
|
||||
|
||||
/**
|
||||
* Agent 运行态远程命令动作。
|
||||
*/
|
||||
public enum AgentRuntimeCommandAction {
|
||||
|
||||
/**
|
||||
* 批准工具执行。
|
||||
*/
|
||||
APPROVE,
|
||||
|
||||
/**
|
||||
* 拒绝工具执行。
|
||||
*/
|
||||
REJECT
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
package tech.easyflow.agent.distributed;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.agent.config.AgentRuntimeProperties;
|
||||
import tech.easyflow.agent.runtime.AgentRunService;
|
||||
import tech.easyflow.common.mq.config.MQProperties;
|
||||
import tech.easyflow.common.mq.core.MQConsumerHandler;
|
||||
import tech.easyflow.common.mq.core.MQMessage;
|
||||
import tech.easyflow.common.mq.core.MQSubscription;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Agent 运行态远程命令消费者。
|
||||
*/
|
||||
@Component
|
||||
public class AgentRuntimeCommandConsumer implements MQConsumerHandler {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(AgentRuntimeCommandConsumer.class);
|
||||
|
||||
private final ObjectMapper objectMapper;
|
||||
private final AgentRuntimeProperties properties;
|
||||
private final MQProperties mqProperties;
|
||||
private final AgentRunService agentRunService;
|
||||
private final AgentRuntimeCommandResultRegistry resultRegistry;
|
||||
|
||||
/**
|
||||
* 创建 Agent 运行态远程命令消费者。
|
||||
*
|
||||
* @param objectMapper JSON 序列化器
|
||||
* @param properties Agent 运行配置
|
||||
* @param mqProperties MQ 配置
|
||||
* @param agentRunService Agent 运行服务
|
||||
* @param resultRegistry 远程命令结果注册表
|
||||
*/
|
||||
public AgentRuntimeCommandConsumer(ObjectMapper objectMapper,
|
||||
AgentRuntimeProperties properties,
|
||||
MQProperties mqProperties,
|
||||
AgentRunService agentRunService,
|
||||
AgentRuntimeCommandResultRegistry resultRegistry) {
|
||||
this.objectMapper = objectMapper;
|
||||
this.properties = properties;
|
||||
this.mqProperties = mqProperties;
|
||||
this.agentRunService = agentRunService;
|
||||
this.resultRegistry = resultRegistry;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MQSubscription subscription() {
|
||||
MQSubscription subscription = new MQSubscription();
|
||||
subscription.setTopic(commandTopic());
|
||||
subscription.setConsumerGroup(commandTopic());
|
||||
subscription.setShardCount(Math.max(mqProperties.getRedis().getChatPersistShardCount(), 1));
|
||||
subscription.setBatchEnabled(false);
|
||||
return subscription;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void handle(List<MQMessage> messages) {
|
||||
if (messages == null || messages.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
for (MQMessage message : messages) {
|
||||
try {
|
||||
handleCommand(message, objectMapper.readValue(message.getBody(), AgentRuntimeCommandMessage.class));
|
||||
} catch (Exception e) {
|
||||
LOG.warn("Agent 远程运行命令解析失败: messageId={}", message.getMessageId(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void handleCommand(MQMessage message, AgentRuntimeCommandMessage command) {
|
||||
if (command == null || command.getAction() == null) {
|
||||
LOG.warn("跳过非法 Agent 远程运行命令: messageId={}", message.getMessageId());
|
||||
return;
|
||||
}
|
||||
if (!properties.getInstanceId().equals(command.getTargetNodeId())) {
|
||||
LOG.warn("跳过非本节点 Agent 远程运行命令: messageId={}, targetNodeId={}, currentNodeId={}",
|
||||
message.getMessageId(), command.getTargetNodeId(), properties.getInstanceId());
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (command.getAction() == AgentRuntimeCommandAction.APPROVE) {
|
||||
agentRunService.approveRuntimeLocal(
|
||||
command.getRequestId(), command.getResumeToken(), command.getOperatorId(), command.getUserId());
|
||||
} else if (command.getAction() == AgentRuntimeCommandAction.REJECT) {
|
||||
agentRunService.rejectRuntimeLocal(
|
||||
command.getRequestId(), command.getResumeToken(), command.getReason(),
|
||||
command.getOperatorId(), command.getUserId());
|
||||
} else {
|
||||
markFailureQuietly(command, new IllegalArgumentException("不支持的 Agent 远程运行命令"));
|
||||
LOG.warn("跳过不支持的 Agent 远程运行命令: messageId={}, commandId={}, action={}",
|
||||
message.getMessageId(), command.getCommandId(), command.getAction());
|
||||
return;
|
||||
}
|
||||
} catch (RuntimeException e) {
|
||||
markFailureQuietly(command, e);
|
||||
LOG.warn("Agent 远程运行命令处理失败: messageId={}, commandId={}",
|
||||
message.getMessageId(), command.getCommandId(), e);
|
||||
return;
|
||||
}
|
||||
markSuccessQuietly(command);
|
||||
}
|
||||
|
||||
private String commandTopic() {
|
||||
return properties.getCommandTopicPrefix() + ":" + properties.getInstanceId();
|
||||
}
|
||||
|
||||
private void markSuccessQuietly(AgentRuntimeCommandMessage command) {
|
||||
try {
|
||||
resultRegistry.markSuccess(command.getCommandId());
|
||||
} catch (RuntimeException e) {
|
||||
LOG.error("Agent 远程运行命令成功结果写入失败: commandId={}", command.getCommandId(), e);
|
||||
}
|
||||
}
|
||||
|
||||
private void markFailureQuietly(AgentRuntimeCommandMessage command, RuntimeException cause) {
|
||||
try {
|
||||
resultRegistry.markFailure(command.getCommandId(), cause.getMessage());
|
||||
} catch (RuntimeException e) {
|
||||
LOG.error("Agent 远程运行命令失败结果写入失败: commandId={}", command.getCommandId(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package tech.easyflow.agent.distributed;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* Agent 运行态远程恢复命令消息。
|
||||
*/
|
||||
public class AgentRuntimeCommandMessage {
|
||||
|
||||
private String commandId;
|
||||
private String requestId;
|
||||
private String resumeToken;
|
||||
private AgentRuntimeCommandAction action;
|
||||
private String reason;
|
||||
private BigInteger operatorId;
|
||||
private String userId;
|
||||
private String targetNodeId;
|
||||
private Date occurredAt;
|
||||
|
||||
public String getCommandId() {
|
||||
return commandId;
|
||||
}
|
||||
|
||||
public void setCommandId(String commandId) {
|
||||
this.commandId = commandId;
|
||||
}
|
||||
|
||||
public String getRequestId() {
|
||||
return requestId;
|
||||
}
|
||||
|
||||
public void setRequestId(String requestId) {
|
||||
this.requestId = requestId;
|
||||
}
|
||||
|
||||
public String getResumeToken() {
|
||||
return resumeToken;
|
||||
}
|
||||
|
||||
public void setResumeToken(String resumeToken) {
|
||||
this.resumeToken = resumeToken;
|
||||
}
|
||||
|
||||
public AgentRuntimeCommandAction getAction() {
|
||||
return action;
|
||||
}
|
||||
|
||||
public void setAction(AgentRuntimeCommandAction action) {
|
||||
this.action = action;
|
||||
}
|
||||
|
||||
public String getReason() {
|
||||
return reason;
|
||||
}
|
||||
|
||||
public void setReason(String reason) {
|
||||
this.reason = reason;
|
||||
}
|
||||
|
||||
public BigInteger getOperatorId() {
|
||||
return operatorId;
|
||||
}
|
||||
|
||||
public void setOperatorId(BigInteger operatorId) {
|
||||
this.operatorId = operatorId;
|
||||
}
|
||||
|
||||
public String getUserId() {
|
||||
return userId;
|
||||
}
|
||||
|
||||
public void setUserId(String userId) {
|
||||
this.userId = userId;
|
||||
}
|
||||
|
||||
public String getTargetNodeId() {
|
||||
return targetNodeId;
|
||||
}
|
||||
|
||||
public void setTargetNodeId(String targetNodeId) {
|
||||
this.targetNodeId = targetNodeId;
|
||||
}
|
||||
|
||||
public Date getOccurredAt() {
|
||||
return occurredAt;
|
||||
}
|
||||
|
||||
public void setOccurredAt(Date occurredAt) {
|
||||
this.occurredAt = occurredAt;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package tech.easyflow.agent.distributed;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tech.easyflow.agent.config.AgentRuntimeProperties;
|
||||
import tech.easyflow.common.mq.core.MQMessage;
|
||||
import tech.easyflow.common.mq.core.MQProducer;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
import java.util.UUID;
|
||||
|
||||
/**
|
||||
* Agent 运行态远程命令生产者。
|
||||
*/
|
||||
@Service
|
||||
public class AgentRuntimeCommandProducer {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(AgentRuntimeCommandProducer.class);
|
||||
|
||||
private final MQProducer mqProducer;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final AgentRuntimeProperties properties;
|
||||
private final AgentRuntimeCommandResultRegistry resultRegistry;
|
||||
|
||||
/**
|
||||
* 测试子类构造器。
|
||||
*/
|
||||
protected AgentRuntimeCommandProducer() {
|
||||
this.mqProducer = null;
|
||||
this.objectMapper = null;
|
||||
this.properties = null;
|
||||
this.resultRegistry = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建 Agent 运行态远程命令生产者。
|
||||
*
|
||||
* @param mqProducer MQ 生产者
|
||||
* @param objectMapper JSON 序列化器
|
||||
* @param properties Agent 运行配置
|
||||
* @param resultRegistry 远程命令结果注册表
|
||||
*/
|
||||
public AgentRuntimeCommandProducer(MQProducer mqProducer,
|
||||
ObjectMapper objectMapper,
|
||||
AgentRuntimeProperties properties,
|
||||
AgentRuntimeCommandResultRegistry resultRegistry) {
|
||||
this.mqProducer = mqProducer;
|
||||
this.objectMapper = objectMapper;
|
||||
this.properties = properties;
|
||||
this.resultRegistry = resultRegistry;
|
||||
}
|
||||
|
||||
/**
|
||||
* 投递远程批准命令。
|
||||
*
|
||||
* @param targetNodeId 目标节点 ID
|
||||
* @param requestId 请求 ID
|
||||
* @param resumeToken 恢复令牌
|
||||
* @param operatorId 操作人 ID
|
||||
* @param userId 用户 ID
|
||||
*/
|
||||
public void sendApprove(String targetNodeId,
|
||||
String requestId,
|
||||
String resumeToken,
|
||||
BigInteger operatorId,
|
||||
String userId) {
|
||||
sendAndWait(targetNodeId, requestId, resumeToken, AgentRuntimeCommandAction.APPROVE, null, operatorId, userId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 投递远程拒绝命令。
|
||||
*
|
||||
* @param targetNodeId 目标节点 ID
|
||||
* @param requestId 请求 ID
|
||||
* @param resumeToken 恢复令牌
|
||||
* @param reason 拒绝原因
|
||||
* @param operatorId 操作人 ID
|
||||
* @param userId 用户 ID
|
||||
*/
|
||||
public void sendReject(String targetNodeId,
|
||||
String requestId,
|
||||
String resumeToken,
|
||||
String reason,
|
||||
BigInteger operatorId,
|
||||
String userId) {
|
||||
sendAndWait(targetNodeId, requestId, resumeToken, AgentRuntimeCommandAction.REJECT, reason, operatorId, userId);
|
||||
}
|
||||
|
||||
private void sendAndWait(String targetNodeId,
|
||||
String requestId,
|
||||
String resumeToken,
|
||||
AgentRuntimeCommandAction action,
|
||||
String reason,
|
||||
BigInteger operatorId,
|
||||
String userId) {
|
||||
if (targetNodeId == null || targetNodeId.isBlank()) {
|
||||
throw new BusinessException("Agent 运行节点不可用,请重新发起对话");
|
||||
}
|
||||
AgentRuntimeCommandMessage command = new AgentRuntimeCommandMessage();
|
||||
command.setCommandId(UUID.randomUUID().toString());
|
||||
command.setRequestId(requestId);
|
||||
command.setResumeToken(resumeToken);
|
||||
command.setAction(action);
|
||||
command.setReason(reason);
|
||||
command.setOperatorId(operatorId);
|
||||
command.setUserId(userId);
|
||||
command.setTargetNodeId(targetNodeId);
|
||||
command.setOccurredAt(new Date());
|
||||
|
||||
MQMessage message = new MQMessage();
|
||||
message.setMessageId(command.getCommandId());
|
||||
message.setTopic(commandTopic(targetNodeId));
|
||||
message.setKey(command.getCommandId());
|
||||
message.setCreatedAt(command.getOccurredAt());
|
||||
try {
|
||||
message.setBody(objectMapper.writeValueAsString(command));
|
||||
String recordId = mqProducer.send(message);
|
||||
LOG.info("Agent 远程运行命令已投递: action={}, requestId={}, targetNodeId={}, recordId={}",
|
||||
action, requestId, targetNodeId, recordId);
|
||||
AgentRuntimeCommandResult result = resultRegistry.waitForResult(command.getCommandId());
|
||||
if (!result.isSuccess()) {
|
||||
throw new BusinessException(result.getMessage());
|
||||
}
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new BusinessException("Agent 运行命令序列化失败");
|
||||
} catch (BusinessException e) {
|
||||
throw e;
|
||||
} catch (RuntimeException e) {
|
||||
LOG.error("Agent 远程运行命令投递失败: action={}, requestId={}, targetNodeId={}",
|
||||
action, requestId, targetNodeId, e);
|
||||
throw new BusinessException("Agent 运行节点不可用,请重新发起对话");
|
||||
} finally {
|
||||
deleteResultQuietly(command.getCommandId());
|
||||
}
|
||||
}
|
||||
|
||||
private String commandTopic(String nodeId) {
|
||||
return properties.getCommandTopicPrefix() + ":" + nodeId;
|
||||
}
|
||||
|
||||
private void deleteResultQuietly(String commandId) {
|
||||
try {
|
||||
resultRegistry.deleteResult(commandId);
|
||||
} catch (RuntimeException e) {
|
||||
LOG.warn("Agent 远程运行命令结果清理失败,等待 TTL 兜底: commandId={}", commandId, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package tech.easyflow.agent.distributed;
|
||||
|
||||
/**
|
||||
* Agent 运行态远程命令结果。
|
||||
*/
|
||||
public class AgentRuntimeCommandResult {
|
||||
|
||||
private boolean success;
|
||||
private String message;
|
||||
|
||||
/**
|
||||
* 判断命令是否执行成功。
|
||||
*
|
||||
* @return true 表示执行成功
|
||||
*/
|
||||
public boolean isSuccess() {
|
||||
return success;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置命令是否执行成功。
|
||||
*
|
||||
* @param success 是否执行成功
|
||||
*/
|
||||
public void setSuccess(boolean success) {
|
||||
this.success = success;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取结果消息。
|
||||
*
|
||||
* @return 结果消息
|
||||
*/
|
||||
public String getMessage() {
|
||||
return message;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置结果消息。
|
||||
*
|
||||
* @param message 结果消息
|
||||
*/
|
||||
public void setMessage(String message) {
|
||||
this.message = message;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
package tech.easyflow.agent.distributed;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.agent.config.AgentRuntimeProperties;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
/**
|
||||
* Agent 运行态远程命令结果注册表。
|
||||
*/
|
||||
@Component
|
||||
public class AgentRuntimeCommandResultRegistry {
|
||||
|
||||
private static final String RESULT_PREFIX = "easyflow:agent:runtime:command-result:";
|
||||
private static final long POLL_INTERVAL_MILLIS = 50L;
|
||||
|
||||
private final StringRedisTemplate stringRedisTemplate;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final AgentRuntimeProperties properties;
|
||||
|
||||
/**
|
||||
* 创建 Agent 运行态远程命令结果注册表。
|
||||
*
|
||||
* @param stringRedisTemplate Redis 字符串模板
|
||||
* @param objectMapper JSON 序列化器
|
||||
* @param properties Agent 运行配置
|
||||
*/
|
||||
public AgentRuntimeCommandResultRegistry(StringRedisTemplate stringRedisTemplate,
|
||||
ObjectMapper objectMapper,
|
||||
AgentRuntimeProperties properties) {
|
||||
this.stringRedisTemplate = stringRedisTemplate;
|
||||
this.objectMapper = objectMapper;
|
||||
this.properties = properties;
|
||||
}
|
||||
|
||||
/**
|
||||
* 写入成功结果。
|
||||
*
|
||||
* @param commandId 命令 ID
|
||||
*/
|
||||
public void markSuccess(String commandId) {
|
||||
AgentRuntimeCommandResult result = new AgentRuntimeCommandResult();
|
||||
result.setSuccess(true);
|
||||
result.setMessage("OK");
|
||||
writeResult(commandId, result);
|
||||
}
|
||||
|
||||
/**
|
||||
* 写入失败结果。
|
||||
*
|
||||
* @param commandId 命令 ID
|
||||
* @param message 失败消息
|
||||
*/
|
||||
public void markFailure(String commandId, String message) {
|
||||
AgentRuntimeCommandResult result = new AgentRuntimeCommandResult();
|
||||
result.setSuccess(false);
|
||||
result.setMessage(message == null || message.isBlank() ? "Agent 运行节点不可用,请重新发起对话" : message);
|
||||
writeResult(commandId, result);
|
||||
}
|
||||
|
||||
/**
|
||||
* 等待远程命令结果。
|
||||
*
|
||||
* @param commandId 命令 ID
|
||||
* @return 命令结果
|
||||
*/
|
||||
public AgentRuntimeCommandResult waitForResult(String commandId) {
|
||||
long deadline = System.nanoTime() + properties.getCommandResultTimeout().toNanos();
|
||||
while (System.nanoTime() <= deadline) {
|
||||
AgentRuntimeCommandResult result = readResult(commandId);
|
||||
if (result != null) {
|
||||
return result;
|
||||
}
|
||||
sleep();
|
||||
}
|
||||
throw new BusinessException("Agent 运行节点响应超时,请稍后重试");
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除远程命令结果。
|
||||
*
|
||||
* @param commandId 命令 ID
|
||||
*/
|
||||
public void deleteResult(String commandId) {
|
||||
if (commandId == null || commandId.isBlank()) {
|
||||
return;
|
||||
}
|
||||
stringRedisTemplate.delete(resultKey(commandId));
|
||||
}
|
||||
|
||||
private AgentRuntimeCommandResult readResult(String commandId) {
|
||||
if (commandId == null || commandId.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
String value = stringRedisTemplate.opsForValue().get(resultKey(commandId));
|
||||
if (value == null || value.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return objectMapper.readValue(value, AgentRuntimeCommandResult.class);
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new BusinessException("Agent 运行命令结果解析失败");
|
||||
}
|
||||
}
|
||||
|
||||
private void writeResult(String commandId, AgentRuntimeCommandResult result) {
|
||||
if (commandId == null || commandId.isBlank()) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
stringRedisTemplate.opsForValue().set(
|
||||
resultKey(commandId),
|
||||
objectMapper.writeValueAsString(result),
|
||||
properties.getCommandResultTtl());
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new IllegalStateException("Agent 运行命令结果序列化失败", e);
|
||||
}
|
||||
}
|
||||
|
||||
private String resultKey(String commandId) {
|
||||
return RESULT_PREFIX + commandId;
|
||||
}
|
||||
|
||||
private void sleep() {
|
||||
try {
|
||||
Thread.sleep(POLL_INTERVAL_MILLIS);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new BusinessException("Agent 运行节点响应等待被中断");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package tech.easyflow.agent.distributed;
|
||||
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
/**
|
||||
* Agent 运行节点心跳维护器。
|
||||
*/
|
||||
@Component
|
||||
public class AgentRuntimeNodeHeartbeat {
|
||||
|
||||
private static final Duration HEARTBEAT_TTL = Duration.ofSeconds(90);
|
||||
|
||||
private final AgentRuntimeRouteRegistry routeRegistry;
|
||||
|
||||
/**
|
||||
* 创建 Agent 运行节点心跳维护器。
|
||||
*
|
||||
* @param routeRegistry Agent 运行态 Redis 路由注册表
|
||||
*/
|
||||
public AgentRuntimeNodeHeartbeat(AgentRuntimeRouteRegistry routeRegistry) {
|
||||
this.routeRegistry = routeRegistry;
|
||||
}
|
||||
|
||||
/**
|
||||
* 启动时立即写入一次当前节点心跳。
|
||||
*/
|
||||
@PostConstruct
|
||||
public void init() {
|
||||
refresh();
|
||||
}
|
||||
|
||||
/**
|
||||
* 定期刷新当前节点心跳。
|
||||
*/
|
||||
@Scheduled(fixedDelayString = "${easyflow.agent.runtime.node-heartbeat-delay:30000}", initialDelay = 30000L)
|
||||
public void refresh() {
|
||||
routeRegistry.heartbeat(HEARTBEAT_TTL);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package tech.easyflow.agent.distributed;
|
||||
|
||||
/**
|
||||
* Agent 运行态 owner 路由。
|
||||
*/
|
||||
public class AgentRuntimeRoute {
|
||||
|
||||
private String nodeId;
|
||||
private String bootId;
|
||||
|
||||
/**
|
||||
* 获取 owner 节点 ID。
|
||||
*
|
||||
* @return owner 节点 ID
|
||||
*/
|
||||
public String getNodeId() {
|
||||
return nodeId;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置 owner 节点 ID。
|
||||
*
|
||||
* @param nodeId owner 节点 ID
|
||||
*/
|
||||
public void setNodeId(String nodeId) {
|
||||
this.nodeId = nodeId;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取 owner 启动代 ID。
|
||||
*
|
||||
* @return 启动代 ID
|
||||
*/
|
||||
public String getBootId() {
|
||||
return bootId;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置 owner 启动代 ID。
|
||||
*
|
||||
* @param bootId 启动代 ID
|
||||
*/
|
||||
public void setBootId(String bootId) {
|
||||
this.bootId = bootId;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
package tech.easyflow.agent.distributed;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.agent.config.AgentRuntimeProperties;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
/**
|
||||
* Agent 运行态 Redis 路由注册表。
|
||||
*/
|
||||
@Component
|
||||
public class AgentRuntimeRouteRegistry {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(AgentRuntimeRouteRegistry.class);
|
||||
|
||||
private static final String REQUEST_ROUTE_PREFIX = "easyflow:agent:runtime:request:";
|
||||
private static final String TOKEN_ROUTE_PREFIX = "easyflow:agent:runtime:resume-token:";
|
||||
private static final String NODE_HEARTBEAT_PREFIX = "easyflow:agent:runtime:node:";
|
||||
|
||||
private final StringRedisTemplate stringRedisTemplate;
|
||||
private final AgentRuntimeProperties properties;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
/**
|
||||
* 创建 Agent 运行态 Redis 路由注册表。
|
||||
*
|
||||
* @param stringRedisTemplate Redis 字符串模板
|
||||
* @param properties Agent 运行配置
|
||||
* @param objectMapper JSON 序列化器
|
||||
*/
|
||||
public AgentRuntimeRouteRegistry(StringRedisTemplate stringRedisTemplate,
|
||||
AgentRuntimeProperties properties,
|
||||
ObjectMapper objectMapper) {
|
||||
this.stringRedisTemplate = stringRedisTemplate;
|
||||
this.properties = properties;
|
||||
this.objectMapper = objectMapper;
|
||||
}
|
||||
|
||||
/**
|
||||
* 注册运行请求 owner 节点。
|
||||
*
|
||||
* @param requestId 请求 ID
|
||||
*/
|
||||
public void registerRun(String requestId) {
|
||||
if (requestId == null || requestId.isBlank()) {
|
||||
return;
|
||||
}
|
||||
stringRedisTemplate.opsForValue().set(requestKey(requestId), serializeRoute(currentRoute()), properties.getRouteTtl());
|
||||
}
|
||||
|
||||
/**
|
||||
* 注册恢复令牌与请求 ID 的关系。
|
||||
*
|
||||
* @param requestId 请求 ID
|
||||
* @param resumeToken 恢复令牌
|
||||
*/
|
||||
public void registerResumeToken(String requestId, String resumeToken) {
|
||||
if (requestId == null || requestId.isBlank() || resumeToken == null || resumeToken.isBlank()) {
|
||||
return;
|
||||
}
|
||||
stringRedisTemplate.opsForValue().set(tokenKey(resumeToken), requestId, properties.getRouteTtl());
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询请求 ID 所属节点。
|
||||
*
|
||||
* @param requestId 请求 ID
|
||||
* @return owner 节点 ID
|
||||
*/
|
||||
public String findOwnerNode(String requestId) {
|
||||
AgentRuntimeRoute route = findOwnerRoute(requestId);
|
||||
return route == null ? null : route.getNodeId();
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询请求 ID 所属路由。
|
||||
*
|
||||
* @param requestId 请求 ID
|
||||
* @return owner 路由
|
||||
*/
|
||||
public AgentRuntimeRoute findOwnerRoute(String requestId) {
|
||||
if (requestId == null || requestId.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
String value = stringRedisTemplate.opsForValue().get(requestKey(requestId));
|
||||
if (value == null || value.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return deserializeRoute(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据恢复令牌查询请求 ID。
|
||||
*
|
||||
* @param resumeToken 恢复令牌
|
||||
* @return 请求 ID
|
||||
*/
|
||||
public String findRequestIdByResumeToken(String resumeToken) {
|
||||
if (resumeToken == null || resumeToken.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return stringRedisTemplate.opsForValue().get(tokenKey(resumeToken));
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除指定运行请求的路由。
|
||||
*
|
||||
* @param requestId 请求 ID
|
||||
*/
|
||||
public void removeRun(String requestId) {
|
||||
if (requestId == null || requestId.isBlank()) {
|
||||
return;
|
||||
}
|
||||
deleteQuietly(requestKey(requestId));
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除指定恢复令牌的路由。
|
||||
*
|
||||
* @param resumeToken 恢复令牌
|
||||
*/
|
||||
public void removeResumeToken(String resumeToken) {
|
||||
if (resumeToken == null || resumeToken.isBlank()) {
|
||||
return;
|
||||
}
|
||||
deleteQuietly(tokenKey(resumeToken));
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取当前节点 ID。
|
||||
*
|
||||
* @return 当前节点 ID
|
||||
*/
|
||||
public String currentNodeId() {
|
||||
return properties.getInstanceId();
|
||||
}
|
||||
|
||||
/**
|
||||
* 刷新当前节点存活心跳。
|
||||
*
|
||||
* @param ttl 心跳 TTL
|
||||
*/
|
||||
public void heartbeat(Duration ttl) {
|
||||
stringRedisTemplate.opsForValue().set(nodeKey(properties.getInstanceId()), properties.getBootId(), ttl);
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询指定节点是否仍有存活心跳。
|
||||
*
|
||||
* @param nodeId 节点 ID
|
||||
* @return true 表示节点心跳仍有效
|
||||
*/
|
||||
public boolean isNodeAlive(String nodeId) {
|
||||
return currentNodeBootId(nodeId) != null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询指定节点当前启动代 ID。
|
||||
*
|
||||
* @param nodeId 节点 ID
|
||||
* @return 启动代 ID
|
||||
*/
|
||||
public String currentNodeBootId(String nodeId) {
|
||||
if (nodeId == null || nodeId.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return stringRedisTemplate.opsForValue().get(nodeKey(nodeId));
|
||||
}
|
||||
|
||||
private String requestKey(String requestId) {
|
||||
return REQUEST_ROUTE_PREFIX + requestId;
|
||||
}
|
||||
|
||||
private String tokenKey(String resumeToken) {
|
||||
return TOKEN_ROUTE_PREFIX + resumeToken;
|
||||
}
|
||||
|
||||
private String nodeKey(String nodeId) {
|
||||
return NODE_HEARTBEAT_PREFIX + nodeId;
|
||||
}
|
||||
|
||||
private AgentRuntimeRoute currentRoute() {
|
||||
AgentRuntimeRoute route = new AgentRuntimeRoute();
|
||||
route.setNodeId(properties.getInstanceId());
|
||||
route.setBootId(properties.getBootId());
|
||||
return route;
|
||||
}
|
||||
|
||||
private String serializeRoute(AgentRuntimeRoute route) {
|
||||
try {
|
||||
return objectMapper.writeValueAsString(route);
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new IllegalStateException("Agent 运行路由序列化失败", e);
|
||||
}
|
||||
}
|
||||
|
||||
private AgentRuntimeRoute deserializeRoute(String value) {
|
||||
try {
|
||||
if (value.trim().startsWith("{")) {
|
||||
return objectMapper.readValue(value, AgentRuntimeRoute.class);
|
||||
}
|
||||
AgentRuntimeRoute legacyRoute = new AgentRuntimeRoute();
|
||||
legacyRoute.setNodeId(value);
|
||||
return legacyRoute;
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new IllegalStateException("Agent 运行路由反序列化失败", e);
|
||||
}
|
||||
}
|
||||
|
||||
private void deleteQuietly(String key) {
|
||||
try {
|
||||
stringRedisTemplate.delete(key);
|
||||
} catch (RuntimeException e) {
|
||||
LOG.warn("清理 Agent 运行态 Redis 路由失败: key={}", key, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6,8 +6,10 @@ import com.easyagents.agent.runtime.event.AgentRuntimeEvent;
|
||||
import com.easyagents.agent.runtime.hitl.AgentResumeToken;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
import reactor.core.Disposable;
|
||||
import tech.easyflow.agent.distributed.AgentRuntimeRouteRegistry;
|
||||
import tech.easyflow.agent.runtime.lock.AgentRunLock;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.core.chat.protocol.sse.ChatSseEmitter;
|
||||
@@ -34,6 +36,17 @@ public class AgentRunRegistry {
|
||||
private final Map<String, String> resumeTokenIndex = new ConcurrentHashMap<>();
|
||||
private final Map<String, Set<String>> requestTokens = new ConcurrentHashMap<>();
|
||||
private final Map<String, RunOwner> owners = new ConcurrentHashMap<>();
|
||||
private AgentRuntimeRouteRegistry routeRegistry;
|
||||
|
||||
/**
|
||||
* 设置 Agent 运行态 Redis 路由注册表。
|
||||
*
|
||||
* @param routeRegistry Redis 路由注册表
|
||||
*/
|
||||
@Autowired(required = false)
|
||||
public void setRouteRegistry(AgentRuntimeRouteRegistry routeRegistry) {
|
||||
this.routeRegistry = routeRegistry;
|
||||
}
|
||||
|
||||
/**
|
||||
* 注册运行态。
|
||||
@@ -57,6 +70,9 @@ public class AgentRunRegistry {
|
||||
throw new BusinessException("当前 Agent 运行请求已存在");
|
||||
}
|
||||
owners.put(context.requestId(), context.owner());
|
||||
if (routeRegistry != null) {
|
||||
routeRegistry.registerRun(context.requestId());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -126,6 +142,9 @@ public class AgentRunRegistry {
|
||||
if (requestId != null && resumeToken != null && !resumeToken.isBlank()) {
|
||||
resumeTokenIndex.put(resumeToken, requestId);
|
||||
requestTokens.computeIfAbsent(requestId, ignored -> ConcurrentHashMap.newKeySet()).add(resumeToken);
|
||||
if (routeRegistry != null) {
|
||||
routeRegistry.registerResumeToken(requestId, resumeToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -147,7 +166,15 @@ public class AgentRunRegistry {
|
||||
owners.remove(requestId);
|
||||
Set<String> tokens = requestTokens.remove(requestId);
|
||||
if (tokens != null) {
|
||||
tokens.forEach(resumeTokenIndex::remove);
|
||||
tokens.forEach(token -> {
|
||||
resumeTokenIndex.remove(token);
|
||||
if (routeRegistry != null) {
|
||||
routeRegistry.removeResumeToken(token);
|
||||
}
|
||||
});
|
||||
}
|
||||
if (routeRegistry != null) {
|
||||
routeRegistry.removeRun(requestId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -257,6 +284,9 @@ public class AgentRunRegistry {
|
||||
tokens.remove(resumeToken);
|
||||
}
|
||||
resumeTokenIndex.remove(resumeToken);
|
||||
if (routeRegistry != null) {
|
||||
routeRegistry.removeResumeToken(resumeToken);
|
||||
}
|
||||
AgentResumeToken token = new AgentResumeToken();
|
||||
token.setValue(resumeToken);
|
||||
AgentResumeRequest request = new AgentResumeRequest();
|
||||
|
||||
@@ -19,6 +19,10 @@ import tech.easyflow.agent.entity.Agent;
|
||||
import tech.easyflow.agent.entity.AgentKnowledgeBinding;
|
||||
import tech.easyflow.agent.entity.AgentToolBinding;
|
||||
import tech.easyflow.agent.enums.AgentToolType;
|
||||
import tech.easyflow.agent.distributed.AgentRuntimeCommandAction;
|
||||
import tech.easyflow.agent.distributed.AgentRuntimeCommandProducer;
|
||||
import tech.easyflow.agent.distributed.AgentRuntimeRoute;
|
||||
import tech.easyflow.agent.distributed.AgentRuntimeRouteRegistry;
|
||||
import tech.easyflow.agent.runtime.event.AgentRunEventRecorder;
|
||||
import tech.easyflow.agent.runtime.hitl.AgentHitlPendingService;
|
||||
import tech.easyflow.agent.runtime.lock.AgentRunLock;
|
||||
@@ -78,6 +82,10 @@ public class AgentRunService {
|
||||
@Resource
|
||||
private AgentRunRegistry agentRunRegistry;
|
||||
@Resource
|
||||
private AgentRuntimeRouteRegistry agentRuntimeRouteRegistry;
|
||||
@Resource
|
||||
private AgentRuntimeCommandProducer agentRuntimeCommandProducer;
|
||||
@Resource
|
||||
private AgentRunLock agentRunLock;
|
||||
@Resource
|
||||
private AgentHitlPendingService agentHitlPendingService;
|
||||
@@ -231,6 +239,22 @@ public class AgentRunService {
|
||||
}
|
||||
|
||||
private void approveRuntime(String requestId, String resumeToken, BigInteger operatorId, String userId) {
|
||||
if (!agentRunRegistry.containsResumeTarget(requestId, resumeToken)) {
|
||||
dispatchRemoteRuntimeCommand(requestId, resumeToken, AgentRuntimeCommandAction.APPROVE, null, operatorId, userId);
|
||||
return;
|
||||
}
|
||||
approveRuntimeLocal(requestId, resumeToken, operatorId, userId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 在当前节点批准工具执行。
|
||||
*
|
||||
* @param requestId 请求 ID
|
||||
* @param resumeToken 恢复令牌
|
||||
* @param operatorId 操作人 ID
|
||||
* @param userId 用户 ID
|
||||
*/
|
||||
public void approveRuntimeLocal(String requestId, String resumeToken, BigInteger operatorId, String userId) {
|
||||
if (agentRunRegistry.isDraftResumeTarget(requestId, resumeToken)) {
|
||||
agentRunRegistry.approve(requestId, resumeToken, userId);
|
||||
return;
|
||||
@@ -252,6 +276,23 @@ public class AgentRunService {
|
||||
}
|
||||
|
||||
private void rejectRuntime(String requestId, String resumeToken, String reason, BigInteger operatorId, String userId) {
|
||||
if (!agentRunRegistry.containsResumeTarget(requestId, resumeToken)) {
|
||||
dispatchRemoteRuntimeCommand(requestId, resumeToken, AgentRuntimeCommandAction.REJECT, reason, operatorId, userId);
|
||||
return;
|
||||
}
|
||||
rejectRuntimeLocal(requestId, resumeToken, reason, operatorId, userId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 在当前节点拒绝工具执行。
|
||||
*
|
||||
* @param requestId 请求 ID
|
||||
* @param resumeToken 恢复令牌
|
||||
* @param reason 拒绝原因
|
||||
* @param operatorId 操作人 ID
|
||||
* @param userId 用户 ID
|
||||
*/
|
||||
public void rejectRuntimeLocal(String requestId, String resumeToken, String reason, BigInteger operatorId, String userId) {
|
||||
if (agentRunRegistry.isDraftResumeTarget(requestId, resumeToken)) {
|
||||
agentRunRegistry.reject(requestId, resumeToken, userId, reason);
|
||||
return;
|
||||
@@ -260,6 +301,46 @@ public class AgentRunService {
|
||||
() -> agentHitlPendingService.reject(resumeToken, operatorId, reason));
|
||||
}
|
||||
|
||||
private void dispatchRemoteRuntimeCommand(String requestId,
|
||||
String resumeToken,
|
||||
AgentRuntimeCommandAction action,
|
||||
String reason,
|
||||
BigInteger operatorId,
|
||||
String userId) {
|
||||
String resolvedRequestId = resolveRequestIdForRemoteDispatch(requestId, resumeToken);
|
||||
AgentRuntimeRoute ownerRoute = agentRuntimeRouteRegistry.findOwnerRoute(resolvedRequestId);
|
||||
String ownerNodeId = ownerRoute == null ? null : ownerRoute.getNodeId();
|
||||
if (ownerNodeId == null || ownerNodeId.isBlank()) {
|
||||
throw new BusinessException("Agent 运行节点不可用,请重新发起对话");
|
||||
}
|
||||
if (ownerNodeId.equals(agentRuntimeRouteRegistry.currentNodeId())) {
|
||||
throw new BusinessException("Agent 运行节点不可用,请重新发起对话");
|
||||
}
|
||||
if (!agentRuntimeRouteRegistry.isNodeAlive(ownerNodeId)) {
|
||||
throw new BusinessException("Agent 运行节点不可用,请重新发起对话");
|
||||
}
|
||||
String currentOwnerBootId = agentRuntimeRouteRegistry.currentNodeBootId(ownerNodeId);
|
||||
if (ownerRoute.getBootId() == null || !ownerRoute.getBootId().equals(currentOwnerBootId)) {
|
||||
throw new BusinessException("Agent 运行节点不可用,请重新发起对话");
|
||||
}
|
||||
if (action == AgentRuntimeCommandAction.APPROVE) {
|
||||
agentRuntimeCommandProducer.sendApprove(ownerNodeId, resolvedRequestId, resumeToken, operatorId, userId);
|
||||
return;
|
||||
}
|
||||
agentRuntimeCommandProducer.sendReject(ownerNodeId, resolvedRequestId, resumeToken, reason, operatorId, userId);
|
||||
}
|
||||
|
||||
private String resolveRequestIdForRemoteDispatch(String requestId, String resumeToken) {
|
||||
if (requestId != null && !requestId.isBlank()) {
|
||||
return requestId;
|
||||
}
|
||||
String resolvedRequestId = agentRuntimeRouteRegistry.findRequestIdByResumeToken(resumeToken);
|
||||
if (resolvedRequestId == null || resolvedRequestId.isBlank()) {
|
||||
throw new BusinessException("Agent 运行节点不可用,请重新发起对话");
|
||||
}
|
||||
return resolvedRequestId;
|
||||
}
|
||||
|
||||
private void startRuntime(Agent agent,
|
||||
String prompt,
|
||||
String requestId,
|
||||
|
||||
@@ -5,6 +5,7 @@ import org.slf4j.LoggerFactory;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.agent.entity.AgentHitlPending;
|
||||
import tech.easyflow.common.cache.DistributedScheduledLock;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@@ -32,6 +33,7 @@ public class AgentHitlPendingExpirationTask {
|
||||
* 定期将超时 pending 标记为 EXPIRED。
|
||||
*/
|
||||
@Scheduled(fixedDelayString = "${easyflow.agent.runtime.hitl-expire-scan-delay:60000}", initialDelay = 60000L)
|
||||
@DistributedScheduledLock(key = "easyflow:schedule:agent-hitl:expire-pending", leaseSeconds = 300L)
|
||||
public void expirePending() {
|
||||
try {
|
||||
List<AgentHitlPending> expired = pendingService.expirePending(BATCH_SIZE);
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
package tech.easyflow.agent.distributed;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import tech.easyflow.agent.config.AgentRuntimeProperties;
|
||||
import tech.easyflow.agent.distributed.AgentRuntimeCommandAction;
|
||||
import tech.easyflow.agent.distributed.AgentRuntimeCommandConsumer;
|
||||
import tech.easyflow.agent.distributed.AgentRuntimeCommandMessage;
|
||||
import tech.easyflow.agent.distributed.AgentRuntimeCommandResultRegistry;
|
||||
import tech.easyflow.agent.runtime.AgentRunService;
|
||||
import tech.easyflow.common.mq.config.MQProperties;
|
||||
import tech.easyflow.common.mq.core.MQMessage;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* {@link AgentRuntimeCommandConsumer} 回归测试。
|
||||
*/
|
||||
public class AgentRuntimeCommandConsumerTest {
|
||||
|
||||
/**
|
||||
* 验证消费者只处理发给当前节点的命令。
|
||||
*
|
||||
* @throws Exception 消息序列化异常
|
||||
*/
|
||||
@Test
|
||||
public void consumerShouldHandleOnlyCurrentNodeCommand() throws Exception {
|
||||
AgentRuntimeProperties properties = new AgentRuntimeProperties();
|
||||
properties.setInstanceId("node-a");
|
||||
MQProperties mqProperties = new MQProperties();
|
||||
mqProperties.getRedis().setChatPersistShardCount(4);
|
||||
RecordingAgentRunService service = new RecordingAgentRunService();
|
||||
RecordingCommandResultRegistry resultRegistry = new RecordingCommandResultRegistry();
|
||||
AgentRuntimeCommandConsumer consumer =
|
||||
new AgentRuntimeCommandConsumer(new ObjectMapper(), properties, mqProperties, service, resultRegistry);
|
||||
|
||||
consumer.handle(List.of(message(command("cmd-1", "node-b")), message(command("cmd-2", "node-a"))));
|
||||
|
||||
Assert.assertEquals(1, service.approveCount);
|
||||
Assert.assertEquals("request-cmd-2", service.lastRequestId);
|
||||
Assert.assertEquals(4, consumer.subscription().getShardCount());
|
||||
Assert.assertFalse(consumer.subscription().isBatchEnabled());
|
||||
Assert.assertEquals("cmd-2", resultRegistry.lastSuccessCommandId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 owner 本机执行失败时写入失败结果,避免 MQ 重试重复消费一次性 token。
|
||||
*
|
||||
* @throws Exception 消息序列化异常
|
||||
*/
|
||||
@Test
|
||||
public void consumerShouldMarkFailureWhenLocalRuntimeFails() throws Exception {
|
||||
AgentRuntimeProperties properties = new AgentRuntimeProperties();
|
||||
properties.setInstanceId("node-a");
|
||||
MQProperties mqProperties = new MQProperties();
|
||||
FailingAgentRunService service = new FailingAgentRunService();
|
||||
RecordingCommandResultRegistry resultRegistry = new RecordingCommandResultRegistry();
|
||||
AgentRuntimeCommandConsumer consumer =
|
||||
new AgentRuntimeCommandConsumer(new ObjectMapper(), properties, mqProperties, service, resultRegistry);
|
||||
|
||||
consumer.handle(List.of(message(command("cmd-1", "node-a"))));
|
||||
|
||||
Assert.assertEquals("cmd-1", resultRegistry.lastFailureCommandId);
|
||||
Assert.assertEquals("runtime missing", resultRegistry.lastFailureMessage);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证成功结果写入失败不会再次执行或改写为失败结果。
|
||||
*
|
||||
* @throws Exception 消息序列化异常
|
||||
*/
|
||||
@Test
|
||||
public void consumerShouldNotMarkFailureWhenSuccessResultWriteFails() throws Exception {
|
||||
AgentRuntimeProperties properties = new AgentRuntimeProperties();
|
||||
properties.setInstanceId("node-a");
|
||||
MQProperties mqProperties = new MQProperties();
|
||||
RecordingAgentRunService service = new RecordingAgentRunService();
|
||||
FailingSuccessResultRegistry resultRegistry = new FailingSuccessResultRegistry();
|
||||
AgentRuntimeCommandConsumer consumer =
|
||||
new AgentRuntimeCommandConsumer(new ObjectMapper(), properties, mqProperties, service, resultRegistry);
|
||||
|
||||
consumer.handle(List.of(message(command("cmd-1", "node-a"))));
|
||||
|
||||
Assert.assertEquals(1, service.approveCount);
|
||||
Assert.assertNull(resultRegistry.lastFailureCommandId);
|
||||
}
|
||||
|
||||
private AgentRuntimeCommandMessage command(String commandId, String targetNodeId) {
|
||||
AgentRuntimeCommandMessage command = new AgentRuntimeCommandMessage();
|
||||
command.setCommandId(commandId);
|
||||
command.setRequestId("request-" + commandId);
|
||||
command.setResumeToken("token-" + commandId);
|
||||
command.setAction(AgentRuntimeCommandAction.APPROVE);
|
||||
command.setOperatorId(BigInteger.ONE);
|
||||
command.setUserId("1");
|
||||
command.setTargetNodeId(targetNodeId);
|
||||
return command;
|
||||
}
|
||||
|
||||
private MQMessage message(AgentRuntimeCommandMessage command) throws Exception {
|
||||
MQMessage message = new MQMessage();
|
||||
message.setMessageId(command.getCommandId());
|
||||
message.setBody(new ObjectMapper().writeValueAsString(command));
|
||||
return message;
|
||||
}
|
||||
|
||||
private static final class RecordingAgentRunService extends AgentRunService {
|
||||
|
||||
private int approveCount;
|
||||
private String lastRequestId;
|
||||
|
||||
@Override
|
||||
public void approveRuntimeLocal(String requestId, String resumeToken, BigInteger operatorId, String userId) {
|
||||
approveCount++;
|
||||
lastRequestId = requestId;
|
||||
}
|
||||
}
|
||||
|
||||
private static class RecordingCommandResultRegistry extends AgentRuntimeCommandResultRegistry {
|
||||
|
||||
private String lastSuccessCommandId;
|
||||
String lastFailureCommandId;
|
||||
private String lastFailureMessage;
|
||||
|
||||
private RecordingCommandResultRegistry() {
|
||||
super(null, null, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void markSuccess(String commandId) {
|
||||
lastSuccessCommandId = commandId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void markFailure(String commandId, String message) {
|
||||
lastFailureCommandId = commandId;
|
||||
lastFailureMessage = message;
|
||||
}
|
||||
}
|
||||
|
||||
private static final class FailingAgentRunService extends AgentRunService {
|
||||
|
||||
@Override
|
||||
public void approveRuntimeLocal(String requestId, String resumeToken, BigInteger operatorId, String userId) {
|
||||
throw new RuntimeException("runtime missing");
|
||||
}
|
||||
}
|
||||
|
||||
private static final class FailingSuccessResultRegistry extends RecordingCommandResultRegistry {
|
||||
|
||||
@Override
|
||||
public void markSuccess(String commandId) {
|
||||
super.markSuccess(commandId);
|
||||
throw new RuntimeException("redis down");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package tech.easyflow.agent.distributed;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.ArgumentMatchers;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.data.redis.core.ValueOperations;
|
||||
import tech.easyflow.agent.config.AgentRuntimeProperties;
|
||||
import tech.easyflow.agent.distributed.AgentRuntimeCommandResult;
|
||||
import tech.easyflow.agent.distributed.AgentRuntimeCommandResultRegistry;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
/**
|
||||
* {@link AgentRuntimeCommandResultRegistry} 回归测试。
|
||||
*/
|
||||
public class AgentRuntimeCommandResultRegistryTest {
|
||||
|
||||
/**
|
||||
* 验证成功结果可被等待方读取。
|
||||
*/
|
||||
@Test
|
||||
public void waitForResultShouldReturnSuccessResult() {
|
||||
StringRedisTemplate redisTemplate = Mockito.mock(StringRedisTemplate.class);
|
||||
@SuppressWarnings("unchecked")
|
||||
ValueOperations<String, String> valueOperations = Mockito.mock(ValueOperations.class);
|
||||
Mockito.when(redisTemplate.opsForValue()).thenReturn(valueOperations);
|
||||
Mockito.when(valueOperations.get("easyflow:agent:runtime:command-result:cmd-1"))
|
||||
.thenReturn("{\"success\":true,\"message\":\"OK\"}");
|
||||
AgentRuntimeCommandResultRegistry registry = registry(redisTemplate);
|
||||
|
||||
AgentRuntimeCommandResult result = registry.waitForResult("cmd-1");
|
||||
|
||||
Assert.assertTrue(result.isSuccess());
|
||||
Assert.assertEquals("OK", result.getMessage());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证写入失败结果时使用配置的 TTL。
|
||||
*/
|
||||
@Test
|
||||
public void markFailureShouldWriteResultWithTtl() {
|
||||
StringRedisTemplate redisTemplate = Mockito.mock(StringRedisTemplate.class);
|
||||
@SuppressWarnings("unchecked")
|
||||
ValueOperations<String, String> valueOperations = Mockito.mock(ValueOperations.class);
|
||||
Mockito.when(redisTemplate.opsForValue()).thenReturn(valueOperations);
|
||||
AgentRuntimeProperties properties = properties();
|
||||
AgentRuntimeCommandResultRegistry registry =
|
||||
new AgentRuntimeCommandResultRegistry(redisTemplate, new ObjectMapper(), properties);
|
||||
|
||||
registry.markFailure("cmd-1", "failed");
|
||||
|
||||
Mockito.verify(valueOperations).set(
|
||||
ArgumentMatchers.eq("easyflow:agent:runtime:command-result:cmd-1"),
|
||||
ArgumentMatchers.contains("\"success\":false"),
|
||||
ArgumentMatchers.eq(properties.getCommandResultTtl()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证等待超时时抛出明确业务异常。
|
||||
*/
|
||||
@Test
|
||||
public void waitForResultShouldThrowBusinessExceptionWhenTimeout() {
|
||||
StringRedisTemplate redisTemplate = Mockito.mock(StringRedisTemplate.class);
|
||||
@SuppressWarnings("unchecked")
|
||||
ValueOperations<String, String> valueOperations = Mockito.mock(ValueOperations.class);
|
||||
Mockito.when(redisTemplate.opsForValue()).thenReturn(valueOperations);
|
||||
Mockito.when(valueOperations.get(ArgumentMatchers.anyString())).thenReturn(null);
|
||||
AgentRuntimeCommandResultRegistry registry = registry(redisTemplate);
|
||||
|
||||
BusinessException exception = Assert.assertThrows(
|
||||
BusinessException.class,
|
||||
() -> registry.waitForResult("cmd-1"));
|
||||
|
||||
Assert.assertEquals("Agent 运行节点响应超时,请稍后重试", exception.getMessage());
|
||||
}
|
||||
|
||||
private AgentRuntimeCommandResultRegistry registry(StringRedisTemplate redisTemplate) {
|
||||
return new AgentRuntimeCommandResultRegistry(redisTemplate, new ObjectMapper(), properties());
|
||||
}
|
||||
|
||||
private AgentRuntimeProperties properties() {
|
||||
AgentRuntimeProperties properties = new AgentRuntimeProperties();
|
||||
properties.setCommandResultTimeout(Duration.ofMillis(10));
|
||||
properties.setCommandResultTtl(Duration.ofMinutes(5));
|
||||
return properties;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package tech.easyflow.agent.distributed;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.ArgumentMatchers;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.data.redis.core.ValueOperations;
|
||||
import tech.easyflow.agent.config.AgentRuntimeProperties;
|
||||
import tech.easyflow.agent.distributed.AgentRuntimeRouteRegistry;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
/**
|
||||
* {@link AgentRuntimeRouteRegistry} 回归测试。
|
||||
*/
|
||||
public class AgentRuntimeRouteRegistryTest {
|
||||
|
||||
/**
|
||||
* 验证注册运行态和恢复令牌时写入 Redis 路由。
|
||||
*/
|
||||
@Test
|
||||
public void registerShouldWriteRunAndTokenRoutes() {
|
||||
StringRedisTemplate redisTemplate = Mockito.mock(StringRedisTemplate.class);
|
||||
@SuppressWarnings("unchecked")
|
||||
ValueOperations<String, String> valueOperations = Mockito.mock(ValueOperations.class);
|
||||
Mockito.when(redisTemplate.opsForValue()).thenReturn(valueOperations);
|
||||
AgentRuntimeProperties properties = properties("node-a");
|
||||
AgentRuntimeRouteRegistry registry = registry(redisTemplate, properties);
|
||||
|
||||
registry.registerRun("request-1");
|
||||
registry.registerResumeToken("request-1", "token-1");
|
||||
|
||||
Mockito.verify(valueOperations).set(
|
||||
ArgumentMatchers.eq("easyflow:agent:runtime:request:request-1"),
|
||||
ArgumentMatchers.contains("\"nodeId\":\"node-a\""),
|
||||
ArgumentMatchers.eq(Duration.ofHours(24)));
|
||||
Mockito.verify(valueOperations).set(
|
||||
"easyflow:agent:runtime:resume-token:token-1", "request-1", Duration.ofHours(24));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证运行结束时清理 Redis 路由。
|
||||
*/
|
||||
@Test
|
||||
public void removeShouldDeleteRunAndTokenRoutes() {
|
||||
StringRedisTemplate redisTemplate = Mockito.mock(StringRedisTemplate.class);
|
||||
AgentRuntimeRouteRegistry registry = registry(redisTemplate, properties("node-a"));
|
||||
|
||||
registry.removeRun("request-1");
|
||||
registry.removeResumeToken("token-1");
|
||||
|
||||
Mockito.verify(redisTemplate).delete("easyflow:agent:runtime:request:request-1");
|
||||
Mockito.verify(redisTemplate).delete("easyflow:agent:runtime:resume-token:token-1");
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证查询 owner 节点和 token 反查请求 ID。
|
||||
*/
|
||||
@Test
|
||||
public void findShouldReadRoutes() {
|
||||
StringRedisTemplate redisTemplate = Mockito.mock(StringRedisTemplate.class);
|
||||
@SuppressWarnings("unchecked")
|
||||
ValueOperations<String, String> valueOperations = Mockito.mock(ValueOperations.class);
|
||||
Mockito.when(redisTemplate.opsForValue()).thenReturn(valueOperations);
|
||||
Mockito.when(valueOperations.get(ArgumentMatchers.eq("easyflow:agent:runtime:request:request-1")))
|
||||
.thenReturn("{\"nodeId\":\"node-a\",\"bootId\":\"boot-a\"}");
|
||||
Mockito.when(valueOperations.get(ArgumentMatchers.eq("easyflow:agent:runtime:resume-token:token-1")))
|
||||
.thenReturn("request-1");
|
||||
AgentRuntimeRouteRegistry registry = registry(redisTemplate, properties("node-a"));
|
||||
|
||||
Assert.assertEquals("node-a", registry.findOwnerNode("request-1"));
|
||||
Assert.assertEquals("boot-a", registry.findOwnerRoute("request-1").getBootId());
|
||||
Assert.assertEquals("request-1", registry.findRequestIdByResumeToken("token-1"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证节点心跳写入和存活查询。
|
||||
*/
|
||||
@Test
|
||||
public void heartbeatShouldWriteAndReadNodeAliveState() {
|
||||
StringRedisTemplate redisTemplate = Mockito.mock(StringRedisTemplate.class);
|
||||
@SuppressWarnings("unchecked")
|
||||
ValueOperations<String, String> valueOperations = Mockito.mock(ValueOperations.class);
|
||||
Mockito.when(redisTemplate.opsForValue()).thenReturn(valueOperations);
|
||||
AgentRuntimeProperties properties = properties("node-a");
|
||||
Mockito.when(valueOperations.get("easyflow:agent:runtime:node:node-a")).thenReturn(properties.getBootId());
|
||||
AgentRuntimeRouteRegistry registry = registry(redisTemplate, properties);
|
||||
|
||||
registry.heartbeat(Duration.ofSeconds(90));
|
||||
|
||||
Mockito.verify(valueOperations).set("easyflow:agent:runtime:node:node-a", properties.getBootId(), Duration.ofSeconds(90));
|
||||
Assert.assertTrue(registry.isNodeAlive("node-a"));
|
||||
Assert.assertEquals(properties.getBootId(), registry.currentNodeBootId("node-a"));
|
||||
}
|
||||
|
||||
private AgentRuntimeProperties properties(String instanceId) {
|
||||
AgentRuntimeProperties properties = new AgentRuntimeProperties();
|
||||
properties.setInstanceId(instanceId);
|
||||
properties.setRouteTtl(Duration.ofHours(24));
|
||||
return properties;
|
||||
}
|
||||
|
||||
private AgentRuntimeRouteRegistry registry(StringRedisTemplate redisTemplate, AgentRuntimeProperties properties) {
|
||||
return new AgentRuntimeRouteRegistry(redisTemplate, properties, new ObjectMapper());
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,9 @@ import tech.easyflow.agent.entity.AgentHitlPending;
|
||||
import tech.easyflow.agent.entity.Agent;
|
||||
import tech.easyflow.agent.entity.AgentKnowledgeBinding;
|
||||
import tech.easyflow.agent.entity.AgentToolBinding;
|
||||
import tech.easyflow.agent.distributed.AgentRuntimeCommandProducer;
|
||||
import tech.easyflow.agent.distributed.AgentRuntimeRoute;
|
||||
import tech.easyflow.agent.distributed.AgentRuntimeRouteRegistry;
|
||||
import tech.easyflow.agent.runtime.event.AgentRunEventRecorder;
|
||||
import tech.easyflow.agent.runtime.hitl.AgentHitlPendingService;
|
||||
import tech.easyflow.agent.runtime.lock.AgentRunLock;
|
||||
@@ -532,6 +535,139 @@ public class AgentRunServiceDraftAndHitlTest {
|
||||
Assert.assertEquals(1, pendingService.approveCount);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证本机存在恢复目标时不投递远程命令。
|
||||
*
|
||||
* @throws Exception 反射调用失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void approveShouldNotDispatchRemoteWhenLocalRuntimeExists() throws Exception {
|
||||
AgentRunService service = new AgentRunService();
|
||||
AgentRunRegistry registry = new AgentRunRegistry();
|
||||
RecordingAgentHitlPendingService pendingService = new RecordingAgentHitlPendingService();
|
||||
RecordingRouteRegistry routeRegistry = new RecordingRouteRegistry("node-a");
|
||||
RecordingCommandProducer commandProducer = new RecordingCommandProducer();
|
||||
setField(service, "agentRunRegistry", registry);
|
||||
setField(service, "agentHitlPendingService", pendingService);
|
||||
setField(service, "agentRuntimeRouteRegistry", routeRegistry);
|
||||
setField(service, "agentRuntimeCommandProducer", commandProducer);
|
||||
|
||||
registry.register(runContext("request-local-approve", "session-local-approve", true));
|
||||
registry.registerResumeToken("request-local-approve", "token-local-approve");
|
||||
invoke(service, "approveRuntime",
|
||||
new Class<?>[]{String.class, String.class, BigInteger.class, String.class},
|
||||
"request-local-approve", "token-local-approve", BigInteger.ONE, "1");
|
||||
|
||||
Assert.assertEquals(1, pendingService.approveCount);
|
||||
Assert.assertEquals(0, commandProducer.approveCount);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证本机无运行态但 Redis owner 存在时投递远程命令。
|
||||
*
|
||||
* @throws Exception 反射调用失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void approveShouldDispatchRemoteWhenOwnerIsRemoteNode() throws Exception {
|
||||
AgentRunService service = new AgentRunService();
|
||||
RecordingRouteRegistry routeRegistry = new RecordingRouteRegistry("node-b");
|
||||
routeRegistry.requestIdByToken = "request-remote-approve";
|
||||
routeRegistry.ownerNode = "node-a";
|
||||
routeRegistry.ownerBootId = "boot-a";
|
||||
routeRegistry.currentOwnerBootId = "boot-a";
|
||||
routeRegistry.nodeAlive = true;
|
||||
RecordingCommandProducer commandProducer = new RecordingCommandProducer();
|
||||
setField(service, "agentRunRegistry", new AgentRunRegistry());
|
||||
setField(service, "agentRuntimeRouteRegistry", routeRegistry);
|
||||
setField(service, "agentRuntimeCommandProducer", commandProducer);
|
||||
|
||||
invoke(service, "approveRuntime",
|
||||
new Class<?>[]{String.class, String.class, BigInteger.class, String.class},
|
||||
null, "token-remote-approve", BigInteger.ONE, "1");
|
||||
|
||||
Assert.assertEquals(1, commandProducer.approveCount);
|
||||
Assert.assertEquals("node-a", commandProducer.lastTargetNodeId);
|
||||
Assert.assertEquals("request-remote-approve", commandProducer.lastRequestId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 owner 缺失时明确失败。
|
||||
*
|
||||
* @throws Exception 反射调用失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void approveShouldFailWhenOwnerRouteMissing() throws Exception {
|
||||
AgentRunService service = new AgentRunService();
|
||||
RecordingRouteRegistry routeRegistry = new RecordingRouteRegistry("node-b");
|
||||
routeRegistry.requestIdByToken = "request-missing-owner";
|
||||
setField(service, "agentRunRegistry", new AgentRunRegistry());
|
||||
setField(service, "agentRuntimeRouteRegistry", routeRegistry);
|
||||
setField(service, "agentRuntimeCommandProducer", new RecordingCommandProducer());
|
||||
|
||||
try {
|
||||
invoke(service, "approveRuntime",
|
||||
new Class<?>[]{String.class, String.class, BigInteger.class, String.class},
|
||||
null, "token-missing-owner", BigInteger.ONE, "1");
|
||||
Assert.fail("expected BusinessException");
|
||||
} catch (Exception e) {
|
||||
Assert.assertTrue(rootCause(e) instanceof BusinessException);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 owner 重启后启动代不匹配会明确失败。
|
||||
*
|
||||
* @throws Exception 反射调用失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void approveShouldFailWhenOwnerBootIdChanged() throws Exception {
|
||||
AgentRunService service = new AgentRunService();
|
||||
RecordingRouteRegistry routeRegistry = new RecordingRouteRegistry("node-b");
|
||||
routeRegistry.requestIdByToken = "request-restarted-owner";
|
||||
routeRegistry.ownerNode = "node-a";
|
||||
routeRegistry.ownerBootId = "boot-old";
|
||||
routeRegistry.currentOwnerBootId = "boot-new";
|
||||
routeRegistry.nodeAlive = true;
|
||||
setField(service, "agentRunRegistry", new AgentRunRegistry());
|
||||
setField(service, "agentRuntimeRouteRegistry", routeRegistry);
|
||||
setField(service, "agentRuntimeCommandProducer", new RecordingCommandProducer());
|
||||
|
||||
try {
|
||||
invoke(service, "approveRuntime",
|
||||
new Class<?>[]{String.class, String.class, BigInteger.class, String.class},
|
||||
null, "token-restarted-owner", BigInteger.ONE, "1");
|
||||
Assert.fail("expected BusinessException");
|
||||
} catch (Exception e) {
|
||||
Assert.assertTrue(rootCause(e) instanceof BusinessException);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 owner 路由存在但节点心跳缺失时明确失败。
|
||||
*
|
||||
* @throws Exception 反射调用失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void approveShouldFailWhenOwnerNodeHeartbeatMissing() throws Exception {
|
||||
AgentRunService service = new AgentRunService();
|
||||
RecordingRouteRegistry routeRegistry = new RecordingRouteRegistry("node-b");
|
||||
routeRegistry.requestIdByToken = "request-offline-owner";
|
||||
routeRegistry.ownerNode = "node-a";
|
||||
routeRegistry.nodeAlive = false;
|
||||
setField(service, "agentRunRegistry", new AgentRunRegistry());
|
||||
setField(service, "agentRuntimeRouteRegistry", routeRegistry);
|
||||
setField(service, "agentRuntimeCommandProducer", new RecordingCommandProducer());
|
||||
|
||||
try {
|
||||
invoke(service, "approveRuntime",
|
||||
new Class<?>[]{String.class, String.class, BigInteger.class, String.class},
|
||||
null, "token-offline-owner", BigInteger.ONE, "1");
|
||||
Assert.fail("expected BusinessException");
|
||||
} catch (Exception e) {
|
||||
Assert.assertTrue(rootCause(e) instanceof BusinessException);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证清理草稿会话只清草稿 store,不触碰 MySQL pending 清理。
|
||||
*
|
||||
@@ -785,6 +921,72 @@ public class AgentRunServiceDraftAndHitlTest {
|
||||
}
|
||||
}
|
||||
|
||||
private static class RecordingRouteRegistry extends AgentRuntimeRouteRegistry {
|
||||
|
||||
private final String currentNodeId;
|
||||
private String ownerNode;
|
||||
private String ownerBootId;
|
||||
private String currentOwnerBootId;
|
||||
private String requestIdByToken;
|
||||
private boolean nodeAlive;
|
||||
|
||||
private RecordingRouteRegistry(String currentNodeId) {
|
||||
super(null, null, null);
|
||||
this.currentNodeId = currentNodeId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String findOwnerNode(String requestId) {
|
||||
return ownerNode;
|
||||
}
|
||||
|
||||
@Override
|
||||
public AgentRuntimeRoute findOwnerRoute(String requestId) {
|
||||
AgentRuntimeRoute route = new AgentRuntimeRoute();
|
||||
route.setNodeId(ownerNode);
|
||||
route.setBootId(ownerBootId);
|
||||
return route;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String findRequestIdByResumeToken(String resumeToken) {
|
||||
return requestIdByToken;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String currentNodeId() {
|
||||
return currentNodeId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isNodeAlive(String nodeId) {
|
||||
return nodeAlive;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String currentNodeBootId(String nodeId) {
|
||||
return currentOwnerBootId;
|
||||
}
|
||||
}
|
||||
|
||||
private static class RecordingCommandProducer extends AgentRuntimeCommandProducer {
|
||||
|
||||
private int approveCount;
|
||||
private String lastTargetNodeId;
|
||||
private String lastRequestId;
|
||||
|
||||
@Override
|
||||
public void sendApprove(String targetNodeId,
|
||||
String requestId,
|
||||
String resumeToken,
|
||||
BigInteger operatorId,
|
||||
String userId) {
|
||||
approveCount++;
|
||||
lastTargetNodeId = targetNodeId;
|
||||
lastRequestId = requestId;
|
||||
}
|
||||
}
|
||||
|
||||
private static class RecordingAgentRuntimeFactory implements AgentRuntimeFactory {
|
||||
|
||||
private final AgentRuntime runtime;
|
||||
|
||||
Reference in New Issue
Block a user