feat: 增强多实例分布式部署兼容
- 增加定时任务分布式锁并覆盖 chatlog、文档导入和 Agent HITL 过期扫描 - 增强 Redis MQ 多实例 consumer 标识、pending reclaim 和单条处理能力 - 增加文档导入状态 Redis 广播和 Agent HITL 跨节点路由确认
This commit is contained in:
@@ -0,0 +1,159 @@
|
||||
package tech.easyflow.agent.distributed;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import tech.easyflow.agent.config.AgentRuntimeProperties;
|
||||
import tech.easyflow.agent.distributed.AgentRuntimeCommandAction;
|
||||
import tech.easyflow.agent.distributed.AgentRuntimeCommandConsumer;
|
||||
import tech.easyflow.agent.distributed.AgentRuntimeCommandMessage;
|
||||
import tech.easyflow.agent.distributed.AgentRuntimeCommandResultRegistry;
|
||||
import tech.easyflow.agent.runtime.AgentRunService;
|
||||
import tech.easyflow.common.mq.config.MQProperties;
|
||||
import tech.easyflow.common.mq.core.MQMessage;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* {@link AgentRuntimeCommandConsumer} 回归测试。
|
||||
*/
|
||||
public class AgentRuntimeCommandConsumerTest {
|
||||
|
||||
/**
|
||||
* 验证消费者只处理发给当前节点的命令。
|
||||
*
|
||||
* @throws Exception 消息序列化异常
|
||||
*/
|
||||
@Test
|
||||
public void consumerShouldHandleOnlyCurrentNodeCommand() throws Exception {
|
||||
AgentRuntimeProperties properties = new AgentRuntimeProperties();
|
||||
properties.setInstanceId("node-a");
|
||||
MQProperties mqProperties = new MQProperties();
|
||||
mqProperties.getRedis().setChatPersistShardCount(4);
|
||||
RecordingAgentRunService service = new RecordingAgentRunService();
|
||||
RecordingCommandResultRegistry resultRegistry = new RecordingCommandResultRegistry();
|
||||
AgentRuntimeCommandConsumer consumer =
|
||||
new AgentRuntimeCommandConsumer(new ObjectMapper(), properties, mqProperties, service, resultRegistry);
|
||||
|
||||
consumer.handle(List.of(message(command("cmd-1", "node-b")), message(command("cmd-2", "node-a"))));
|
||||
|
||||
Assert.assertEquals(1, service.approveCount);
|
||||
Assert.assertEquals("request-cmd-2", service.lastRequestId);
|
||||
Assert.assertEquals(4, consumer.subscription().getShardCount());
|
||||
Assert.assertFalse(consumer.subscription().isBatchEnabled());
|
||||
Assert.assertEquals("cmd-2", resultRegistry.lastSuccessCommandId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 owner 本机执行失败时写入失败结果，避免 MQ 重试重复消费一次性 token。
|
||||
*
|
||||
* @throws Exception 消息序列化异常
|
||||
*/
|
||||
@Test
|
||||
public void consumerShouldMarkFailureWhenLocalRuntimeFails() throws Exception {
|
||||
AgentRuntimeProperties properties = new AgentRuntimeProperties();
|
||||
properties.setInstanceId("node-a");
|
||||
MQProperties mqProperties = new MQProperties();
|
||||
FailingAgentRunService service = new FailingAgentRunService();
|
||||
RecordingCommandResultRegistry resultRegistry = new RecordingCommandResultRegistry();
|
||||
AgentRuntimeCommandConsumer consumer =
|
||||
new AgentRuntimeCommandConsumer(new ObjectMapper(), properties, mqProperties, service, resultRegistry);
|
||||
|
||||
consumer.handle(List.of(message(command("cmd-1", "node-a"))));
|
||||
|
||||
Assert.assertEquals("cmd-1", resultRegistry.lastFailureCommandId);
|
||||
Assert.assertEquals("runtime missing", resultRegistry.lastFailureMessage);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证成功结果写入失败不会再次执行或改写为失败结果。
|
||||
*
|
||||
* @throws Exception 消息序列化异常
|
||||
*/
|
||||
@Test
|
||||
public void consumerShouldNotMarkFailureWhenSuccessResultWriteFails() throws Exception {
|
||||
AgentRuntimeProperties properties = new AgentRuntimeProperties();
|
||||
properties.setInstanceId("node-a");
|
||||
MQProperties mqProperties = new MQProperties();
|
||||
RecordingAgentRunService service = new RecordingAgentRunService();
|
||||
FailingSuccessResultRegistry resultRegistry = new FailingSuccessResultRegistry();
|
||||
AgentRuntimeCommandConsumer consumer =
|
||||
new AgentRuntimeCommandConsumer(new ObjectMapper(), properties, mqProperties, service, resultRegistry);
|
||||
|
||||
consumer.handle(List.of(message(command("cmd-1", "node-a"))));
|
||||
|
||||
Assert.assertEquals(1, service.approveCount);
|
||||
Assert.assertNull(resultRegistry.lastFailureCommandId);
|
||||
}
|
||||
|
||||
private AgentRuntimeCommandMessage command(String commandId, String targetNodeId) {
|
||||
AgentRuntimeCommandMessage command = new AgentRuntimeCommandMessage();
|
||||
command.setCommandId(commandId);
|
||||
command.setRequestId("request-" + commandId);
|
||||
command.setResumeToken("token-" + commandId);
|
||||
command.setAction(AgentRuntimeCommandAction.APPROVE);
|
||||
command.setOperatorId(BigInteger.ONE);
|
||||
command.setUserId("1");
|
||||
command.setTargetNodeId(targetNodeId);
|
||||
return command;
|
||||
}
|
||||
|
||||
private MQMessage message(AgentRuntimeCommandMessage command) throws Exception {
|
||||
MQMessage message = new MQMessage();
|
||||
message.setMessageId(command.getCommandId());
|
||||
message.setBody(new ObjectMapper().writeValueAsString(command));
|
||||
return message;
|
||||
}
|
||||
|
||||
private static final class RecordingAgentRunService extends AgentRunService {
|
||||
|
||||
private int approveCount;
|
||||
private String lastRequestId;
|
||||
|
||||
@Override
|
||||
public void approveRuntimeLocal(String requestId, String resumeToken, BigInteger operatorId, String userId) {
|
||||
approveCount++;
|
||||
lastRequestId = requestId;
|
||||
}
|
||||
}
|
||||
|
||||
private static class RecordingCommandResultRegistry extends AgentRuntimeCommandResultRegistry {
|
||||
|
||||
private String lastSuccessCommandId;
|
||||
String lastFailureCommandId;
|
||||
private String lastFailureMessage;
|
||||
|
||||
private RecordingCommandResultRegistry() {
|
||||
super(null, null, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void markSuccess(String commandId) {
|
||||
lastSuccessCommandId = commandId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void markFailure(String commandId, String message) {
|
||||
lastFailureCommandId = commandId;
|
||||
lastFailureMessage = message;
|
||||
}
|
||||
}
|
||||
|
||||
private static final class FailingAgentRunService extends AgentRunService {
|
||||
|
||||
@Override
|
||||
public void approveRuntimeLocal(String requestId, String resumeToken, BigInteger operatorId, String userId) {
|
||||
throw new RuntimeException("runtime missing");
|
||||
}
|
||||
}
|
||||
|
||||
private static final class FailingSuccessResultRegistry extends RecordingCommandResultRegistry {
|
||||
|
||||
@Override
|
||||
public void markSuccess(String commandId) {
|
||||
super.markSuccess(commandId);
|
||||
throw new RuntimeException("redis down");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package tech.easyflow.agent.distributed;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.ArgumentMatchers;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.data.redis.core.ValueOperations;
|
||||
import tech.easyflow.agent.config.AgentRuntimeProperties;
|
||||
import tech.easyflow.agent.distributed.AgentRuntimeCommandResult;
|
||||
import tech.easyflow.agent.distributed.AgentRuntimeCommandResultRegistry;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
/**
|
||||
* {@link AgentRuntimeCommandResultRegistry} 回归测试。
|
||||
*/
|
||||
public class AgentRuntimeCommandResultRegistryTest {
|
||||
|
||||
/**
|
||||
* 验证成功结果可被等待方读取。
|
||||
*/
|
||||
@Test
|
||||
public void waitForResultShouldReturnSuccessResult() {
|
||||
StringRedisTemplate redisTemplate = Mockito.mock(StringRedisTemplate.class);
|
||||
@SuppressWarnings("unchecked")
|
||||
ValueOperations<String, String> valueOperations = Mockito.mock(ValueOperations.class);
|
||||
Mockito.when(redisTemplate.opsForValue()).thenReturn(valueOperations);
|
||||
Mockito.when(valueOperations.get("easyflow:agent:runtime:command-result:cmd-1"))
|
||||
.thenReturn("{\"success\":true,\"message\":\"OK\"}");
|
||||
AgentRuntimeCommandResultRegistry registry = registry(redisTemplate);
|
||||
|
||||
AgentRuntimeCommandResult result = registry.waitForResult("cmd-1");
|
||||
|
||||
Assert.assertTrue(result.isSuccess());
|
||||
Assert.assertEquals("OK", result.getMessage());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证写入失败结果时使用配置的 TTL。
|
||||
*/
|
||||
@Test
|
||||
public void markFailureShouldWriteResultWithTtl() {
|
||||
StringRedisTemplate redisTemplate = Mockito.mock(StringRedisTemplate.class);
|
||||
@SuppressWarnings("unchecked")
|
||||
ValueOperations<String, String> valueOperations = Mockito.mock(ValueOperations.class);
|
||||
Mockito.when(redisTemplate.opsForValue()).thenReturn(valueOperations);
|
||||
AgentRuntimeProperties properties = properties();
|
||||
AgentRuntimeCommandResultRegistry registry =
|
||||
new AgentRuntimeCommandResultRegistry(redisTemplate, new ObjectMapper(), properties);
|
||||
|
||||
registry.markFailure("cmd-1", "failed");
|
||||
|
||||
Mockito.verify(valueOperations).set(
|
||||
ArgumentMatchers.eq("easyflow:agent:runtime:command-result:cmd-1"),
|
||||
ArgumentMatchers.contains("\"success\":false"),
|
||||
ArgumentMatchers.eq(properties.getCommandResultTtl()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证等待超时时抛出明确业务异常。
|
||||
*/
|
||||
@Test
|
||||
public void waitForResultShouldThrowBusinessExceptionWhenTimeout() {
|
||||
StringRedisTemplate redisTemplate = Mockito.mock(StringRedisTemplate.class);
|
||||
@SuppressWarnings("unchecked")
|
||||
ValueOperations<String, String> valueOperations = Mockito.mock(ValueOperations.class);
|
||||
Mockito.when(redisTemplate.opsForValue()).thenReturn(valueOperations);
|
||||
Mockito.when(valueOperations.get(ArgumentMatchers.anyString())).thenReturn(null);
|
||||
AgentRuntimeCommandResultRegistry registry = registry(redisTemplate);
|
||||
|
||||
BusinessException exception = Assert.assertThrows(
|
||||
BusinessException.class,
|
||||
() -> registry.waitForResult("cmd-1"));
|
||||
|
||||
Assert.assertEquals("Agent 运行节点响应超时，请稍后重试", exception.getMessage());
|
||||
}
|
||||
|
||||
private AgentRuntimeCommandResultRegistry registry(StringRedisTemplate redisTemplate) {
|
||||
return new AgentRuntimeCommandResultRegistry(redisTemplate, new ObjectMapper(), properties());
|
||||
}
|
||||
|
||||
private AgentRuntimeProperties properties() {
|
||||
AgentRuntimeProperties properties = new AgentRuntimeProperties();
|
||||
properties.setCommandResultTimeout(Duration.ofMillis(10));
|
||||
properties.setCommandResultTtl(Duration.ofMinutes(5));
|
||||
return properties;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package tech.easyflow.agent.distributed;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.ArgumentMatchers;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.data.redis.core.ValueOperations;
|
||||
import tech.easyflow.agent.config.AgentRuntimeProperties;
|
||||
import tech.easyflow.agent.distributed.AgentRuntimeRouteRegistry;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
/**
|
||||
* {@link AgentRuntimeRouteRegistry} 回归测试。
|
||||
*/
|
||||
public class AgentRuntimeRouteRegistryTest {
|
||||
|
||||
/**
|
||||
* 验证注册运行态和恢复令牌时写入 Redis 路由。
|
||||
*/
|
||||
@Test
|
||||
public void registerShouldWriteRunAndTokenRoutes() {
|
||||
StringRedisTemplate redisTemplate = Mockito.mock(StringRedisTemplate.class);
|
||||
@SuppressWarnings("unchecked")
|
||||
ValueOperations<String, String> valueOperations = Mockito.mock(ValueOperations.class);
|
||||
Mockito.when(redisTemplate.opsForValue()).thenReturn(valueOperations);
|
||||
AgentRuntimeProperties properties = properties("node-a");
|
||||
AgentRuntimeRouteRegistry registry = registry(redisTemplate, properties);
|
||||
|
||||
registry.registerRun("request-1");
|
||||
registry.registerResumeToken("request-1", "token-1");
|
||||
|
||||
Mockito.verify(valueOperations).set(
|
||||
ArgumentMatchers.eq("easyflow:agent:runtime:request:request-1"),
|
||||
ArgumentMatchers.contains("\"nodeId\":\"node-a\""),
|
||||
ArgumentMatchers.eq(Duration.ofHours(24)));
|
||||
Mockito.verify(valueOperations).set(
|
||||
"easyflow:agent:runtime:resume-token:token-1", "request-1", Duration.ofHours(24));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证运行结束时清理 Redis 路由。
|
||||
*/
|
||||
@Test
|
||||
public void removeShouldDeleteRunAndTokenRoutes() {
|
||||
StringRedisTemplate redisTemplate = Mockito.mock(StringRedisTemplate.class);
|
||||
AgentRuntimeRouteRegistry registry = registry(redisTemplate, properties("node-a"));
|
||||
|
||||
registry.removeRun("request-1");
|
||||
registry.removeResumeToken("token-1");
|
||||
|
||||
Mockito.verify(redisTemplate).delete("easyflow:agent:runtime:request:request-1");
|
||||
Mockito.verify(redisTemplate).delete("easyflow:agent:runtime:resume-token:token-1");
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证查询 owner 节点和 token 反查请求 ID。
|
||||
*/
|
||||
@Test
|
||||
public void findShouldReadRoutes() {
|
||||
StringRedisTemplate redisTemplate = Mockito.mock(StringRedisTemplate.class);
|
||||
@SuppressWarnings("unchecked")
|
||||
ValueOperations<String, String> valueOperations = Mockito.mock(ValueOperations.class);
|
||||
Mockito.when(redisTemplate.opsForValue()).thenReturn(valueOperations);
|
||||
Mockito.when(valueOperations.get(ArgumentMatchers.eq("easyflow:agent:runtime:request:request-1")))
|
||||
.thenReturn("{\"nodeId\":\"node-a\",\"bootId\":\"boot-a\"}");
|
||||
Mockito.when(valueOperations.get(ArgumentMatchers.eq("easyflow:agent:runtime:resume-token:token-1")))
|
||||
.thenReturn("request-1");
|
||||
AgentRuntimeRouteRegistry registry = registry(redisTemplate, properties("node-a"));
|
||||
|
||||
Assert.assertEquals("node-a", registry.findOwnerNode("request-1"));
|
||||
Assert.assertEquals("boot-a", registry.findOwnerRoute("request-1").getBootId());
|
||||
Assert.assertEquals("request-1", registry.findRequestIdByResumeToken("token-1"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证节点心跳写入和存活查询。
|
||||
*/
|
||||
@Test
|
||||
public void heartbeatShouldWriteAndReadNodeAliveState() {
|
||||
StringRedisTemplate redisTemplate = Mockito.mock(StringRedisTemplate.class);
|
||||
@SuppressWarnings("unchecked")
|
||||
ValueOperations<String, String> valueOperations = Mockito.mock(ValueOperations.class);
|
||||
Mockito.when(redisTemplate.opsForValue()).thenReturn(valueOperations);
|
||||
AgentRuntimeProperties properties = properties("node-a");
|
||||
Mockito.when(valueOperations.get("easyflow:agent:runtime:node:node-a")).thenReturn(properties.getBootId());
|
||||
AgentRuntimeRouteRegistry registry = registry(redisTemplate, properties);
|
||||
|
||||
registry.heartbeat(Duration.ofSeconds(90));
|
||||
|
||||
Mockito.verify(valueOperations).set("easyflow:agent:runtime:node:node-a", properties.getBootId(), Duration.ofSeconds(90));
|
||||
Assert.assertTrue(registry.isNodeAlive("node-a"));
|
||||
Assert.assertEquals(properties.getBootId(), registry.currentNodeBootId("node-a"));
|
||||
}
|
||||
|
||||
private AgentRuntimeProperties properties(String instanceId) {
|
||||
AgentRuntimeProperties properties = new AgentRuntimeProperties();
|
||||
properties.setInstanceId(instanceId);
|
||||
properties.setRouteTtl(Duration.ofHours(24));
|
||||
return properties;
|
||||
}
|
||||
|
||||
private AgentRuntimeRouteRegistry registry(StringRedisTemplate redisTemplate, AgentRuntimeProperties properties) {
|
||||
return new AgentRuntimeRouteRegistry(redisTemplate, properties, new ObjectMapper());
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,9 @@ import tech.easyflow.agent.entity.AgentHitlPending;
|
||||
import tech.easyflow.agent.entity.Agent;
|
||||
import tech.easyflow.agent.entity.AgentKnowledgeBinding;
|
||||
import tech.easyflow.agent.entity.AgentToolBinding;
|
||||
import tech.easyflow.agent.distributed.AgentRuntimeCommandProducer;
|
||||
import tech.easyflow.agent.distributed.AgentRuntimeRoute;
|
||||
import tech.easyflow.agent.distributed.AgentRuntimeRouteRegistry;
|
||||
import tech.easyflow.agent.runtime.event.AgentRunEventRecorder;
|
||||
import tech.easyflow.agent.runtime.hitl.AgentHitlPendingService;
|
||||
import tech.easyflow.agent.runtime.lock.AgentRunLock;
|
||||
@@ -532,6 +535,139 @@ public class AgentRunServiceDraftAndHitlTest {
|
||||
Assert.assertEquals(1, pendingService.approveCount);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证本机存在恢复目标时不投递远程命令。
|
||||
*
|
||||
* @throws Exception 反射调用失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void approveShouldNotDispatchRemoteWhenLocalRuntimeExists() throws Exception {
|
||||
AgentRunService service = new AgentRunService();
|
||||
AgentRunRegistry registry = new AgentRunRegistry();
|
||||
RecordingAgentHitlPendingService pendingService = new RecordingAgentHitlPendingService();
|
||||
RecordingRouteRegistry routeRegistry = new RecordingRouteRegistry("node-a");
|
||||
RecordingCommandProducer commandProducer = new RecordingCommandProducer();
|
||||
setField(service, "agentRunRegistry", registry);
|
||||
setField(service, "agentHitlPendingService", pendingService);
|
||||
setField(service, "agentRuntimeRouteRegistry", routeRegistry);
|
||||
setField(service, "agentRuntimeCommandProducer", commandProducer);
|
||||
|
||||
registry.register(runContext("request-local-approve", "session-local-approve", true));
|
||||
registry.registerResumeToken("request-local-approve", "token-local-approve");
|
||||
invoke(service, "approveRuntime",
|
||||
new Class<?>[]{String.class, String.class, BigInteger.class, String.class},
|
||||
"request-local-approve", "token-local-approve", BigInteger.ONE, "1");
|
||||
|
||||
Assert.assertEquals(1, pendingService.approveCount);
|
||||
Assert.assertEquals(0, commandProducer.approveCount);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证本机无运行态但 Redis owner 存在时投递远程命令。
|
||||
*
|
||||
* @throws Exception 反射调用失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void approveShouldDispatchRemoteWhenOwnerIsRemoteNode() throws Exception {
|
||||
AgentRunService service = new AgentRunService();
|
||||
RecordingRouteRegistry routeRegistry = new RecordingRouteRegistry("node-b");
|
||||
routeRegistry.requestIdByToken = "request-remote-approve";
|
||||
routeRegistry.ownerNode = "node-a";
|
||||
routeRegistry.ownerBootId = "boot-a";
|
||||
routeRegistry.currentOwnerBootId = "boot-a";
|
||||
routeRegistry.nodeAlive = true;
|
||||
RecordingCommandProducer commandProducer = new RecordingCommandProducer();
|
||||
setField(service, "agentRunRegistry", new AgentRunRegistry());
|
||||
setField(service, "agentRuntimeRouteRegistry", routeRegistry);
|
||||
setField(service, "agentRuntimeCommandProducer", commandProducer);
|
||||
|
||||
invoke(service, "approveRuntime",
|
||||
new Class<?>[]{String.class, String.class, BigInteger.class, String.class},
|
||||
null, "token-remote-approve", BigInteger.ONE, "1");
|
||||
|
||||
Assert.assertEquals(1, commandProducer.approveCount);
|
||||
Assert.assertEquals("node-a", commandProducer.lastTargetNodeId);
|
||||
Assert.assertEquals("request-remote-approve", commandProducer.lastRequestId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 owner 缺失时明确失败。
|
||||
*
|
||||
* @throws Exception 反射调用失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void approveShouldFailWhenOwnerRouteMissing() throws Exception {
|
||||
AgentRunService service = new AgentRunService();
|
||||
RecordingRouteRegistry routeRegistry = new RecordingRouteRegistry("node-b");
|
||||
routeRegistry.requestIdByToken = "request-missing-owner";
|
||||
setField(service, "agentRunRegistry", new AgentRunRegistry());
|
||||
setField(service, "agentRuntimeRouteRegistry", routeRegistry);
|
||||
setField(service, "agentRuntimeCommandProducer", new RecordingCommandProducer());
|
||||
|
||||
try {
|
||||
invoke(service, "approveRuntime",
|
||||
new Class<?>[]{String.class, String.class, BigInteger.class, String.class},
|
||||
null, "token-missing-owner", BigInteger.ONE, "1");
|
||||
Assert.fail("expected BusinessException");
|
||||
} catch (Exception e) {
|
||||
Assert.assertTrue(rootCause(e) instanceof BusinessException);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 owner 重启后启动代不匹配会明确失败。
|
||||
*
|
||||
* @throws Exception 反射调用失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void approveShouldFailWhenOwnerBootIdChanged() throws Exception {
|
||||
AgentRunService service = new AgentRunService();
|
||||
RecordingRouteRegistry routeRegistry = new RecordingRouteRegistry("node-b");
|
||||
routeRegistry.requestIdByToken = "request-restarted-owner";
|
||||
routeRegistry.ownerNode = "node-a";
|
||||
routeRegistry.ownerBootId = "boot-old";
|
||||
routeRegistry.currentOwnerBootId = "boot-new";
|
||||
routeRegistry.nodeAlive = true;
|
||||
setField(service, "agentRunRegistry", new AgentRunRegistry());
|
||||
setField(service, "agentRuntimeRouteRegistry", routeRegistry);
|
||||
setField(service, "agentRuntimeCommandProducer", new RecordingCommandProducer());
|
||||
|
||||
try {
|
||||
invoke(service, "approveRuntime",
|
||||
new Class<?>[]{String.class, String.class, BigInteger.class, String.class},
|
||||
null, "token-restarted-owner", BigInteger.ONE, "1");
|
||||
Assert.fail("expected BusinessException");
|
||||
} catch (Exception e) {
|
||||
Assert.assertTrue(rootCause(e) instanceof BusinessException);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 owner 路由存在但节点心跳缺失时明确失败。
|
||||
*
|
||||
* @throws Exception 反射调用失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void approveShouldFailWhenOwnerNodeHeartbeatMissing() throws Exception {
|
||||
AgentRunService service = new AgentRunService();
|
||||
RecordingRouteRegistry routeRegistry = new RecordingRouteRegistry("node-b");
|
||||
routeRegistry.requestIdByToken = "request-offline-owner";
|
||||
routeRegistry.ownerNode = "node-a";
|
||||
routeRegistry.nodeAlive = false;
|
||||
setField(service, "agentRunRegistry", new AgentRunRegistry());
|
||||
setField(service, "agentRuntimeRouteRegistry", routeRegistry);
|
||||
setField(service, "agentRuntimeCommandProducer", new RecordingCommandProducer());
|
||||
|
||||
try {
|
||||
invoke(service, "approveRuntime",
|
||||
new Class<?>[]{String.class, String.class, BigInteger.class, String.class},
|
||||
null, "token-offline-owner", BigInteger.ONE, "1");
|
||||
Assert.fail("expected BusinessException");
|
||||
} catch (Exception e) {
|
||||
Assert.assertTrue(rootCause(e) instanceof BusinessException);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证清理草稿会话只清草稿 store，不触碰 MySQL pending 清理。
|
||||
*
|
||||
@@ -785,6 +921,72 @@ public class AgentRunServiceDraftAndHitlTest {
|
||||
}
|
||||
}
|
||||
|
||||
private static class RecordingRouteRegistry extends AgentRuntimeRouteRegistry {
|
||||
|
||||
private final String currentNodeId;
|
||||
private String ownerNode;
|
||||
private String ownerBootId;
|
||||
private String currentOwnerBootId;
|
||||
private String requestIdByToken;
|
||||
private boolean nodeAlive;
|
||||
|
||||
private RecordingRouteRegistry(String currentNodeId) {
|
||||
super(null, null, null);
|
||||
this.currentNodeId = currentNodeId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String findOwnerNode(String requestId) {
|
||||
return ownerNode;
|
||||
}
|
||||
|
||||
@Override
|
||||
public AgentRuntimeRoute findOwnerRoute(String requestId) {
|
||||
AgentRuntimeRoute route = new AgentRuntimeRoute();
|
||||
route.setNodeId(ownerNode);
|
||||
route.setBootId(ownerBootId);
|
||||
return route;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String findRequestIdByResumeToken(String resumeToken) {
|
||||
return requestIdByToken;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String currentNodeId() {
|
||||
return currentNodeId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isNodeAlive(String nodeId) {
|
||||
return nodeAlive;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String currentNodeBootId(String nodeId) {
|
||||
return currentOwnerBootId;
|
||||
}
|
||||
}
|
||||
|
||||
private static class RecordingCommandProducer extends AgentRuntimeCommandProducer {
|
||||
|
||||
private int approveCount;
|
||||
private String lastTargetNodeId;
|
||||
private String lastRequestId;
|
||||
|
||||
@Override
|
||||
public void sendApprove(String targetNodeId,
|
||||
String requestId,
|
||||
String resumeToken,
|
||||
BigInteger operatorId,
|
||||
String userId) {
|
||||
approveCount++;
|
||||
lastTargetNodeId = targetNodeId;
|
||||
lastRequestId = requestId;
|
||||
}
|
||||
}
|
||||
|
||||
private static class RecordingAgentRuntimeFactory implements AgentRuntimeFactory {
|
||||
|
||||
private final AgentRuntime runtime;
|
||||
|
||||
Reference in New Issue
Block a user