feat: 增强多实例分布式部署兼容

- 增加定时任务分布式锁并覆盖 chatlog、文档导入和 Agent HITL 过期扫描 - 增强 Redis MQ 多实例 consumer 标识、pending reclaim 和单条处理能力 - 增加文档导入状态 Redis 广播和 Agent HITL 跨节点路由确认
2026-05-29 18:27:46 +08:00
parent cc3bb9cff0
commit 0f4d10c43c
39 changed files with 2703 additions and 17 deletions
--- a/easyflow-commons/easyflow-common-cache/pom.xml
+++ b/easyflow-commons/easyflow-common-cache/pom.xml
@@ -39,7 +39,23 @@
            <artifactId>fastjson</artifactId>
        </dependency>

+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-aop</artifactId>
+        </dependency>

+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>${junit.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <version>5.12.0</version>
+            <scope>test</scope>
+        </dependency>

    </dependencies>

--- a/easyflow-commons/easyflow-common-cache/src/main/java/tech/easyflow/common/cache/DistributedScheduledLock.java
+++ b/easyflow-commons/easyflow-common-cache/src/main/java/tech/easyflow/common/cache/DistributedScheduledLock.java
@@ -0,0 +1,35 @@
+package tech.easyflow.common.cache;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Spring 定时任务 Redis 分布式锁。
+ */
+@Target(ElementType.METHOD)
+@Retention(RetentionPolicy.RUNTIME)
+public @interface DistributedScheduledLock {
+
+    /**
+     * 获取锁使用的 Redis key。
+     *
+     * @return Redis 锁 key
+     */
+    String key();
+
+    /**
+     * 等待锁的秒数。
+     *
+     * @return 等待锁的秒数
+     */
+    long waitSeconds() default 0L;
+
+    /**
+     * 锁租约秒数。
+     *
+     * @return 锁租约秒数
+     */
+    long leaseSeconds() default 300L;
+}
--- a/easyflow-commons/easyflow-common-cache/src/main/java/tech/easyflow/common/cache/DistributedScheduledLockAspect.java
+++ b/easyflow-commons/easyflow-common-cache/src/main/java/tech/easyflow/common/cache/DistributedScheduledLockAspect.java
@@ -0,0 +1,111 @@
+package tech.easyflow.common.cache;
+
+import org.aspectj.lang.ProceedingJoinPoint;
+import org.aspectj.lang.annotation.Around;
+import org.aspectj.lang.annotation.Aspect;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import jakarta.annotation.PreDestroy;
+import org.springframework.stereotype.Component;
+
+import java.time.Duration;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * 定时任务分布式锁切面。
+ */
+@Aspect
+@Component
+public class DistributedScheduledLockAspect {
+
+    private static final Logger LOG = LoggerFactory.getLogger(DistributedScheduledLockAspect.class);
+
+    private final RedisLockExecutor redisLockExecutor;
+    private final ScheduledExecutorService renewExecutor;
+
+    /**
+     * 创建定时任务分布式锁切面。
+     *
+     * @param redisLockExecutor Redis 分布式锁执行器
+     */
+    public DistributedScheduledLockAspect(RedisLockExecutor redisLockExecutor) {
+        this.redisLockExecutor = redisLockExecutor;
+        this.renewExecutor = Executors.newScheduledThreadPool(
+            1,
+            new DistributedScheduledLockThreadFactory()
+        );
+    }
+
+    /**
+     * 拦截带分布式调度锁的定时任务。
+     *
+     * @param joinPoint 切点
+     * @param lock 锁注解
+     * @return 原方法返回值；未抢到锁时返回 null
+     * @throws Throwable 原方法执行异常或 Redis 访问异常
+     */
+    @Around("@annotation(lock)")
+    public Object around(ProceedingJoinPoint joinPoint, DistributedScheduledLock lock) throws Throwable {
+        Duration waitTimeout = Duration.ofSeconds(Math.max(lock.waitSeconds(), 0L));
+        Duration leaseTimeout = Duration.ofSeconds(Math.max(lock.leaseSeconds(), 1L));
+        RedisLockExecutor.LockHandle handle = redisLockExecutor.tryAcquire(lock.key(), waitTimeout, leaseTimeout);
+        if (handle == null) {
+            LOG.info("定时任务分布式锁已被其他实例持有，跳过本轮执行: lockKey={}, method={}",
+                lock.key(), joinPoint.getSignature().toShortString());
+            return null;
+        }
+        ScheduledFuture<?> renewTask = scheduleRenew(lock.key(), handle, leaseTimeout);
+        try {
+            return joinPoint.proceed();
+        } finally {
+            renewTask.cancel(false);
+            handle.release();
+        }
+    }
+
+    private ScheduledFuture<?> scheduleRenew(String lockKey,
+                                             RedisLockExecutor.LockHandle handle,
+                                             Duration leaseTimeout) {
+        long renewIntervalMillis = Math.max(leaseTimeout.toMillis() / 3L, 1000L);
+        return renewExecutor.scheduleWithFixedDelay(() -> {
+            if (!handle.renew()) {
+                LOG.warn("定时任务分布式锁续期失败: lockKey={}", lockKey);
+            }
+        }, renewIntervalMillis, renewIntervalMillis, TimeUnit.MILLISECONDS);
+    }
+
+    /**
+     * 关闭调度锁续期线程池。
+     */
+    @PreDestroy
+    public void destroy() {
+        renewExecutor.shutdownNow();
+    }
+
+    /**
+     * 调度锁续期线程工厂。
+     */
+    private static final class DistributedScheduledLockThreadFactory implements ThreadFactory {
+
+        private final AtomicInteger index = new AtomicInteger(1);
+
+        /**
+         * 创建续期线程。
+         *
+         * @param runnable 线程任务
+         * @return 续期线程
+         */
+        @Override
+        public Thread newThread(Runnable runnable) {
+            Thread thread = new Thread(runnable);
+            thread.setName("distributed-scheduled-lock-renew-" + index.getAndIncrement());
+            thread.setDaemon(true);
+            return thread;
+        }
+    }
+}
--- a/easyflow-commons/easyflow-common-cache/src/main/java/tech/easyflow/common/cache/RedisLockExecutor.java
+++ b/easyflow-commons/easyflow-common-cache/src/main/java/tech/easyflow/common/cache/RedisLockExecutor.java
@@ -12,6 +12,9 @@ import java.util.Collections;
 import java.util.UUID;
 import java.util.function.Supplier;

+/**
+ * Redis 分布式锁执行器。
+ */
@Component
 public class RedisLockExecutor {

@@ -42,6 +45,14 @@ public class RedisLockExecutor {
    @Autowired
    private StringRedisTemplate stringRedisTemplate;

+    /**
+     * 在分布式锁保护下执行无返回任务。
+     *
+     * @param lockKey 锁 key
+     * @param waitTimeout 等待锁的最大时间
+     * @param leaseTimeout 锁租约时间
+     * @param task 业务任务
+     */
    public void executeWithLock(String lockKey, Duration waitTimeout, Duration leaseTimeout, Runnable task) {
        executeWithLock(lockKey, waitTimeout, leaseTimeout, () -> {
            task.run();
@@ -49,6 +60,16 @@ public class RedisLockExecutor {
        });
    }

+    /**
+     * 在分布式锁保护下执行有返回任务。
+     *
+     * @param lockKey 锁 key
+     * @param waitTimeout 等待锁的最大时间
+     * @param leaseTimeout 锁租约时间
+     * @param task 业务任务
+     * @param <T> 返回类型
+     * @return 任务返回值
+     */
    public <T> T executeWithLock(String lockKey, Duration waitTimeout, Duration leaseTimeout, Supplier<T> task) {
        LockHandle handle = acquire(lockKey, waitTimeout, leaseTimeout);
        try {
@@ -70,24 +91,46 @@ public class RedisLockExecutor {
     * @return 锁句柄
     */
    public LockHandle acquire(String lockKey, Duration waitTimeout, Duration leaseTimeout) {
+        LockHandle handle = tryAcquire(lockKey, waitTimeout, leaseTimeout);
+        if (handle == null) {
+            throw new IllegalStateException("获取分布式锁失败，请稍后重试，lockKey=" + lockKey);
+        }
+        return handle;
+    }
+
+    /**
+     * 尝试获取显式释放的分布式锁句柄。
+     *
+     * <p>返回 {@code null} 表示锁当前被其他节点持有。Redis 访问失败或等待过程被中断仍会抛出异常，
+     * 调用方可据此区分“正常跳过”和“基础设施异常”。</p>
+     *
+     * @param lockKey 锁 key
+     * @param waitTimeout 等待时间
+     * @param leaseTimeout 租约时间
+     * @return 获取成功时返回锁句柄，否则返回 null
+     */
+    public LockHandle tryAcquire(String lockKey, Duration waitTimeout, Duration leaseTimeout) {
        String lockValue = UUID.randomUUID().toString();
        boolean acquired = false;
        long deadline = System.nanoTime() + waitTimeout.toNanos();
        try {
-            while (System.nanoTime() <= deadline) {
+            do {
                Boolean success = stringRedisTemplate.opsForValue().setIfAbsent(lockKey, lockValue, leaseTimeout);
                if (Boolean.TRUE.equals(success)) {
                    acquired = true;
                    break;
                }
+                if (System.nanoTime() >= deadline) {
+                    break;
+                }
                Thread.sleep(RETRY_INTERVAL_MILLIS);
-            }
+            } while (System.nanoTime() <= deadline);
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new IllegalStateException("等待分布式锁被中断，lockKey=" + lockKey, e);
        }
        if (!acquired) {
-            throw new IllegalStateException("获取分布式锁失败，请稍后重试，lockKey=" + lockKey);
+            return null;
        }
        return new LockHandle(lockKey, lockValue, leaseTimeout);
    }
--- a/easyflow-commons/easyflow-common-cache/src/test/java/tech/easyflow/common/cache/DistributedScheduledLockAspectTest.java
+++ b/easyflow-commons/easyflow-common-cache/src/test/java/tech/easyflow/common/cache/DistributedScheduledLockAspectTest.java
@@ -0,0 +1,108 @@
+package tech.easyflow.common.cache;
+
+import org.aspectj.lang.ProceedingJoinPoint;
+import org.aspectj.lang.Signature;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mockito;
+import org.springframework.data.redis.core.StringRedisTemplate;
+import org.springframework.data.redis.core.ValueOperations;
+import org.springframework.data.redis.core.script.RedisScript;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.time.Duration;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * {@link DistributedScheduledLockAspect} 回归测试。
+ */
+public class DistributedScheduledLockAspectTest {
+
+    /**
+     * 验证未抢到调度锁时跳过原方法。
+     *
+     * @throws Throwable 切面执行异常
+     */
+    @Test
+    public void aroundShouldSkipTaskWhenLockIsHeld() throws Throwable {
+        RedisLockExecutor executor = createExecutor(false);
+        DistributedScheduledLockAspect aspect = new DistributedScheduledLockAspect(executor);
+        AtomicInteger proceedCount = new AtomicInteger();
+
+        Object result = aspect.around(
+            mockJoinPoint(proceedCount),
+            annotatedMethod("lockedTask").getAnnotation(DistributedScheduledLock.class)
+        );
+
+        Assert.assertNull(result);
+        Assert.assertEquals(0, proceedCount.get());
+    }
+
+    /**
+     * 验证抢到调度锁时执行原方法并释放锁。
+     *
+     * @throws Throwable 切面执行异常
+     */
+    @Test
+    public void aroundShouldProceedAndReleaseWhenLockAcquired() throws Throwable {
+        RedisLockExecutor executor = createExecutor(true);
+        DistributedScheduledLockAspect aspect = new DistributedScheduledLockAspect(executor);
+        AtomicInteger proceedCount = new AtomicInteger();
+
+        Object result = aspect.around(
+            mockJoinPoint(proceedCount),
+            annotatedMethod("lockedTask").getAnnotation(DistributedScheduledLock.class)
+        );
+
+        Assert.assertEquals("ok", result);
+        Assert.assertEquals(1, proceedCount.get());
+    }
+
+    @DistributedScheduledLock(key = "easyflow:test:scheduled", leaseSeconds = 30L)
+    private void lockedTask() {
+    }
+
+    private Method annotatedMethod(String methodName) throws NoSuchMethodException {
+        Method method = DistributedScheduledLockAspectTest.class.getDeclaredMethod(methodName);
+        method.setAccessible(true);
+        return method;
+    }
+
+    private ProceedingJoinPoint mockJoinPoint(AtomicInteger proceedCount) throws Throwable {
+        ProceedingJoinPoint joinPoint = Mockito.mock(ProceedingJoinPoint.class);
+        Signature signature = Mockito.mock(Signature.class);
+        Mockito.when(signature.toShortString()).thenReturn("lockedTask()");
+        Mockito.when(joinPoint.getSignature()).thenReturn(signature);
+        Mockito.when(joinPoint.proceed()).thenAnswer(invocation -> {
+            proceedCount.incrementAndGet();
+            return "ok";
+        });
+        return joinPoint;
+    }
+
+    @SuppressWarnings("unchecked")
+    private RedisLockExecutor createExecutor(boolean acquired) throws Exception {
+        StringRedisTemplate redisTemplate = Mockito.mock(StringRedisTemplate.class);
+        ValueOperations<String, String> valueOperations = Mockito.mock(ValueOperations.class);
+        Mockito.when(valueOperations.setIfAbsent(
+            ArgumentMatchers.anyString(),
+            ArgumentMatchers.anyString(),
+            ArgumentMatchers.any(Duration.class)
+        )).thenReturn(acquired);
+        Mockito.when(redisTemplate.opsForValue()).thenReturn(valueOperations);
+        Mockito.when(redisTemplate.execute(
+            ArgumentMatchers.<RedisScript<Long>>any(),
+            ArgumentMatchers.<List<String>>any(),
+            ArgumentMatchers.<Object[]>any()
+        )).thenReturn(1L);
+
+        RedisLockExecutor executor = new RedisLockExecutor();
+        Field field = RedisLockExecutor.class.getDeclaredField("stringRedisTemplate");
+        field.setAccessible(true);
+        field.set(executor, redisTemplate);
+        return executor;
+    }
+}
--- a/easyflow-commons/easyflow-common-cache/src/test/java/tech/easyflow/common/cache/RedisLockExecutorTest.java
+++ b/easyflow-commons/easyflow-common-cache/src/test/java/tech/easyflow/common/cache/RedisLockExecutorTest.java
@@ -0,0 +1,98 @@
+package tech.easyflow.common.cache;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mockito;
+import org.springframework.data.redis.core.StringRedisTemplate;
+import org.springframework.data.redis.core.ValueOperations;
+import org.springframework.data.redis.core.script.RedisScript;
+
+import java.lang.reflect.Field;
+import java.time.Duration;
+import java.util.List;
+
+/**
+ * {@link RedisLockExecutor} 回归测试。
+ */
+public class RedisLockExecutorTest {
+
+    /**
+     * 验证锁被占用时返回 null，便于调度任务跳过本轮执行。
+     *
+     * @throws Exception 反射注入异常
+     */
+    @Test
+    public void tryAcquireShouldReturnNullWhenLockIsHeld() throws Exception {
+        StringRedisTemplate redisTemplate = Mockito.mock(StringRedisTemplate.class);
+        ValueOperations<String, String> valueOperations = mockValueOperations(false);
+        Mockito.when(redisTemplate.opsForValue()).thenReturn(valueOperations);
+
+        RedisLockExecutor executor = new RedisLockExecutor();
+        setRedisTemplate(executor, redisTemplate);
+
+        RedisLockExecutor.LockHandle handle = executor.tryAcquire(
+            "easyflow:test:lock",
+            Duration.ZERO,
+            Duration.ofSeconds(30)
+        );
+
+        Assert.assertNull(handle);
+        Mockito.verify(valueOperations).setIfAbsent(
+            ArgumentMatchers.eq("easyflow:test:lock"),
+            ArgumentMatchers.anyString(),
+            ArgumentMatchers.eq(Duration.ofSeconds(30))
+        );
+    }
+
+    /**
+     * 验证锁获取成功后释放会执行 owner token 校验脚本。
+     *
+     * @throws Exception 反射注入异常
+     */
+    @Test
+    public void acquiredHandleShouldReleaseLockWithOwnerToken() throws Exception {
+        StringRedisTemplate redisTemplate = Mockito.mock(StringRedisTemplate.class);
+        ValueOperations<String, String> valueOperations = mockValueOperations(true);
+        Mockito.when(redisTemplate.opsForValue()).thenReturn(valueOperations);
+        Mockito.when(redisTemplate.execute(
+            ArgumentMatchers.<RedisScript<Long>>any(),
+            ArgumentMatchers.<List<String>>any(),
+            ArgumentMatchers.<Object[]>any()
+        )).thenReturn(1L);
+
+        RedisLockExecutor executor = new RedisLockExecutor();
+        setRedisTemplate(executor, redisTemplate);
+
+        RedisLockExecutor.LockHandle handle = executor.tryAcquire(
+            "easyflow:test:lock",
+            Duration.ZERO,
+            Duration.ofSeconds(30)
+        );
+
+        Assert.assertNotNull(handle);
+        handle.release();
+        Mockito.verify(redisTemplate).execute(
+            ArgumentMatchers.<RedisScript<Long>>any(),
+            ArgumentMatchers.eq(List.of("easyflow:test:lock")),
+            ArgumentMatchers.<Object[]>any()
+        );
+    }
+
+    @SuppressWarnings("unchecked")
+    private ValueOperations<String, String> mockValueOperations(boolean acquired) {
+        ValueOperations<String, String> valueOperations = Mockito.mock(ValueOperations.class);
+        Mockito.when(valueOperations.setIfAbsent(
+            ArgumentMatchers.anyString(),
+            ArgumentMatchers.anyString(),
+            ArgumentMatchers.any(Duration.class)
+        )).thenReturn(acquired);
+        return valueOperations;
+    }
+
+    private void setRedisTemplate(RedisLockExecutor executor, StringRedisTemplate redisTemplate) throws Exception {
+        Field field = RedisLockExecutor.class.getDeclaredField("stringRedisTemplate");
+        field.setAccessible(true);
+        field.set(executor, redisTemplate);
+    }
+}