diff --git a/easyflow-commons/easyflow-common-ai/src/main/java/tech/easyflow/common/ai/plugin/PluginHttpClient.java b/easyflow-commons/easyflow-common-ai/src/main/java/tech/easyflow/common/ai/plugin/PluginHttpClient.java index cb36228c..42d3ce2b 100644 --- a/easyflow-commons/easyflow-common-ai/src/main/java/tech/easyflow/common/ai/plugin/PluginHttpClient.java +++ b/easyflow-commons/easyflow-common-ai/src/main/java/tech/easyflow/common/ai/plugin/PluginHttpClient.java @@ -1,182 +1,461 @@ package tech.easyflow.common.ai.plugin; -import cn.hutool.core.util.ObjectUtil; -import cn.hutool.http.*; import cn.hutool.json.JSONObject; import cn.hutool.json.JSONUtil; +import okhttp3.Dispatcher; +import okhttp3.HttpUrl; +import okhttp3.MediaType; +import okhttp3.MultipartBody; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import okhttp3.ResponseBody; +import okio.BufferedSink; import org.springframework.web.multipart.MultipartFile; -import java.util.*; -import java.util.concurrent.atomic.AtomicBoolean; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; -public class PluginHttpClient { +/** + * 使用共享连接池、有界并发和实际响应字节限制调用 HTTP 插件。 + */ +public final class PluginHttpClient { - private static final int TIMEOUT = 10_000; + private static final int TIMEOUT_MILLIS = 10_000; + private static final long DEFAULT_MAX_RESPONSE_BYTES = + 64L * 1024L * 1024L; + private static final int MAX_TRACKED_HOSTS = 512; + private static final Semaphore GLOBAL_PERMITS = + new Semaphore(32, true); + private static final Semaphore OVERFLOW_HOST_PERMITS = + new Semaphore(8, true); + private static final Map HOST_PERMITS = + new ConcurrentHashMap<>(); + private static final Object HOST_REGISTRY_LOCK = new Object(); + private static final OkHttpClient CLIENT = buildClient(true); + private static final OkHttpClient NO_RETRY_CLIENT = buildClient(false); - public static JSONObject sendRequest(String url, String method, - Map headers, - List pluginParams) { - // 1. 处理路径参数 - String processedUrl = replacePathVariables(url, pluginParams); - - // 2. 初始化请求 - Method httpMethod = Method.valueOf(method.toUpperCase()); - HttpRequest request = HttpRequest.of(processedUrl) - .method(httpMethod) - .timeout(TIMEOUT); - // 3. 处理请求头（合并默认头和参数头） - processHeaders(request, headers, pluginParams); - - // 4. 处理查询参数和请求体 - processQueryAndBodyParams(request, httpMethod, pluginParams); - - // 5. 执行请求 - HttpResponse response = request.execute(); - return JSONUtil.parseObj(response.body()); + private PluginHttpClient() { } /** - * 处理请求头（合并默认头和参数头） + * 发送插件请求。 + * + * @param url 插件 URL + * @param method HTTP 方法 + * @param headers 默认请求头 + * @param pluginParams 插件参数 + * @return JSON 响应 */ - private static void processHeaders(HttpRequest request, - Map defaultHeaders, - List params) { - // 添加默认头 - if (ObjectUtil.isNotEmpty(defaultHeaders)) { - defaultHeaders.forEach((k, v) -> request.header(k, v.toString())); - } + public static JSONObject sendRequest( + String url, + String method, + Map headers, + List pluginParams) { + String normalizedMethod = method == null + ? "GET" + : method.trim().toUpperCase(); + List safeParams = pluginParams == null + ? List.of() + : pluginParams; + HttpUrl processedUrl = buildUrl(url, safeParams); + Request.Builder requestBuilder = new Request.Builder().url(processedUrl); + applyHeaders(requestBuilder, headers, safeParams); + RequestBody requestBody = buildRequestBody( + normalizedMethod, safeParams); + applyMethod(requestBuilder, normalizedMethod, requestBody); - // 添加参数中指定的头 - params.stream() - .filter(p -> "header".equalsIgnoreCase(p.getMethod()) && p.isEnabled()) - .forEach(p -> request.header(p.getName(), p.getDefaultValue().toString())); - } - - /** - * 处理查询参数和请求体 - */ - /** - * 处理查询参数和请求体（新增文件参数支持） - */ - private static void processQueryAndBodyParams(HttpRequest request, - Method httpMethod, - List params) { - Map queryParams = new HashMap<>(); - Map bodyParams = new HashMap<>(); - // 标记是否包含文件参数 - AtomicBoolean hasMultipartFile = new AtomicBoolean(false); - - // 分类参数（同时检测是否有文件） - params.stream() - .filter(PluginParam::isEnabled) - .forEach(p -> { - String methodType = p.getMethod().toLowerCase(); - Object paramValue = buildNestedParamValue(p); - - // 检测是否为文件参数（MultipartFile 类型） - if (paramValue instanceof org.springframework.web.multipart.MultipartFile) { - hasMultipartFile.set(true); - } - - switch (methodType) { - case "query": - queryParams.put(p.getName(), paramValue); - break; - case "body": - bodyParams.put(p.getName(), paramValue); - break; - } - }); - - // 1. 设置查询参数（原有逻辑不变） - if (!queryParams.isEmpty()) { - request.form(queryParams); - } - - // 2. 设置请求体（分两种情况：有文件 vs 无文件） - if (!bodyParams.isEmpty() && (httpMethod == Method.POST || httpMethod == Method.PUT)) { - if (hasMultipartFile.get()) { - // 2.1 包含文件参数 → 用 multipart/form-data 格式 - processMultipartBody(request, bodyParams); - } else { - // 2.2 无文件参数 → 保持原有 JSON 格式 - request.body(JSONUtil.toJsonStr(bodyParams)) - .header(Header.CONTENT_TYPE, ContentType.JSON.getValue()); + Semaphore hostPermits = hostSemaphore(processedUrl.host()); + boolean hostAcquired = false; + boolean globalAcquired = false; + try { + hostAcquired = hostPermits.tryAcquire( + 1L, TimeUnit.SECONDS); + if (!hostAcquired) { + throw new IllegalStateException( + "插件目标并发已达上限: " + processedUrl.host()); + } + globalAcquired = GLOBAL_PERMITS.tryAcquire( + 1L, TimeUnit.SECONDS); + if (!globalAcquired) { + throw new IllegalStateException( + "插件 HTTP 并发已达上限"); + } + OkHttpClient client = isIdempotent(normalizedMethod) + ? CLIENT + : NO_RETRY_CLIENT; + try (Response response = + client.newCall(requestBuilder.build()).execute()) { + ResponseBody body = response.body(); + if (body == null) { + throw new IllegalStateException("插件响应内容为空"); + } + long maxBytes = Long.getLong( + "easyflow.plugin.http.max-response-bytes", + DEFAULT_MAX_RESPONSE_BYTES); + String responseText = readBounded(body, maxBytes); + return JSONUtil.parseObj(responseText); + } + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new IllegalStateException( + "插件 HTTP 请求被中断", error); + } catch (IOException error) { + throw new IllegalStateException( + "插件 HTTP 请求失败", error); + } finally { + if (globalAcquired) { + GLOBAL_PERMITS.release(); + } + if (hostAcquired) { + hostPermits.release(); } } } /** - * 递归构建嵌套参数值 - * @param param 当前参数 - * @return 如果是 Object 类型，返回 Map；否则返回 defaultValue + * 创建共享 OkHttp 客户端。 + * + * @param retryOnConnectionFailure 是否允许连接级自动恢复 + * @return 共享客户端 */ - private static Object buildNestedParamValue(PluginParam param) { - // 如果不是 Object 类型，直接返回默认值 + private static OkHttpClient buildClient( + boolean retryOnConnectionFailure) { + Dispatcher dispatcher = new Dispatcher(); + dispatcher.setMaxRequests(32); + dispatcher.setMaxRequestsPerHost(8); + return new OkHttpClient.Builder() + .dispatcher(dispatcher) + .connectTimeout(TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) + .readTimeout(TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) + .writeTimeout(TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) + .callTimeout(TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) + .retryOnConnectionFailure(retryOnConnectionFailure) + .build(); + } + + /** + * 构造带路径参数和查询参数的 URL。 + * + * @param url 原始 URL + * @param params 插件参数 + * @return 完整 URL + */ + private static HttpUrl buildUrl( + String url, List params) { + String processed = replacePathVariables(url, params); + HttpUrl parsed = HttpUrl.parse(processed); + if (parsed == null) { + throw new IllegalArgumentException( + "插件 URL 不合法: " + processed); + } + HttpUrl.Builder builder = parsed.newBuilder(); + for (PluginParam param : params) { + if (param.isEnabled() + && "query".equalsIgnoreCase(param.getMethod()) + && param.getDefaultValue() != null) { + builder.addQueryParameter( + param.getName(), + stringify(param.getDefaultValue())); + } + } + return builder.build(); + } + + /** + * 合并默认请求头和参数请求头。 + * + * @param builder 请求构造器 + * @param headers 默认请求头 + * @param params 插件参数 + */ + private static void applyHeaders( + Request.Builder builder, + Map headers, + List params) { + if (headers != null) { + headers.forEach((name, value) -> { + if (name != null && value != null) { + builder.header(name, String.valueOf(value)); + } + }); + } + for (PluginParam param : params) { + if (param.isEnabled() + && "header".equalsIgnoreCase(param.getMethod()) + && param.getDefaultValue() != null) { + builder.header( + param.getName(), + String.valueOf(param.getDefaultValue())); + } + } + } + + /** + * 构造 JSON 或 multipart 请求体。 + * + * @param method HTTP 方法 + * @param params 插件参数 + * @return 请求体；无请求体时为 {@code null} + */ + private static RequestBody buildRequestBody( + String method, List params) { + if (!supportsRequestBody(method)) { + return null; + } + Map bodyValues = new HashMap<>(); + boolean multipart = false; + for (PluginParam param : params) { + if (!param.isEnabled() + || !"body".equalsIgnoreCase(param.getMethod())) { + continue; + } + Object value = buildNestedParamValue(param); + bodyValues.put(param.getName(), value); + multipart |= value instanceof MultipartFile; + } + if (bodyValues.isEmpty()) { + return RequestBody.create( + new byte[0], null); + } + if (!multipart) { + return RequestBody.create( + JSONUtil.toJsonStr(bodyValues), + MediaType.parse("application/json; charset=utf-8")); + } + MultipartBody.Builder builder = + new MultipartBody.Builder().setType(MultipartBody.FORM); + for (Map.Entry entry : bodyValues.entrySet()) { + Object value = entry.getValue(); + if (value instanceof MultipartFile) { + MultipartFile file = (MultipartFile) value; + MediaType contentType = MediaType.parse( + file.getContentType() == null + ? "application/octet-stream" + : file.getContentType()); + builder.addFormDataPart( + entry.getKey(), + Objects.toString( + file.getOriginalFilename(), + entry.getKey()), + streamingFileBody(file, contentType)); + } else { + builder.addFormDataPart( + entry.getKey(), stringify(value)); + } + } + return builder.build(); + } + + /** + * 创建从 MultipartFile 流式读取的请求体，避免 getBytes 二次复制。 + * + * @param file 文件参数 + * @param mediaType 内容类型 + * @return 流式请求体 + */ + private static RequestBody streamingFileBody( + MultipartFile file, MediaType mediaType) { + return new RequestBody() { + @Override + public MediaType contentType() { + return mediaType; + } + + @Override + public long contentLength() { + return file.getSize(); + } + + @Override + public void writeTo(BufferedSink sink) throws IOException { + try (InputStream inputStream = file.getInputStream()) { + byte[] buffer = new byte[64 * 1024]; + int read; + while ((read = inputStream.read(buffer)) != -1) { + sink.write(buffer, 0, read); + } + } + } + }; + } + + /** + * 应用 HTTP 方法。 + * + * @param builder 请求构造器 + * @param method HTTP 方法 + * @param body 请求体 + */ + private static void applyMethod( + Request.Builder builder, + String method, + RequestBody body) { + if ("GET".equals(method)) { + builder.get(); + } else if ("HEAD".equals(method)) { + builder.head(); + } else if ("DELETE".equals(method) && body == null) { + builder.delete(); + } else { + builder.method(method, body); + } + } + + /** + * 按实际字节读取响应。 + * + * @param body 响应体 + * @param maxBytes 最大字节数 + * @return UTF-8 响应 + * @throws IOException 读取失败或响应超限 + */ + private static String readBounded( + ResponseBody body, long maxBytes) throws IOException { + if (maxBytes > 0L + && body.contentLength() > maxBytes) { + throw new IOException( + "插件响应超过字节上限: " + maxBytes); + } + try (InputStream inputStream = body.byteStream(); + ByteArrayOutputStream outputStream = + new ByteArrayOutputStream(8 * 1024)) { + byte[] buffer = new byte[64 * 1024]; + long total = 0L; + int read; + while ((read = inputStream.read(buffer)) != -1) { + total += read; + if (maxBytes > 0L && total > maxBytes) { + throw new IOException( + "插件响应超过字节上限: " + maxBytes); + } + outputStream.write(buffer, 0, read); + } + return outputStream.toString(StandardCharsets.UTF_8); + } + } + + /** + * 递归构造对象参数。 + * + * @param param 参数 + * @return 参数值 + */ + private static Object buildNestedParamValue( + PluginParam param) { if (!"Object".equalsIgnoreCase(param.getType())) { return param.getDefaultValue(); } - - // 如果是 Object 类型，递归处理子参数 - Map nestedParams = new HashMap<>(); + Map nested = new HashMap<>(); if (param.getChildren() != null) { - param.getChildren().stream() - .filter(PluginParam::isEnabled) - .forEach(child -> { - Object childValue = buildNestedParamValue(child); // 递归处理子参数 - nestedParams.put(child.getName(), childValue); - }); + for (PluginParam child : param.getChildren()) { + if (child.isEnabled()) { + nested.put( + child.getName(), + buildNestedParamValue(child)); + } + } } - return nestedParams; + return nested; } /** - * 替换URL中的路径变量 {xxx} + * 替换 URL 路径变量。 + * + * @param url 原始 URL + * @param params 插件参数 + * @return 替换后的 URL */ - private static String replacePathVariables(String url, List params) { - String result = url; - - // 收集路径参数 - Map pathParams = new HashMap<>(); - params.stream() - .filter(p -> "path".equalsIgnoreCase(p.getMethod()) && p.isEnabled()) - .forEach(p -> pathParams.put(p.getName(), p.getDefaultValue())); - - // 替换变量 - for (Map.Entry entry : pathParams.entrySet()) { - result = result.replaceAll("\\{" + entry.getKey() + "\\}", - entry.getValue().toString()); + private static String replacePathVariables( + String url, List params) { + String result = Objects.requireNonNull( + url, "插件 URL 不能为空"); + for (PluginParam param : params) { + if (param.isEnabled() + && "path".equalsIgnoreCase(param.getMethod()) + && param.getDefaultValue() != null) { + result = result.replace( + "{" + param.getName() + "}", + String.valueOf(param.getDefaultValue())); + } } - return result; } - private static void processMultipartBody(HttpRequest request, Map bodyParams) { - // 手动设置 Content-Type 为 multipart/form-data - request.header(Header.CONTENT_TYPE, "multipart/form-data"); - for (Map.Entry entry : bodyParams.entrySet()) { - String paramName = entry.getKey(); - Object paramValue = entry.getValue(); - - if (paramValue instanceof MultipartFile) { - MultipartFile file = (MultipartFile) paramValue; - try { - request.form(paramName, file.getBytes(), file.getOriginalFilename()); - } catch (Exception e) { - throw new RuntimeException(String.format("文件参数处理失败：参数名=%s，文件名=%s", - paramName, file.getOriginalFilename()), e); - } - } else { - // 处理普通参数 - String valueStr; - if (paramValue instanceof String || paramValue instanceof Number || paramValue instanceof Boolean) { - valueStr = paramValue.toString(); - } else { - valueStr = JSONUtil.toJsonStr(paramValue); - } - request.form(paramName, valueStr); + /** + * 获取主机隔离许可并严格限制注册表体积。 + * + * @param host 主机名 + * @return 主机许可 + */ + private static Semaphore hostSemaphore(String host) { + Semaphore existing = HOST_PERMITS.get(host); + if (existing != null) { + return existing; + } + synchronized (HOST_REGISTRY_LOCK) { + existing = HOST_PERMITS.get(host); + if (existing != null) { + return existing; } + if (HOST_PERMITS.size() >= MAX_TRACKED_HOSTS) { + return OVERFLOW_HOST_PERMITS; + } + Semaphore created = new Semaphore(8, true); + HOST_PERMITS.put(host, created); + return created; } } + /** + * 判断方法是否幂等。 + * + * @param method HTTP 方法 + * @return 是否允许连接级恢复 + */ + private static boolean isIdempotent(String method) { + return "GET".equals(method) + || "HEAD".equals(method) + || "OPTIONS".equals(method); + } + + /** + * 判断方法是否支持请求体。 + * + * @param method HTTP 方法 + * @return 是否支持请求体 + */ + private static boolean supportsRequestBody(String method) { + return "POST".equals(method) + || "PUT".equals(method) + || "PATCH".equals(method); + } + + /** + * 将普通或嵌套值转换为表单字符串。 + * + * @param value 参数值 + * @return 表单值 + */ + private static String stringify(Object value) { + if (value == null) { + return ""; + } + if (value instanceof String + || value instanceof Number + || value instanceof Boolean) { + return String.valueOf(value); + } + return JSONUtil.toJsonStr(value); + } } diff --git a/easyflow-commons/easyflow-common-base/src/main/java/tech/easyflow/common/constant/CacheKey.java b/easyflow-commons/easyflow-common-base/src/main/java/tech/easyflow/common/constant/CacheKey.java index 538a9f6e..66ea80c2 100644 --- a/easyflow-commons/easyflow-common-base/src/main/java/tech/easyflow/common/constant/CacheKey.java +++ b/easyflow-commons/easyflow-common-base/src/main/java/tech/easyflow/common/constant/CacheKey.java @@ -8,7 +8,19 @@ public interface CacheKey { String CHAIN_STATUS_CACHE_KEY = "chain:status:"; String CHAIN_CACHE_KEY = "chainState:"; + /** + * 工作流实例状态 CAS key 前缀；同实例状态使用 hash tag 保持同槽。 + */ + String CHAIN_STATE_CAS_KEY = "workflowState:"; + String CHAIN_DEFINITION_SNAPSHOT_CACHE_KEY = "chainDefinitionSnapshot:"; + String CHAIN_LOCK_KEY = "chainLock:"; String NODE_CACHE_KEY = "nodeState:"; + String LOOP_RESULT_CACHE_KEY = "loopResult:"; + String TRIGGER_DATA_KEY = "workflowTrigger:data:"; + String TRIGGER_CLAIM_KEY = "workflowTrigger:claim:"; + String TRIGGER_DEAD_LETTER_KEY = "workflowTrigger:dead:"; + String TRIGGER_PENDING_KEY = "workflowTrigger:pending"; + String WORKFLOW_DEFINITION_VERSION_KEY = "workflowDefinition:version:"; String OAUTH_STATE_KEY = "oauth:state:"; } diff --git a/easyflow-commons/easyflow-common-cache/src/main/java/tech/easyflow/common/cache/RedisIdempotencyExecutor.java b/easyflow-commons/easyflow-common-cache/src/main/java/tech/easyflow/common/cache/RedisIdempotencyExecutor.java new file mode 100644 index 00000000..e09778cc --- /dev/null +++ b/easyflow-commons/easyflow-common-cache/src/main/java/tech/easyflow/common/cache/RedisIdempotencyExecutor.java @@ -0,0 +1,296 @@ +package tech.easyflow.common.cache; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.data.redis.core.script.DefaultRedisScript; +import org.springframework.stereotype.Component; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Duration; +import java.util.Collections; +import java.util.HexFormat; +import java.util.Objects; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * 基于 Redis owner token 的通用幂等执行器。 + * + *

该执行器用于降低持久化触发器重复投递造成的副作用重复执行窗口。处理中凭证和完成 + * 凭证均有界过期，异常时仅允许当前 owner 释放凭证。

+ */ +@Component +public class RedisIdempotencyExecutor { + + private static final Logger log = + LoggerFactory.getLogger(RedisIdempotencyExecutor.class); + private static final String KEY_PREFIX = "idempotency:"; + private static final String PROCESSING_PREFIX = "P:"; + private static final String COMPLETED_PREFIX = "C:"; + private static final Duration DEFAULT_PROCESSING_TTL = + Duration.ofMinutes(5); + private static final Duration DEFAULT_COMPLETED_TTL = + Duration.ofDays(3); + private static final ScheduledExecutorService LEASE_RENEWER = + Executors.newSingleThreadScheduledExecutor(runnable -> { + Thread thread = new Thread( + runnable, "redis-idempotency-lease-renewer"); + thread.setDaemon(true); + return thread; + }); + private static final DefaultRedisScript CLAIM_SCRIPT = script( + "local current = redis.call('get', KEYS[1]); " + + "if current == ARGV[3] then return 2 end; " + + "if current and string.sub(current, 3, 66) ~= ARGV[4] then return -1 end; " + + "if current then return 0 end; " + + "redis.call('psetex', KEYS[1], ARGV[2], ARGV[1]); return 1"); + private static final DefaultRedisScript COMPLETE_SCRIPT = script( + "if redis.call('get', KEYS[1]) == ARGV[1] then " + + "redis.call('psetex', KEYS[1], ARGV[2], ARGV[3]); return 1 " + + "else return 0 end"); + private static final DefaultRedisScript RELEASE_SCRIPT = script( + "if redis.call('get', KEYS[1]) == ARGV[1] then " + + "return redis.call('del', KEYS[1]) else return 0 end"); + private static final DefaultRedisScript RENEW_SCRIPT = script( + "if redis.call('get', KEYS[1]) == ARGV[1] then " + + "return redis.call('pexpire', KEYS[1], ARGV[2]) " + + "else return 0 end"); + + private final StringRedisTemplate redisTemplate; + private final Duration processingTtl; + private final Duration completedTtl; + + /** + * 创建 Redis 幂等执行器。 + * + * @param redisTemplate Redis 字符串模板 + */ + @Autowired + public RedisIdempotencyExecutor(StringRedisTemplate redisTemplate) { + this( + redisTemplate, + DEFAULT_PROCESSING_TTL, + DEFAULT_COMPLETED_TTL); + } + + /** + * 创建可指定租约周期的 Redis 幂等执行器。 + * + * @param redisTemplate Redis 字符串模板 + * @param processingTtl 处理中 owner 租约周期 + * @param completedTtl 完成凭证保留周期 + */ + RedisIdempotencyExecutor( + StringRedisTemplate redisTemplate, + Duration processingTtl, + Duration completedTtl) { + this.redisTemplate = Objects.requireNonNull( + redisTemplate, "redisTemplate must not be null"); + this.processingTtl = requirePositive( + processingTtl, "processingTtl"); + this.completedTtl = requirePositive( + completedTtl, "completedTtl"); + } + + /** + * 以稳定幂等键至多执行一次当前可观测操作。 + * + *

返回 {@code false} 表示该键已有成功记录。另一个 owner 仍在执行时抛出明确异常， + * 由上层持久化触发器稍后重试，避免提前返回虚假成功。

+ * + * @param idempotencyKey 稳定业务幂等键 + * @param action 副作用操作 + * @return 本次实际执行操作时为 {@code true}，已有成功记录时为 {@code false} + * @throws IdempotentOperationInProgressException 同一操作仍由其他 owner 执行时抛出 + * @throws IllegalStateException 操作完成后无法确认幂等凭证时抛出 + */ + public boolean executeOnce(String idempotencyKey, Runnable action) { + return executeOnce(idempotencyKey, sha256(""), action); + } + + /** + * 以稳定幂等键和负载摘要至多执行一次当前可观测操作。 + * + * @param idempotencyKey 稳定业务幂等键 + * @param payloadHash 负载摘要，用于拒绝同键不同数据 + * @param action 副作用操作 + * @return 本次实际执行操作时为 {@code true}，已有成功记录时为 {@code false} + * @throws IdempotentOperationInProgressException 同一操作仍在执行 + * @throws IdempotencyPayloadMismatchException 同一幂等键对应不同负载 + */ + public boolean executeOnce(String idempotencyKey, String payloadHash, Runnable action) { + if (idempotencyKey == null || idempotencyKey.isBlank()) { + Objects.requireNonNull(action, "action must not be null").run(); + return true; + } + Objects.requireNonNull(action, "action must not be null"); + String normalizedPayloadHash = Objects.requireNonNull( + payloadHash, "payloadHash must not be null"); + String key = KEY_PREFIX + sha256(idempotencyKey); + String ownerToken = UUID.randomUUID().toString(); + String processingValue = PROCESSING_PREFIX + normalizedPayloadHash + ":" + ownerToken; + String completedValue = COMPLETED_PREFIX + normalizedPayloadHash; + Long claim = redisTemplate.execute( + CLAIM_SCRIPT, + Collections.singletonList(key), + processingValue, + String.valueOf(processingTtl.toMillis()), + completedValue, + normalizedPayloadHash); + if (Long.valueOf(2L).equals(claim)) { + return false; + } + if (Long.valueOf(-1L).equals(claim)) { + throw new IdempotencyPayloadMismatchException(idempotencyKey); + } + if (!Long.valueOf(1L).equals(claim)) { + throw new IdempotentOperationInProgressException(idempotencyKey); + } + + AtomicBoolean ownershipLost = new AtomicBoolean(); + long renewIntervalMillis = Math.max( + 1L, processingTtl.toMillis() / 3L); + ScheduledFuture renewal = LEASE_RENEWER.scheduleAtFixedRate( + () -> renewLease( + key, processingValue, ownershipLost), + renewIntervalMillis, + renewIntervalMillis, + TimeUnit.MILLISECONDS); + try { + action.run(); + } catch (RuntimeException | Error error) { + renewal.cancel(false); + redisTemplate.execute( + RELEASE_SCRIPT, Collections.singletonList(key), processingValue); + throw error; + } + renewal.cancel(false); + if (ownershipLost.get()) { + throw new IllegalStateException( + "Idempotency ownership lost while operation was running"); + } + Long completed = redisTemplate.execute( + COMPLETE_SCRIPT, + Collections.singletonList(key), + processingValue, + String.valueOf(completedTtl.toMillis()), + completedValue); + if (!Long.valueOf(1L).equals(completed)) { + // 副作用已经完成时保留处理中凭证，避免确认异常立即打开重复执行窗口。 + throw new IllegalStateException( + "Idempotency ownership lost after operation completed"); + } + return true; + } + + /** + * 续期仍由当前 owner 持有的处理中凭证。 + * + * @param key Redis 幂等键 + * @param processingValue 当前 owner 完整凭证 + * @param ownershipLost owner 丢失标记 + */ + private void renewLease( + String key, + String processingValue, + AtomicBoolean ownershipLost) { + try { + Long renewed = redisTemplate.execute( + RENEW_SCRIPT, + Collections.singletonList(key), + processingValue, + String.valueOf(processingTtl.toMillis())); + if (!Long.valueOf(1L).equals(renewed)) { + ownershipLost.set(true); + } + } catch (RuntimeException error) { + // 短暂 Redis 故障由后续周期继续续期；最终完成脚本仍会校验 owner token。 + log.warn("Redis idempotency lease renewal failed", error); + } + } + + /** + * 校验正数时长配置。 + * + * @param duration 配置值 + * @param name 配置名 + * @return 原配置值 + */ + private Duration requirePositive( + Duration duration, String name) { + Duration value = Objects.requireNonNull( + duration, name + " must not be null"); + if (value.isZero() || value.isNegative()) { + throw new IllegalArgumentException( + name + " must be positive"); + } + return value; + } + + /** + * 计算有界 Redis key 摘要。 + * + * @param value 原始幂等键 + * @return SHA-256 十六进制摘要 + */ + private String sha256(String value) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + return HexFormat.of().formatHex(digest.digest( + value.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException error) { + throw new IllegalStateException("SHA-256 is unavailable", error); + } + } + + /** + * 创建返回 Long 的 Redis Lua 脚本。 + * + * @param text 脚本文本 + * @return Redis 脚本 + */ + private static DefaultRedisScript script(String text) { + DefaultRedisScript script = new DefaultRedisScript<>(); + script.setScriptText(text); + script.setResultType(Long.class); + return script; + } + + /** + * 同一幂等操作仍在执行时的冲突异常。 + */ + public static final class IdempotentOperationInProgressException extends IllegalStateException { + + /** + * 创建幂等执行冲突异常。 + * + * @param idempotencyKey 冲突的业务幂等键 + */ + public IdempotentOperationInProgressException(String idempotencyKey) { + super("Idempotent operation is already in progress: " + idempotencyKey); + } + } + + /** + * 同一幂等键被不同负载复用时的冲突异常。 + */ + public static final class IdempotencyPayloadMismatchException extends IllegalStateException { + + /** + * 创建负载冲突异常。 + * + * @param idempotencyKey 冲突的业务幂等键 + */ + public IdempotencyPayloadMismatchException(String idempotencyKey) { + super("Idempotency payload mismatch: " + idempotencyKey); + } + } +} diff --git a/easyflow-commons/easyflow-common-cache/src/main/java/tech/easyflow/common/cache/RedisLockExecutor.java b/easyflow-commons/easyflow-common-cache/src/main/java/tech/easyflow/common/cache/RedisLockExecutor.java index 54911283..17d040bc 100644 --- a/easyflow-commons/easyflow-common-cache/src/main/java/tech/easyflow/common/cache/RedisLockExecutor.java +++ b/easyflow-commons/easyflow-common-cache/src/main/java/tech/easyflow/common/cache/RedisLockExecutor.java @@ -24,6 +24,8 @@ public class RedisLockExecutor { private static final DefaultRedisScript RELEASE_LOCK_SCRIPT; private static final DefaultRedisScript RENEW_LOCK_SCRIPT; + private static final DefaultRedisScript NEXT_FENCING_TOKEN_SCRIPT; + private static final DefaultRedisScript ACQUIRE_FENCED_LOCK_SCRIPT; static { RELEASE_LOCK_SCRIPT = new DefaultRedisScript<>(); @@ -40,6 +42,19 @@ public class RedisLockExecutor { "else return 0 end" ); RENEW_LOCK_SCRIPT.setResultType(Long.class); + NEXT_FENCING_TOKEN_SCRIPT = new DefaultRedisScript<>(); + NEXT_FENCING_TOKEN_SCRIPT.setScriptText( + "local token = redis.call('hincrby', KEYS[1], 'version', 1); " + + "redis.call('pexpire', KEYS[1], ARGV[1]); return token" + ); + NEXT_FENCING_TOKEN_SCRIPT.setResultType(Long.class); + ACQUIRE_FENCED_LOCK_SCRIPT = new DefaultRedisScript<>(); + ACQUIRE_FENCED_LOCK_SCRIPT.setScriptText( + "if redis.call('exists', KEYS[1]) == 1 then return 0 end; " + + "redis.call('psetex', KEYS[1], ARGV[2], ARGV[1]); " + + "local token = redis.call('hincrby', KEYS[2], 'version', 1); " + + "redis.call('pexpire', KEYS[2], ARGV[3]); return token"); + ACQUIRE_FENCED_LOCK_SCRIPT.setResultType(Long.class); } @Autowired @@ -132,7 +147,53 @@ public class RedisLockExecutor { if (!acquired) { return null; } - return new LockHandle(lockKey, lockValue, leaseTimeout); + return new LockHandle(lockKey, lockValue, leaseTimeout, 0L); + } + + /** + * 原子获取互斥锁并分配 fencing token。 + * + *

锁键和 fencing 键必须位于同一 Redis Cluster slot。成功的 SET 与 token + * 递增在同一 Lua 脚本内完成，消除新 owner 已取得锁但 token 尚未推进的窗口。

+ * + * @param lockKey 互斥锁键 + * @param fenceKey fencing token 哈希键 + * @param waitTimeout 等待时间 + * @param leaseTimeout 锁租约时间 + * @param fenceTtl fencing token 有效期 + * @return 获取成功时返回带 token 的锁句柄，否则返回 {@code null} + */ + public LockHandle tryAcquireFenced( + String lockKey, + String fenceKey, + Duration waitTimeout, + Duration leaseTimeout, + Duration fenceTtl) { + String lockValue = UUID.randomUUID().toString(); + long deadline = System.nanoTime() + waitTimeout.toNanos(); + try { + do { + Long token = stringRedisTemplate.execute( + ACQUIRE_FENCED_LOCK_SCRIPT, + java.util.List.of(lockKey, fenceKey), + lockValue, + String.valueOf(Math.max(1L, leaseTimeout.toMillis())), + String.valueOf(Math.max(1L, fenceTtl.toMillis()))); + if (token != null && token > 0L) { + return new LockHandle( + lockKey, lockValue, leaseTimeout, token); + } + if (System.nanoTime() >= deadline) { + return null; + } + Thread.sleep(RETRY_INTERVAL_MILLIS); + } while (System.nanoTime() <= deadline); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new IllegalStateException( + "等待 fenced 分布式锁被中断，lockKey=" + lockKey, error); + } + return null; } /** @@ -168,6 +229,25 @@ public class RedisLockExecutor { } } + /** + * 为已经取得互斥锁的实例分配单调递增 fencing token。 + * + * @param fenceKey fencing token 哈希键 + * @param ttl token 有效期 + * @return 大于零的 fencing token + * @throws IllegalStateException Redis 未返回有效 token 时抛出 + */ + public long nextFencingToken(String fenceKey, Duration ttl) { + Long token = stringRedisTemplate.execute( + NEXT_FENCING_TOKEN_SCRIPT, + Collections.singletonList(fenceKey), + String.valueOf(Math.max(1L, ttl.toMillis()))); + if (token == null || token <= 0L) { + throw new IllegalStateException("分配 fencing token 失败，fenceKey=" + fenceKey); + } + return token; + } + /** * 显式分布式锁句柄。 */ @@ -176,12 +256,27 @@ public class RedisLockExecutor { private final String lockKey; private final String lockValue; private final Duration leaseTimeout; + private final long fencingToken; private volatile boolean released; - private LockHandle(String lockKey, String lockValue, Duration leaseTimeout) { + private LockHandle( + String lockKey, + String lockValue, + Duration leaseTimeout, + long fencingToken) { this.lockKey = lockKey; this.lockValue = lockValue; this.leaseTimeout = leaseTimeout; + this.fencingToken = fencingToken; + } + + /** + * 获取原子分配的 fencing token。 + * + * @return 普通锁为 {@code 0}，fenced 锁为正数 + */ + public long getFencingToken() { + return fencingToken; } /** diff --git a/easyflow-commons/easyflow-common-cache/src/main/java/tech/easyflow/common/cache/RedisVersionedObjectStore.java b/easyflow-commons/easyflow-common-cache/src/main/java/tech/easyflow/common/cache/RedisVersionedObjectStore.java new file mode 100644 index 00000000..8b2ae0ac --- /dev/null +++ b/easyflow-commons/easyflow-common-cache/src/main/java/tech/easyflow/common/cache/RedisVersionedObjectStore.java @@ -0,0 +1,870 @@ +package tech.easyflow.common.cache; + +import com.alicp.jetcache.support.JavaValueEncoder; +import org.springframework.data.redis.connection.RedisConnection; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.connection.ReturnType; +import org.springframework.stereotype.Component; + +import java.io.Serializable; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * 基于 Redis Hash 与 Lua 的原子版本对象存储。 + * + *

对象继续使用项目既有的 Java 序列化协议，避免状态内多态值在迁移后改变类型。 + * payload 与 version 保存在同一 Redis Hash 中，创建、版本比较、写入和 TTL 刷新均由 + * 单次 Lua 脚本原子完成。

+ */ +@Component +public class RedisVersionedObjectStore implements VersionedObjectStore { + + private static final byte[] PAYLOAD_FIELD = bytes("payload"); + private static final byte[] CREATE_SCRIPT = bytes( + "if redis.call('exists', KEYS[1]) == 1 then return 0 end; " + + "redis.call('hset', KEYS[1], 'version', ARGV[1], 'payload', ARGV[2]); " + + "redis.call('pexpire', KEYS[1], ARGV[3]); return 1"); + private static final byte[] GUARDED_CREATE_SCRIPT = bytes( + "local guard = redis.call('hget', KEYS[2], 'version'); " + + "if not guard or guard ~= ARGV[2] then return -1 end; " + + "if redis.call('exists', KEYS[1]) == 1 then return 0 end; " + + "redis.call('hset', KEYS[1], 'version', ARGV[1], 'payload', ARGV[3]); " + + "redis.call('pexpire', KEYS[1], ARGV[4]); return 1"); + private static final byte[] DOUBLE_GUARDED_CREATE_SCRIPT = bytes( + "local guard = redis.call('hget', KEYS[2], 'version'); " + + "local secondary = redis.call('hget', KEYS[3], 'version'); " + + "if not guard or guard ~= ARGV[2] " + + "or not secondary or secondary ~= ARGV[3] then return -1 end; " + + "if redis.call('exists', KEYS[1]) == 1 then return 0 end; " + + "redis.call('hset', KEYS[1], 'version', ARGV[1], 'payload', ARGV[4]); " + + "redis.call('pexpire', KEYS[1], ARGV[5]); return 1"); + private static final byte[] CAS_SCRIPT = bytes( + "local current = redis.call('hget', KEYS[1], 'version'); " + + "if not current then return -1 end; " + + "if current ~= ARGV[1] then return 0 end; " + + "redis.call('hset', KEYS[1], 'version', ARGV[2], 'payload', ARGV[3]); " + + "redis.call('pexpire', KEYS[1], ARGV[4]); return 1"); + private static final byte[] GUARDED_CAS_SCRIPT = bytes( + "local guard = redis.call('hget', KEYS[2], 'version'); " + + "if not guard or guard ~= ARGV[3] then return -2 end; " + + "local current = redis.call('hget', KEYS[1], 'version'); " + + "if not current then return -1 end; " + + "if current ~= ARGV[1] then return 0 end; " + + "redis.call('hset', KEYS[1], 'version', ARGV[2], 'payload', ARGV[4]); " + + "redis.call('pexpire', KEYS[1], ARGV[5]); return 1"); + private static final byte[] DOUBLE_GUARDED_CAS_SCRIPT = bytes( + "local guard = redis.call('hget', KEYS[2], 'version'); " + + "local secondary = redis.call('hget', KEYS[3], 'version'); " + + "if not guard or guard ~= ARGV[3] " + + "or not secondary or secondary ~= ARGV[4] then return -2 end; " + + "local current = redis.call('hget', KEYS[1], 'version'); " + + "if not current then return -1 end; " + + "if current ~= ARGV[1] then return 0 end; " + + "redis.call('hset', KEYS[1], 'version', ARGV[2], 'payload', ARGV[5]); " + + "redis.call('pexpire', KEYS[1], ARGV[6]); return 1"); + private static final byte[] DOUBLE_GUARDED_DELETE_ALL_SCRIPT = bytes( + "local guard = redis.call('hget', KEYS[#KEYS - 1], 'version'); " + + "local secondary = redis.call('hget', KEYS[#KEYS], 'version'); " + + "if not guard or guard ~= ARGV[1] " + + "or not secondary or secondary ~= ARGV[2] then return -1 end; " + + "for i = 1, #KEYS - 2 do redis.call('del', KEYS[i]); end; " + + "return 1"); + private static final byte[] CREATE_FIELDS_SCRIPT = bytes( + "if redis.call('exists', KEYS[1]) == 1 then return 0 end; " + + "redis.call('hset', KEYS[1], 'version', ARGV[1]); " + + "for i = 2, #ARGV - 1, 2 do redis.call('hset', KEYS[1], ARGV[i], ARGV[i + 1]); end; " + + "redis.call('pexpire', KEYS[1], ARGV[#ARGV]); return 1"); + private static final byte[] GUARDED_CREATE_FIELDS_SCRIPT = bytes( + "local guard = redis.call('hget', KEYS[2], 'version'); " + + "if not guard or guard ~= ARGV[2] then return -1 end; " + + "if redis.call('exists', KEYS[1]) == 1 then return 0 end; " + + "redis.call('hset', KEYS[1], 'version', ARGV[1]); " + + "for i = 3, #ARGV - 1, 2 do redis.call('hset', KEYS[1], ARGV[i], ARGV[i + 1]); end; " + + "redis.call('pexpire', KEYS[1], ARGV[#ARGV]); return 1"); + private static final byte[] DOUBLE_GUARDED_CREATE_FIELDS_SCRIPT = bytes( + "local guard = redis.call('hget', KEYS[2], 'version'); " + + "local secondary = redis.call('hget', KEYS[3], 'version'); " + + "if not guard or guard ~= ARGV[2] " + + "or not secondary or secondary ~= ARGV[3] then return -1 end; " + + "if redis.call('exists', KEYS[1]) == 1 then return 0 end; " + + "redis.call('hset', KEYS[1], 'version', ARGV[1]); " + + "for i = 4, #ARGV - 1, 2 do " + + "redis.call('hset', KEYS[1], ARGV[i], ARGV[i + 1]); end; " + + "redis.call('pexpire', KEYS[1], ARGV[#ARGV]); return 1"); + private static final byte[] TRIPLE_GUARDED_CREATE_FIELDS_SCRIPT = bytes( + "local guard = redis.call('hget', KEYS[2], 'version'); " + + "local secondary = redis.call('hget', KEYS[3], 'version'); " + + "local tertiary = redis.call('hget', KEYS[4], 'version'); " + + "if not guard or guard ~= ARGV[2] " + + "or not secondary or secondary ~= ARGV[3] " + + "or not tertiary or tertiary ~= ARGV[4] then return -1 end; " + + "if redis.call('exists', KEYS[1]) == 1 then return 0 end; " + + "redis.call('hset', KEYS[1], 'version', ARGV[1]); " + + "for i = 5, #ARGV - 1, 2 do " + + "redis.call('hset', KEYS[1], ARGV[i], ARGV[i + 1]); end; " + + "redis.call('pexpire', KEYS[1], ARGV[#ARGV]); return 1"); + private static final byte[] CAS_FIELDS_SCRIPT = bytes( + "local current = redis.call('hget', KEYS[1], 'version'); " + + "if not current then return -1 end; " + + "if current ~= ARGV[1] then return 0 end; " + + "redis.call('hset', KEYS[1], 'version', ARGV[2]); " + + "for i = 3, #ARGV - 1, 2 do redis.call('hset', KEYS[1], ARGV[i], ARGV[i + 1]); end; " + + "redis.call('pexpire', KEYS[1], ARGV[#ARGV]); return 1"); + private static final byte[] GUARDED_CAS_FIELDS_SCRIPT = bytes( + "local guard = redis.call('hget', KEYS[2], 'version'); " + + "if not guard or guard ~= ARGV[3] then return -2 end; " + + "local current = redis.call('hget', KEYS[1], 'version'); " + + "if not current then return -1 end; " + + "if current ~= ARGV[1] then return 0 end; " + + "redis.call('hset', KEYS[1], 'version', ARGV[2]); " + + "for i = 4, #ARGV - 1, 2 do redis.call('hset', KEYS[1], ARGV[i], ARGV[i + 1]); end; " + + "redis.call('pexpire', KEYS[1], ARGV[#ARGV]); return 1"); + private static final byte[] DOUBLE_GUARDED_CAS_FIELDS_SCRIPT = bytes( + "local guard = redis.call('hget', KEYS[2], 'version'); " + + "local secondary = redis.call('hget', KEYS[3], 'version'); " + + "if not guard or guard ~= ARGV[3] " + + "or not secondary or secondary ~= ARGV[4] then return -2 end; " + + "local current = redis.call('hget', KEYS[1], 'version'); " + + "if not current then return -1 end; " + + "if current ~= ARGV[1] then return 0 end; " + + "redis.call('hset', KEYS[1], 'version', ARGV[2]); " + + "for i = 5, #ARGV - 1, 2 do " + + "redis.call('hset', KEYS[1], ARGV[i], ARGV[i + 1]); end; " + + "redis.call('pexpire', KEYS[1], ARGV[#ARGV]); return 1"); + private static final byte[] TRIPLE_GUARDED_CAS_FIELDS_SCRIPT = bytes( + "local guard = redis.call('hget', KEYS[2], 'version'); " + + "local secondary = redis.call('hget', KEYS[3], 'version'); " + + "local tertiary = redis.call('hget', KEYS[4], 'version'); " + + "if not guard or guard ~= ARGV[3] " + + "or not secondary or secondary ~= ARGV[4] " + + "or not tertiary or tertiary ~= ARGV[5] then return -2 end; " + + "local current = redis.call('hget', KEYS[1], 'version'); " + + "if not current then return -1 end; " + + "if current ~= ARGV[1] then return 0 end; " + + "redis.call('hset', KEYS[1], 'version', ARGV[2]); " + + "for i = 6, #ARGV - 1, 2 do " + + "redis.call('hset', KEYS[1], ARGV[i], ARGV[i + 1]); end; " + + "redis.call('pexpire', KEYS[1], ARGV[#ARGV]); return 1"); + private static final byte[] DOUBLE_GUARDED_CAS_FIELDS_REFRESH_SCRIPT = bytes( + "local guard = redis.call('hget', KEYS[2], 'version'); " + + "local secondary = redis.call('hget', KEYS[3], 'version'); " + + "if not guard or guard ~= ARGV[3] " + + "or not secondary or secondary ~= ARGV[4] then return -2 end; " + + "local current = redis.call('hget', KEYS[1], 'version'); " + + "if not current then return -1 end; " + + "if current ~= ARGV[1] then return 0 end; " + + "redis.call('hset', KEYS[1], 'version', ARGV[2]); " + + "for i = 5, #ARGV - 2, 2 do " + + "redis.call('hset', KEYS[1], ARGV[i], ARGV[i + 1]); end; " + + "redis.call('pexpire', KEYS[1], ARGV[#ARGV - 1]); " + + "if redis.call('exists', KEYS[4]) == 0 then " + + "redis.call('hset', KEYS[4], 'version', '0'); end; " + + "redis.call('pexpire', KEYS[4], ARGV[#ARGV]); return 1"); + private static final byte[] REWRITE_FIELDS_SCRIPT = bytes( + "local current = redis.call('hget', KEYS[1], 'version'); " + + "if not current or current ~= ARGV[1] then return 0 end; " + + "redis.call('del', KEYS[1]); " + + "redis.call('hset', KEYS[1], 'version', ARGV[1]); " + + "for i = 2, #ARGV - 1, 2 do redis.call('hset', KEYS[1], ARGV[i], ARGV[i + 1]); end; " + + "redis.call('pexpire', KEYS[1], ARGV[#ARGV]); return 1"); + + private final RedisConnectionFactory connectionFactory; + private final ApplicationClassLoaderJavaValueDecoder valueDecoder = + new ApplicationClassLoaderJavaValueDecoder(); + + /** + * 创建 Redis 版本对象存储。 + * + * @param connectionFactory Redis 连接工厂 + */ + public RedisVersionedObjectStore(RedisConnectionFactory connectionFactory) { + this.connectionFactory = Objects.requireNonNull( + connectionFactory, "connectionFactory must not be null"); + } + + /** + * {@inheritDoc} + */ + @Override + public T load(String key, Class type) { + requireKey(key); + Objects.requireNonNull(type, "type must not be null"); + try (RedisConnection connection = connectionFactory.getConnection()) { + byte[] payload = connection.hashCommands().hGet(bytes(key), PAYLOAD_FIELD); + if (payload == null) { + return null; + } + return type.cast(valueDecoder.apply(payload)); + } + } + + /** + * {@inheritDoc} + */ + @Override + public List loadAll(List keys, Class type) { + Objects.requireNonNull(keys, "keys must not be null"); + Objects.requireNonNull(type, "type must not be null"); + if (keys.isEmpty()) { + return List.of(); + } + try (RedisConnection connection = connectionFactory.getConnection()) { + connection.openPipeline(); + for (String key : keys) { + connection.hashCommands().hGet(bytes(requireKey(key)), PAYLOAD_FIELD); + } + List encodedValues = connection.closePipeline(); + List values = new ArrayList<>(keys.size()); + for (int index = 0; index < keys.size(); index++) { + Object encoded = encodedValues != null && index < encodedValues.size() + ? encodedValues.get(index) + : null; + values.add(encoded instanceof byte[] + ? type.cast(valueDecoder.apply((byte[]) encoded)) + : null); + } + return values; + } + } + + /** + * {@inheritDoc} + */ + @Override + public void deleteAll(List keys) { + Objects.requireNonNull(keys, "keys must not be null"); + if (keys.isEmpty()) { + return; + } + byte[][] encodedKeys = new byte[keys.size()][]; + for (int index = 0; index < keys.size(); index++) { + encodedKeys[index] = bytes(requireKey(keys.get(index))); + } + try (RedisConnection connection = connectionFactory.getConnection()) { + connection.keyCommands().del(encodedKeys); + } + } + + /** + * {@inheritDoc} + */ + @Override + public boolean deleteAll( + List keys, + String guardKey, + long guardVersion, + String secondaryGuardKey, + long secondaryGuardVersion) { + Objects.requireNonNull(keys, "keys must not be null"); + if (keys.isEmpty()) { + return true; + } + List arguments = new ArrayList<>(keys.size() + 4); + for (String key : keys) { + arguments.add(bytes(requireKey(key))); + } + arguments.add(bytes(requireKey(guardKey))); + arguments.add(bytes(requireKey(secondaryGuardKey))); + arguments.add(bytes(guardVersion)); + arguments.add(bytes(secondaryGuardVersion)); + Long result = eval( + DOUBLE_GUARDED_DELETE_ALL_SCRIPT, + keys.size() + 2, + arguments.toArray(new byte[0][])); + return Long.valueOf(1L).equals(result); + } + + /** + * {@inheritDoc} + */ + @Override + public void refreshExpirations(List keys, Duration ttl) { + Objects.requireNonNull(keys, "keys must not be null"); + if (keys.isEmpty()) { + return; + } + long ttlMillis = ttlMillis(ttl); + try (RedisConnection connection = connectionFactory.getConnection()) { + connection.openPipeline(); + for (String key : keys) { + connection.keyCommands().pExpire( + bytes(requireKey(key)), ttlMillis); + } + connection.closePipeline(); + } + } + + /** + * {@inheritDoc} + */ + @Override + public boolean createIfAbsent(String key, + Serializable value, + long version, + Duration ttl) { + Long result = eval( + CREATE_SCRIPT, + 1, + bytes(requireKey(key)), + bytes(version), + encode(value), + bytes(ttlMillis(ttl))); + return Long.valueOf(1L).equals(result); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean createIfAbsent( + String key, + Serializable value, + long version, + String guardKey, + long guardVersion, + String secondaryGuardKey, + long secondaryGuardVersion, + Duration ttl) { + Long result = eval( + DOUBLE_GUARDED_CREATE_SCRIPT, + 3, + bytes(requireKey(key)), + bytes(requireKey(guardKey)), + bytes(requireKey(secondaryGuardKey)), + bytes(version), + bytes(guardVersion), + bytes(secondaryGuardVersion), + encode(value), + bytes(ttlMillis(ttl))); + return Long.valueOf(1L).equals(result); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean createIfAbsent(String key, + Serializable value, + long version, + String guardKey, + long guardVersion, + Duration ttl) { + Long result = eval( + GUARDED_CREATE_SCRIPT, + 2, + bytes(requireKey(key)), + bytes(requireKey(guardKey)), + bytes(version), + bytes(guardVersion), + encode(value), + bytes(ttlMillis(ttl))); + return Long.valueOf(1L).equals(result); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean compareAndSet(String key, + long expectedVersion, + Serializable value, + long newVersion, + Duration ttl) { + Long result = eval( + CAS_SCRIPT, + 1, + bytes(requireKey(key)), + bytes(expectedVersion), + bytes(newVersion), + encode(value), + bytes(ttlMillis(ttl))); + return Long.valueOf(1L).equals(result); + } + + /** + * {@inheritDoc} + */ + @Override + public VersionedFields loadFields(String key) { + requireKey(key); + try (RedisConnection connection = connectionFactory.getConnection()) { + Map encoded = connection.hashCommands().hGetAll(bytes(key)); + if (encoded == null || encoded.isEmpty()) { + return null; + } + Long version = null; + Map fields = new LinkedHashMap<>(); + for (Map.Entry entry : encoded.entrySet()) { + String fieldName = new String(entry.getKey(), StandardCharsets.UTF_8); + if ("version".equals(fieldName)) { + version = Long.parseLong(new String(entry.getValue(), StandardCharsets.UTF_8)); + continue; + } + Object value = valueDecoder.apply(entry.getValue()); + fields.put(fieldName, value == NullFieldValue.INSTANCE ? null : value); + } + return version == null ? null : new VersionedFields(version, fields); + } + } + + /** + * {@inheritDoc} + */ + @Override + public Long loadVersion(String key) { + requireKey(key); + try (RedisConnection connection = connectionFactory.getConnection()) { + byte[] encoded = connection.hashCommands().hGet(bytes(key), bytes("version")); + return encoded == null + ? null + : Long.parseLong(new String(encoded, StandardCharsets.UTF_8)); + } + } + + /** + * {@inheritDoc} + */ + @Override + public boolean createFieldsIfAbsent(String key, + Map fields, + long version, + Duration ttl) { + Long result = eval( + CREATE_FIELDS_SCRIPT, + 1, + fieldArguments( + List.of(bytes(requireKey(key)), bytes(version)), + fields, + ttl)); + return Long.valueOf(1L).equals(result); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean createFieldsIfAbsent( + String key, + Map fields, + long version, + String guardKey, + long guardVersion, + String secondaryGuardKey, + long secondaryGuardVersion, + Duration ttl) { + Long result = eval( + DOUBLE_GUARDED_CREATE_FIELDS_SCRIPT, + 3, + fieldArguments( + List.of( + bytes(requireKey(key)), + bytes(requireKey(guardKey)), + bytes(requireKey(secondaryGuardKey)), + bytes(version), + bytes(guardVersion), + bytes(secondaryGuardVersion)), + fields, + ttl)); + return Long.valueOf(1L).equals(result); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean createFieldsIfAbsent( + String key, + Map fields, + long version, + String guardKey, + long guardVersion, + String secondaryGuardKey, + long secondaryGuardVersion, + String tertiaryGuardKey, + long tertiaryGuardVersion, + Duration ttl) { + Long result = eval( + TRIPLE_GUARDED_CREATE_FIELDS_SCRIPT, + 4, + fieldArguments( + List.of( + bytes(requireKey(key)), + bytes(requireKey(guardKey)), + bytes(requireKey(secondaryGuardKey)), + bytes(requireKey(tertiaryGuardKey)), + bytes(version), + bytes(guardVersion), + bytes(secondaryGuardVersion), + bytes(tertiaryGuardVersion)), + fields, + ttl)); + return Long.valueOf(1L).equals(result); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean createFieldsIfAbsent(String key, + Map fields, + long version, + String guardKey, + long guardVersion, + Duration ttl) { + Long result = eval( + GUARDED_CREATE_FIELDS_SCRIPT, + 2, + fieldArguments( + List.of( + bytes(requireKey(key)), + bytes(requireKey(guardKey)), + bytes(version), + bytes(guardVersion)), + fields, + ttl)); + return Long.valueOf(1L).equals(result); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean compareAndSetFields(String key, + long expectedVersion, + Map fields, + long newVersion, + Duration ttl) { + Long result = eval( + CAS_FIELDS_SCRIPT, + 1, + fieldArguments( + List.of( + bytes(requireKey(key)), + bytes(expectedVersion), + bytes(newVersion)), + fields, + ttl)); + return Long.valueOf(1L).equals(result); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean compareAndSetFields( + String key, + long expectedVersion, + Map fields, + long newVersion, + String guardKey, + long guardVersion, + String secondaryGuardKey, + long secondaryGuardVersion, + Duration ttl) { + Long result = eval( + DOUBLE_GUARDED_CAS_FIELDS_SCRIPT, + 3, + fieldArguments( + List.of( + bytes(requireKey(key)), + bytes(requireKey(guardKey)), + bytes(requireKey(secondaryGuardKey)), + bytes(expectedVersion), + bytes(newVersion), + bytes(guardVersion), + bytes(secondaryGuardVersion)), + fields, + ttl)); + return Long.valueOf(1L).equals(result); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean compareAndSetFields( + String key, + long expectedVersion, + Map fields, + long newVersion, + String guardKey, + long guardVersion, + String secondaryGuardKey, + long secondaryGuardVersion, + String tertiaryGuardKey, + long tertiaryGuardVersion, + Duration ttl) { + Long result = eval( + TRIPLE_GUARDED_CAS_FIELDS_SCRIPT, + 4, + fieldArguments( + List.of( + bytes(requireKey(key)), + bytes(requireKey(guardKey)), + bytes(requireKey(secondaryGuardKey)), + bytes(requireKey(tertiaryGuardKey)), + bytes(expectedVersion), + bytes(newVersion), + bytes(guardVersion), + bytes(secondaryGuardVersion), + bytes(tertiaryGuardVersion)), + fields, + ttl)); + return Long.valueOf(1L).equals(result); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean compareAndSetFieldsAndRefresh( + String key, + long expectedVersion, + Map fields, + long newVersion, + String guardKey, + long guardVersion, + String secondaryGuardKey, + long secondaryGuardVersion, + Duration ttl, + String refreshKey, + Duration refreshTtl) { + Long result = eval( + DOUBLE_GUARDED_CAS_FIELDS_REFRESH_SCRIPT, + 4, + fieldArgumentsWithRefreshTtl( + List.of( + bytes(requireKey(key)), + bytes(requireKey(guardKey)), + bytes(requireKey(secondaryGuardKey)), + bytes(requireKey(refreshKey)), + bytes(expectedVersion), + bytes(newVersion), + bytes(guardVersion), + bytes(secondaryGuardVersion)), + fields, + ttl, + refreshTtl)); + return Long.valueOf(1L).equals(result); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean compareAndSetFields(String key, + long expectedVersion, + Map fields, + long newVersion, + String guardKey, + long guardVersion, + Duration ttl) { + Long result = eval( + GUARDED_CAS_FIELDS_SCRIPT, + 2, + fieldArguments( + List.of( + bytes(requireKey(key)), + bytes(requireKey(guardKey)), + bytes(expectedVersion), + bytes(newVersion), + bytes(guardVersion)), + fields, + ttl)); + return Long.valueOf(1L).equals(result); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean rewriteAsFields(String key, + long expectedVersion, + Map fields, + Duration ttl) { + Long result = eval( + REWRITE_FIELDS_SCRIPT, + 1, + fieldArguments( + List.of(bytes(requireKey(key)), bytes(expectedVersion)), + fields, + ttl)); + return Long.valueOf(1L).equals(result); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean compareAndSet(String key, + long expectedVersion, + Serializable value, + long newVersion, + String guardKey, + long guardVersion, + Duration ttl) { + Long result = eval( + GUARDED_CAS_SCRIPT, + 2, + bytes(requireKey(key)), + bytes(requireKey(guardKey)), + bytes(expectedVersion), + bytes(newVersion), + bytes(guardVersion), + encode(value), + bytes(ttlMillis(ttl))); + return Long.valueOf(1L).equals(result); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean compareAndSet( + String key, + long expectedVersion, + Serializable value, + long newVersion, + String guardKey, + long guardVersion, + String secondaryGuardKey, + long secondaryGuardVersion, + Duration ttl) { + Long result = eval( + DOUBLE_GUARDED_CAS_SCRIPT, + 3, + bytes(requireKey(key)), + bytes(requireKey(guardKey)), + bytes(requireKey(secondaryGuardKey)), + bytes(expectedVersion), + bytes(newVersion), + bytes(guardVersion), + bytes(secondaryGuardVersion), + encode(value), + bytes(ttlMillis(ttl))); + return Long.valueOf(1L).equals(result); + } + + /** + * 执行返回整数的 Redis Lua 脚本。 + * + * @param script Lua 脚本 + * @param keyCount 参数中的 Redis key 数量 + * @param keysAndArgs key 与脚本参数 + * @return Redis 整数结果 + */ + private Long eval(byte[] script, int keyCount, byte[]... keysAndArgs) { + try (RedisConnection connection = connectionFactory.getConnection()) { + return connection.scriptingCommands().eval( + script, ReturnType.INTEGER, keyCount, keysAndArgs); + } + } + + /** + * 使用项目既有 Java 协议编码对象。 + * + * @param value 待编码对象 + * @return 编码后的二进制 payload + */ + private byte[] encode(Serializable value) { + return JavaValueEncoder.INSTANCE.apply( + Objects.requireNonNull(value, "value must not be null")); + } + + /** + * 组装动态字段脚本参数，每个字段只编码一次。 + * + * @param prefix Redis key 与固定参数 + * @param fields 待写字段 + * @param ttl 有效期 + * @return EVAL 的完整二进制参数 + */ + private byte[][] fieldArguments(List prefix, + Map fields, + Duration ttl) { + Objects.requireNonNull(fields, "fields must not be null"); + List arguments = new ArrayList<>(prefix.size() + fields.size() * 2 + 1); + arguments.addAll(prefix); + for (Map.Entry entry : fields.entrySet()) { + String fieldName = requireKey(entry.getKey()); + if ("version".equals(fieldName)) { + throw new IllegalArgumentException("version is a reserved field"); + } + Serializable value = entry.getValue() == null ? NullFieldValue.INSTANCE : entry.getValue(); + arguments.add(bytes(fieldName)); + arguments.add(encode(value)); + } + arguments.add(bytes(ttlMillis(ttl))); + return arguments.toArray(new byte[0][]); + } + + /** + * 组装需要同时刷新关联键 TTL 的字段脚本参数。 + * + * @param prefix Redis key 与固定参数 + * @param fields 待写字段 + * @param ttl 主对象有效期 + * @param refreshTtl 关联键有效期 + * @return EVAL 的完整二进制参数 + */ + private byte[][] fieldArgumentsWithRefreshTtl( + List prefix, + Map fields, + Duration ttl, + Duration refreshTtl) { + byte[][] base = fieldArguments(prefix, fields, ttl); + byte[][] arguments = java.util.Arrays.copyOf(base, base.length + 1); + arguments[base.length] = bytes(ttlMillis(refreshTtl)); + return arguments; + } + + /** + * 校验并换算有效期。 + * + * @param ttl 有效期 + * @return 至少一毫秒的有效期 + */ + private long ttlMillis(Duration ttl) { + Objects.requireNonNull(ttl, "ttl must not be null"); + if (ttl.isNegative() || ttl.isZero()) { + throw new IllegalArgumentException("ttl must be positive"); + } + return Math.max(1L, ttl.toMillis()); + } + + /** + * 校验 Redis key。 + * + * @param key Redis key + * @return 原 key + */ + private String requireKey(String key) { + if (key == null || key.isBlank()) { + throw new IllegalArgumentException("key must not be blank"); + } + return key; + } + + /** + * 将文本转为 Redis 二进制参数。 + * + * @param value 文本 + * @return UTF-8 字节 + */ + private static byte[] bytes(Object value) { + return String.valueOf(value).getBytes(StandardCharsets.UTF_8); + } + + /** + * Redis Hash 中显式保存的空字段占位。 + */ + private enum NullFieldValue { + INSTANCE + } +} diff --git a/easyflow-commons/easyflow-common-cache/src/main/java/tech/easyflow/common/cache/VersionedFields.java b/easyflow-commons/easyflow-common-cache/src/main/java/tech/easyflow/common/cache/VersionedFields.java new file mode 100644 index 00000000..3fc9e1e0 --- /dev/null +++ b/easyflow-commons/easyflow-common-cache/src/main/java/tech/easyflow/common/cache/VersionedFields.java @@ -0,0 +1,45 @@ +package tech.easyflow.common.cache; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Redis 字段化版本状态快照。 + */ +public final class VersionedFields { + + private final long version; + private final Map fields; + + /** + * 创建字段化状态快照。 + * + * @param version 当前版本 + * @param fields 字段值 + */ + public VersionedFields(long version, Map fields) { + this.version = version; + this.fields = fields == null + ? Collections.emptyMap() + : Collections.unmodifiableMap(new LinkedHashMap<>(fields)); + } + + /** + * 获取状态版本。 + * + * @return 当前版本 + */ + public long getVersion() { + return version; + } + + /** + * 获取字段快照。 + * + * @return 不可变字段映射 + */ + public Map getFields() { + return fields; + } +} diff --git a/easyflow-commons/easyflow-common-cache/src/main/java/tech/easyflow/common/cache/VersionedObjectStore.java b/easyflow-commons/easyflow-common-cache/src/main/java/tech/easyflow/common/cache/VersionedObjectStore.java new file mode 100644 index 00000000..2275cfa6 --- /dev/null +++ b/easyflow-commons/easyflow-common-cache/src/main/java/tech/easyflow/common/cache/VersionedObjectStore.java @@ -0,0 +1,479 @@ +package tech.easyflow.common.cache; + +import java.io.Serializable; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * 支持原子版本比较的对象存储。 + */ +public interface VersionedObjectStore { + + /** + * 加载对象。 + * + * @param key 存储键 + * @param type 对象类型 + * @param 对象类型 + * @return 已存在的对象；不存在时返回 {@code null} + */ + T load(String key, Class type); + + /** + * 批量加载对象并保持输入键顺序。 + * + * @param keys 存储键 + * @param type 对象类型 + * @param 对象类型 + * @return 与输入键等长的对象列表；缺失位置为 {@code null} + */ + default List loadAll(List keys, Class type) { + List values = new ArrayList<>(keys.size()); + for (String key : keys) { + values.add(load(key, type)); + } + return values; + } + + /** + * 批量删除对象。 + * + * @param keys 存储键 + */ + default void deleteAll(List keys) { + throw new UnsupportedOperationException("Batch delete is not supported"); + } + + /** + * 两个守卫版本均匹配时原子批量删除对象。 + * + * @param keys 待删除对象键 + * @param guardKey 第一守卫键 + * @param guardVersion 第一守卫版本 + * @param secondaryGuardKey 第二守卫键 + * @param secondaryGuardVersion 第二守卫版本 + * @return 守卫匹配并执行删除时为 {@code true} + */ + default boolean deleteAll( + List keys, + String guardKey, + long guardVersion, + String secondaryGuardKey, + long secondaryGuardVersion) { + throw new UnsupportedOperationException( + "Guarded batch delete is not supported"); + } + + /** + * 批量刷新对象有效期。 + * + * @param keys 存储键 + * @param ttl 新有效期 + */ + default void refreshExpirations(List keys, Duration ttl) { + throw new UnsupportedOperationException("Expiration refresh is not supported"); + } + + /** + * 原子创建对象。 + * + * @param key 存储键 + * @param value 初始对象 + * @param version 初始版本 + * @param ttl 有效期 + * @return 创建成功时为 {@code true}，对象已存在时为 {@code false} + */ + boolean createIfAbsent(String key, Serializable value, long version, Duration ttl); + + /** + * 在守卫对象版本匹配时原子创建对象。 + * + * @param key 存储键 + * @param value 初始对象 + * @param version 初始版本 + * @param guardKey 守卫对象键 + * @param guardVersion 期望的守卫版本 + * @param ttl 有效期 + * @return 创建成功时为 {@code true}，对象已存在或守卫版本冲突时为 {@code false} + */ + boolean createIfAbsent(String key, + Serializable value, + long version, + String guardKey, + long guardVersion, + Duration ttl); + + /** + * 两个守卫版本均匹配时原子创建对象。 + * + * @param key 存储键 + * @param value 初始对象 + * @param version 初始版本 + * @param guardKey 第一守卫键 + * @param guardVersion 第一守卫版本 + * @param secondaryGuardKey 第二守卫键 + * @param secondaryGuardVersion 第二守卫版本 + * @param ttl 有效期 + * @return 创建成功时为 {@code true} + */ + default boolean createIfAbsent( + String key, + Serializable value, + long version, + String guardKey, + long guardVersion, + String secondaryGuardKey, + long secondaryGuardVersion, + Duration ttl) { + return createIfAbsent(key, value, version, guardKey, guardVersion, ttl); + } + + /** + * 原子比较并更新对象。 + * + * @param key 存储键 + * @param expectedVersion 期望的当前版本 + * @param value 新对象 + * @param newVersion 新版本 + * @param ttl 有效期 + * @return 更新成功时为 {@code true}，对象缺失或版本冲突时为 {@code false} + */ + boolean compareAndSet(String key, + long expectedVersion, + Serializable value, + long newVersion, + Duration ttl); + + /** + * 在守卫对象版本匹配时原子比较并更新对象。 + * + * @param key 存储键 + * @param expectedVersion 期望的当前版本 + * @param value 新对象 + * @param newVersion 新版本 + * @param guardKey 守卫对象键 + * @param guardVersion 期望的守卫版本 + * @param ttl 有效期 + * @return 更新成功时为 {@code true}，对象缺失或任一版本冲突时为 {@code false} + */ + boolean compareAndSet(String key, + long expectedVersion, + Serializable value, + long newVersion, + String guardKey, + long guardVersion, + Duration ttl); + + /** + * 两个守卫版本均匹配时原子比较并更新对象。 + * + * @param key 存储键 + * @param expectedVersion 期望的当前版本 + * @param value 新对象 + * @param newVersion 新版本 + * @param guardKey 第一守卫键 + * @param guardVersion 第一守卫版本 + * @param secondaryGuardKey 第二守卫键 + * @param secondaryGuardVersion 第二守卫版本 + * @param ttl 有效期 + * @return 更新成功时为 {@code true} + */ + default boolean compareAndSet( + String key, + long expectedVersion, + Serializable value, + long newVersion, + String guardKey, + long guardVersion, + String secondaryGuardKey, + long secondaryGuardVersion, + Duration ttl) { + return compareAndSet( + key, expectedVersion, value, newVersion, guardKey, guardVersion, ttl); + } + + /** + * 一次读取字段化状态及其版本。 + * + * @param key 存储键 + * @return 字段化状态；不存在时返回 {@code null} + */ + default VersionedFields loadFields(String key) { + throw new UnsupportedOperationException("Field storage is not supported"); + } + + /** + * 只读取对象版本。 + * + * @param key 存储键 + * @return 当前版本；对象不存在或缺少版本时返回 {@code null} + */ + default Long loadVersion(String key) { + VersionedFields fields = loadFields(key); + return fields == null ? null : fields.getVersion(); + } + + /** + * 原子创建字段化状态。 + * + * @param key 存储键 + * @param fields 初始字段 + * @param version 初始版本 + * @param ttl 有效期 + * @return 创建成功时为 {@code true} + */ + default boolean createFieldsIfAbsent(String key, + Map fields, + long version, + Duration ttl) { + throw new UnsupportedOperationException("Field storage is not supported"); + } + + /** + * 守卫版本匹配时原子创建字段化状态。 + * + * @param key 存储键 + * @param fields 初始字段 + * @param version 初始版本 + * @param guardKey 守卫状态键 + * @param guardVersion 期望守卫版本 + * @param ttl 有效期 + * @return 创建成功时为 {@code true} + */ + default boolean createFieldsIfAbsent(String key, + Map fields, + long version, + String guardKey, + long guardVersion, + Duration ttl) { + throw new UnsupportedOperationException("Field storage is not supported"); + } + + /** + * 两个守卫版本均匹配时原子创建字段化状态。 + * + * @param key 存储键 + * @param fields 初始字段 + * @param version 初始版本 + * @param guardKey 第一守卫键 + * @param guardVersion 第一守卫版本 + * @param secondaryGuardKey 第二守卫键 + * @param secondaryGuardVersion 第二守卫版本 + * @param ttl 有效期 + * @return 创建成功时为 {@code true} + */ + default boolean createFieldsIfAbsent( + String key, + Map fields, + long version, + String guardKey, + long guardVersion, + String secondaryGuardKey, + long secondaryGuardVersion, + Duration ttl) { + return createFieldsIfAbsent(key, fields, version, guardKey, guardVersion, ttl); + } + + /** + * 三个守卫版本均匹配时原子创建字段化状态。 + * + * @param key 存储键 + * @param fields 初始字段 + * @param version 初始版本 + * @param guardKey 第一守卫键 + * @param guardVersion 第一守卫版本 + * @param secondaryGuardKey 第二守卫键 + * @param secondaryGuardVersion 第二守卫版本 + * @param tertiaryGuardKey 第三守卫键 + * @param tertiaryGuardVersion 第三守卫版本 + * @param ttl 有效期 + * @return 创建成功时为 {@code true} + */ + default boolean createFieldsIfAbsent( + String key, + Map fields, + long version, + String guardKey, + long guardVersion, + String secondaryGuardKey, + long secondaryGuardVersion, + String tertiaryGuardKey, + long tertiaryGuardVersion, + Duration ttl) { + return createFieldsIfAbsent( + key, + fields, + version, + guardKey, + guardVersion, + secondaryGuardKey, + secondaryGuardVersion, + ttl); + } + + /** + * 原子比较版本并仅提交变化字段。 + * + * @param key 存储键 + * @param expectedVersion 期望版本 + * @param fields 变化字段 + * @param newVersion 新版本 + * @param ttl 有效期 + * @return 更新成功时为 {@code true} + */ + default boolean compareAndSetFields(String key, + long expectedVersion, + Map fields, + long newVersion, + Duration ttl) { + throw new UnsupportedOperationException("Field storage is not supported"); + } + + /** + * 两个守卫版本均匹配时原子提交变化字段。 + * + * @param key 存储键 + * @param expectedVersion 当前对象期望版本 + * @param fields 变化字段 + * @param newVersion 新版本 + * @param guardKey 第一守卫键 + * @param guardVersion 第一守卫版本 + * @param secondaryGuardKey 第二守卫键 + * @param secondaryGuardVersion 第二守卫版本 + * @param ttl 有效期 + * @return 提交成功时为 {@code true} + */ + default boolean compareAndSetFields( + String key, + long expectedVersion, + Map fields, + long newVersion, + String guardKey, + long guardVersion, + String secondaryGuardKey, + long secondaryGuardVersion, + Duration ttl) { + return compareAndSetFields( + key, expectedVersion, fields, newVersion, guardKey, guardVersion, ttl); + } + + /** + * 三个守卫版本均匹配时原子提交变化字段。 + * + * @param key 存储键 + * @param expectedVersion 当前对象期望版本 + * @param fields 变化字段 + * @param newVersion 新版本 + * @param guardKey 第一守卫键 + * @param guardVersion 第一守卫版本 + * @param secondaryGuardKey 第二守卫键 + * @param secondaryGuardVersion 第二守卫版本 + * @param tertiaryGuardKey 第三守卫键 + * @param tertiaryGuardVersion 第三守卫版本 + * @param ttl 有效期 + * @return 提交成功时为 {@code true} + */ + default boolean compareAndSetFields( + String key, + long expectedVersion, + Map fields, + long newVersion, + String guardKey, + long guardVersion, + String secondaryGuardKey, + long secondaryGuardVersion, + String tertiaryGuardKey, + long tertiaryGuardVersion, + Duration ttl) { + return compareAndSetFields( + key, + expectedVersion, + fields, + newVersion, + guardKey, + guardVersion, + secondaryGuardKey, + secondaryGuardVersion, + ttl); + } + + /** + * 两个守卫版本均匹配时提交字段，并在同一原子操作中刷新关联键有效期。 + * + * @param key 存储键 + * @param expectedVersion 当前对象期望版本 + * @param fields 变化字段 + * @param newVersion 新版本 + * @param guardKey 第一守卫键 + * @param guardVersion 第一守卫版本 + * @param secondaryGuardKey 第二守卫键 + * @param secondaryGuardVersion 第二守卫版本 + * @param ttl 有效期 + * @param refreshKey 需要刷新有效期的关联键 + * @param refreshTtl 关联键有效期 + * @return 提交成功时为 {@code true} + */ + default boolean compareAndSetFieldsAndRefresh( + String key, + long expectedVersion, + Map fields, + long newVersion, + String guardKey, + long guardVersion, + String secondaryGuardKey, + long secondaryGuardVersion, + Duration ttl, + String refreshKey, + Duration refreshTtl) { + return compareAndSetFields( + key, + expectedVersion, + fields, + newVersion, + guardKey, + guardVersion, + secondaryGuardKey, + secondaryGuardVersion, + ttl); + } + + /** + * 守卫版本匹配时原子比较版本并仅提交变化字段。 + * + * @param key 存储键 + * @param expectedVersion 期望版本 + * @param fields 变化字段 + * @param newVersion 新版本 + * @param guardKey 守卫状态键 + * @param guardVersion 期望守卫版本 + * @param ttl 有效期 + * @return 更新成功时为 {@code true} + */ + default boolean compareAndSetFields(String key, + long expectedVersion, + Map fields, + long newVersion, + String guardKey, + long guardVersion, + Duration ttl) { + throw new UnsupportedOperationException("Field storage is not supported"); + } + + /** + * 将同版本完整对象原子改写为字段化状态。 + * + * @param key 存储键 + * @param expectedVersion 期望版本 + * @param fields 完整字段 + * @param ttl 有效期 + * @return 改写成功时为 {@code true} + */ + default boolean rewriteAsFields(String key, + long expectedVersion, + Map fields, + Duration ttl) { + throw new UnsupportedOperationException("Field storage is not supported"); + } +} diff --git a/easyflow-commons/easyflow-common-cache/src/test/java/tech/easyflow/common/cache/RedisIdempotencyExecutorTest.java b/easyflow-commons/easyflow-common-cache/src/test/java/tech/easyflow/common/cache/RedisIdempotencyExecutorTest.java new file mode 100644 index 00000000..0fd59f8b --- /dev/null +++ b/easyflow-commons/easyflow-common-cache/src/test/java/tech/easyflow/common/cache/RedisIdempotencyExecutorTest.java @@ -0,0 +1,185 @@ +package tech.easyflow.common.cache; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.data.redis.core.script.RedisScript; + +import java.lang.reflect.Constructor; +import java.time.Duration; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * {@link RedisIdempotencyExecutor} 状态转换回归测试。 + */ +public class RedisIdempotencyExecutorTest { + + /** + * 验证存在测试构造器时生产构造器仍能被 Spring 明确选择。 + * + * @throws Exception 生产构造器不存在时抛出 + */ + @Test + public void productionConstructorShouldBeAutowired() + throws Exception { + Constructor constructor = + RedisIdempotencyExecutor.class.getConstructor( + StringRedisTemplate.class); + + Assert.assertNotNull( + constructor.getAnnotation(Autowired.class)); + } + + /** + * 验证首次认领会执行操作并写入完成凭证。 + */ + @Test + public void executeOnceShouldRunAndCompleteForNewKey() { + StringRedisTemplate redisTemplate = redisTemplateReturning(1L, 1L); + RedisIdempotencyExecutor executor = new RedisIdempotencyExecutor(redisTemplate); + AtomicInteger executions = new AtomicInteger(); + + boolean executed = executor.executeOnce( + "workflow:instance:node:trigger", executions::incrementAndGet); + + Assert.assertTrue(executed); + Assert.assertEquals(1, executions.get()); + Mockito.verify(redisTemplate).execute( + ArgumentMatchers.>any(), + ArgumentMatchers.>any(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString()); + Mockito.verify(redisTemplate).execute( + ArgumentMatchers.>any(), + ArgumentMatchers.>any(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString()); + } + + /** + * 验证已有完成凭证时直接跳过副作用操作。 + */ + @Test + public void executeOnceShouldSkipCompletedKey() { + StringRedisTemplate redisTemplate = redisTemplateReturning(2L); + RedisIdempotencyExecutor executor = new RedisIdempotencyExecutor(redisTemplate); + AtomicInteger executions = new AtomicInteger(); + + boolean executed = executor.executeOnce( + "workflow:instance:node:trigger", executions::incrementAndGet); + + Assert.assertFalse(executed); + Assert.assertEquals(0, executions.get()); + } + + /** + * 验证其他 owner 仍在处理时返回明确冲突。 + */ + @Test + public void executeOnceShouldRejectInProgressKey() { + StringRedisTemplate redisTemplate = redisTemplateReturning(0L); + RedisIdempotencyExecutor executor = new RedisIdempotencyExecutor(redisTemplate); + + try { + executor.executeOnce("workflow:instance:node:trigger", () -> { + }); + Assert.fail("in-progress operation must be rejected"); + } catch (RedisIdempotencyExecutor.IdempotentOperationInProgressException expected) { + Assert.assertTrue(expected.getMessage().contains("workflow:instance")); + } + } + + /** + * 验证同一幂等键绑定不同负载时明确拒绝，避免把不同业务写入误判为已完成。 + */ + @Test + public void executeOnceShouldRejectPayloadMismatch() { + StringRedisTemplate redisTemplate = redisTemplateReturning(-1L); + RedisIdempotencyExecutor executor = new RedisIdempotencyExecutor(redisTemplate); + + try { + executor.executeOnce( + "workflow:instance:node:trigger", + "different-payload-hash", + () -> { + }); + Assert.fail("payload mismatch must be rejected"); + } catch (RedisIdempotencyExecutor.IdempotencyPayloadMismatchException expected) { + Assert.assertTrue(expected.getMessage().contains("workflow:instance")); + } + } + + /** + * 验证长操作会在处理中凭证到期前持续续期。 + * + * @throws Exception 测试等待被中断 + */ + @Test + public void executeOnceShouldRenewLeaseForLongOperation() + throws Exception { + StringRedisTemplate redisTemplate = + redisTemplateReturning(1L, 1L); + Mockito.when(redisTemplate.execute( + ArgumentMatchers.>any(), + ArgumentMatchers.>any(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString())) + .thenReturn(1L); + RedisIdempotencyExecutor executor = + new RedisIdempotencyExecutor( + redisTemplate, + Duration.ofMillis(60L), + Duration.ofMinutes(1L)); + + Assert.assertTrue(executor.executeOnce( + "workflow:long-operation", + () -> { + try { + Thread.sleep(90L); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new IllegalStateException( + "test interrupted", exception); + } + })); + + Mockito.verify(redisTemplate, Mockito.atLeastOnce()).execute( + ArgumentMatchers.>any(), + ArgumentMatchers.>any(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString()); + } + + /** + * 创建按顺序返回脚本结果的 Redis 模板。 + * + * @param results 脚本返回值 + * @return Redis 模板 + */ + private StringRedisTemplate redisTemplateReturning(Long... results) { + StringRedisTemplate redisTemplate = Mockito.mock(StringRedisTemplate.class); + Mockito.when(redisTemplate.execute( + ArgumentMatchers.>any(), + ArgumentMatchers.>any(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString() + )).thenReturn(results[0]); + Mockito.when(redisTemplate.execute( + ArgumentMatchers.>any(), + ArgumentMatchers.>any(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString() + )).thenReturn(results[Math.min(1, results.length - 1)]); + return redisTemplate; + } +} diff --git a/easyflow-commons/easyflow-common-cache/src/test/java/tech/easyflow/common/cache/RedisLockExecutorTest.java b/easyflow-commons/easyflow-common-cache/src/test/java/tech/easyflow/common/cache/RedisLockExecutorTest.java index 95fc7b31..3f286a53 100644 --- a/easyflow-commons/easyflow-common-cache/src/test/java/tech/easyflow/common/cache/RedisLockExecutorTest.java +++ b/easyflow-commons/easyflow-common-cache/src/test/java/tech/easyflow/common/cache/RedisLockExecutorTest.java @@ -79,6 +79,74 @@ public class RedisLockExecutorTest { ); } + /** + * 验证 fencing token 通过 Redis 原子脚本分配并返回。 + * + * @throws Exception 反射注入异常 + */ + @Test + public void nextFencingTokenShouldReturnAtomicSequenceValue() throws Exception { + StringRedisTemplate redisTemplate = Mockito.mock(StringRedisTemplate.class); + Mockito.when(redisTemplate.execute( + ArgumentMatchers.>any(), + ArgumentMatchers.>any(), + ArgumentMatchers.any() + )).thenReturn(9L); + + RedisLockExecutor executor = new RedisLockExecutor(); + setRedisTemplate(executor, redisTemplate); + + long token = executor.nextFencingToken( + "workflowState:{instance}:fence", + Duration.ofDays(4)); + + Assert.assertEquals(9L, token); + Mockito.verify(redisTemplate).execute( + ArgumentMatchers.>any(), + ArgumentMatchers.eq(List.of("workflowState:{instance}:fence")), + ArgumentMatchers.eq(String.valueOf(Duration.ofDays(4).toMillis())) + ); + } + + /** + * 验证互斥锁与 fencing token 通过同一个 Redis 脚本原子获取。 + * + * @throws Exception 反射注入异常 + */ + @Test + public void tryAcquireFencedShouldSetLockAndAdvanceTokenAtomically() throws Exception { + StringRedisTemplate redisTemplate = Mockito.mock(StringRedisTemplate.class); + Mockito.when(redisTemplate.execute( + ArgumentMatchers.>any(), + ArgumentMatchers.>any(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString() + )).thenReturn(17L); + + RedisLockExecutor executor = new RedisLockExecutor(); + setRedisTemplate(executor, redisTemplate); + + RedisLockExecutor.LockHandle handle = executor.tryAcquireFenced( + "chainLock:{instance}", + "workflowState:{instance}:fence", + Duration.ZERO, + Duration.ofSeconds(30), + Duration.ofDays(4)); + + Assert.assertNotNull(handle); + Assert.assertEquals(17L, handle.getFencingToken()); + Mockito.verify(redisTemplate).execute( + ArgumentMatchers.>any(), + ArgumentMatchers.eq(List.of( + "chainLock:{instance}", + "workflowState:{instance}:fence")), + ArgumentMatchers.anyString(), + ArgumentMatchers.eq("30000"), + ArgumentMatchers.eq( + String.valueOf(Duration.ofDays(4).toMillis()))); + } + @SuppressWarnings("unchecked") private ValueOperations mockValueOperations(boolean acquired) { ValueOperations valueOperations = Mockito.mock(ValueOperations.class); diff --git a/easyflow-commons/easyflow-common-cache/src/test/java/tech/easyflow/common/cache/RedisVersionedObjectStoreTest.java b/easyflow-commons/easyflow-common-cache/src/test/java/tech/easyflow/common/cache/RedisVersionedObjectStoreTest.java new file mode 100644 index 00000000..c4328ba9 --- /dev/null +++ b/easyflow-commons/easyflow-common-cache/src/test/java/tech/easyflow/common/cache/RedisVersionedObjectStoreTest.java @@ -0,0 +1,141 @@ +package tech.easyflow.common.cache; + +import com.alicp.jetcache.support.JavaValueEncoder; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; +import org.springframework.data.redis.connection.RedisConnection; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.connection.RedisHashCommands; +import org.springframework.data.redis.connection.RedisScriptingCommands; +import org.springframework.data.redis.connection.ReturnType; + +import java.time.Duration; +import java.util.Map; + +/** + * {@link RedisVersionedObjectStore} 二进制协议与脚本调用回归测试。 + */ +public class RedisVersionedObjectStoreTest { + + /** + * 验证状态对象继续按项目 Java 序列化协议读取。 + */ + @Test + public void loadShouldDecodeExistingJavaPayload() { + RedisConnectionFactory connectionFactory = Mockito.mock(RedisConnectionFactory.class); + RedisConnection connection = Mockito.mock(RedisConnection.class); + RedisHashCommands hashCommands = Mockito.mock(RedisHashCommands.class); + Mockito.when(connectionFactory.getConnection()).thenReturn(connection); + Mockito.when(connection.hashCommands()).thenReturn(hashCommands); + Mockito.when(hashCommands.hGet( + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.any(byte[].class) + )).thenReturn(JavaValueEncoder.INSTANCE.apply("state-value")); + RedisVersionedObjectStore store = new RedisVersionedObjectStore(connectionFactory); + + String loaded = store.load("workflowState:{instance}:chain", String.class); + + Assert.assertEquals("state-value", loaded); + Mockito.verify(connection).close(); + } + + /** + * 验证 CAS 更新通过单次 Redis Lua 调用完成。 + */ + @Test + public void compareAndSetShouldUseSingleAtomicScript() { + RedisConnectionFactory connectionFactory = Mockito.mock(RedisConnectionFactory.class); + RedisConnection connection = Mockito.mock(RedisConnection.class); + RedisScriptingCommands scriptingCommands = Mockito.mock(RedisScriptingCommands.class); + Mockito.when(connectionFactory.getConnection()).thenReturn(connection); + Mockito.when(connection.scriptingCommands()).thenReturn(scriptingCommands); + Mockito.when(scriptingCommands.eval( + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.eq(ReturnType.INTEGER), + ArgumentMatchers.eq(1), + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.any(byte[].class) + )).thenReturn(1L); + RedisVersionedObjectStore store = new RedisVersionedObjectStore(connectionFactory); + + boolean updated = store.compareAndSet( + "workflowState:{instance}:chain", + 3L, + "state-value", + 4L, + Duration.ofDays(3)); + + Assert.assertTrue(updated); + Mockito.verify(scriptingCommands, Mockito.times(1)).eval( + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.eq(ReturnType.INTEGER), + ArgumentMatchers.eq(1), + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.any(byte[].class)); + Mockito.verify(connection).close(); + } + + /** + * 验证节点状态更新在同一脚本内校验链版本和 fencing token。 + */ + @Test + public void compareAndSetFieldsShouldUseDoubleGuardedAtomicScript() { + RedisConnectionFactory connectionFactory = Mockito.mock(RedisConnectionFactory.class); + RedisConnection connection = Mockito.mock(RedisConnection.class); + RedisScriptingCommands scriptingCommands = Mockito.mock(RedisScriptingCommands.class); + Mockito.when(connectionFactory.getConnection()).thenReturn(connection); + Mockito.when(connection.scriptingCommands()).thenReturn(scriptingCommands); + Mockito.when(scriptingCommands.eval( + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.eq(ReturnType.INTEGER), + ArgumentMatchers.eq(3), + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.any(byte[].class) + )).thenReturn(1L); + RedisVersionedObjectStore store = new RedisVersionedObjectStore(connectionFactory); + + boolean updated = store.compareAndSetFields( + "workflowState:{instance}:node:node-1", + 2L, + Map.of("status", "RUNNING"), + 3L, + "workflowState:{instance}:chain", + 8L, + "workflowState:{instance}:fence", + 12L, + Duration.ofDays(3)); + + Assert.assertTrue(updated); + Mockito.verify(scriptingCommands).eval( + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.eq(ReturnType.INTEGER), + ArgumentMatchers.eq(3), + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.any(byte[].class), + ArgumentMatchers.any(byte[].class)); + Mockito.verify(connection).close(); + } +} diff --git a/easyflow-commons/easyflow-common-mq/src/main/java/tech/easyflow/common/mq/redis/RedisMQProducer.java b/easyflow-commons/easyflow-common-mq/src/main/java/tech/easyflow/common/mq/redis/RedisMQProducer.java index 0732fc8e..4b2a2248 100644 --- a/easyflow-commons/easyflow-common-mq/src/main/java/tech/easyflow/common/mq/redis/RedisMQProducer.java +++ b/easyflow-commons/easyflow-common-mq/src/main/java/tech/easyflow/common/mq/redis/RedisMQProducer.java @@ -51,7 +51,7 @@ public class RedisMQProducer implements MQProducer { int shardCount = Math.max(properties.getRedis().getChatPersistShardCount(), 1); int shard = keySupport.resolveShard(message.getKey(), shardCount); String streamKey = keySupport.streamKey(message.getTopic(), shard); - LOG.info("MQ 开始投递消息: topic={}, messageId={}, key={}, shard={}, streamKey={}", + LOG.debug("MQ 开始投递消息: topic={}, messageId={}, key={}, shard={}, streamKey={}", message.getTopic(), message.getMessageId(), message.getKey(), shard, streamKey); RecordId recordId = stringRedisTemplate.opsForStream().add( StreamRecords.string(Map.of("payload", messageConverter.serialize(message))).withStreamKey(streamKey) @@ -59,7 +59,7 @@ public class RedisMQProducer implements MQProducer { if (recordId == null) { throw new MQException("MQ 消息投递失败"); } - LOG.info("MQ 消息投递完成: topic={}, messageId={}, key={}, shard={}, streamKey={}, recordId={}", + LOG.debug("MQ 消息投递完成: topic={}, messageId={}, key={}, shard={}, streamKey={}, recordId={}", message.getTopic(), message.getMessageId(), message.getKey(), shard, streamKey, recordId.getValue()); return recordId.getValue(); } diff --git a/easyflow-modules/easyflow-module-ai/pom.xml b/easyflow-modules/easyflow-module-ai/pom.xml index f11d064a..0f713e43 100644 --- a/easyflow-modules/easyflow-module-ai/pom.xml +++ b/easyflow-modules/easyflow-module-ai/pom.xml @@ -120,6 +120,11 @@ spring-boot-actuator ${spring-boot.version} + + io.micrometer + micrometer-core + 1.15.7 + com.easyagents diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagents/tool/PluginTool.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagents/tool/PluginTool.java index 2a07fa5b..423b50cb 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagents/tool/PluginTool.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagents/tool/PluginTool.java @@ -14,6 +14,7 @@ import org.slf4j.LoggerFactory; import tech.easyflow.ai.easyagents.CustomMultipartFile; import tech.easyflow.ai.entity.Plugin; import tech.easyflow.ai.entity.PluginItem; +import tech.easyflow.ai.node.TemporaryFileMultipartFile; import tech.easyflow.ai.mapper.PluginMapper; import tech.easyflow.ai.service.PluginItemService; import tech.easyflow.common.ai.plugin.NestedParamConverter; @@ -23,10 +24,13 @@ import tech.easyflow.common.ai.plugin.PluginParamConverter; import tech.easyflow.common.filestorage.FileStorageManager; import tech.easyflow.common.filestorage.FileStorageService; import tech.easyflow.common.util.SpringContextUtil; +import com.easyagents.flow.core.util.IoBulkhead; import java.io.*; import java.lang.reflect.Array; import java.math.BigInteger; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.*; public class PluginTool extends BaseTool { @@ -36,6 +40,8 @@ public class PluginTool extends BaseTool { private String name; private String description; private Parameter[] parameters; + private transient PluginItem pluginItemSnapshot; + private transient Plugin pluginSnapshot; private static final Logger logger = LoggerFactory.getLogger(PluginTool.class); public PluginTool() { @@ -43,9 +49,21 @@ public class PluginTool extends BaseTool { } public PluginTool(PluginItem pluginItem) { + this(pluginItem, null); + } + + /** + * 使用已加载实体快照创建插件工具。 + * + * @param pluginItem 插件项快照 + * @param plugin 插件快照，可为空 + */ + public PluginTool(PluginItem pluginItem, Plugin plugin) { this.name = pluginItem.getEnglishName(); this.description = pluginItem.getDescription(); this.pluginToolId = pluginItem.getId(); + this.pluginItemSnapshot = pluginItem; + this.pluginSnapshot = plugin; this.parameters = getDefaultParameters(pluginItem.getInputData()); } @@ -80,18 +98,7 @@ public class PluginTool extends BaseTool { } private Parameter[] getDefaultParameters(String inputData) { - PluginItemService pluginToolService = SpringContextUtil.getBean(PluginItemService.class); - QueryWrapper queryAiPluginToolWrapper = QueryWrapper.create() - .select("*") - .from("tb_plugin_item") - .where("id = ? ", this.pluginToolId); - PluginItem pluginItem = pluginToolService.getMapper().selectOneByQuery(queryAiPluginToolWrapper); - List> dataList = null; - if (pluginItem == null || pluginItem.getInputData() == null){ - dataList = getDataList(inputData); - } else { - dataList = getDataList(pluginItem.getInputData()); - } + List> dataList = getDataList(inputData); Parameter[] params = new Parameter[dataList.size()]; for (int i = 0; i < dataList.size(); i++) { Map item = dataList.get(i); @@ -147,14 +154,16 @@ public class PluginTool extends BaseTool { } public Object runPluginTool(Map argsMap, String inputData, BigInteger pluginId){ - PluginItemService pluginToolService = SpringContextUtil.getBean(PluginItemService.class); - QueryWrapper queryAiPluginToolWrapper = QueryWrapper.create() - .select("*") - .from("tb_plugin_item") - .where("id = ? ", pluginId); - PluginItem pluginItem = pluginToolService.getMapper().selectOneByQuery(queryAiPluginToolWrapper); + PluginItem pluginItem = pluginItemSnapshot != null + && Objects.equals(pluginItemSnapshot.getId(), pluginId) + ? pluginItemSnapshot + : loadPluginItem(pluginId); String method = pluginItem.getRequestMethod().toUpperCase(); - Plugin plugin = getAiPlugin(pluginItem.getPluginId()); + Plugin plugin = pluginSnapshot != null + && Objects.equals( + pluginSnapshot.getId(), pluginItem.getPluginId()) + ? pluginSnapshot + : getAiPlugin(pluginItem.getPluginId()); String url; if (!StrUtil.isEmpty(pluginItem.getBasePath())) { @@ -200,6 +209,8 @@ public class PluginTool extends BaseTool { List pathParams = new ArrayList<>(); Map nestedParams = NestedParamConverter.convertToNestedParamMap(pluginParams); + List temporaryFiles = new ArrayList<>(); + try { // 遍历嵌套参数 for (Map.Entry entry : nestedParams.entrySet()) { String paramName = entry.getKey(); @@ -234,15 +245,37 @@ public class PluginTool extends BaseTool { // 如果是文件类型 if (originalParam.getType().equals("File")){ try { - FileStorageService fileStorageService = SpringContextUtil.getBean(FileStorageManager.class); - InputStream inputStream = fileStorageService.readStream((String)originalParam.getDefaultValue()); - requestParam.setType("MultipartFile"); - byte[] bytes = inputStreamToBytes(inputStream); - String contentType = FileTypeUtil.getType(new ByteArrayInputStream(bytes)); - String fileUrl = (String) originalParam.getDefaultValue(); + FileStorageService fileStorageService = + SpringContextUtil.getBean( + FileStorageManager.class); + String fileUrl = + (String) originalParam.getDefaultValue(); int lastSlashIndex = fileUrl.lastIndexOf("/"); - String fileName = fileUrl.substring(lastSlashIndex + 1); - requestParam.setDefaultValue(new CustomMultipartFile(bytes, originalParam.getName(), fileName, contentType)); + String fileName = + fileUrl.substring(lastSlashIndex + 1); + Path temporaryFile = Files.createTempFile( + "easyflow-plugin-", ".upload"); + try (IoBulkhead.Permit ignored = + IoBulkhead.storage().acquire( + "storage:plugin-read"); + InputStream inputStream = + fileStorageService.readStream(fileUrl); + OutputStream outputStream = + Files.newOutputStream(temporaryFile)) { + copyBounded( + inputStream, + outputStream, + Long.getLong( + "easyflow.plugin.file.max-bytes", + 256L * 1024L * 1024L)); + } + temporaryFiles.add(temporaryFile); + requestParam.setType("MultipartFile"); + requestParam.setDefaultValue( + new TemporaryFileMultipartFile( + fileName, + temporaryFile, + null)); } catch (IOException e) { throw new RuntimeException(e); } @@ -283,6 +316,59 @@ public class PluginTool extends BaseTool { logger.error(result.get("error").toString()); } return result; + } finally { + for (Path temporaryFile : temporaryFiles) { + try { + Files.deleteIfExists(temporaryFile); + } catch (IOException cleanupError) { + logger.warn( + "清理插件上传临时文件失败，path={}", + temporaryFile, + cleanupError); + } + } + } + } + + /** + * 按 ID 加载插件项。 + * + * @param pluginId 插件项 ID + * @return 插件项 + */ + private PluginItem loadPluginItem(BigInteger pluginId) { + PluginItemService pluginToolService = + SpringContextUtil.getBean(PluginItemService.class); + QueryWrapper query = QueryWrapper.create() + .select("*") + .from("tb_plugin_item") + .where("id = ? ", pluginId); + return pluginToolService.getMapper().selectOneByQuery(query); + } + + /** + * 使用固定缓冲区复制文件并校验实际字节数。 + * + * @param inputStream 输入流 + * @param outputStream 输出流 + * @param maxBytes 最大字节数 + * @throws IOException 读取失败或超限 + */ + private void copyBounded( + InputStream inputStream, + OutputStream outputStream, + long maxBytes) throws IOException { + byte[] buffer = new byte[64 * 1024]; + long total = 0L; + int read; + while ((read = inputStream.read(buffer)) != -1) { + total += read; + if (maxBytes > 0L && total > maxBytes) { + throw new IOException( + "插件文件超过字节上限: " + maxBytes); + } + outputStream.write(buffer, 0, read); + } } // 辅助方法：根据参数名查找原始参数定义 diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/code/PythonRuntimeEngine.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/code/PythonRuntimeEngine.java index 1d14423a..56f7dded 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/code/PythonRuntimeEngine.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/code/PythonRuntimeEngine.java @@ -3,6 +3,7 @@ package tech.easyflow.ai.easyagentsflow.code; import com.alibaba.fastjson2.JSON; import com.alibaba.fastjson2.JSONObject; import com.easyagents.flow.core.chain.Chain; +import com.easyagents.flow.core.chain.ChainState; import com.easyagents.flow.core.code.CodeRuntimeEngine; import com.easyagents.flow.core.node.CodeNode; import com.easyagents.flow.core.util.StringUtil; @@ -218,20 +219,24 @@ public class PythonRuntimeEngine implements CodeRuntimeEngine { private Map buildContext(Chain chain, CodeNode node) { Map context = new HashMap<>(); + ChainState chainState = + chain.getExecutionState(); - Map all = chain.getState().getMemory(); + Map all = + chainState.getMemory(); all.forEach((key, value) -> { if (!key.contains(".")) { context.put(key, value); } }); - Map parameterValues = chain.getState().resolveParameters(node); + Map parameterValues = + chainState.resolveParameters(node); if (parameterValues != null && !parameterValues.isEmpty()) { context.putAll(parameterValues); } - context.put("_env", chain.getState().getEnvMap()); + context.put("_env", chainState.getEnvMap()); return context; } diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/config/ChainExecutorConfig.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/config/ChainExecutorConfig.java index e0a64b3f..6f68aef5 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/config/ChainExecutorConfig.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/config/ChainExecutorConfig.java @@ -1,9 +1,13 @@ package tech.easyflow.ai.easyagentsflow.config; import com.easyagents.flow.core.chain.repository.ChainDefinitionRepository; +import com.easyagents.flow.core.chain.repository.ChainDefinitionSnapshotRepository; import com.easyagents.flow.core.chain.repository.ChainStateRepository; +import com.easyagents.flow.core.chain.repository.LoopResultRepository; import com.easyagents.flow.core.chain.repository.NodeStateRepository; import com.easyagents.flow.core.chain.runtime.ChainExecutor; +import com.easyagents.flow.core.chain.runtime.TriggerScheduler; +import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import tech.easyflow.ai.easyagentsflow.listener.ChainErrorListenerForSave; @@ -11,8 +15,10 @@ import tech.easyflow.ai.easyagentsflow.listener.ChainEventListenerForSave; import tech.easyflow.ai.easyagentsflow.listener.NodeErrorListenerForSave; import javax.annotation.Resource; +import java.time.Duration; @Configuration +@EnableConfigurationProperties(WorkflowExecutionBudgetProperties.class) public class ChainExecutorConfig { @Resource @@ -22,14 +28,51 @@ public class ChainExecutorConfig { @Resource private NodeStateRepository nodeStateRepository; @Resource + private LoopResultRepository loopResultRepository; + @Resource + private ChainDefinitionSnapshotRepository chainDefinitionSnapshotRepository; + @Resource + private TriggerScheduler triggerScheduler; + @Resource private ChainEventListenerForSave chainEventListenerForSave; + @Resource + private WorkflowExecutionBudgetProperties workflowExecutionBudgetProperties; + @Resource + private WorkflowRuntimeProperties workflowRuntimeProperties; @Bean(name = "chainExecutor") public ChainExecutor chainExecutor() { ChainExecutor chainExecutor = new ChainExecutor(chainDefinitionRepository, chainStateRepository, - nodeStateRepository); + nodeStateRepository, + loopResultRepository, + chainDefinitionSnapshotRepository, + triggerScheduler, + workflowExecutionBudgetProperties.toExecutionBudget()); + int laneMaxDepth = workflowRuntimeProperties + .getChildWorkflowLaneMaxDepth(); + int budgetMaxDepth = workflowExecutionBudgetProperties + .getMaxNestedDepth(); + if (laneMaxDepth <= 0 + || budgetMaxDepth <= 0 + || budgetMaxDepth > laneMaxDepth) { + throw new IllegalStateException( + "easyflow.workflow.execution-budget.max-nested-depth " + + "must be positive and not exceed " + + "easyflow.workflow.runtime." + + "child-workflow-lane-max-depth"); + } + Duration pollInterval = workflowRuntimeProperties + .getChildWorkflowPollInterval(); + long pollMillis = pollInterval == null + ? 500L + : Math.max(100L, pollInterval.toMillis()); + chainExecutor.configureChildWorkflowRuntime( + Math.max(1, workflowRuntimeProperties + .getChildWorkflowRootPermits()), + pollMillis, + laneMaxDepth); saveStepsListeners(chainExecutor); diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/config/WorkflowExecutionBudgetProperties.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/config/WorkflowExecutionBudgetProperties.java new file mode 100644 index 00000000..962383bc --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/config/WorkflowExecutionBudgetProperties.java @@ -0,0 +1,144 @@ +package tech.easyflow.ai.easyagentsflow.config; + +import com.easyagents.flow.core.chain.runtime.ExecutionBudget; +import org.springframework.boot.context.properties.ConfigurationProperties; + +import java.time.Duration; + +/** + * 工作流执行资源保护预算配置。 + */ +@ConfigurationProperties(prefix = "easyflow.workflow.execution-budget") +public class WorkflowExecutionBudgetProperties { + + private long maxIterations = ExecutionBudget.DEFAULT_MAX_ITERATIONS; + private Duration maxDuration = Duration.ofMillis(ExecutionBudget.DEFAULT_MAX_DURATION_MILLIS); + private long maxChildExecutions = ExecutionBudget.DEFAULT_MAX_CHILD_EXECUTIONS; + private long maxAccumulatedBytes = ExecutionBudget.DEFAULT_MAX_ACCUMULATED_BYTES; + private int maxNestedDepth = ExecutionBudget.DEFAULT_MAX_NESTED_DEPTH; + private long maxHotStateBytes = ExecutionBudget.DEFAULT_MAX_HOT_STATE_BYTES; + + /** + * 转换为工作流引擎使用的不可变执行预算。 + * + * @return 执行预算 + */ + public ExecutionBudget toExecutionBudget() { + long maxDurationMillis = maxDuration == null ? 0L : maxDuration.toMillis(); + return new ExecutionBudget( + maxIterations, + maxDurationMillis, + maxChildExecutions, + maxAccumulatedBytes, + maxNestedDepth, + maxHotStateBytes); + } + + /** + * 获取单个执行实例允许的最大循环次数。 + * + * @return 最大循环次数 + */ + public long getMaxIterations() { + return maxIterations; + } + + /** + * 设置单个执行实例允许的最大循环次数。 + * + * @param maxIterations 最大循环次数；小于等于零表示不限制 + */ + public void setMaxIterations(long maxIterations) { + this.maxIterations = maxIterations; + } + + /** + * 获取单个执行实例允许的最大运行时间。 + * + * @return 最大运行时间 + */ + public Duration getMaxDuration() { + return maxDuration; + } + + /** + * 设置单个执行实例允许的最大运行时间。 + * + * @param maxDuration 最大运行时间 + */ + public void setMaxDuration(Duration maxDuration) { + this.maxDuration = maxDuration; + } + + /** + * 获取允许调度的最大子节点执行次数。 + * + * @return 最大子节点执行次数 + */ + public long getMaxChildExecutions() { + return maxChildExecutions; + } + + /** + * 设置允许调度的最大子节点执行次数。 + * + * @param maxChildExecutions 最大子节点执行次数；小于等于零表示不限制 + */ + public void setMaxChildExecutions(long maxChildExecutions) { + this.maxChildExecutions = maxChildExecutions; + } + + /** + * 获取允许累计的结果估算字节数。 + * + * @return 最大累计结果字节数 + */ + public long getMaxAccumulatedBytes() { + return maxAccumulatedBytes; + } + + /** + * 设置允许累计的结果估算字节数。 + * + * @param maxAccumulatedBytes 最大累计结果字节数；小于等于零表示不限制 + */ + public void setMaxAccumulatedBytes(long maxAccumulatedBytes) { + this.maxAccumulatedBytes = maxAccumulatedBytes; + } + + /** + * 获取循环允许的最大嵌套深度。 + * + * @return 最大嵌套深度 + */ + public int getMaxNestedDepth() { + return maxNestedDepth; + } + + /** + * 设置循环允许的最大嵌套深度。 + * + * @param maxNestedDepth 最大嵌套深度；小于等于零表示不限制 + */ + public void setMaxNestedDepth(int maxNestedDepth) { + this.maxNestedDepth = maxNestedDepth; + } + + /** + * 获取单个热状态允许的最大估算字节数。 + * + * @return 最大热状态字节数 + */ + public long getMaxHotStateBytes() { + return maxHotStateBytes; + } + + /** + * 设置单个热状态允许的最大估算字节数。 + * + * @param maxHotStateBytes 最大热状态字节数；小于等于零表示不限制 + */ + public void setMaxHotStateBytes(long maxHotStateBytes) { + this.maxHotStateBytes = maxHotStateBytes; + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/config/WorkflowIoBulkheadMetrics.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/config/WorkflowIoBulkheadMetrics.java new file mode 100644 index 00000000..02e2e886 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/config/WorkflowIoBulkheadMetrics.java @@ -0,0 +1,93 @@ +package tech.easyflow.ai.easyagentsflow.config; + +import com.easyagents.flow.core.util.IoBulkhead; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; +import org.springframework.stereotype.Component; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.function.ToDoubleFunction; + +/** + * 将工作流各类 I/O 隔离器运行状态接入 Micrometer。 + */ +@Component +public class WorkflowIoBulkheadMetrics { + + /** + * 注册工作流 I/O 隔离器指标。 + * + * @param meterRegistry Micrometer 注册表 + * @param properties 工作流 I/O 配置 + */ + public WorkflowIoBulkheadMetrics( + MeterRegistry meterRegistry, + WorkflowIoProperties properties) { + IoBulkhead.configure( + properties.getHttp().toSettings(), + properties.getDataset().toSettings(), + properties.getStorage().toSettings(), + properties.getDocumentParse().toSettings(), + properties.getResponseAggregation().toSettings()); + Map lanes = new LinkedHashMap<>(); + lanes.put("http", IoBulkhead.shared()); + lanes.put("dataset", IoBulkhead.dataset()); + lanes.put("storage", IoBulkhead.storage()); + lanes.put("document_parse", IoBulkhead.documentParse()); + lanes.put( + "response_aggregation", + IoBulkhead.responseAggregation()); + lanes.forEach((lane, bulkhead) -> registerLane( + meterRegistry, lane, bulkhead)); + } + + /** + * 注册一个资源 lane 的核心容量和等待指标。 + * + * @param registry 指标注册表 + * @param lane lane 名 + * @param bulkhead 隔离器 + */ + private void registerLane( + MeterRegistry registry, + String lane, + IoBulkhead bulkhead) { + gauge(registry, lane, bulkhead, "in_flight", + snapshot -> snapshot.inFlightCount()); + gauge(registry, lane, bulkhead, "acquired_total", + snapshot -> snapshot.acquiredCount()); + gauge(registry, lane, bulkhead, "rejected_total", + snapshot -> snapshot.rejectedCount()); + gauge(registry, lane, bulkhead, "wait_nanos_total", + snapshot -> snapshot.totalWaitNanos()); + gauge(registry, lane, bulkhead, "available_permits", + snapshot -> snapshot.availableGlobalPermits()); + gauge(registry, lane, bulkhead, "tracked_targets", + snapshot -> snapshot.trackedTargetCount()); + } + + /** + * 注册从快照读取的 Gauge。 + * + * @param registry 指标注册表 + * @param lane lane 名 + * @param bulkhead 隔离器 + * @param metric 指标后缀 + * @param valueFunction 快照取值函数 + */ + private void gauge( + MeterRegistry registry, + String lane, + IoBulkhead bulkhead, + String metric, + ToDoubleFunction valueFunction) { + Gauge.builder( + "easyflow.workflow.io." + metric, + bulkhead, + value -> valueFunction.applyAsDouble( + value.snapshot())) + .tag("lane", lane) + .register(registry); + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/config/WorkflowIoProperties.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/config/WorkflowIoProperties.java new file mode 100644 index 00000000..68843476 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/config/WorkflowIoProperties.java @@ -0,0 +1,236 @@ +package tech.easyflow.ai.easyagentsflow.config; + +import com.easyagents.flow.core.util.IoBulkhead; +import org.springframework.boot.context.properties.ConfigurationProperties; + +import java.time.Duration; + +/** + * 工作流阻塞 I/O 隔离配置。 + * + *

全部参数均提供宽松默认值，应用无需新增配置即可保持现有业务行为。

+ */ +@ConfigurationProperties(prefix = "easyflow.workflow.io") +public class WorkflowIoProperties { + + private Lane http = new Lane(64, 16, Duration.ofSeconds(1), 1_024); + private Lane dataset = new Lane(32, 8, Duration.ofSeconds(1), 512); + private Lane storage = new Lane(24, 12, Duration.ofSeconds(2), 256); + private Lane documentParse = new Lane(8, 4, Duration.ofSeconds(2), 128); + private Lane responseAggregation = + new Lane(8, 4, Duration.ofSeconds(2), 1_024); + + /** + * 获取 HTTP 隔离配置。 + * + * @return HTTP 配置 + */ + public Lane getHttp() { + return http; + } + + /** + * 设置 HTTP 隔离配置。 + * + * @param http HTTP 配置 + */ + public void setHttp(Lane http) { + this.http = http; + } + + /** + * 获取数据集隔离配置。 + * + * @return 数据集配置 + */ + public Lane getDataset() { + return dataset; + } + + /** + * 设置数据集隔离配置。 + * + * @param dataset 数据集配置 + */ + public void setDataset(Lane dataset) { + this.dataset = dataset; + } + + /** + * 获取对象存储隔离配置。 + * + * @return 对象存储配置 + */ + public Lane getStorage() { + return storage; + } + + /** + * 设置对象存储隔离配置。 + * + * @param storage 对象存储配置 + */ + public void setStorage(Lane storage) { + this.storage = storage; + } + + /** + * 获取文档解析隔离配置。 + * + * @return 文档解析配置 + */ + public Lane getDocumentParse() { + return documentParse; + } + + /** + * 设置文档解析隔离配置。 + * + * @param documentParse 文档解析配置 + */ + public void setDocumentParse(Lane documentParse) { + this.documentParse = documentParse; + } + + /** + * 获取响应聚合隔离配置。 + * + * @return 响应聚合配置 + */ + public Lane getResponseAggregation() { + return responseAggregation; + } + + /** + * 设置响应聚合隔离配置。 + * + * @param responseAggregation 响应聚合配置 + */ + public void setResponseAggregation(Lane responseAggregation) { + this.responseAggregation = responseAggregation; + } + + /** + * 单类阻塞 I/O 的容量配置。 + */ + public static class Lane { + + private int maxConcurrency; + private int perTargetMaxConcurrency; + private Duration acquireTimeout; + private int maxTrackedTargets; + + /** + * 创建供 Spring 绑定使用的空配置对象。 + */ + public Lane() { + } + + /** + * 创建带默认值的隔离配置。 + * + * @param maxConcurrency 总并发 + * @param perTargetMaxConcurrency 单目标并发 + * @param acquireTimeout 许可等待时间 + * @param maxTrackedTargets 最大目标数 + */ + public Lane( + int maxConcurrency, + int perTargetMaxConcurrency, + Duration acquireTimeout, + int maxTrackedTargets) { + this.maxConcurrency = maxConcurrency; + this.perTargetMaxConcurrency = perTargetMaxConcurrency; + this.acquireTimeout = acquireTimeout; + this.maxTrackedTargets = maxTrackedTargets; + } + + /** + * 转换为引擎配置并完成启动期校验。 + * + * @return 引擎隔离配置 + * @throws IllegalArgumentException 配置值无效时抛出 + */ + public IoBulkhead.Settings toSettings() { + return new IoBulkhead.Settings( + maxConcurrency, + perTargetMaxConcurrency, + acquireTimeout, + maxTrackedTargets); + } + + /** + * 获取总并发。 + * + * @return 总并发 + */ + public int getMaxConcurrency() { + return maxConcurrency; + } + + /** + * 设置总并发。 + * + * @param maxConcurrency 总并发 + */ + public void setMaxConcurrency(int maxConcurrency) { + this.maxConcurrency = maxConcurrency; + } + + /** + * 获取单目标并发。 + * + * @return 单目标并发 + */ + public int getPerTargetMaxConcurrency() { + return perTargetMaxConcurrency; + } + + /** + * 设置单目标并发。 + * + * @param perTargetMaxConcurrency 单目标并发 + */ + public void setPerTargetMaxConcurrency( + int perTargetMaxConcurrency) { + this.perTargetMaxConcurrency = + perTargetMaxConcurrency; + } + + /** + * 获取许可等待时间。 + * + * @return 等待时间 + */ + public Duration getAcquireTimeout() { + return acquireTimeout; + } + + /** + * 设置许可等待时间。 + * + * @param acquireTimeout 等待时间 + */ + public void setAcquireTimeout(Duration acquireTimeout) { + this.acquireTimeout = acquireTimeout; + } + + /** + * 获取最大目标数。 + * + * @return 最大目标数 + */ + public int getMaxTrackedTargets() { + return maxTrackedTargets; + } + + /** + * 设置最大目标数。 + * + * @param maxTrackedTargets 最大目标数 + */ + public void setMaxTrackedTargets(int maxTrackedTargets) { + this.maxTrackedTargets = maxTrackedTargets; + } + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/config/WorkflowRuntimeProperties.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/config/WorkflowRuntimeProperties.java new file mode 100644 index 00000000..1bbcb8cc --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/config/WorkflowRuntimeProperties.java @@ -0,0 +1,291 @@ +package tech.easyflow.ai.easyagentsflow.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +import java.time.Duration; + +/** + * 工作流调度运行时配置。 + */ +@ConfigurationProperties(prefix = "easyflow.workflow.runtime") +public class WorkflowRuntimeProperties { + + private Duration triggerScanInterval = Duration.ofSeconds(5); + private int schedulerThreads = 2; + private int workerCoreSize = 32; + private int workerMaxSize = 256; + private int workerQueueCapacity = 256; + private int childWorkflowLaneMaxDepth = 32; + private int childWorkflowLaneMaxThreads = 16; + private int childWorkflowRootPermits = 32; + private Duration childWorkflowPollInterval = + Duration.ofMillis(500); + private int dataWriteBatchSize = 200; + private long downloadMaxBytes = 2L * 1024L * 1024L * 1024L; + private int definitionCacheMaxEntries = 512; + private long definitionCacheMaxBytes = + 256L * 1024L * 1024L; + private Duration definitionCacheExpireAfterAccess = Duration.ofMinutes(30); + + /** + * 获取触发器补偿扫描间隔。 + * + * @return 扫描间隔 + */ + public Duration getTriggerScanInterval() { + return triggerScanInterval; + } + + /** + * 设置触发器补偿扫描间隔。 + * + * @param triggerScanInterval 扫描间隔 + */ + public void setTriggerScanInterval(Duration triggerScanInterval) { + this.triggerScanInterval = triggerScanInterval; + } + + /** + * 获取调度线程数。 + * + * @return 调度线程数 + */ + public int getSchedulerThreads() { + return schedulerThreads; + } + + /** + * 设置调度线程数。 + * + * @param schedulerThreads 调度线程数 + */ + public void setSchedulerThreads(int schedulerThreads) { + this.schedulerThreads = schedulerThreads; + } + + /** + * 获取工作线程核心数。 + * + * @return 核心线程数 + */ + public int getWorkerCoreSize() { + return workerCoreSize; + } + + /** + * 设置工作线程核心数。 + * + * @param workerCoreSize 核心线程数 + */ + public void setWorkerCoreSize(int workerCoreSize) { + this.workerCoreSize = workerCoreSize; + } + + /** + * 获取工作线程最大数。 + * + * @return 最大线程数 + */ + public int getWorkerMaxSize() { + return workerMaxSize; + } + + /** + * 设置工作线程最大数。 + * + * @param workerMaxSize 最大线程数 + */ + public void setWorkerMaxSize(int workerMaxSize) { + this.workerMaxSize = workerMaxSize; + } + + /** + * 获取工作队列容量。 + * + * @return 队列容量 + */ + public int getWorkerQueueCapacity() { + return workerQueueCapacity; + } + + /** + * 设置工作队列容量。 + * + * @param workerQueueCapacity 队列容量 + */ + public void setWorkerQueueCapacity(int workerQueueCapacity) { + this.workerQueueCapacity = workerQueueCapacity; + } + + /** + * 获取子工作流独立通道覆盖的最大嵌套深度。 + * + * @return 最大深度 + */ + public int getChildWorkflowLaneMaxDepth() { + return childWorkflowLaneMaxDepth; + } + + /** + * 设置子工作流独立通道覆盖的最大嵌套深度。 + * + * @param childWorkflowLaneMaxDepth 最大深度 + */ + public void setChildWorkflowLaneMaxDepth( + int childWorkflowLaneMaxDepth) { + this.childWorkflowLaneMaxDepth = + childWorkflowLaneMaxDepth; + } + + /** + * 获取每个子工作流深度通道的最大线程数。 + * + * @return 最大线程数 + */ + public int getChildWorkflowLaneMaxThreads() { + return childWorkflowLaneMaxThreads; + } + + /** + * 设置每个子工作流深度通道的最大线程数。 + * + * @param childWorkflowLaneMaxThreads 最大线程数 + */ + public void setChildWorkflowLaneMaxThreads( + int childWorkflowLaneMaxThreads) { + this.childWorkflowLaneMaxThreads = + childWorkflowLaneMaxThreads; + } + + /** + * 获取根级同步子工作流并发许可数。 + * + * @return 并发许可数 + */ + public int getChildWorkflowRootPermits() { + return childWorkflowRootPermits; + } + + /** + * 设置根级同步子工作流并发许可数。 + * + * @param childWorkflowRootPermits 并发许可数 + */ + public void setChildWorkflowRootPermits( + int childWorkflowRootPermits) { + this.childWorkflowRootPermits = + childWorkflowRootPermits; + } + + /** + * 获取同步等待持久终态的轮询间隔。 + * + * @return 轮询间隔 + */ + public Duration getChildWorkflowPollInterval() { + return childWorkflowPollInterval; + } + + /** + * 设置同步等待持久终态的轮询间隔。 + * + * @param childWorkflowPollInterval 轮询间隔 + */ + public void setChildWorkflowPollInterval( + Duration childWorkflowPollInterval) { + this.childWorkflowPollInterval = + childWorkflowPollInterval; + } + + /** + * 获取数据写入节点单批最大行数。 + * + * @return 单批最大行数 + */ + public int getDataWriteBatchSize() { + return dataWriteBatchSize; + } + + /** + * 设置数据写入节点单批最大行数。 + * + * @param dataWriteBatchSize 单批最大行数 + */ + public void setDataWriteBatchSize(int dataWriteBatchSize) { + this.dataWriteBatchSize = dataWriteBatchSize; + } + + /** + * 获取下载节点允许的最大文件字节数。 + * + * @return 最大文件字节数，小于等于 0 表示不限制 + */ + public long getDownloadMaxBytes() { + return downloadMaxBytes; + } + + /** + * 设置下载节点允许的最大文件字节数。 + * + * @param downloadMaxBytes 最大文件字节数，小于等于 0 表示不限制 + */ + public void setDownloadMaxBytes(long downloadMaxBytes) { + this.downloadMaxBytes = downloadMaxBytes; + } + + /** + * 获取本地编译定义缓存最大条目数。 + * + * @return 最大条目数 + */ + public int getDefinitionCacheMaxEntries() { + return definitionCacheMaxEntries; + } + + /** + * 设置本地编译定义缓存最大条目数。 + * + * @param definitionCacheMaxEntries 最大条目数 + */ + public void setDefinitionCacheMaxEntries(int definitionCacheMaxEntries) { + this.definitionCacheMaxEntries = definitionCacheMaxEntries; + } + + /** + * 获取本地编译定义缓存的最大估算字节数。 + * + * @return 最大字节数 + */ + public long getDefinitionCacheMaxBytes() { + return definitionCacheMaxBytes; + } + + /** + * 设置本地编译定义缓存的最大估算字节数。 + * + * @param definitionCacheMaxBytes 最大字节数 + */ + public void setDefinitionCacheMaxBytes( + long definitionCacheMaxBytes) { + this.definitionCacheMaxBytes = + definitionCacheMaxBytes; + } + + /** + * 获取编译定义缓存访问过期时间。 + * + * @return 访问过期时间 + */ + public Duration getDefinitionCacheExpireAfterAccess() { + return definitionCacheExpireAfterAccess; + } + + /** + * 设置编译定义缓存访问过期时间。 + * + * @param definitionCacheExpireAfterAccess 访问过期时间 + */ + public void setDefinitionCacheExpireAfterAccess(Duration definitionCacheExpireAfterAccess) { + this.definitionCacheExpireAfterAccess = definitionCacheExpireAfterAccess; + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/config/WorkflowTriggerSchedulerConfig.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/config/WorkflowTriggerSchedulerConfig.java new file mode 100644 index 00000000..d4fe0014 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/config/WorkflowTriggerSchedulerConfig.java @@ -0,0 +1,100 @@ +package tech.easyflow.ai.easyagentsflow.config; + +import com.easyagents.flow.core.chain.runtime.ChainExecutor; +import com.easyagents.flow.core.chain.runtime.TriggerScheduler; +import com.easyagents.flow.core.chain.runtime.TriggerStore; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.time.Duration; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * 工作流持久化触发调度器配置。 + */ +@Configuration +@EnableConfigurationProperties({ + WorkflowRuntimeProperties.class, + WorkflowIoProperties.class +}) +public class WorkflowTriggerSchedulerConfig { + + /** + * 创建使用 Redis 触发器仓储的独立调度器。 + * + * @param triggerStore 持久化触发器仓储 + * @param properties 调度运行时配置 + * @return 工作流触发调度器 + */ + @Bean(destroyMethod = "shutdown") + public TriggerScheduler workflowTriggerScheduler( + TriggerStore triggerStore, WorkflowRuntimeProperties properties) { + int schedulerThreads = Math.max(1, properties.getSchedulerThreads()); + int workerCoreSize = Math.max(1, properties.getWorkerCoreSize()); + int workerMaxSize = Math.max(workerCoreSize, properties.getWorkerMaxSize()); + int queueCapacity = Math.max(1, properties.getWorkerQueueCapacity()); + Duration scanInterval = properties.getTriggerScanInterval(); + long scanIntervalMillis = scanInterval == null ? 5000L : Math.max(1000L, scanInterval.toMillis()); + + ScheduledThreadPoolExecutor scheduler = new ScheduledThreadPoolExecutor( + schedulerThreads, namedThreadFactory("workflow-trigger-scheduler")); + scheduler.setRemoveOnCancelPolicy(true); + ThreadPoolExecutor worker = new ThreadPoolExecutor( + workerCoreSize, + workerMaxSize, + 60L, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>(queueCapacity), + namedThreadFactory("workflow-node-worker"), + new ThreadPoolExecutor.AbortPolicy()); + TriggerScheduler triggerScheduler = new TriggerScheduler( + triggerStore, scheduler, worker, scanIntervalMillis); + int childLaneMaxDepth = Math.max( + 1, properties.getChildWorkflowLaneMaxDepth()); + int childLaneMaxThreads = Math.max( + 1, properties.getChildWorkflowLaneMaxThreads()); + /* + * 不同嵌套深度使用独立小通道。depth N 的 WorkflowNode 即使全部同步等待， + * depth N+1 的触发器仍有独立容量，不会形成同池递归饥饿。 + */ + for (int depth = 1; depth <= childLaneMaxDepth; depth++) { + ThreadPoolExecutor childWorkflowWorker = + new ThreadPoolExecutor( + 0, + childLaneMaxThreads, + 60L, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + namedThreadFactory( + "workflow-child-" + depth), + new ThreadPoolExecutor.AbortPolicy()); + childWorkflowWorker.allowCoreThreadTimeOut(true); + triggerScheduler.registerWorker( + ChainExecutor.childExecutionLane(depth), + childWorkflowWorker); + } + return triggerScheduler; + } + + /** + * 创建带稳定前缀的守护线程工厂。 + * + * @param prefix 线程名前缀 + * @return 线程工厂 + */ + private ThreadFactory namedThreadFactory(String prefix) { + AtomicInteger sequence = new AtomicInteger(); + return task -> { + Thread thread = new Thread(task, prefix + "-" + sequence.incrementAndGet()); + thread.setDaemon(true); + return thread; + }; + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/event/WorkflowDefinitionChangedEvent.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/event/WorkflowDefinitionChangedEvent.java new file mode 100644 index 00000000..c9ee1dc9 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/event/WorkflowDefinitionChangedEvent.java @@ -0,0 +1,9 @@ +package tech.easyflow.ai.easyagentsflow.event; + +/** + * 工作流定义内容或发布快照发生变化的本地事件。 + * + * @param workflowId 工作流 ID + */ +public record WorkflowDefinitionChangedEvent(String workflowId) { +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/event/WorkflowExecutionAuditConsumer.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/event/WorkflowExecutionAuditConsumer.java new file mode 100644 index 00000000..e8d8340d --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/event/WorkflowExecutionAuditConsumer.java @@ -0,0 +1,261 @@ +package tech.easyflow.ai.easyagentsflow.event; + +import com.alibaba.fastjson2.JSON; +import com.easyagents.flow.core.chain.repository.LoopInputReference; +import com.easyagents.flow.core.chain.repository.LoopResultReference; +import com.easyagents.flow.core.chain.repository.LoopResultRepository; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.stereotype.Component; +import tech.easyflow.ai.entity.WorkflowExecResult; +import tech.easyflow.ai.entity.WorkflowExecStep; +import tech.easyflow.ai.service.WorkflowExecResultService; +import tech.easyflow.ai.service.WorkflowExecStepService; +import tech.easyflow.common.mq.core.MQConsumerHandler; +import tech.easyflow.common.mq.core.MQMessage; +import tech.easyflow.common.mq.core.MQSubscription; +import tech.easyflow.common.mq.config.MQProperties; + +import java.util.List; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * 工作流执行审计事件消费者。 + */ +@Component +public class WorkflowExecutionAuditConsumer implements MQConsumerHandler { + + private final WorkflowExecResultService workflowExecResultService; + private final WorkflowExecStepService workflowExecStepService; + private final MQProperties mqProperties; + private final LoopResultRepository loopResultRepository; + + /** + * 创建工作流执行审计事件消费者。 + * + * @param workflowExecResultService 工作流执行记录服务 + * @param workflowExecStepService 节点执行步骤服务 + * @param mqProperties MQ 配置 + * @param loopResultRepository 循环与大型查询结果仓储 + */ + public WorkflowExecutionAuditConsumer(WorkflowExecResultService workflowExecResultService, + WorkflowExecStepService workflowExecStepService, + MQProperties mqProperties, + LoopResultRepository loopResultRepository) { + this.workflowExecResultService = workflowExecResultService; + this.workflowExecStepService = workflowExecStepService; + this.mqProperties = mqProperties; + this.loopResultRepository = + loopResultRepository; + } + + /** + * {@inheritDoc} + */ + @Override + public MQSubscription subscription() { + MQSubscription subscription = new MQSubscription(); + subscription.setTopic(WorkflowExecutionAuditMqConstants.TOPIC); + subscription.setConsumerGroup(WorkflowExecutionAuditMqConstants.CONSUMER_GROUP); + subscription.setShardCount(Math.max( + 1, + mqProperties.getRedis().getChatPersistShardCount())); + subscription.setBatchEnabled(true); + return subscription; + } + + /** + * {@inheritDoc} + */ + @Override + public void handle(List messages) { + if (messages == null || messages.isEmpty()) { + return; + } + for (MQMessage message : messages) { + WorkflowExecutionAuditEvent event = JSON.parseObject( + message.getBody(), WorkflowExecutionAuditEvent.class); + if (event == null || event.getType() == null) { + throw new IllegalArgumentException("Invalid workflow execution audit event"); + } + apply(event); + } + } + + /** + * 按事件顺序幂等写入执行记录。 + * + * @param event 审计事件 + */ + private void apply(WorkflowExecutionAuditEvent event) { + switch (event.getType()) { + case CHAIN_STARTED -> createExecution(event); + case CHAIN_ENDED -> finishExecution(event); + case NODE_STARTED -> createStep(event); + case NODE_ENDED -> finishStep(event); + default -> throw new IllegalArgumentException( + "Unsupported workflow execution audit event: " + event.getType()); + } + } + + /** + * 创建工作流执行记录。 + * + * @param event 启动事件 + */ + private void createExecution(WorkflowExecutionAuditEvent event) { + WorkflowExecResult incoming = requireResult(event); + try { + workflowExecResultService.save(incoming); + } catch (DuplicateKeyException ignored) { + // MQ 至少一次投递下的重复启动事件按 exec_key 幂等处理。 + } + } + + /** + * 完成工作流执行记录。 + * + * @param event 结束事件 + */ + private void finishExecution(WorkflowExecutionAuditEvent event) { + WorkflowExecResult incoming = requireResult(event); + incoming.setOutput(resolveAuditOutput( + incoming.getOutput())); + if (workflowExecResultService.updateByExecKey(incoming) != 1) { + throw new IllegalStateException( + "Unable to update workflow execution record: " + incoming.getExecKey()); + } + } + + /** + * 创建节点执行步骤。 + * + * @param event 节点启动事件 + */ + private void createStep(WorkflowExecutionAuditEvent event) { + WorkflowExecStep incoming = requireStep(event); + WorkflowExecResult record = workflowExecResultService.getByExecKey(event.getInstanceId()); + if (record == null) { + throw new IllegalStateException( + "Workflow execution record not found: " + event.getInstanceId()); + } + incoming.setRecordId(record.getId()); + incoming.setInput(resolveAuditOutput( + incoming.getInput())); + try { + workflowExecStepService.save(incoming); + } catch (DuplicateKeyException ignored) { + // 同一基础设施触发器恢复后重复投递时按稳定 exec_key 幂等处理。 + } + } + + /** + * 完成节点执行步骤。 + * + * @param event 节点结束事件 + */ + private void finishStep(WorkflowExecutionAuditEvent event) { + WorkflowExecStep incoming = requireStep(event); + incoming.setOutput(resolveAuditOutput( + incoming.getOutput())); + if (workflowExecStepService.updateByExecKey(incoming) != 1) { + throw new IllegalStateException( + "Unable to update workflow execution step: " + incoming.getExecKey()); + } + } + + /** + * 获取事件中的工作流执行记录。 + * + * @param event 审计事件 + * @return 工作流执行记录 + */ + private WorkflowExecResult requireResult(WorkflowExecutionAuditEvent event) { + if (event.getResult() == null || event.getResult().getExecKey() == null) { + throw new IllegalArgumentException("Workflow execution audit result is required"); + } + return event.getResult(); + } + + /** + * 获取事件中的节点执行步骤。 + * + * @param event 审计事件 + * @return 节点执行步骤 + */ + private WorkflowExecStep requireStep(WorkflowExecutionAuditEvent event) { + if (event.getStep() == null || event.getStep().getExecKey() == null) { + throw new IllegalArgumentException("Workflow execution audit step is required"); + } + return event.getStep(); + } + + /** + * 在审计消费线程还原轻量引用，保持执行记录既有完整 JSON 语义。 + * + * @param output 可能包含内部引用的 JSON + * @return 已还原的完整 JSON + */ + private String resolveAuditOutput( + String output) { + if (output == null || output.isBlank()) { + return output; + } + Object parsed = JSON.parse(output); + Object references = restoreReferences(parsed); + Object resolved = + loopResultRepository.resolveReferences( + references); + return JSON.toJSONString(resolved); + } + + /** + * 将异步消息中的稳定引用标记恢复为引擎引用对象。 + * + * @param value JSON 值 + * @return 引擎可解析值 + */ + private Object restoreReferences(Object value) { + if (value instanceof Map map) { + if ("easyflow.loop-input.v1".equals( + map.get("referenceType")) + && map.get("resultId") != null + && map.get("itemCount") + instanceof Number) { + return new LoopInputReference( + String.valueOf( + map.get("resultId")), + ((Number) map.get( + "itemCount")).intValue()); + } + if ("easyflow.loop-result.v1".equals( + map.get("referenceType")) + && map.get("resultId") != null + && map.get("iterationCount") + instanceof Number + && map.get("outputName") != null) { + return new LoopResultReference( + String.valueOf( + map.get("resultId")), + ((Number) map.get( + "iterationCount")) + .intValue(), + String.valueOf( + map.get("outputName"))); + } + Map restored = + new LinkedHashMap<>(); + map.forEach((key, item) -> + restored.put( + key, + restoreReferences(item))); + return restored; + } + if (value instanceof List list) { + return list.stream() + .map(this::restoreReferences) + .toList(); + } + return value; + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/event/WorkflowExecutionAuditEvent.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/event/WorkflowExecutionAuditEvent.java new file mode 100644 index 00000000..7b3cea1f --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/event/WorkflowExecutionAuditEvent.java @@ -0,0 +1,158 @@ +package tech.easyflow.ai.easyagentsflow.event; + +import tech.easyflow.ai.entity.WorkflowExecResult; +import tech.easyflow.ai.entity.WorkflowExecStep; + +import java.io.Serializable; +import java.util.Date; + +/** + * 工作流执行记录异步持久化事件。 + */ +public class WorkflowExecutionAuditEvent implements Serializable { + + private static final long serialVersionUID = 1L; + + /** + * 审计事件类型。 + */ + public enum Type { + CHAIN_STARTED, + CHAIN_ENDED, + NODE_STARTED, + NODE_ENDED + } + + /** + * 事件幂等 ID。 + */ + private String eventId; + /** + * 工作流实例 ID，同时作为同实例事件顺序键。 + */ + private String instanceId; + /** + * 事件类型。 + */ + private Type type; + /** + * 事件发生时间。 + */ + private Date occurredAt; + /** + * 工作流执行记录快照。 + */ + private WorkflowExecResult result; + /** + * 节点执行步骤快照。 + */ + private WorkflowExecStep step; + + /** + * 获取事件幂等 ID。 + * + * @return 事件幂等 ID + */ + public String getEventId() { + return eventId; + } + + /** + * 设置事件幂等 ID。 + * + * @param eventId 事件幂等 ID + */ + public void setEventId(String eventId) { + this.eventId = eventId; + } + + /** + * 获取工作流实例 ID。 + * + * @return 工作流实例 ID + */ + public String getInstanceId() { + return instanceId; + } + + /** + * 设置工作流实例 ID。 + * + * @param instanceId 工作流实例 ID + */ + public void setInstanceId(String instanceId) { + this.instanceId = instanceId; + } + + /** + * 获取事件类型。 + * + * @return 事件类型 + */ + public Type getType() { + return type; + } + + /** + * 设置事件类型。 + * + * @param type 事件类型 + */ + public void setType(Type type) { + this.type = type; + } + + /** + * 获取事件发生时间。 + * + * @return 事件发生时间 + */ + public Date getOccurredAt() { + return occurredAt; + } + + /** + * 设置事件发生时间。 + * + * @param occurredAt 事件发生时间 + */ + public void setOccurredAt(Date occurredAt) { + this.occurredAt = occurredAt; + } + + /** + * 获取工作流执行记录快照。 + * + * @return 工作流执行记录快照 + */ + public WorkflowExecResult getResult() { + return result; + } + + /** + * 设置工作流执行记录快照。 + * + * @param result 工作流执行记录快照 + */ + public void setResult(WorkflowExecResult result) { + this.result = result; + } + + /** + * 获取节点执行步骤快照。 + * + * @return 节点执行步骤快照 + */ + public WorkflowExecStep getStep() { + return step; + } + + /** + * 设置节点执行步骤快照。 + * + * @param step 节点执行步骤快照 + */ + public void setStep(WorkflowExecStep step) { + this.step = step; + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/event/WorkflowExecutionAuditMqConstants.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/event/WorkflowExecutionAuditMqConstants.java new file mode 100644 index 00000000..7bc40ad8 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/event/WorkflowExecutionAuditMqConstants.java @@ -0,0 +1,19 @@ +package tech.easyflow.ai.easyagentsflow.event; + +/** + * 工作流执行审计 MQ 常量。 + */ +public final class WorkflowExecutionAuditMqConstants { + + /** + * 工作流执行审计主题。 + */ + public static final String TOPIC = "workflow-execution-audit"; + /** + * 工作流执行审计消费组。 + */ + public static final String CONSUMER_GROUP = "workflow-execution-audit-writer"; + + private WorkflowExecutionAuditMqConstants() { + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/event/WorkflowExecutionAuditProducer.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/event/WorkflowExecutionAuditProducer.java new file mode 100644 index 00000000..0af5986e --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/event/WorkflowExecutionAuditProducer.java @@ -0,0 +1,713 @@ +package tech.easyflow.ai.easyagentsflow.event; + +import com.alibaba.fastjson2.JSON; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import tech.easyflow.common.mq.core.MQDeadLetterService; +import tech.easyflow.common.mq.core.MQMessage; +import tech.easyflow.common.mq.core.MQProducer; + +import javax.annotation.PreDestroy; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Date; +import java.util.Deque; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * 工作流执行审计事件生产者。 + * + *

少量固定 lane 保证同一实例 FIFO，跨 lane 并行发送和重试，避免单个异常 + * 实例阻塞全部工作流。全局条数与字节预算共同约束本地重试内存。

+ */ +@Service +public class WorkflowExecutionAuditProducer { + + private static final Logger log = + LoggerFactory.getLogger(WorkflowExecutionAuditProducer.class); + private static final int DEFAULT_LANE_COUNT = 8; + private static final int MAX_LOCAL_BACKLOG = 10_000; + private static final long MAX_MESSAGE_BYTES = + 64L * 1024L * 1024L; + private static final long MAX_BACKLOG_BYTES = + 512L * 1024L * 1024L; + private static final long SHUTDOWN_FLUSH_MILLIS = + TimeUnit.SECONDS.toMillis(5L); + private static final int MAX_DRAIN_BATCH = 256; + private static final int MAX_SEND_ATTEMPTS = 16; + private static final long MAX_RETRY_DELAY_MILLIS = + TimeUnit.MINUTES.toMillis(1); + + private final MQProducer mqProducer; + private final MQDeadLetterService deadLetterService; + private final List lanes; + private final Object admissionLock = + new Object(); + private final ScheduledExecutorService retryExecutor; + private final AtomicBoolean closed = + new AtomicBoolean(); + private final int maxLocalBacklog; + private final long maxMessageBytes; + private final long maxBacklogBytes; + private final long shutdownFlushMillis; + private int backlogCount; + private long backlogBytes; + + /** + * 创建工作流执行审计事件生产者。 + * + * @param mqProducer 通用 MQ 生产者 + * @param deadLetterService 通用 MQ 死信服务 + */ + @Autowired + public WorkflowExecutionAuditProducer( + MQProducer mqProducer, + MQDeadLetterService deadLetterService) { + this( + mqProducer, + deadLetterService, + DEFAULT_LANE_COUNT, + MAX_LOCAL_BACKLOG, + MAX_MESSAGE_BYTES, + MAX_BACKLOG_BYTES, + SHUTDOWN_FLUSH_MILLIS); + } + + /** + * 创建带测试预算的工作流审计生产者。 + * + * @param mqProducer 通用 MQ 生产者 + * @param deadLetterService 通用 MQ 死信服务 + * @param laneCount 固定发送 lane 数 + * @param maxLocalBacklog 最大本地积压条数 + * @param maxMessageBytes 单条消息最大字节数 + * @param maxBacklogBytes 本地积压最大总字节数 + * @param shutdownFlushMillis 关闭时最大收口时间 + */ + WorkflowExecutionAuditProducer( + MQProducer mqProducer, + MQDeadLetterService deadLetterService, + int laneCount, + int maxLocalBacklog, + long maxMessageBytes, + long maxBacklogBytes, + long shutdownFlushMillis) { + this.mqProducer = mqProducer; + this.deadLetterService = deadLetterService; + this.maxLocalBacklog = + Math.max(1, maxLocalBacklog); + this.maxMessageBytes = + Math.max(1L, maxMessageBytes); + this.maxBacklogBytes = + Math.max(this.maxMessageBytes, + maxBacklogBytes); + this.shutdownFlushMillis = + Math.max(0L, shutdownFlushMillis); + int effectiveLaneCount = + Math.max(1, laneCount); + this.lanes = + new ArrayList<>(effectiveLaneCount); + for (int index = 0; + index < effectiveLaneCount; + index++) { + lanes.add(new DeliveryLane()); + } + this.retryExecutor = + Executors.newScheduledThreadPool( + Math.min( + effectiveLaneCount, 4), + runnable -> { + Thread thread = + new Thread( + runnable, + "workflow-audit-producer-retry"); + thread.setDaemon(true); + return thread; + }); + for (DeliveryLane lane : lanes) { + retryExecutor.scheduleWithFixedDelay( + () -> drainReadyBatch( + lane, false), + 10L, + 10L, + TimeUnit.MILLISECONDS); + } + } + + /** + * 投递工作流执行审计事件。 + * + *

同一实例稳定落在同一发送 lane 和 MQ 分片并保持事件顺序；不同 lane + * 独立发送和退避。

+ * + * @param event 工作流执行审计事件 + * @return Redis Stream 记录 ID，或本地排队标识 + */ + public String send( + WorkflowExecutionAuditEvent event) { + if (closed.get()) { + throw new IllegalStateException( + "Workflow audit producer is closed"); + } + if (event == null + || event.getType() == null) { + throw new IllegalArgumentException( + "Workflow execution audit event is required"); + } + Date occurredAt = + event.getOccurredAt() == null + ? new Date() + : event.getOccurredAt(); + event.setOccurredAt(occurredAt); + + MQMessage message = + new MQMessage(); + message.setMessageId(event.getEventId()); + message.setTopic( + WorkflowExecutionAuditMqConstants.TOPIC); + message.setKey(event.getInstanceId()); + message.setCreatedAt(occurredAt); + message.setBody( + JSON.toJSONString(event)); + long messageBytes = + messageBytes(message); + ensureMessageSize(message, messageBytes); + + DeliveryLane lane = + laneFor(message.getKey()); + synchronized (lane.lock) { + if (closed.get()) { + throw new IllegalStateException( + "Workflow audit producer is closed"); + } + if (lane.sending + || !lane.backlog.isEmpty()) { + enqueueLast( + lane, + new PendingDelivery( + message, + 0, + 0L, + messageBytes)); + return queuedId(message); + } + lane.sending = true; + } + try { + return mqProducer.send(message); + } catch (RuntimeException sendError) { + boolean closing; + synchronized (lane.lock) { + closing = closed.get(); + if (!closing) { + enqueueFirst( + lane, + new PendingDelivery( + message, + 1, + System.currentTimeMillis() + + retryDelayMillis(1), + messageBytes)); + } + } + if (closing) { + try { + deadLetterService.deadLetter( + message, + "producer closed during send failure"); + } catch (RuntimeException deadLetterError) { + sendError.addSuppressed( + deadLetterError); + } + throw new IllegalStateException( + "Workflow audit producer closed during send", + sendError); + } + log.warn( + "工作流审计 MQ 暂时不可用，事件已进入有界重试队列，eventId={}", + message.getMessageId()); + return queuedId(message); + } finally { + synchronized (lane.lock) { + lane.sending = false; + } + } + } + + /** + * 在单个 lane 中按原顺序批量重试。 + * + * @param lane 发送 lane + * @param ignoreRetryTime 关闭收口时是否忽略退避时间 + */ + private void drainReadyBatch( + DeliveryLane lane, + boolean ignoreRetryTime) { + for (int index = 0; + index < MAX_DRAIN_BATCH; + index++) { + if (!drainOne( + lane, ignoreRetryTime)) { + return; + } + } + } + + /** + * 重试一个 lane 的队首消息。 + * + * @param lane 发送 lane + * @param ignoreRetryTime 是否忽略退避时间 + * @return 队首已移除且可继续排空时为 {@code true} + */ + private boolean drainOne( + DeliveryLane lane, + boolean ignoreRetryTime) { + PendingDelivery pending; + synchronized (lane.lock) { + if (lane.sending + || lane.backlog.isEmpty()) { + return false; + } + pending = lane.backlog.peekFirst(); + if (pending == null + || (!ignoreRetryTime + && pending.nextAttemptAtMillis() + > System.currentTimeMillis())) { + return false; + } + lane.sending = true; + } + boolean removed = false; + try { + mqProducer.send(pending.message()); + removed = removeHead( + lane, pending); + } catch (RuntimeException sendError) { + int nextAttempt = + pending.attempt() + 1; + if (nextAttempt + >= MAX_SEND_ATTEMPTS) { + if (deadLetter( + pending.message(), + sendError)) { + removed = removeHead( + lane, pending); + } else { + replaceHead( + lane, + pending, + pending.retryAt( + nextAttempt, + System.currentTimeMillis() + + MAX_RETRY_DELAY_MILLIS)); + } + } else { + replaceHead( + lane, + pending, + pending.retryAt( + nextAttempt, + System.currentTimeMillis() + + retryDelayMillis( + nextAttempt))); + } + } finally { + synchronized (lane.lock) { + lane.sending = false; + } + } + return removed; + } + + /** + * 将耗尽生产重试的事件写入通用死信流。 + * + * @param message MQ 消息 + * @param failure 最终发送异常 + * @return 死信写入成功时为 {@code true} + */ + private boolean deadLetter( + MQMessage message, + RuntimeException failure) { + try { + deadLetterService.deadLetter( + message, + "producer send attempts exhausted: " + + failure.getClass().getName() + + ": " + + failure.getMessage()); + return true; + } catch (RuntimeException deadLetterError) { + log.error( + "工作流审计生产失败且死信写入失败，eventId={}", + message.getMessageId(), + deadLetterError); + return false; + } + } + + /** + * 删除仍位于 lane 队首的消息并归还全局积压预算。 + * + * @param lane 发送 lane + * @param expected 期望队首 + * @return 成功删除时为 {@code true} + */ + private boolean removeHead( + DeliveryLane lane, + PendingDelivery expected) { + synchronized (lane.lock) { + if (lane.backlog.peekFirst() + != expected) { + return false; + } + lane.backlog.removeFirst(); + releaseAdmission(expected); + return true; + } + } + + /** + * 原子替换仍位于 lane 队首的消息。 + * + * @param lane 发送 lane + * @param expected 当前队首 + * @param replacement 替换项 + */ + private void replaceHead( + DeliveryLane lane, + PendingDelivery expected, + PendingDelivery replacement) { + synchronized (lane.lock) { + if (lane.backlog.peekFirst() + == expected) { + lane.backlog.removeFirst(); + lane.backlog.addFirst( + replacement); + } + } + } + + /** + * 入队到指定 lane 尾部。 + * + * @param lane 发送 lane + * @param pending 待投递事件 + */ + private void enqueueLast( + DeliveryLane lane, + PendingDelivery pending) { + reserveAdmission(pending); + lane.backlog.addLast(pending); + } + + /** + * 入队到指定 lane 头部。 + * + * @param lane 发送 lane + * @param pending 待投递事件 + */ + private void enqueueFirst( + DeliveryLane lane, + PendingDelivery pending) { + reserveAdmission(pending); + lane.backlog.addFirst(pending); + } + + /** + * 预占全局积压条数和字节预算。 + * + * @param pending 待入队消息 + */ + private void reserveAdmission( + PendingDelivery pending) { + boolean rejected; + synchronized (admissionLock) { + rejected = backlogCount + >= maxLocalBacklog + || pending.messageBytes() + > maxBacklogBytes + - backlogBytes; + if (!rejected) { + backlogCount++; + backlogBytes += + pending.messageBytes(); + } + } + if (rejected) { + deadLetterService.deadLetter( + pending.message(), + "producer retry queue is full"); + throw new IllegalStateException( + "Workflow audit producer retry queue is full"); + } + } + + /** + * 归还一条积压消息占用的全局预算。 + * + * @param pending 已移除消息 + */ + private void releaseAdmission( + PendingDelivery pending) { + synchronized (admissionLock) { + backlogCount = + Math.max(0, backlogCount - 1); + backlogBytes = + Math.max( + 0L, + backlogBytes + - pending.messageBytes()); + } + } + + /** + * 校验单条消息字节上限。 + * + * @param message MQ 消息 + * @param bytes 消息估算字节数 + */ + private void ensureMessageSize( + MQMessage message, + long bytes) { + if (bytes <= maxMessageBytes) { + return; + } + deadLetterService.deadLetter( + message, + "producer message exceeds byte limit"); + throw new IllegalArgumentException( + "Workflow audit message exceeds byte limit"); + } + + /** + * 估算 MQ 消息本地持有字节数。 + * + * @param message MQ 消息 + * @return UTF-8 负载和关键元数据字节数 + */ + private long messageBytes( + MQMessage message) { + return utf8Bytes(message.getBody()) + + utf8Bytes(message.getMessageId()) + + utf8Bytes(message.getKey()) + + 128L; + } + + /** + * 计算字符串 UTF-8 字节数。 + * + * @param value 字符串 + * @return 字节数 + */ + private long utf8Bytes(String value) { + if (value == null) { + return 0L; + } + long bytes = 0L; + for (int index = 0; + index < value.length(); + index++) { + char current = + value.charAt(index); + if (current <= 0x7F) { + bytes++; + } else if (current <= 0x7FF) { + bytes += 2L; + } else if (Character.isHighSurrogate( + current) + && index + 1 < value.length() + && Character.isLowSurrogate( + value.charAt(index + 1))) { + bytes += 4L; + index++; + } else { + bytes += 3L; + } + } + return bytes; + } + + /** + * 按稳定 key 选择固定发送 lane。 + * + * @param key 工作流实例键 + * @return 发送 lane + */ + private DeliveryLane laneFor(String key) { + int hash = key == null + ? 0 + : key.hashCode(); + return lanes.get( + Math.floorMod(hash, lanes.size())); + } + + /** + * 构造本地排队返回标识。 + * + * @param message 已排队消息 + * @return 排队标识 + */ + private String queuedId( + MQMessage message) { + return "queued:" + + message.getMessageId(); + } + + /** + * 计算生产者重试退避。 + * + * @param attempt 已失败次数 + * @return 退避毫秒数 + */ + private long retryDelayMillis(int attempt) { + int shift = + Math.min( + 16, + Math.max(0, attempt - 1)); + return Math.min( + MAX_RETRY_DELAY_MILLIS, + 100L << shift); + } + + /** + * 关闭生产者重试线程，并在有界时间内发送或死信收口积压事件。 + */ + @PreDestroy + public void close() { + if (!closed.compareAndSet( + false, true)) { + return; + } + retryExecutor.shutdownNow(); + long deadline = + System.nanoTime() + + TimeUnit.MILLISECONDS.toNanos( + shutdownFlushMillis); + while (hasBacklog() + && System.nanoTime() < deadline) { + boolean progressed = false; + for (DeliveryLane lane : lanes) { + int before = laneSize(lane); + drainReadyBatch(lane, true); + progressed |= laneSize(lane) + < before; + } + if (!progressed) { + try { + Thread.sleep(10L); + } catch (InterruptedException error) { + Thread.currentThread() + .interrupt(); + break; + } + } + } + deadLetterRemainingBacklog(); + } + + /** + * 判断是否仍有本地积压消息。 + * + * @return 有积压时为 {@code true} + */ + private boolean hasBacklog() { + synchronized (admissionLock) { + return backlogCount > 0; + } + } + + /** + * 获取 lane 当前积压条数。 + * + * @param lane 发送 lane + * @return 积压条数 + */ + private int laneSize(DeliveryLane lane) { + synchronized (lane.lock) { + return lane.backlog.size(); + } + } + + /** + * 将关闭期限后剩余消息转入死信，避免进程内静默丢失。 + */ + private void deadLetterRemainingBacklog() { + for (DeliveryLane lane : lanes) { + List remaining = + new ArrayList<>(); + synchronized (lane.lock) { + while (!lane.backlog.isEmpty()) { + PendingDelivery pending = + lane.backlog.removeFirst(); + remaining.add(pending); + releaseAdmission(pending); + } + } + for (PendingDelivery pending : + remaining) { + try { + deadLetterService.deadLetter( + pending.message(), + "producer shutdown flush timeout"); + } catch (RuntimeException error) { + log.error( + "工作流审计关闭收口死信失败，eventId={}", + pending.message() + .getMessageId(), + error); + } + } + } + } + + /** + * 一条有序待投递审计消息。 + * + * @param message MQ 消息 + * @param attempt 已失败次数 + * @param nextAttemptAtMillis 下次允许重试时间 + * @param messageBytes 本地持有字节数 + */ + private record PendingDelivery( + MQMessage message, + int attempt, + long nextAttemptAtMillis, + long messageBytes) { + + /** + * 创建下一次重试记录。 + * + * @param nextAttempt 下一次尝试次数 + * @param retryAt 下次允许发送时间 + * @return 保留原消息与字节大小的新记录 + */ + private PendingDelivery retryAt( + int nextAttempt, + long retryAt) { + return new PendingDelivery( + message, + nextAttempt, + retryAt, + messageBytes); + } + } + + /** + * 一个独立 FIFO 发送 lane。 + */ + private static final class DeliveryLane { + private final Object lock = + new Object(); + private final Deque backlog = + new ArrayDeque<>(); + private boolean sending; + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/listener/ChainEventListenerForSave.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/listener/ChainEventListenerForSave.java index 4b89b1e8..755b228e 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/listener/ChainEventListenerForSave.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/listener/ChainEventListenerForSave.java @@ -1,28 +1,27 @@ package tech.easyflow.ai.easyagentsflow.listener; -import cn.hutool.core.util.IdUtil; import cn.hutool.core.util.StrUtil; import com.alibaba.fastjson2.JSON; import com.easyagents.flow.core.chain.*; import com.easyagents.flow.core.chain.event.*; import com.easyagents.flow.core.chain.listener.ChainEventListener; -import com.easyagents.flow.core.chain.repository.NodeStateField; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.dao.DuplicateKeyException; import org.springframework.stereotype.Component; +import tech.easyflow.ai.easyagentsflow.event.WorkflowExecutionAuditEvent; +import tech.easyflow.ai.easyagentsflow.event.WorkflowExecutionAuditProducer; import tech.easyflow.ai.easyagentsflow.support.PublishedWorkflowDefinitionIds; +import tech.easyflow.ai.easyagentsflow.support.WorkflowExecutionStepKey; import tech.easyflow.ai.entity.Workflow; import tech.easyflow.ai.entity.WorkflowExecResult; import tech.easyflow.ai.entity.WorkflowExecStep; import tech.easyflow.ai.service.WorkflowExecResultService; -import tech.easyflow.ai.service.WorkflowExecStepService; import tech.easyflow.ai.service.WorkflowService; import tech.easyflow.ai.utils.WorkFlowUtil; import javax.annotation.Resource; import java.util.Date; -import java.util.EnumSet; @Component public class ChainEventListenerForSave implements ChainEventListener { @@ -35,7 +34,7 @@ public class ChainEventListenerForSave implements ChainEventListener { @Resource private WorkflowExecResultService workflowExecResultService; @Resource - private WorkflowExecStepService workflowExecStepService; + private WorkflowExecutionAuditProducer auditProducer; @Override public void onEvent(Event event, Chain chain) { @@ -60,19 +59,19 @@ public class ChainEventListenerForSave implements ChainEventListener { } private void handleChainStartEvent(ChainStartEvent event, Chain chain) { - log.info("ChainStartEvent: {}", event); ChainDefinition definition = chain.getDefinition(); ChainState state = chain.getState(); + log.info( + "workflow event type=chain-started, instanceId={}, definitionId={}, variableCount={}", + state.getInstanceId(), + definition == null ? null : definition.getId(), + event.getVariables() == null ? 0 : event.getVariables().size()); Workflow workflow = resolveWorkflow(definition); if (workflow == null) { log.error("ChainStartEvent: workflow not found, definitionId={}", definition.getId()); return; } String instanceId = state.getInstanceId(); - WorkflowExecResult existed = workflowExecResultService.getByExecKey(instanceId); - if (existed != null) { - return; - } WorkflowExecResult record = new WorkflowExecResult(); record.setExecKey(instanceId); record.setWorkflowId(workflow.getId()); @@ -84,102 +83,157 @@ public class ChainEventListenerForSave implements ChainEventListener { record.setStatus(state.getStatus().getValue()); record.setCreatedKey(WorkFlowUtil.getCreatedKey(chain)); record.setCreatedBy(WorkFlowUtil.getOperator(chain).getId().toString()); + // 启动记录保留同步确认，避免执行接口返回后立即查询时记录尚不可见。 try { workflowExecResultService.save(record); - } catch (DuplicateKeyException e) { - // 多节点重试时可能并发写同一 exec_key，按幂等处理。 - log.debug("exec result already exists, execKey={}", instanceId, e); + } catch (DuplicateKeyException duplicate) { + // 重复启动或恢复按 exec_key 幂等处理。 + log.debug("exec result already exists, execKey={}", instanceId, duplicate); } } private void handleChainEndEvent(ChainEndEvent event, Chain chain) { - log.info("ChainEndEvent: {}", event); ChainState state = chain.getState(); String instanceId = state.getInstanceId(); - WorkflowExecResult record = workflowExecResultService.getByExecKey(instanceId); - if (record == null) { - log.error("ChainEndEvent: record not found: {}", instanceId); - } else { - record.setEndTime(new Date()); - record.setStatus(state.getStatus().getValue()); - record.setOutput(JSON.toJSONString(state.getExecuteResult())); - ExceptionSummary error = state.getError(); - if (error != null) { - record.setErrorInfo(error.getRootCauseClass() + " --> " + error.getRootCauseMessage()); - } - workflowExecResultService.updateById(record); + log.info( + "workflow event type=chain-ended, instanceId={}, status={}", + instanceId, + state.getStatus()); + WorkflowExecResult record = new WorkflowExecResult(); + record.setExecKey(instanceId); + record.setEndTime(new Date()); + record.setStatus(state.getStatus().getValue()); + // 大型引用由审计消费者异步还原，避免阻塞工作流终态提交。 + record.setOutput(JSON.toJSONString( + state.getExecuteResult())); + ExceptionSummary error = state.getError(); + if (error != null) { + record.setErrorInfo(error.getRootCauseClass() + " --> " + error.getRootCauseMessage()); } + sendAuditEvent( + WorkflowExecutionAuditEvent.Type.CHAIN_ENDED, + instanceId + ":chain-ended", + instanceId, + record, + null); } private void handleNodeStartEvent(NodeStartEvent event, Chain chain) { - log.info("NodeStartEvent: {}", event); Node node = event.getNode(); - ChainState ancestorState = findAncestorState(chain.getState(), chain); + String auditInstanceId = + event.getAuditInstanceId(); + ChainState ancestorState = + StrUtil.isBlank(auditInstanceId) + || auditInstanceId.equals( + chain.getStateInstanceId()) + ? chain.getExecutionState() + : chain.getChainStateRepository() + .load(auditInstanceId); + if (ancestorState == null) { + throw new IllegalStateException( + "Workflow audit state not found: " + + auditInstanceId); + } String instanceId = ancestorState.getInstanceId(); - NodeState nodeState = chain.getNodeState(node.getId()); - - String execKey = IdUtil.fastSimpleUUID(); - chain.updateNodeStateSafely(node.getId(), state -> { - state.getMemory().put("executeId", execKey); - return EnumSet.of(NodeStateField.MEMORY); - }); - - WorkflowExecResult record = workflowExecResultService.getByExecKey(instanceId); - if (record == null) { - log.error("NodeStartEvent: record not found: {}", instanceId); - } else { - WorkflowExecStep step = new WorkflowExecStep(); - step.setRecordId(record.getId()); - step.setExecKey(execKey); - step.setNodeId(node.getId()); - step.setNodeName(node.getName()); - step.setInput(JSON.toJSONString(ancestorState.resolveParameters(node))); - step.setNodeData(JSON.toJSONString(node)); - step.setStartTime(new Date()); - step.setStatus(nodeState.getStatus().getValue()); - workflowExecStepService.save(step); + NodeStatus nodeStatus = event.getStatus(); + if (nodeStatus == null) { + NodeState nodeState = chain.getNodeState(node.getId()); + nodeStatus = nodeState.getStatus(); } + log.info( + "workflow event type=node-started, instanceId={}, nodeId={}, nodeType={}, status={}", + instanceId, + node.getId(), + node.getClass().getSimpleName(), + nodeStatus); + + String execKey = currentStepExecKey( + event.getExecutionAttemptKey(), + chain, + node); + WorkflowExecStep step = new WorkflowExecStep(); + step.setExecKey(execKey); + step.setNodeId(node.getId()); + step.setNodeName(node.getName()); + // 业务线程保留大型引用，完整审计输入由 MQ 消费线程异步还原。 + step.setInput(JSON.toJSONString( + ancestorState + .resolveParametersPreservingReferences( + node))); + step.setNodeData(JSON.toJSONString(node)); + step.setStartTime(new Date()); + step.setStatus(nodeStatus.getValue()); + sendAuditEvent( + WorkflowExecutionAuditEvent.Type.NODE_STARTED, + execKey + ":started", + instanceId, + null, + step); } private void handleNodeEndEvent(NodeEndEvent event, Chain chain) { - log.info("NodeEndEvent: {}", event); Node node = event.getNode(); - NodeState nodeState = chain.getNodeState(node.getId()); - String execKey = nodeState.getMemory().get("executeId").toString(); - WorkflowExecStep step = workflowExecStepService.getByExecKey(execKey); - if (step == null) { - log.error("NodeEndEvent: step not found: {}", execKey); - } else { - step.setOutput(JSON.toJSONString(event.getResult())); - step.setEndTime(new Date()); - step.setStatus(nodeState.getStatus().getValue()); - ExceptionSummary error = nodeState.getError(); - if (error != null) { - step.setErrorInfo(error.getRootCauseClass() + " --> " + error.getRootCauseMessage()); - } - workflowExecStepService.updateById(step); + String auditInstanceId = + chain.getAuditInstanceId(); + NodeState legacyNodeState = null; + NodeStatus nodeStatus = + event.getStatus(); + if (nodeStatus == null) { + // 兼容旧版引擎未携带不可变终态的事件。 + legacyNodeState = + chain.getNodeState( + node.getId()); + nodeStatus = + legacyNodeState.getStatus(); } + log.info( + "workflow event type=node-ended, instanceId={}, nodeId={}, nodeType={}, status={}, resultFieldCount={}", + auditInstanceId, + node.getId(), + node.getClass().getSimpleName(), + nodeStatus, + event.getResult() == null ? 0 : event.getResult().size()); + String execKey = currentStepExecKey( + event.getExecutionAttemptKey(), + chain, + node); + WorkflowExecStep step = new WorkflowExecStep(); + step.setExecKey(execKey); + // 节点线程只投递轻量引用，完整执行记录仍由审计消费者透明还原。 + step.setOutput(JSON.toJSONString( + event.getResult())); + step.setEndTime(new Date()); + step.setStatus(nodeStatus.getValue()); + ExceptionSummary error = + event.getError() == null + ? (legacyNodeState == null + ? null + : legacyNodeState.getError()) + : new ExceptionSummary( + event.getError()); + if (error != null) { + step.setErrorInfo(error.getRootCauseClass() + " --> " + error.getRootCauseMessage()); + } + sendAuditEvent( + WorkflowExecutionAuditEvent.Type.NODE_ENDED, + execKey + ":ended", + auditInstanceId, + null, + step); } private void handleChainStatusChangeEvent(ChainStatusChangeEvent event, Chain chain) { - log.info("ChainStatusChangeEvent: {}", event); + log.info( + "workflow event type=chain-status-changed, instanceId={}, status={}", + chain.getStateInstanceId(), + event.getStatus()); } private void handleChainResumeEvent(ChainResumeEvent event, Chain chain) { - log.info("ChainResumeEvent: {}", event); - } - - /** - * 递归查找顶级状态 - */ - private ChainState findAncestorState(ChainState state, Chain chain) { - String parentInstanceId = state.getParentInstanceId(); - if (StrUtil.isEmpty(parentInstanceId)) { - return state; - } - ChainState chainState = chain.getChainStateRepository().load(parentInstanceId); - return findAncestorState(chainState, chain); + log.info( + "workflow event type=chain-resumed, instanceId={}", + chain.getStateInstanceId()); } /** @@ -203,4 +257,60 @@ public class ChainEventListenerForSave implements ChainEventListener { return null; } } + + /** + * 生成节点本次业务尝试的稳定执行步骤键。 + * + * @param capturedAttemptKey 事件创建时捕获的业务尝试键 + * @param chain 当前工作流 + * @param node 当前节点 + * @return 长度固定的执行步骤键 + */ + private String currentStepExecKey( + String capturedAttemptKey, + Chain chain, + Node node) { + String attemptKey = capturedAttemptKey; + if (StrUtil.isBlank(attemptKey)) { + // 兼容旧版引擎直接构造、尚未携带不可变尝试键的事件。 + NodeState nodeState = + chain.getNodeState(node.getId()); + attemptKey = nodeState == null + ? null + : nodeState + .getExecutionAttemptKey(); + } + String execKey = + WorkflowExecutionStepKey.resolve( + attemptKey); + if (StrUtil.isBlank(execKey)) { + throw new IllegalStateException( + "Workflow execution attempt key is unavailable: " + node.getId()); + } + return execKey; + } + + /** + * 投递工作流执行审计事件。 + * + * @param type 事件类型 + * @param eventId 事件幂等 ID + * @param instanceId 顶级工作流实例 ID + * @param result 工作流执行记录快照 + * @param step 节点执行步骤快照 + */ + private void sendAuditEvent(WorkflowExecutionAuditEvent.Type type, + String eventId, + String instanceId, + WorkflowExecResult result, + WorkflowExecStep step) { + WorkflowExecutionAuditEvent auditEvent = new WorkflowExecutionAuditEvent(); + auditEvent.setType(type); + auditEvent.setEventId(eventId); + auditEvent.setInstanceId(instanceId); + auditEvent.setOccurredAt(new Date()); + auditEvent.setResult(result); + auditEvent.setStep(step); + auditProducer.send(auditEvent); + } } diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/BaseRepository.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/BaseRepository.java index 6b65e4cc..e9b1107e 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/BaseRepository.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/BaseRepository.java @@ -57,6 +57,22 @@ public class BaseRepository { return clazz.cast(value); } + /** + * 删除并校验工作流运行状态缓存。 + * + * @param key 缓存键 + */ + protected void removeCache(String key) { + CacheResult result = cache.REMOVE(key); + CacheResultCode resultCode = result.getResultCode(); + if (resultCode == CacheResultCode.NOT_EXISTS || resultCode == CacheResultCode.EXPIRED) { + return; + } + if (!result.isSuccess()) { + throw cacheOperationException("删除", key, result); + } + } + /** * 构建包含缓存操作上下文的异常。 * diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/ChainDefinitionRepositoryImpl.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/ChainDefinitionRepositoryImpl.java index 5781d91e..6231ca27 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/ChainDefinitionRepositoryImpl.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/ChainDefinitionRepositoryImpl.java @@ -20,14 +20,29 @@ public class ChainDefinitionRepositoryImpl implements ChainDefinitionRepository private ChainParser chainParser; @Resource private WorkflowDatacenterContentService workflowDatacenterContentService; + @Resource + private WorkflowDefinitionCache workflowDefinitionCache; @Override public ChainDefinition getChainDefinitionById(String id) { + return workflowDefinitionCache.get(id, () -> loadAndCompile(id)); + } + + /** + * 从持久层加载工作流并编译定义。 + * + * @param id 定义 ID + * @return 已编译工作流定义 + */ + private ChainDefinition loadAndCompile(String id) { boolean publishedDefinition = PublishedWorkflowDefinitionIds.isPublished(id); String workflowId = PublishedWorkflowDefinitionIds.unwrap(id); Workflow workflow = publishedDefinition ? workflowService.getPublishedById(new java.math.BigInteger(workflowId)) : workflowService.getById(workflowId); + if (workflow == null) { + throw new IllegalStateException("Workflow not found: " + workflowId); + } String json = workflowDatacenterContentService.prepareContent(workflow.getContent()); ChainDefinition chainDefinition = chainParser.parse(json); chainDefinition.setId(id); diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/ChainDefinitionSnapshotRepositoryImpl.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/ChainDefinitionSnapshotRepositoryImpl.java new file mode 100644 index 00000000..2ccd7dc9 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/ChainDefinitionSnapshotRepositoryImpl.java @@ -0,0 +1,158 @@ +package tech.easyflow.ai.easyagentsflow.repository; + +import com.easyagents.flow.core.chain.ChainDefinition; +import com.easyagents.flow.core.chain.repository.ChainDefinitionSnapshotRepository; +import org.springframework.stereotype.Component; +import tech.easyflow.common.cache.VersionedObjectStore; +import tech.easyflow.common.constant.CacheKey; + +import javax.annotation.Resource; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectOutputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Duration; +import java.util.Collections; +import java.util.HexFormat; +import java.util.List; +import java.util.Map; +import java.util.WeakHashMap; + +/** + * 基于 Redis 版本对象存储的工作流实例定义快照仓储。 + */ +@Component +public class ChainDefinitionSnapshotRepositoryImpl extends BaseRepository + implements ChainDefinitionSnapshotRepository { + + /** + * 快照由工作流终态显式删除；长 TTL 仅用于异常中断后的孤儿兜底清理。 + */ + private static final Duration SNAPSHOT_TTL = Duration.ofDays(7); + private static final Map CONTENT_HASH_CACHE = + Collections.synchronizedMap(new WeakHashMap<>()); + + @Resource + private VersionedObjectStore versionedObjectStore; + + /** + * {@inheritDoc} + */ + @Override + public void save(String instanceId, ChainDefinition definition) { + String contentHash = contentHash(definition); + String immutableContentKey = contentKey(contentHash); + versionedObjectStore.createIfAbsent( + immutableContentKey, + definition, + 0L, + SNAPSHOT_TTL); + // 每个新实例都延长不可变内容寿命，保证引用 TTL 内不会悬空。 + versionedObjectStore.refreshExpirations( + List.of(immutableContentKey), SNAPSHOT_TTL); + versionedObjectStore.createIfAbsent( + stateKey(instanceId), + contentHash, + 0L, + SNAPSHOT_TTL); + } + + /** + * {@inheritDoc} + */ + @Override + public ChainDefinition load(String instanceId) { + Object reference = versionedObjectStore.load( + stateKey(instanceId), Object.class); + if (reference instanceof String) { + ChainDefinition snapshot = versionedObjectStore.load( + contentKey((String) reference), + ChainDefinition.class); + if (snapshot != null) { + return snapshot; + } + } else if (reference instanceof ChainDefinition) { + // 兼容 XL12 之前按实例保存完整定义的运行中实例。 + return (ChainDefinition) reference; + } + ChainDefinition legacy = getCache(legacyKey(instanceId), ChainDefinition.class); + if (legacy != null) { + save(instanceId, legacy); + } + return legacy; + } + + /** + * {@inheritDoc} + */ + @Override + public void remove(String instanceId) { + versionedObjectStore.deleteAll(List.of(stateKey(instanceId))); + removeCache(legacyKey(instanceId)); + } + + /** + * 构建定义快照缓存键。 + * + * @param instanceId 工作流实例 ID + * @return 缓存键 + */ + private String legacyKey(String instanceId) { + return CacheKey.CHAIN_DEFINITION_SNAPSHOT_CACHE_KEY + instanceId; + } + + /** + * 构建不会随短期运行状态 TTL 漂移的快照键。 + * + * @param instanceId 工作流实例 ID + * @return Redis 快照键 + */ + private String stateKey(String instanceId) { + return CacheKey.CHAIN_STATE_CAS_KEY + "{" + instanceId + "}:definition"; + } + + /** + * 构建按定义内容复用的快照键。 + * + * @param contentHash 定义内容摘要 + * @return Redis 内容键 + */ + private String contentKey(String contentHash) { + return CacheKey.CHAIN_DEFINITION_SNAPSHOT_CACHE_KEY + + "content:" + + contentHash; + } + + /** + * 计算定义序列化内容摘要；同一编译定义对象仅计算一次。 + * + * @param definition 编译后的定义 + * @return SHA-256 十六进制摘要 + */ + private String contentHash(ChainDefinition definition) { + if (definition == null) { + throw new IllegalArgumentException( + "Chain definition required"); + } + String cached = CONTENT_HASH_CACHE.get(definition); + if (cached != null) { + return cached; + } + try (ByteArrayOutputStream output = + new ByteArrayOutputStream(); + ObjectOutputStream objects = + new ObjectOutputStream(output)) { + objects.writeObject(definition); + objects.flush(); + String calculated = HexFormat.of().formatHex( + MessageDigest.getInstance("SHA-256") + .digest(output.toByteArray())); + CONTENT_HASH_CACHE.put(definition, calculated); + return calculated; + } catch (IOException | NoSuchAlgorithmException error) { + throw new IllegalStateException( + "Failed to hash chain definition", error); + } + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/ChainStateRepositoryImpl.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/ChainStateRepositoryImpl.java index 25595c8e..4030fab7 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/ChainStateRepositoryImpl.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/ChainStateRepositoryImpl.java @@ -1,32 +1,447 @@ package tech.easyflow.ai.easyagentsflow.repository; import com.easyagents.flow.core.chain.ChainState; +import com.easyagents.flow.core.chain.repository.ChainLock; import com.easyagents.flow.core.chain.repository.ChainStateField; import com.easyagents.flow.core.chain.repository.ChainStateRepository; import org.springframework.stereotype.Component; +import tech.easyflow.common.cache.RedisLockExecutor; +import tech.easyflow.common.cache.VersionedFields; +import tech.easyflow.common.cache.VersionedObjectStore; import tech.easyflow.common.constant.CacheKey; +import javax.annotation.Resource; +import java.io.Serializable; +import java.time.Duration; import java.util.EnumSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +/** + * 基于 Redis 字段化 CAS、认领守卫和短期实例锁的工作流状态仓储。 + */ @Component public class ChainStateRepositoryImpl extends BaseRepository implements ChainStateRepository { - @Override - public ChainState load(String instanceId) { - String key = CacheKey.CHAIN_CACHE_KEY + instanceId; - ChainState chainState = getCache(key, ChainState.class); - if (chainState == null) { - chainState = new ChainState(); - chainState.setInstanceId(instanceId); - putCache(key, chainState); - } - return chainState; + private static final Duration STATE_TTL = Duration.ofDays(3); + private static final Duration MIGRATION_MARKER_TTL = Duration.ofDays(4); + private static final Duration FENCING_COUNTER_TTL = Duration.ofDays(4); + private static final Duration MIN_LOCK_LEASE = Duration.ofSeconds(30); + private static final ScheduledThreadPoolExecutor LOCK_RENEW_EXECUTOR = + createLockRenewExecutor(); + private final Set legacyInstances = java.util.concurrent.ConcurrentHashMap.newKeySet(); + + @Resource + private RedisLockExecutor redisLockExecutor; + @Resource + private VersionedObjectStore versionedObjectStore; + + /** + * 创建会主动移除已取消任务的锁续期线程池。 + * + *

绝大多数实例锁仅持有数毫秒；开启 remove-on-cancel 可避免高吞吐场景下， + * 已取消的十秒延迟续期任务在队列中短时堆积。

+ * + * @return 小型多线程锁续期调度器 + */ + private static ScheduledThreadPoolExecutor createLockRenewExecutor() { + ScheduledThreadPoolExecutor executor = + new ScheduledThreadPoolExecutor(2, new LockRenewThreadFactory()); + executor.setRemoveOnCancelPolicy(true); + executor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); + return executor; } + /** + * {@inheritDoc} + */ + @Override + public ChainState load(String instanceId) { + String stateKey = stateKey(instanceId); + VersionedFields snapshot = versionedObjectStore.loadFields(stateKey); + if (WorkflowStateFields.isFieldFormat(snapshot)) { + legacyInstances.remove(instanceId); + return WorkflowStateFields.decodeChain(snapshot); + } + if (snapshot != null) { + ChainState payloadState = versionedObjectStore.load(stateKey, ChainState.class); + if (payloadState != null) { + rewritePayloadState(stateKey, payloadState); + return payloadState; + } + } + if (hasMigrationMarker(instanceId)) { + legacyInstances.remove(instanceId); + return null; + } + + String legacyKey = CacheKey.CHAIN_CACHE_KEY + instanceId; + ChainState legacyState = getCache(legacyKey, ChainState.class); + if (legacyState == null) { + return null; + } + // 活跃旧实例继续沿用旧写路径，避免滚动升级期间两个格式同时推进。 + if (legacyState.getStatus() == null || !legacyState.getStatus().isTerminal()) { + legacyInstances.add(instanceId); + return legacyState; + } + migrateTerminalLegacyState(instanceId, legacyKey, legacyState); + VersionedFields migrated = versionedObjectStore.loadFields(stateKey); + if (!WorkflowStateFields.isFieldFormat(migrated)) { + throw new IllegalStateException("Workflow state migration failed: " + instanceId); + } + return WorkflowStateFields.decodeChain(migrated); + } + + /** + * {@inheritDoc} + */ + @Override + public Long loadVersion(String instanceId) { + if (!legacyInstances.contains(instanceId)) { + Long version = versionedObjectStore.loadVersion(stateKey(instanceId)); + if (version != null) { + return version; + } + } + ChainState state = load(instanceId); + return state == null ? null : state.getVersion(); + } + + /** + * {@inheritDoc} + */ + @Override + public ChainState create(String instanceId) { + ChainState existing = load(instanceId); + if (existing != null) { + return existing; + } + ChainState created = new ChainState(); + created.setInstanceId(instanceId); + if (versionedObjectStore.createFieldsIfAbsent( + stateKey(instanceId), + WorkflowStateFields.allChainFields(created), + created.getVersion(), + STATE_TTL)) { + touchMigrationMarker(instanceId); + return created; + } + VersionedFields concurrent = versionedObjectStore.loadFields(stateKey(instanceId)); + if (!WorkflowStateFields.isFieldFormat(concurrent)) { + throw new IllegalStateException("Unable to initialize workflow state: " + instanceId); + } + return WorkflowStateFields.decodeChain(concurrent); + } + + /** + * {@inheritDoc} + */ @Override public boolean tryUpdate(ChainState newState, EnumSet fields) { - String key = CacheKey.CHAIN_CACHE_KEY + newState.getInstanceId(); - putCache(key, newState); - return true; + return tryUpdate(newState, fields, 0L); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean tryUpdate( + ChainState newState, EnumSet fields, long fencingToken) { + return tryUpdate(newState, fields, fencingToken, null, 0L); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean tryUpdate( + ChainState newState, + EnumSet fields, + long lockFencingToken, + String claimId, + long claimGeneration) { + String instanceId = newState.getInstanceId(); + if (legacyInstances.contains(instanceId)) { + String legacyKey = CacheKey.CHAIN_CACHE_KEY + instanceId; + putCache(legacyKey, newState); + if (newState.getStatus() != null && newState.getStatus().isTerminal()) { + migrateTerminalLegacyState(instanceId, legacyKey, newState); + } + return true; + } + long newVersion = newState.getVersion(); + if (newVersion <= 0L) { + throw new IllegalArgumentException("newState.version must be positive"); + } + requireClaimId(claimId, claimGeneration); + boolean updated; + if (lockFencingToken > 0L) { + String claimGuardKey = claimGeneration > 0L + ? executionGuardKey(instanceId, claimId) + : lockFenceKey(instanceId); + long effectiveClaimGeneration = claimGeneration > 0L + ? claimGeneration + : lockFencingToken; + updated = versionedObjectStore.compareAndSetFieldsAndRefresh( + stateKey(instanceId), + newVersion - 1L, + WorkflowStateFields.chainFields(newState, fields), + newVersion, + lockFenceKey(instanceId), + lockFencingToken, + claimGuardKey, + effectiveClaimGeneration, + STATE_TTL, + markerKey(instanceId), + MIGRATION_MARKER_TTL); + } else { + updated = versionedObjectStore.compareAndSetFields( + stateKey(instanceId), + newVersion - 1L, + WorkflowStateFields.chainFields(newState, fields), + newVersion, + STATE_TTL); + } + return updated; + } + + /** + * 将当前 Redis Hash 中的旧完整 payload 原地改写为字段化状态。 + * + * @param stateKey 状态键 + * @param state 旧完整状态 + */ + private void rewritePayloadState(String stateKey, ChainState state) { + boolean rewritten = versionedObjectStore.rewriteAsFields( + stateKey, + state.getVersion(), + WorkflowStateFields.allChainFields(state), + STATE_TTL); + if (rewritten) { + touchMigrationMarker(state.getInstanceId()); + } + } + + /** + * 在旧实例终态后完成受控迁移并留下 tombstone。 + * + * @param instanceId 实例 ID + * @param legacyKey 旧 JetCache 键 + * @param state 终态状态 + */ + private void migrateTerminalLegacyState( + String instanceId, String legacyKey, ChainState state) { + boolean created = versionedObjectStore.createFieldsIfAbsent( + stateKey(instanceId), + WorkflowStateFields.allChainFields(state), + state.getVersion(), + STATE_TTL); + VersionedFields existing = versionedObjectStore.loadFields(stateKey(instanceId)); + if (!created && !WorkflowStateFields.isFieldFormat(existing)) { + throw new IllegalStateException("Workflow state migration conflict: " + instanceId); + } + touchMigrationMarker(instanceId); + removeCache(legacyKey); + legacyInstances.remove(instanceId); + } + + /** + * 判断实例是否已经切换到字段化格式。 + * + * @param instanceId 实例 ID + * @return 已存在迁移 tombstone 时为 {@code true} + */ + private boolean hasMigrationMarker(String instanceId) { + return versionedObjectStore.loadFields(markerKey(instanceId)) != null; + } + + /** + * 创建或刷新迁移 tombstone，防止新状态过期后复活旧缓存。 + * + * @param instanceId 实例 ID + */ + private void touchMigrationMarker(String instanceId) { + Map marker = Map.of( + WorkflowStateFields.FORMAT_FIELD, WorkflowStateFields.FORMAT_VERSION); + if (!versionedObjectStore.createFieldsIfAbsent( + markerKey(instanceId), marker, 0L, MIGRATION_MARKER_TTL)) { + versionedObjectStore.compareAndSetFields( + markerKey(instanceId), 0L, marker, 0L, MIGRATION_MARKER_TTL); + } + } + + /** + * 构建工作流状态 CAS key。 + * + * @param instanceId 工作流实例 ID + * @return Redis 状态 key + */ + private String stateKey(String instanceId) { + return CacheKey.CHAIN_STATE_CAS_KEY + "{" + instanceId + "}:chain"; + } + + /** + * 构建状态格式 tombstone 键。 + * + * @param instanceId 实例 ID + * @return Redis marker 键 + */ + private String markerKey(String instanceId) { + return CacheKey.CHAIN_STATE_CAS_KEY + "{" + instanceId + "}:format"; + } + + /** + * 构建本次触发器认领的执行守卫键。 + * + * @param instanceId 工作流实例 ID + * @param claimId 触发器 ID + * @return 执行守卫键 + */ + private String executionGuardKey(String instanceId, String claimId) { + return CacheKey.CHAIN_STATE_CAS_KEY + "{" + instanceId + "}:claim:" + claimId; + } + + /** + * 校验分布式提交所需的 claim ID。 + * + * @param claimId 触发器 ID + * @param fencingToken 当前认领 token + */ + private void requireClaimId(String claimId, long fencingToken) { + if (fencingToken > 0L && (claimId == null || claimId.trim().isEmpty())) { + throw new IllegalArgumentException("claimId is required with fencingToken"); + } + } + + /** + * 获取工作流实例级 Redis 分布式锁。 + * + * @param instanceId 工作流实例 ID + * @param timeout 等待锁的最大时间 + * @param unit 时间单位 + * @return 分布式锁句柄 + */ + @Override + public ChainLock getLock(String instanceId, long timeout, TimeUnit unit) { + if (instanceId == null || instanceId.trim().isEmpty()) { + throw new IllegalArgumentException("instanceId must not be blank"); + } + if (redisLockExecutor == null) { + throw new IllegalStateException("RedisLockExecutor is not configured"); + } + Duration waitTimeout = Duration.ofMillis(Math.max(1L, unit.toMillis(timeout))); + Duration leaseTimeout = waitTimeout.compareTo(MIN_LOCK_LEASE) > 0 ? waitTimeout : MIN_LOCK_LEASE; + RedisLockExecutor.LockHandle handle = redisLockExecutor.tryAcquireFenced( + CacheKey.CHAIN_LOCK_KEY + "{" + instanceId + "}", + lockFenceKey(instanceId), + waitTimeout, + leaseTimeout, + FENCING_COUNTER_TTL); + if (handle == null) { + return new RedisChainLock(null, 0L); + } + return new RedisChainLock(handle, handle.getFencingToken()); + } + + /** + * 构建实例锁 fencing token 键。 + * + * @param instanceId 工作流实例 ID + * @return fencing token 哈希键 + */ + private String lockFenceKey(String instanceId) { + return CacheKey.CHAIN_STATE_CAS_KEY + "{" + instanceId + "}:fence"; + } + + /** + * Redis 工作流锁适配器。 + */ + private static final class RedisChainLock implements ChainLock { + + private final RedisLockExecutor.LockHandle handle; + private final long fencingToken; + private final AtomicBoolean valid = new AtomicBoolean(true); + private final ScheduledFuture renewTask; + + /** + * 创建锁适配器。 + * + * @param handle Redis 锁句柄；为空表示未获取 + * @param fencingToken 本次锁持有期的 fencing token + */ + private RedisChainLock( + RedisLockExecutor.LockHandle handle, long fencingToken) { + this.handle = handle; + this.fencingToken = fencingToken; + this.renewTask = handle == null ? null : LOCK_RENEW_EXECUTOR.scheduleWithFixedDelay( + () -> { + if (!handle.renew()) { + valid.set(false); + } + }, + MIN_LOCK_LEASE.toMillis() / 3L, + MIN_LOCK_LEASE.toMillis() / 3L, + TimeUnit.MILLISECONDS); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean isAcquired() { + return handle != null && valid.get(); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean isValid() { + return isAcquired(); + } + + /** + * {@inheritDoc} + */ + @Override + public long getFencingToken() { + return fencingToken; + } + + /** + * {@inheritDoc} + */ + @Override + public void close() { + valid.set(false); + if (renewTask != null) { + renewTask.cancel(false); + } + if (handle != null) { + handle.release(); + } + } + } + + /** + * 创建守护型工作流状态锁续期线程。 + */ + private static final class LockRenewThreadFactory implements ThreadFactory { + + /** + * {@inheritDoc} + */ + @Override + public Thread newThread(Runnable runnable) { + Thread thread = new Thread(runnable, "workflow-state-lock-renew"); + thread.setDaemon(true); + return thread; + } } } diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/LoopResultRepositoryImpl.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/LoopResultRepositoryImpl.java new file mode 100644 index 00000000..d15cde63 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/LoopResultRepositoryImpl.java @@ -0,0 +1,1501 @@ +package tech.easyflow.ai.easyagentsflow.repository; + +import com.alicp.jetcache.support.JavaValueEncoder; +import com.easyagents.flow.core.chain.repository.LoopInputReference; +import com.easyagents.flow.core.chain.repository.LoopResultRepository; +import com.easyagents.flow.core.chain.runtime.ExecutionBudgetExceededException; +import com.easyagents.flow.core.chain.runtime.TriggerClaimLostException; +import org.springframework.stereotype.Component; +import tech.easyflow.common.cache.ApplicationClassLoaderJavaValueDecoder; +import tech.easyflow.common.cache.VersionedObjectStore; +import tech.easyflow.common.constant.CacheKey; + +import javax.annotation.Resource; +import java.io.Serializable; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * 基于分块缓存的循环累计结果仓储。 + *

+ * 每轮只重写当前固定大小分块，避免循环历史结果随节点热状态反复序列化。 + */ +@Component +public class LoopResultRepositoryImpl extends BaseRepository implements LoopResultRepository { + + static final int CHUNK_SIZE = 128; + private static final Duration RESULT_TTL = Duration.ofDays(7); + private static final long REFRESH_INTERVAL_MILLIS = + Duration.ofHours(12).toMillis(); + private static final int MAX_CACHED_INPUT_RESULTS = 256; + private static final int MAX_CACHED_OUTPUT_RESULTS = 256; + private static final long MAX_INPUT_CACHE_WEIGHT_BYTES = + 32L * 1024L * 1024L; + private static final long MAX_OUTPUT_CACHE_WEIGHT_BYTES = + 64L * 1024L * 1024L; + private static final long MAX_CACHE_ENTRY_WEIGHT_BYTES = + 8L * 1024L * 1024L; + private static final long EMPTY_CHUNK_WEIGHT_BYTES = 256L; + private static final ApplicationClassLoaderJavaValueDecoder + CACHE_VALUE_DECODER = + new ApplicationClassLoaderJavaValueDecoder(); + + @Resource + private VersionedObjectStore versionedObjectStore; + /** + * 每个活跃循环只保留最近读取的一块，避免同一 128 项分块重复反序列化 128 次。 + */ + private final BoundedWeightedCache + inputChunkCache = + new BoundedWeightedCache<>( + MAX_CACHED_INPUT_RESULTS, + MAX_INPUT_CACHE_WEIGHT_BYTES, + MAX_CACHE_ENTRY_WEIGHT_BYTES); + /** + * 每个循环只缓存最近成功提交的活动输出分块，跨分块时自动替换历史块。 + */ + private final BoundedWeightedCache + outputChunkCache = + new BoundedWeightedCache<>( + MAX_CACHED_OUTPUT_RESULTS, + MAX_OUTPUT_CACHE_WEIGHT_BYTES, + MAX_CACHE_ENTRY_WEIGHT_BYTES); + + /** + * {@inheritDoc} + */ + @Override + public int storeInput(String resultId, Iterable items) { + return storeInputInternal( + null, 0L, null, 0L, resultId, items); + } + + /** + * {@inheritDoc} + */ + @Override + public int storeInput( + String instanceId, + long lockFencingToken, + String claimId, + long claimGeneration, + String resultId, + Iterable items, + long maxItems) { + return storeProducedInputInternal( + instanceId, + lockFencingToken, + claimId, + claimGeneration, + resultId, + sink -> items.forEach(sink), + maxItems); + } + + /** + * 流式分块保存循环输入。 + * + * @param instanceId 工作流实例 ID + * @param lockFencingToken 当前实例锁 token + * @param claimId 当前触发器 ID + * @param claimGeneration 当前认领代际 + * @param resultId 循环结果 ID + * @param items 已应用预算的输入 + * @return 输入元素数量 + */ + private int storeInputInternal( + String instanceId, + long lockFencingToken, + String claimId, + long claimGeneration, + String resultId, + Iterable items) { + return storeProducedInputInternal( + instanceId, + lockFencingToken, + claimId, + claimGeneration, + resultId, + sink -> items.forEach(sink), + 0L); + } + + /** + * {@inheritDoc} + */ + @Override + public int storeProducedInput( + String instanceId, + long lockFencingToken, + String claimId, + long claimGeneration, + String resultId, + InputProducer producer, + long maxItems) { + return storeProducedInputInternal( + instanceId, + lockFencingToken, + claimId, + claimGeneration, + resultId, + producer, + maxItems); + } + + /** + * 接收生产者推送并直接写入固定大小分块。 + * + * @param instanceId 工作流实例 ID + * @param lockFencingToken 当前实例锁 token + * @param claimId 当前触发器 ID + * @param claimGeneration 当前触发器认领代际 + * @param resultId 循环结果 ID + * @param producer 输入生产者 + * @param maxItems 最大元素数 + * @return 已写入元素数量 + */ + private int storeProducedInputInternal( + String instanceId, + long lockFencingToken, + String claimId, + long claimGeneration, + String resultId, + InputProducer producer, + long maxItems) { + InputChunkWriter writer = new InputChunkWriter( + instanceId, + lockFencingToken, + claimId, + claimGeneration, + resultId, + maxItems); + try { + producer.produce(writer::append); + return writer.finish(); + } catch (RuntimeException | Error error) { + if (writer.versionedInput) { + try { + // resultId 每次物化均唯一，失锁后也可安全清理本 owner 已写的孤立分块。 + versionedObjectStore.deleteAll( + writer.writtenKeys); + } catch (RuntimeException cleanupError) { + error.addSuppressed(cleanupError); + } + } else { + for (String writtenKey + : writer.writtenKeys) { + removeCache(writtenKey); + } + } + throw error; + } + } + + /** + * 单次推送式输入的分块写入状态。 + */ + private final class InputChunkWriter { + + private final String instanceId; + private final long lockFencingToken; + private final String claimId; + private final long claimGeneration; + private final String resultId; + private final long maxItems; + private final boolean versionedInput; + private final List chunk = + new ArrayList<>(CHUNK_SIZE); + private final List writtenKeys = + new ArrayList<>(); + private int count; + private int chunkIndex; + + /** + * 创建一次分块写入。 + * + * @param instanceId 工作流实例 ID + * @param lockFencingToken 当前实例锁 token + * @param claimId 当前触发器 ID + * @param claimGeneration 当前触发器认领代际 + * @param resultId 循环结果 ID + * @param maxItems 最大元素数 + */ + private InputChunkWriter( + String instanceId, + long lockFencingToken, + String claimId, + long claimGeneration, + String resultId, + long maxItems) { + this.instanceId = instanceId; + this.lockFencingToken = lockFencingToken; + this.claimId = claimId; + this.claimGeneration = claimGeneration; + this.resultId = resultId; + this.maxItems = maxItems; + this.versionedInput = + versionedObjectStore != null + && isVersionedResult(resultId); + } + + /** + * 接收一个元素，达到固定分块大小时立即写入。 + * + * @param item 输入元素 + */ + private void append(Object item) { + if (maxItems > 0L && count >= maxItems) { + throw new ExecutionBudgetExceededException( + "Loop iteration budget exceeded while storing input " + + resultId + + ": more than " + + maxItems); + } + chunk.add(item); + count++; + if (chunk.size() == CHUNK_SIZE) { + flushChunk(); + } + } + + /** + * 写入尾块和输入元数据。 + * + * @return 输入元素数量 + */ + private int finish() { + if (!chunk.isEmpty()) { + flushChunk(); + } + String sizeKey = inputSizeKey(resultId); + write(sizeKey, count); + if (versionedInput + && lockFencingToken > 0L) { + write( + refreshMarkerKey(resultId), + new RefreshMarker( + 0L, + System.currentTimeMillis())); + } + return count; + } + + /** + * 写入当前数据分块。 + */ + private void flushChunk() { + write( + inputChunkKey( + resultId, chunkIndex++), + new ArrayList<>(chunk)); + chunk.clear(); + } + + /** + * 写入一个受 claim/lock 保护的值。 + * + * @param key 存储键 + * @param value 存储值 + */ + private void write( + String key, + Serializable value) { + storeInputValue( + key, + value, + versionedInput, + instanceId, + lockFencingToken, + claimId, + claimGeneration); + writtenKeys.add(key); + } + } + + /** + * 保存一个循环输入分块或元数据。 + * + * @param key 存储键 + * @param value 输入值 + * @param versioned 是否使用版本对象存储 + * @param instanceId 工作流实例 ID + * @param lockFencingToken 当前实例锁 token + * @param claimId 当前触发器 ID + * @param claimGeneration 当前认领代际 + */ + private void storeInputValue( + String key, + Serializable value, + boolean versioned, + String instanceId, + long lockFencingToken, + String claimId, + long claimGeneration) { + if (versioned) { + boolean created; + if (lockFencingToken > 0L && instanceId != null) { + String claimGuardKey = claimGeneration > 0L + ? executionGuardKey(instanceId, claimId) + : lockFenceKey(instanceId); + long effectiveClaimGeneration = claimGeneration > 0L + ? claimGeneration + : lockFencingToken; + created = versionedObjectStore.createIfAbsent( + key, + value, + 0L, + lockFenceKey(instanceId), + lockFencingToken, + claimGuardKey, + effectiveClaimGeneration, + RESULT_TTL); + } else if (claimGeneration > 0L + && instanceId != null + && claimId != null) { + created = versionedObjectStore.createIfAbsent( + key, + value, + 0L, + executionGuardKey(instanceId, claimId), + claimGeneration, + RESULT_TTL); + } else { + created = versionedObjectStore.createIfAbsent( + key, value, 0L, RESULT_TTL); + } + if (!created) { + throw new TriggerClaimLostException("loop-input:" + key); + } + return; + } + putCache(key, value); + } + + /** + * {@inheritDoc} + */ + @Override + @SuppressWarnings("unchecked") + public Object loadInputItem(String resultId, int index) { + if (index < 0) { + throw new IllegalArgumentException("index must not be negative"); + } + int chunkIndex = index / CHUNK_SIZE; + boolean versionedInput = + versionedObjectStore != null + && isVersionedResult(resultId); + CachedInputChunk cached = versionedInput + ? inputChunkCache.get(resultId) + : null; + List chunk; + if (cached != null && cached.chunkIndex == chunkIndex) { + chunk = cached.values; + } else { + String key = inputChunkKey(resultId, chunkIndex); + chunk = versionedInput + ? versionedObjectStore.load(key, List.class) + : getCache(key, List.class); + if (versionedInput && chunk != null) { + inputChunkCache.put( + resultId, + new CachedInputChunk( + chunkIndex, + chunk, + serializedWeight(chunk))); + } + } + if (chunk == null || index % CHUNK_SIZE >= chunk.size()) { + throw new IllegalStateException("Loop input item not found: " + resultId + ":" + index); + } + Object value = chunk.get(index % CHUNK_SIZE); + return versionedInput + ? snapshotValue(value).value + : value; + } + + /** + * {@inheritDoc} + * + *

版本化输入按 128 项分块批量读取并直接展平。每个分块由对象存储独立 + * 反序列化，不进入共享活动缓存，因此既保持调用方可变值隔离，也避免逐元素 + * 创建 ObjectStream。

+ */ + @Override + @SuppressWarnings({"unchecked", "rawtypes"}) + public List loadInput( + LoopInputReference reference) { + if (reference == null) { + throw new IllegalArgumentException( + "loop input reference required"); + } + int itemCount = + reference.getItemCount(); + if (itemCount < 0) { + throw new IllegalArgumentException( + "itemCount must not be negative"); + } + if (itemCount == 0) { + return new ArrayList<>(); + } + String resultId = + reference.getResultId(); + if (versionedObjectStore == null + || !isVersionedResult(resultId)) { + return LoopResultRepository.super + .loadInput(reference); + } + + int chunkCount = + (itemCount + CHUNK_SIZE - 1) + / CHUNK_SIZE; + List keys = + new ArrayList<>(chunkCount); + for (int chunkIndex = 0; + chunkIndex < chunkCount; + chunkIndex++) { + keys.add(inputChunkKey( + resultId, chunkIndex)); + } + List> chunks = + (List) versionedObjectStore.loadAll( + keys, List.class); + if (chunks == null + || chunks.size() != chunkCount) { + throw new IllegalStateException( + "Incomplete loop input chunks: " + + resultId); + } + + List items = + new ArrayList<>(itemCount); + for (int chunkIndex = 0; + chunkIndex < chunkCount; + chunkIndex++) { + List chunk = + chunks.get(chunkIndex); + if (chunk == null) { + throw new IllegalStateException( + "Loop input chunk not found: " + + inputChunkKey( + resultId, + chunkIndex)); + } + int remaining = + itemCount - items.size(); + int expectedChunkSize = + Math.min(CHUNK_SIZE, remaining); + if (chunk.size() + != expectedChunkSize) { + throw new IllegalStateException( + "Incomplete loop input chunk: " + + inputChunkKey( + resultId, + chunkIndex)); + } + items.addAll(chunk); + } + return items; + } + + /** + * {@inheritDoc} + */ + @Override + public void removeInput(String resultId) { + removeInput(null, 0L, null, 0L, resultId); + } + + /** + * {@inheritDoc} + */ + @Override + public void removeInput( + String instanceId, + long lockFencingToken, + String claimId, + long claimGeneration, + String resultId) { + inputChunkCache.remove(resultId); + outputChunkCache.remove(resultId); + boolean versionedInput = + versionedObjectStore != null && isVersionedResult(resultId); + String sizeKey = inputSizeKey(resultId); + Integer count = versionedInput + ? versionedObjectStore.load(sizeKey, Integer.class) + : getCache(sizeKey, Integer.class); + if (count == null) { + return; + } + int chunkCount = (count + CHUNK_SIZE - 1) / CHUNK_SIZE; + List keys = new ArrayList<>(chunkCount + 1); + for (int chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++) { + keys.add(inputChunkKey(resultId, chunkIndex)); + } + keys.add(sizeKey); + if (versionedInput) { + keys.add(refreshMarkerKey(resultId)); + } + if (versionedInput) { + guardedDelete( + keys, + instanceId, + lockFencingToken, + claimId, + claimGeneration); + return; + } + for (String key : keys) { + removeCache(key); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void releaseActiveCache(String resultId) { + inputChunkCache.remove(resultId); + outputChunkCache.remove(resultId); + } + + /** + * {@inheritDoc} + */ + @Override + public void append(String resultId, int iterationIndex, Map outputValues) { + append(null, 0L, null, 0L, resultId, iterationIndex, outputValues); + } + + /** + * {@inheritDoc} + */ + @Override + public void append( + String instanceId, + long fencingToken, + String resultId, + int iterationIndex, + Map outputValues) { + append(instanceId, fencingToken, null, 0L, resultId, iterationIndex, outputValues); + } + + /** + * {@inheritDoc} + */ + @Override + public void append( + String instanceId, + long lockFencingToken, + String claimId, + long claimGeneration, + String resultId, + int iterationIndex, + Map outputValues) { + if (iterationIndex < 0) { + throw new IllegalArgumentException("iterationIndex must not be negative"); + } + + int chunkIndex = iterationIndex / CHUNK_SIZE; + int chunkOffset = iterationIndex % CHUNK_SIZE; + boolean guardedVersionedWrite = + versionedObjectStore != null && lockFencingToken > 0L && instanceId != null; + if (guardedVersionedWrite && chunkOffset == 0 && chunkIndex > 0) { + // 续期与是否声明输出无关，纯副作用循环同样需要保留早期输入分块。 + refreshActiveChunkExpirations( + instanceId, + lockFencingToken, + claimId, + claimGeneration, + resultId, + chunkIndex); + } + if (outputValues == null || outputValues.isEmpty()) { + return; + } + + String key = chunkKey(resultId, chunkIndex); + if (guardedVersionedWrite) { + if (claimGeneration > 0L && (claimId == null || claimId.trim().isEmpty())) { + throw new IllegalArgumentException( + "claimId is required with claimGeneration"); + } + appendVersioned( + instanceId, + lockFencingToken, + claimId, + claimGeneration, + resultId, + chunkIndex, + key, + chunkOffset, + outputValues); + return; + } + LoopResultChunk chunk = getCache(key, LoopResultChunk.class); + if (chunk == null) { + chunk = new LoopResultChunk(); + } + + for (Map.Entry entry : outputValues.entrySet()) { + List values = chunk.values.computeIfAbsent(entry.getKey(), ignored -> new ArrayList<>()); + if (values.size() < chunkOffset) { + throw new IllegalStateException("Loop result chunk is incomplete: " + key); + } + if (values.size() == chunkOffset) { + values.add(entry.getValue()); + continue; + } + Object existing = values.get(chunkOffset); + if (!java.util.Objects.deepEquals(existing, entry.getValue())) { + throw new IllegalStateException("Conflicting loop result replay: " + key); + } + } + putCache(key, chunk); + } + + /** + * 使用版本比较和 fencing token 原子追加循环分块。 + * + * @param instanceId 工作流实例 ID + * @param lockFencingToken 当前实例锁 token + * @param claimId 当前触发器 ID + * @param claimGeneration 当前触发器认领代际 + * @param resultId 循环结果 ID + * @param chunkIndex 当前分块序号 + * @param key 分块键 + * @param chunkOffset 分块内偏移 + * @param outputValues 本轮输出 + */ + private void appendVersioned( + String instanceId, + long lockFencingToken, + String claimId, + long claimGeneration, + String resultId, + int chunkIndex, + String key, + int chunkOffset, + Map outputValues) { + String claimGuardKey = claimGeneration > 0L + ? executionGuardKey(instanceId, claimId) + : lockFenceKey(instanceId); + long effectiveClaimGeneration = claimGeneration > 0L + ? claimGeneration + : lockFencingToken; + for (int attempt = 0; attempt < 16; attempt++) { + CachedOutputChunk cached = + outputChunkCache.get(resultId); + if (cached != null + && cached.chunkIndex + != chunkIndex) { + outputChunkCache.remove(resultId); + cached = null; + } + LoopResultChunk chunk = cached == null + ? versionedObjectStore.load(key, LoopResultChunk.class) + : cached.chunk.copy(); + long chunkWeight = cached == null + ? serializedWeightOrDefault( + chunk, + EMPTY_CHUNK_WEIGHT_BYTES) + : cached.weight(); + if (chunk == null) { + LoopResultChunk created = new LoopResultChunk(); + OutputMutation mutation = applyOutput( + created, + key, + chunkOffset, + outputValues); + if (versionedObjectStore.createIfAbsent( + key, + created, + created.version, + lockFenceKey(instanceId), + lockFencingToken, + claimGuardKey, + effectiveClaimGeneration, + RESULT_TTL)) { + outputChunkCache.put( + resultId, + new CachedOutputChunk( + chunkIndex, + created.copy(), + saturatingAdd( + EMPTY_CHUNK_WEIGHT_BYTES, + mutation.addedWeight))); + return; + } + outputChunkCache.remove(resultId); + continue; + } + long expectedVersion = chunk.version; + OutputMutation mutation = applyOutput( + chunk, + key, + chunkOffset, + outputValues); + if (!mutation.changed) { + if (cached == null) { + outputChunkCache.put( + resultId, + new CachedOutputChunk( + chunkIndex, + chunk.copy(), + chunkWeight)); + } + return; + } + chunk.version = expectedVersion + 1L; + if (versionedObjectStore.compareAndSet( + key, + expectedVersion, + chunk, + chunk.version, + lockFenceKey(instanceId), + lockFencingToken, + claimGuardKey, + effectiveClaimGeneration, + RESULT_TTL)) { + outputChunkCache.put( + resultId, + new CachedOutputChunk( + chunkIndex, + chunk.copy(), + saturatingAdd( + chunkWeight, + mutation.addedWeight))); + return; + } + outputChunkCache.remove(resultId); + } + throw new TriggerClaimLostException("loop-result:" + key); + } + + /** + * 在进入新结果分块时批量续期既有结果和输入分块。 + * + *

每 128 轮最多执行一次 Redis pipeline，保证长循环早期分块不会先于活跃实例 + * 过期，同时避免每轮逐块续期。

+ * + * @param resultId 循环结果 ID + * @param currentChunkIndex 当前新分块序号 + */ + private void refreshActiveChunkExpirations( + String instanceId, + long lockFencingToken, + String claimId, + long claimGeneration, + String resultId, + int currentChunkIndex) { + String markerKey = refreshMarkerKey(resultId); + RefreshMarker marker = + versionedObjectStore.load(markerKey, RefreshMarker.class); + long now = System.currentTimeMillis(); + if (marker != null + && now - marker.refreshedAt < REFRESH_INTERVAL_MILLIS) { + return; + } + String claimGuardKey = claimGeneration > 0L + ? executionGuardKey(instanceId, claimId) + : lockFenceKey(instanceId); + long effectiveClaimGeneration = claimGeneration > 0L + ? claimGeneration + : lockFencingToken; + boolean markerUpdated; + if (marker == null) { + markerUpdated = versionedObjectStore.createIfAbsent( + markerKey, + new RefreshMarker(0L, now), + 0L, + lockFenceKey(instanceId), + lockFencingToken, + claimGuardKey, + effectiveClaimGeneration, + RESULT_TTL); + } else { + long expectedVersion = marker.version; + marker.version = expectedVersion + 1L; + marker.refreshedAt = now; + markerUpdated = versionedObjectStore.compareAndSet( + markerKey, + expectedVersion, + marker, + marker.version, + lockFenceKey(instanceId), + lockFencingToken, + claimGuardKey, + effectiveClaimGeneration, + RESULT_TTL); + } + if (!markerUpdated) { + throw new TriggerClaimLostException( + "loop-refresh:" + resultId); + } + List keys = new ArrayList<>(); + for (int chunkIndex = 0; chunkIndex < currentChunkIndex; chunkIndex++) { + keys.add(chunkKey(resultId, chunkIndex)); + } + Integer inputSize = versionedObjectStore.load( + inputSizeKey(resultId), Integer.class); + if (inputSize != null) { + int inputChunkCount = (inputSize + CHUNK_SIZE - 1) / CHUNK_SIZE; + for (int inputChunkIndex = 0; + inputChunkIndex < inputChunkCount; + inputChunkIndex++) { + keys.add(inputChunkKey(resultId, inputChunkIndex)); + } + keys.add(inputSizeKey(resultId)); + } + keys.add(markerKey); + versionedObjectStore.refreshExpirations(keys, RESULT_TTL); + } + + /** + * 将一轮输出合并到分块并校验幂等重放。 + * + * @param chunk 当前分块 + * @param key 分块键 + * @param chunkOffset 分块内偏移 + * @param outputValues 本轮输出 + * @return 本次变更及新增缓存重量 + */ + private OutputMutation applyOutput( + LoopResultChunk chunk, + String key, + int chunkOffset, + Map outputValues) { + boolean changed = false; + long addedWeight = 0L; + for (Map.Entry entry : outputValues.entrySet()) { + boolean newOutput = + !chunk.values.containsKey( + entry.getKey()); + List values = + chunk.values.computeIfAbsent(entry.getKey(), ignored -> new ArrayList<>()); + if (values.size() < chunkOffset) { + throw new IllegalStateException("Loop result chunk is incomplete: " + key); + } + if (values.size() == chunkOffset) { + SnapshotValue snapshot = + snapshotValue(entry.getValue()); + values.add(snapshot.value); + changed = true; + addedWeight = saturatingAdd( + addedWeight, + snapshot.weight + + (newOutput + ? estimateStringWeight( + entry.getKey()) + : 0L) + + 32L); + continue; + } + Object existing = values.get(chunkOffset); + if (!java.util.Objects.deepEquals(existing, entry.getValue())) { + throw new IllegalStateException("Conflicting loop result replay: " + key); + } + } + return new OutputMutation( + changed, addedWeight); + } + + /** + * {@inheritDoc} + */ + @Override + public Map load(String resultId, int iterationCount, List outputNames) { + Map result = new LinkedHashMap<>(); + if (iterationCount == 0 || outputNames == null || outputNames.isEmpty()) { + return result; + } + if (iterationCount < 0) { + throw new IllegalArgumentException("iterationCount must not be negative"); + } + + Map> collected = new LinkedHashMap<>(); + for (String outputName : outputNames) { + collected.put(outputName, new ArrayList<>(iterationCount)); + } + + int chunkCount = (iterationCount + CHUNK_SIZE - 1) / CHUNK_SIZE; + List versionedChunks = null; + if (versionedObjectStore != null && isVersionedResult(resultId)) { + List keys = new ArrayList<>(chunkCount); + for (int chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++) { + keys.add(chunkKey(resultId, chunkIndex)); + } + versionedChunks = versionedObjectStore.loadAll( + keys, LoopResultChunk.class); + } + for (int chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++) { + String key = chunkKey(resultId, chunkIndex); + LoopResultChunk chunk = versionedChunks != null + ? versionedChunks.get(chunkIndex) + : getCache(key, LoopResultChunk.class); + if (chunk == null) { + throw new IllegalStateException("Loop result chunk not found: " + key); + } + for (String outputName : outputNames) { + List values = chunk.values.get(outputName); + if (values == null) { + throw new IllegalStateException("Loop output not found: " + outputName); + } + collected.get(outputName).addAll(values); + } + } + + for (Map.Entry> entry : collected.entrySet()) { + if (entry.getValue().size() != iterationCount) { + throw new IllegalStateException("Incomplete loop result: " + entry.getKey()); + } + result.put(entry.getKey(), entry.getValue()); + } + return result; + } + + /** + * 构建结果分块缓存键。 + * + * @param resultId 循环结果 ID + * @param chunkIndex 分块序号 + * @return 缓存键 + */ + private String chunkKey(String resultId, int chunkIndex) { + if (isVersionedResult(resultId)) { + String instanceId = resultId.substring(0, resultId.indexOf(':')); + String localResultId = resultId.substring(resultId.indexOf(':') + 1); + return CacheKey.CHAIN_STATE_CAS_KEY + "{" + instanceId + "}:loop:" + + localResultId + ":" + chunkIndex; + } + return CacheKey.LOOP_RESULT_CACHE_KEY + resultId + ":" + chunkIndex; + } + + /** + * 判断结果 ID 是否为包含实例前缀的新格式。 + * + * @param resultId 循环结果 ID + * @return 新格式时为 {@code true} + */ + private boolean isVersionedResult(String resultId) { + return resultId != null && resultId.indexOf(':') > 0; + } + + /** + * 构建实例 fencing token 键。 + * + * @param instanceId 工作流实例 ID + * @return fencing token 键 + */ + private String executionGuardKey(String instanceId, String claimId) { + return CacheKey.CHAIN_STATE_CAS_KEY + "{" + instanceId + "}:claim:" + claimId; + } + + /** + * 构建实例锁 fencing token 键。 + * + * @param instanceId 工作流实例 ID + * @return fencing token 键 + */ + private String lockFenceKey(String instanceId) { + return CacheKey.CHAIN_STATE_CAS_KEY + "{" + instanceId + "}:fence"; + } + + /** + * 构建输入分块缓存键。 + * + * @param resultId 循环结果 ID + * @param chunkIndex 分块序号 + * @return 缓存键 + */ + private String inputChunkKey(String resultId, int chunkIndex) { + if (isVersionedResult(resultId)) { + String instanceId = resultId.substring(0, resultId.indexOf(':')); + String localResultId = resultId.substring(resultId.indexOf(':') + 1); + return CacheKey.CHAIN_STATE_CAS_KEY + "{" + instanceId + "}:loop:" + + localResultId + ":input:" + chunkIndex; + } + return CacheKey.LOOP_RESULT_CACHE_KEY + resultId + ":input:" + chunkIndex; + } + + /** + * 构建输入数量缓存键。 + * + * @param resultId 循环结果 ID + * @return 缓存键 + */ + private String inputSizeKey(String resultId) { + if (isVersionedResult(resultId)) { + String instanceId = resultId.substring(0, resultId.indexOf(':')); + String localResultId = resultId.substring(resultId.indexOf(':') + 1); + return CacheKey.CHAIN_STATE_CAS_KEY + "{" + instanceId + "}:loop:" + + localResultId + ":input:size"; + } + return CacheKey.LOOP_RESULT_CACHE_KEY + resultId + ":input:size"; + } + + /** + * 构建循环分块生命周期刷新标记键。 + * + * @param resultId 循环结果 ID + * @return 刷新标记键 + */ + private String refreshMarkerKey(String resultId) { + if (isVersionedResult(resultId)) { + String instanceId = resultId.substring(0, resultId.indexOf(':')); + String localResultId = + resultId.substring(resultId.indexOf(':') + 1); + return CacheKey.CHAIN_STATE_CAS_KEY + "{" + instanceId + "}:loop:" + + localResultId + ":refresh"; + } + return CacheKey.LOOP_RESULT_CACHE_KEY + resultId + ":refresh"; + } + + /** + * 仅在当前双守卫仍有效时删除循环输入。 + * + * @param keys 待删除键 + * @param instanceId 工作流实例 ID + * @param lockFencingToken 当前实例锁 token + * @param claimId 当前触发器 ID + * @param claimGeneration 当前认领代际 + */ + private void guardedDelete( + List keys, + String instanceId, + long lockFencingToken, + String claimId, + long claimGeneration) { + if (keys.isEmpty()) { + return; + } + if (lockFencingToken <= 0L || instanceId == null) { + versionedObjectStore.deleteAll(keys); + return; + } + String claimGuardKey = claimGeneration > 0L + ? executionGuardKey(instanceId, claimId) + : lockFenceKey(instanceId); + long effectiveClaimGeneration = claimGeneration > 0L + ? claimGeneration + : lockFencingToken; + versionedObjectStore.deleteAll( + keys, + lockFenceKey(instanceId), + lockFencingToken, + claimGuardKey, + effectiveClaimGeneration); + } + + /** + * 创建与调用方可变对象图隔离的缓存快照。 + * + *

持久化路径本身使用相同 Java 序列化协议；这里只对可能可变的单个业务值复制， + * 避免每轮重新复制整个 128 项分块。

+ * + * @param value 业务值 + * @return 隔离值及其序列化重量 + */ + private static SnapshotValue snapshotValue( + Object value) { + if (isKnownImmutable(value)) { + return new SnapshotValue( + value, + estimateImmutableWeight(value)); + } + byte[] encoded = JavaValueEncoder.INSTANCE.apply( + value); + return new SnapshotValue( + CACHE_VALUE_DECODER.apply(encoded), + encoded.length); + } + + /** + * 计算完整已持久化对象的序列化重量。 + * + * @param value 已持久化对象 + * @return 字节重量；不可序列化时返回最大值以拒绝缓存 + */ + private static long serializedWeight( + Object value) { + if (!(value instanceof Serializable)) { + return Long.MAX_VALUE; + } + return JavaValueEncoder.INSTANCE.apply( + value).length; + } + + /** + * 对可空对象计算序列化重量。 + * + * @param value 对象 + * @param defaultWeight 空对象默认重量 + * @return 缓存重量 + */ + private static long serializedWeightOrDefault( + Object value, + long defaultWeight) { + return value == null + ? defaultWeight + : serializedWeight(value); + } + + /** + * 判断无需复制即可安全共享的基础不可变值。 + * + * @param value 值 + * @return 已知不可变时为 {@code true} + */ + private static boolean isKnownImmutable( + Object value) { + return value == null + || value instanceof String + || value instanceof Boolean + || value instanceof Character + || value instanceof Byte + || value instanceof Short + || value instanceof Integer + || value instanceof Long + || value instanceof Float + || value instanceof Double + || value instanceof java.math.BigInteger + || value instanceof java.math.BigDecimal + || value instanceof Enum + || value instanceof java.util.UUID; + } + + /** + * 估算基础不可变值的堆重量。 + * + * @param value 不可变值 + * @return 保守字节估算 + */ + private static long estimateImmutableWeight( + Object value) { + if (value == null) { + return 8L; + } + if (value instanceof String) { + return estimateStringWeight( + (String) value); + } + return 32L; + } + + /** + * 估算字符串及其字符存储重量。 + * + * @param value 字符串 + * @return 保守字节估算 + */ + private static long estimateStringWeight( + String value) { + return value == null + ? 8L + : 40L + (long) value.length() * 2L; + } + + /** + * 饱和相加两个非负重量。 + * + * @param left 左值 + * @param right 右值 + * @return 相加结果；溢出时为 {@link Long#MAX_VALUE} + */ + private static long saturatingAdd( + long left, + long right) { + if (left >= Long.MAX_VALUE - right) { + return Long.MAX_VALUE; + } + return left + right; + } + + /** + * 可计量缓存值。 + */ + private interface WeightedCacheValue { + + /** + * 获取缓存重量。 + * + * @return 非负字节重量 + */ + long weight(); + } + + /** + * 同时限制条目数、单条重量和总重量的 LRU 缓存。 + * + * @param 键类型 + * @param 可计量值类型 + */ + private static final class BoundedWeightedCache< + K, V extends WeightedCacheValue> { + + private final int maxEntries; + private final long maxTotalWeight; + private final long maxEntryWeight; + private final LinkedHashMap values = + new LinkedHashMap<>(16, 0.75F, true); + private long totalWeight; + + /** + * 创建有界缓存。 + * + * @param maxEntries 最大条目数 + * @param maxTotalWeight 最大总重量 + * @param maxEntryWeight 最大单条重量 + */ + private BoundedWeightedCache( + int maxEntries, + long maxTotalWeight, + long maxEntryWeight) { + this.maxEntries = maxEntries; + this.maxTotalWeight = maxTotalWeight; + this.maxEntryWeight = maxEntryWeight; + } + + /** + * 获取并提升条目热度。 + * + * @param key 键 + * @return 缓存值；未命中时为 {@code null} + */ + private synchronized V get(K key) { + return values.get(key); + } + + /** + * 写入缓存；超大条目直接拒绝，并按 LRU 淘汰到双上限以内。 + * + * @param key 键 + * @param value 值 + */ + private synchronized void put( + K key, + V value) { + V previous = values.remove(key); + if (previous != null) { + totalWeight -= previous.weight(); + } + long valueWeight = + Math.max(0L, value.weight()); + if (valueWeight > maxEntryWeight + || valueWeight + > maxTotalWeight) { + return; + } + while (!values.isEmpty() + && (values.size() >= maxEntries + || totalWeight + > maxTotalWeight + - valueWeight)) { + Iterator> iterator = + values.entrySet().iterator(); + Map.Entry eldest = + iterator.next(); + totalWeight -= eldest.getValue().weight(); + iterator.remove(); + } + values.put(key, value); + totalWeight += valueWeight; + } + + /** + * 删除缓存条目。 + * + * @param key 键 + */ + private synchronized void remove(K key) { + V removed = values.remove(key); + if (removed != null) { + totalWeight -= removed.weight(); + } + } + } + + /** + * 单个隔离业务值及其缓存重量。 + */ + private static final class SnapshotValue { + + private final Object value; + private final long weight; + + /** + * 创建业务值快照。 + * + * @param value 隔离值 + * @param weight 字节重量 + */ + private SnapshotValue( + Object value, + long weight) { + this.value = value; + this.weight = weight; + } + } + + /** + * 一次输出合并结果。 + */ + private static final class OutputMutation { + + private final boolean changed; + private final long addedWeight; + + /** + * 创建输出变更结果。 + * + * @param changed 分块是否变化 + * @param addedWeight 新增重量 + */ + private OutputMutation( + boolean changed, + long addedWeight) { + this.changed = changed; + this.addedWeight = addedWeight; + } + } + + /** + * 一个循环最近读取的输入分块。 + */ + private static final class CachedInputChunk + implements WeightedCacheValue { + + private final int chunkIndex; + private final List values; + private final long weight; + + /** + * 创建输入分块缓存。 + * + * @param chunkIndex 分块序号 + * @param values 分块值 + * @param weight 缓存重量 + */ + private CachedInputChunk( + int chunkIndex, + List values, + long weight) { + this.chunkIndex = chunkIndex; + this.values = values; + this.weight = weight; + } + + /** + * {@inheritDoc} + */ + @Override + public long weight() { + return weight; + } + } + + /** + * 一个循环最近成功提交的活动输出分块。 + */ + private static final class CachedOutputChunk + implements WeightedCacheValue { + + private final int chunkIndex; + private final LoopResultChunk chunk; + private final long weight; + + /** + * 创建活动输出缓存。 + * + * @param chunkIndex 分块序号 + * @param chunk 私有分块快照 + * @param weight 缓存重量 + */ + private CachedOutputChunk( + int chunkIndex, + LoopResultChunk chunk, + long weight) { + this.chunkIndex = chunkIndex; + this.chunk = chunk; + this.weight = weight; + } + + /** + * {@inheritDoc} + */ + @Override + public long weight() { + return weight; + } + } + + /** + * 循环分块批量续期节流标记。 + */ + static final class RefreshMarker implements Serializable { + + private long version; + private long refreshedAt; + + /** + * 创建刷新标记。 + * + * @param version 对象版本 + * @param refreshedAt 最近刷新时间 + */ + RefreshMarker(long version, long refreshedAt) { + this.version = version; + this.refreshedAt = refreshedAt; + } + } + + /** + * 单个循环结果分块。 + */ + static final class LoopResultChunk implements Serializable { + + private long version; + private final Map> values = new LinkedHashMap<>(); + + /** + * 创建空结果分块。 + */ + LoopResultChunk() { + } + + /** + * 创建与源分块容器相互隔离的私有快照。 + * + *

业务值在首次进入活动缓存时已经逐值序列化隔离，后续只复制 Map/List + * 容器即可避免追加线程相互污染。

+ * + * @param source 源分块 + */ + private LoopResultChunk(LoopResultChunk source) { + this.version = source.version; + source.values.forEach((name, items) -> + this.values.put(name, new ArrayList<>(items))); + } + + /** + * 复制当前分块，避免并发失锁执行者污染共享缓存。 + * + * @return 独立分块快照 + */ + private LoopResultChunk copy() { + return new LoopResultChunk(this); + } + + /** + * 获取当前分块输出。 + * + * @return 输出名称到值列表的映射 + */ + Map> getValues() { + return values; + } + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/NodeStateRepositoryImpl.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/NodeStateRepositoryImpl.java index 9eb783ee..db7f701a 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/NodeStateRepositoryImpl.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/NodeStateRepositoryImpl.java @@ -1,33 +1,320 @@ package tech.easyflow.ai.easyagentsflow.repository; +import com.easyagents.flow.core.chain.ChainState; import com.easyagents.flow.core.chain.NodeState; import com.easyagents.flow.core.chain.repository.NodeStateField; import com.easyagents.flow.core.chain.repository.NodeStateRepository; import org.springframework.stereotype.Component; +import tech.easyflow.common.cache.VersionedObjectStore; +import tech.easyflow.common.cache.VersionedFields; import tech.easyflow.common.constant.CacheKey; +import javax.annotation.Resource; +import java.time.Duration; import java.util.EnumSet; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +/** + * 基于 Redis 原子版本存储的节点状态仓储。 + */ @Component public class NodeStateRepositoryImpl extends BaseRepository implements NodeStateRepository { + private static final Duration STATE_TTL = Duration.ofDays(3); + + @Resource + private VersionedObjectStore versionedObjectStore; + private final ConcurrentMap legacyFormats = new ConcurrentHashMap<>(); + + /** + * {@inheritDoc} + */ @Override public NodeState load(String instanceId, String nodeId) { - String key = CacheKey.NODE_CACHE_KEY + instanceId + ":" + nodeId; - NodeState nodeState = getCache(key, NodeState.class); - if (nodeState == null) { - nodeState = new NodeState(); - nodeState.setChainInstanceId(instanceId); - nodeState.setNodeId(nodeId); - putCache(key, nodeState); + String stateKey = stateKey(instanceId, nodeId); + VersionedFields snapshot = versionedObjectStore.loadFields(stateKey); + if (WorkflowStateFields.isFieldFormat(snapshot)) { + return WorkflowStateFields.decodeNode(snapshot); } - return nodeState; + if (snapshot != null) { + NodeState payloadState = versionedObjectStore.load(stateKey, NodeState.class); + if (payloadState != null) { + WorkflowStateFields.normalizeNode( + payloadState); + versionedObjectStore.rewriteAsFields( + stateKey, + payloadState.getVersion(), + WorkflowStateFields.allNodeFields(payloadState), + STATE_TTL); + return payloadState; + } + } + + String legacyKey = legacyStateKey(instanceId, nodeId); + NodeState legacyState = getCache(legacyKey, NodeState.class); + if (legacyState == null) { + return null; + } + WorkflowStateFields.normalizeNode( + legacyState); + if (isLegacyInstance(instanceId)) { + return legacyState; + } + versionedObjectStore.createFieldsIfAbsent( + stateKey, + WorkflowStateFields.allNodeFields(legacyState), + legacyState.getVersion(), + STATE_TTL); + VersionedFields migrated = versionedObjectStore.loadFields(stateKey); + if (!WorkflowStateFields.isFieldFormat(migrated)) { + throw new IllegalStateException( + "Workflow node state migration failed: " + instanceId + "/" + nodeId); + } + removeCache(legacyKey); + return WorkflowStateFields.decodeNode(migrated); } + /** + * {@inheritDoc} + */ + @Override + public NodeState create(String instanceId, String nodeId, long chainStateVersion) { + return create(instanceId, nodeId, chainStateVersion, 0L); + } + + /** + * {@inheritDoc} + */ + @Override + public NodeState create( + String instanceId, + String nodeId, + long chainStateVersion, + long fencingToken) { + return create(instanceId, nodeId, chainStateVersion, fencingToken, null, 0L); + } + + /** + * {@inheritDoc} + */ + @Override + public NodeState create( + String instanceId, + String nodeId, + long chainStateVersion, + long lockFencingToken, + String claimId, + long claimGeneration) { + NodeState existing = load(instanceId, nodeId); + if (existing != null) { + return existing; + } + NodeState created = new NodeState(); + created.setChainInstanceId(instanceId); + created.setNodeId(nodeId); + if (isLegacyInstance(instanceId)) { + putCache(legacyStateKey(instanceId, nodeId), created); + return created; + } + requireClaimId(claimId, claimGeneration); + String claimGuardKey = claimGeneration > 0L + ? executionGuardKey(instanceId, claimId) + : lockFenceKey(instanceId); + long effectiveClaimGeneration = claimGeneration > 0L + ? claimGeneration + : lockFencingToken; + boolean createdNow = lockFencingToken > 0L + ? versionedObjectStore.createFieldsIfAbsent( + stateKey(instanceId, nodeId), + WorkflowStateFields.allNodeFields(created), + created.getVersion(), + chainStateKey(instanceId), + chainStateVersion, + lockFenceKey(instanceId), + lockFencingToken, + claimGuardKey, + effectiveClaimGeneration, + STATE_TTL) + : versionedObjectStore.createFieldsIfAbsent( + stateKey(instanceId, nodeId), + WorkflowStateFields.allNodeFields(created), + created.getVersion(), + chainStateKey(instanceId), + chainStateVersion, + STATE_TTL); + if (createdNow) { + return created; + } + VersionedFields concurrent = versionedObjectStore.loadFields( + stateKey(instanceId, nodeId)); + return WorkflowStateFields.isFieldFormat(concurrent) + ? WorkflowStateFields.decodeNode(concurrent) + : null; + } + + /** + * {@inheritDoc} + */ @Override public boolean tryUpdate(NodeState newState, EnumSet fields, long chainStateVersion) { - String key = CacheKey.NODE_CACHE_KEY + newState.getChainInstanceId() + ":" + newState.getNodeId(); - putCache(key, newState); - return true; + return tryUpdate(newState, fields, chainStateVersion, 0L); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean tryUpdate( + NodeState newState, + EnumSet fields, + long chainStateVersion, + long fencingToken) { + return tryUpdate( + newState, fields, chainStateVersion, fencingToken, null, 0L); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean tryUpdate( + NodeState newState, + EnumSet fields, + long chainStateVersion, + long lockFencingToken, + String claimId, + long claimGeneration) { + String instanceId = newState.getChainInstanceId(); + if (isLegacyInstance(instanceId)) { + putCache(legacyStateKey(instanceId, newState.getNodeId()), newState); + return true; + } + long newVersion = newState.getVersion(); + if (newVersion <= 0L) { + throw new IllegalArgumentException("newState.version must be positive"); + } + requireClaimId(claimId, claimGeneration); + String claimGuardKey = claimGeneration > 0L + ? executionGuardKey(instanceId, claimId) + : lockFenceKey(instanceId); + long effectiveClaimGeneration = claimGeneration > 0L + ? claimGeneration + : lockFencingToken; + return lockFencingToken > 0L + ? versionedObjectStore.compareAndSetFields( + stateKey(instanceId, newState.getNodeId()), + newVersion - 1L, + WorkflowStateFields.nodeFields(newState, fields), + newVersion, + chainStateKey(instanceId), + chainStateVersion, + lockFenceKey(instanceId), + lockFencingToken, + claimGuardKey, + effectiveClaimGeneration, + STATE_TTL) + : versionedObjectStore.compareAndSetFields( + stateKey(instanceId, newState.getNodeId()), + newVersion - 1L, + WorkflowStateFields.nodeFields(newState, fields), + newVersion, + chainStateKey(instanceId), + chainStateVersion, + STATE_TTL); + } + + /** + * 判断实例是否仍由旧 JetCache 状态推进。 + * + * @param instanceId 实例 ID + * @return 活跃旧格式实例时为 {@code true} + */ + private boolean isLegacyInstance(String instanceId) { + return legacyFormats.computeIfAbsent(instanceId, this::detectLegacyInstance); + } + + /** + * 从持久化状态检测实例格式。 + * + * @param instanceId 实例 ID + * @return 活跃旧格式实例时为 {@code true} + */ + private boolean detectLegacyInstance(String instanceId) { + VersionedFields chainSnapshot = versionedObjectStore.loadFields( + chainStateKey(instanceId)); + if (WorkflowStateFields.isFieldFormat(chainSnapshot)) { + return false; + } + VersionedFields marker = versionedObjectStore.loadFields( + CacheKey.CHAIN_STATE_CAS_KEY + "{" + instanceId + "}:format"); + if (marker != null) { + return false; + } + return getCache(CacheKey.CHAIN_CACHE_KEY + instanceId, ChainState.class) != null; + } + + /** + * 构建旧节点状态键。 + * + * @param instanceId 实例 ID + * @param nodeId 节点 ID + * @return JetCache 键 + */ + private String legacyStateKey(String instanceId, String nodeId) { + return CacheKey.NODE_CACHE_KEY + instanceId + ":" + nodeId; + } + + /** + * 构建节点状态 CAS key。 + * + * @param instanceId 工作流实例 ID + * @param nodeId 节点 ID + * @return Redis 状态 key + */ + private String stateKey(String instanceId, String nodeId) { + return CacheKey.CHAIN_STATE_CAS_KEY + "{" + instanceId + "}:node:" + nodeId; + } + + /** + * 构建节点状态提交所依赖的工作流状态 key。 + * + * @param instanceId 工作流实例 ID + * @return Redis 工作流状态 key + */ + private String chainStateKey(String instanceId) { + return CacheKey.CHAIN_STATE_CAS_KEY + "{" + instanceId + "}:chain"; + } + + /** + * 构建实例锁 fencing token 键。 + * + * @param instanceId 工作流实例 ID + * @return fencing token 键 + */ + private String lockFenceKey(String instanceId) { + return CacheKey.CHAIN_STATE_CAS_KEY + "{" + instanceId + "}:fence"; + } + + /** + * 构建实例 fencing token 键。 + * + * @param instanceId 工作流实例 ID + * @return fencing token 键 + */ + private String executionGuardKey(String instanceId, String claimId) { + return CacheKey.CHAIN_STATE_CAS_KEY + "{" + instanceId + "}:claim:" + claimId; + } + + /** + * 校验分布式提交所需的 claim ID。 + * + * @param claimId 触发器 ID + * @param fencingToken 当前认领 token + */ + private void requireClaimId(String claimId, long fencingToken) { + if (fencingToken > 0L && (claimId == null || claimId.trim().isEmpty())) { + throw new IllegalArgumentException("claimId is required with fencingToken"); + } } } diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/RedisTriggerStore.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/RedisTriggerStore.java new file mode 100644 index 00000000..8c9f4770 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/RedisTriggerStore.java @@ -0,0 +1,785 @@ +package tech.easyflow.ai.easyagentsflow.repository; + +import com.easyagents.flow.core.chain.runtime.Trigger; +import com.easyagents.flow.core.chain.runtime.TriggerClaimLostException; +import com.easyagents.flow.core.chain.runtime.TriggerStore; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.data.redis.core.script.DefaultRedisScript; +import org.springframework.stereotype.Component; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import tech.easyflow.common.constant.CacheKey; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +/** + * 基于 Redis 有序集合和租约认领的工作流触发器仓储。 + * + *

当前实现的待执行集合与触发器数据使用同一 Lua 操作，派生触发器保存时会同时 + * 校验实例锁 fencing token 和父触发器认领代际。部署约束为 Redis Standalone 或 + * Sentinel；不支持 Redis Cluster。

+ */ +@Component +public class RedisTriggerStore implements TriggerStore { + + private static final Logger log = LoggerFactory.getLogger(RedisTriggerStore.class); + private static final Duration TRIGGER_TTL = Duration.ofDays(3); + private static final Duration CLAIM_GENERATION_TTL = Duration.ofDays(4); + /** + * 与底层调度器本地 Future 容量对齐，避免已预热任务反复占据扫描窗口。 + */ + private static final int DUE_BATCH_SIZE = 1024; + private static final int RECOVERY_BATCH_SIZE = 1000; + private static final DefaultRedisScript SAVE_SCRIPT = longScript( + "redis.call('psetex', KEYS[1], ARGV[1], ARGV[2]); " + + "redis.call('zadd', KEYS[2], ARGV[3], ARGV[4]); return 1"); + private static final DefaultRedisScript SAVE_IF_ABSENT_SCRIPT = + longScript( + "if redis.call('exists', KEYS[1]) == 1 then return 0 end; " + + "redis.call('psetex', KEYS[1], ARGV[1], ARGV[2]); " + + "redis.call('zadd', KEYS[2], ARGV[3], ARGV[4]); return 1"); + private static final DefaultRedisScript LOCK_GUARDED_SAVE_SCRIPT = longScript( + "local fence = redis.call('hget', KEYS[3], 'version'); " + + "if not fence or fence ~= ARGV[5] then return 0 end; " + + "redis.call('psetex', KEYS[1], ARGV[1], ARGV[2]); " + + "redis.call('zadd', KEYS[2], ARGV[3], ARGV[4]); return 1"); + private static final DefaultRedisScript + LOCK_GUARDED_SAVE_IF_ABSENT_SCRIPT = + longScript( + "local fence = redis.call('hget', KEYS[3], 'version'); " + + "if not fence or fence ~= ARGV[5] then return -1 end; " + + "if redis.call('exists', KEYS[1]) == 1 then return 0 end; " + + "redis.call('psetex', KEYS[1], ARGV[1], ARGV[2]); " + + "redis.call('zadd', KEYS[2], ARGV[3], ARGV[4]); return 1"); + private static final DefaultRedisScript DOUBLE_GUARDED_SAVE_SCRIPT = longScript( + "local fence = redis.call('hget', KEYS[3], 'version'); " + + "local claim = redis.call('hget', KEYS[4], 'version'); " + + "if not fence or fence ~= ARGV[5] " + + "or not claim or claim ~= ARGV[6] then return 0 end; " + + "redis.call('psetex', KEYS[1], ARGV[1], ARGV[2]); " + + "redis.call('zadd', KEYS[2], ARGV[3], ARGV[4]); return 1"); + private static final DefaultRedisScript + DOUBLE_GUARDED_SAVE_IF_ABSENT_SCRIPT = + longScript( + "local fence = redis.call('hget', KEYS[3], 'version'); " + + "local claim = redis.call('hget', KEYS[4], 'version'); " + + "if not fence or fence ~= ARGV[5] " + + "or not claim or claim ~= ARGV[6] then return -1 end; " + + "if redis.call('exists', KEYS[1]) == 1 then return 0 end; " + + "redis.call('psetex', KEYS[1], ARGV[1], ARGV[2]); " + + "redis.call('zadd', KEYS[2], ARGV[3], ARGV[4]); return 1"); + private static final DefaultRedisScript CLAIM_SCRIPT = stringScript( + "if redis.call('exists', KEYS[1]) == 0 then " + + "redis.call('zrem', KEYS[3], ARGV[3]); return nil end; " + + "local claimed = redis.call('set', KEYS[2], ARGV[1], 'PX', ARGV[2], 'NX'); " + + "if not claimed then return nil end; " + + "local payload = redis.call('get', KEYS[1]); " + + "local generation = redis.call('hincrby', KEYS[5], 'version', 1); " + + "redis.call('pexpire', KEYS[5], ARGV[5]); " + + "redis.call('hset', KEYS[4], 'version', generation); " + + "redis.call('pexpire', KEYS[4], ARGV[2]); " + + "redis.call('zadd', KEYS[3], ARGV[4], ARGV[3]); " + + "return tostring(generation) .. '\\n' .. payload"); + private static final DefaultRedisScript ACK_SCRIPT = longScript( + "if redis.call('get', KEYS[1]) == ARGV[1] then " + + "redis.call('del', KEYS[1]); redis.call('del', KEYS[2]); " + + "redis.call('del', KEYS[4]); " + + "redis.call('zrem', KEYS[3], ARGV[2]); return 1 else return 0 end"); + private static final DefaultRedisScript RELEASE_SCRIPT = longScript( + "if redis.call('get', KEYS[1]) == ARGV[1] then " + + "redis.call('del', KEYS[1]); redis.call('del', KEYS[4]); " + + "redis.call('psetex', KEYS[3], ARGV[4], ARGV[5]); " + + "redis.call('zadd', KEYS[2], ARGV[3], ARGV[2]); " + + "return 1 else return 0 end"); + private static final DefaultRedisScript MARK_DEAD_LETTER_PENDING_SCRIPT = + longScript( + "if redis.call('get', KEYS[1]) == ARGV[1] then " + + "redis.call('psetex', KEYS[2], ARGV[2], ARGV[3]); " + + "return 1 else return 0 end"); + private static final DefaultRedisScript DEAD_LETTER_SCRIPT = longScript( + "if redis.call('get', KEYS[1]) == ARGV[1] then " + + "redis.call('psetex', KEYS[4], ARGV[3], ARGV[4]); " + + "redis.call('del', KEYS[5]); " + + "redis.call('del', KEYS[1]); redis.call('del', KEYS[2]); " + + "redis.call('zrem', KEYS[3], ARGV[2]); return 1 else return 0 end"); + private static final DefaultRedisScript RENEW_SCRIPT = longScript( + "local guard = redis.call('hget', KEYS[3], 'version'); " + + "if redis.call('get', KEYS[1]) == ARGV[1] " + + "and guard and guard == ARGV[5] then " + + "redis.call('pexpire', KEYS[1], ARGV[2]); " + + "redis.call('pexpire', KEYS[3], ARGV[2]); " + + "redis.call('zadd', KEYS[2], ARGV[4], ARGV[3]); " + + "return 1 else return 0 end"); + private static final DefaultRedisScript REMOVE_SCRIPT = longScript( + "redis.call('del', KEYS[1]); redis.call('del', KEYS[2]); " + + "return redis.call('zrem', KEYS[3], ARGV[1])"); + + private final StringRedisTemplate redisTemplate; + private final ObjectMapper objectMapper; + private final Map claimContexts = + Collections.synchronizedMap(new IdentityHashMap<>()); + + /** + * 创建 Redis 触发器仓储。 + * + * @param redisTemplate Redis 字符串模板 + * @param objectMapper JSON 序列化器 + */ + public RedisTriggerStore(StringRedisTemplate redisTemplate, + ObjectMapper objectMapper) { + this.redisTemplate = redisTemplate; + this.objectMapper = objectMapper; + } + + /** + * {@inheritDoc} + */ + @Override + public Trigger save(Trigger trigger) { + if (trigger.getId() == null) { + trigger.setId(UUID.randomUUID().toString()); + } + long ttlMillis = Math.max( + TRIGGER_TTL.toMillis(), + Math.max(0L, trigger.getTriggerAt() - System.currentTimeMillis()) + + TRIGGER_TTL.toMillis()); + List keys = new ArrayList<>(); + keys.add(dataKey(trigger.getId())); + keys.add(CacheKey.TRIGGER_PENDING_KEY); + Long saved; + long requiredLockFencingToken = trigger.getRequiredLockFencingToken(); + long requiredClaimGeneration = trigger.getRequiredFencingToken(); + if (requiredLockFencingToken > 0L && requiredClaimGeneration > 0L) { + String requiredClaimId = requireText( + trigger.getRequiredFencingClaimId(), + "required fencing claim ID"); + keys.add(lockFenceKey(trigger.getStateInstanceId())); + keys.add(executionGuardKey(trigger.getStateInstanceId(), requiredClaimId)); + saved = redisTemplate.execute( + DOUBLE_GUARDED_SAVE_SCRIPT, + keys, + String.valueOf(ttlMillis), + serialize(trigger), + String.valueOf(trigger.getTriggerAt()), + trigger.getId(), + String.valueOf(requiredLockFencingToken), + String.valueOf(requiredClaimGeneration)); + } else if (requiredLockFencingToken > 0L) { + keys.add(lockFenceKey(trigger.getStateInstanceId())); + saved = redisTemplate.execute( + LOCK_GUARDED_SAVE_SCRIPT, + keys, + String.valueOf(ttlMillis), + serialize(trigger), + String.valueOf(trigger.getTriggerAt()), + trigger.getId(), + String.valueOf(requiredLockFencingToken)); + } else if (requiredClaimGeneration > 0L) { + throw new IllegalArgumentException( + "required lock fencing token is required with claim generation"); + } else { + saved = redisTemplate.execute( + SAVE_SCRIPT, + keys, + String.valueOf(ttlMillis), + serialize(trigger), + String.valueOf(trigger.getTriggerAt()), + trigger.getId()); + } + if (!Long.valueOf(1L).equals(saved)) { + throw new TriggerClaimLostException(trigger.getId()); + } + return trigger; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean saveIfAbsent(Trigger trigger) { + String triggerId = + requireText(trigger.getId(), + "stable trigger ID"); + long ttlMillis = Math.max( + TRIGGER_TTL.toMillis(), + Math.max(0L, + trigger.getTriggerAt() + - System.currentTimeMillis()) + + TRIGGER_TTL.toMillis()); + List keys = new ArrayList<>(); + keys.add(dataKey(triggerId)); + keys.add(CacheKey.TRIGGER_PENDING_KEY); + Long saved; + long requiredLockFencingToken = + trigger.getRequiredLockFencingToken(); + long requiredClaimGeneration = + trigger.getRequiredFencingToken(); + if (requiredLockFencingToken > 0L + && requiredClaimGeneration > 0L) { + String requiredClaimId = requireText( + trigger.getRequiredFencingClaimId(), + "required fencing claim ID"); + keys.add(lockFenceKey( + trigger.getStateInstanceId())); + keys.add(executionGuardKey( + trigger.getStateInstanceId(), + requiredClaimId)); + saved = redisTemplate.execute( + DOUBLE_GUARDED_SAVE_IF_ABSENT_SCRIPT, + keys, + String.valueOf(ttlMillis), + serialize(trigger), + String.valueOf(trigger.getTriggerAt()), + triggerId, + String.valueOf(requiredLockFencingToken), + String.valueOf(requiredClaimGeneration)); + } else if (requiredLockFencingToken > 0L) { + keys.add(lockFenceKey( + trigger.getStateInstanceId())); + saved = redisTemplate.execute( + LOCK_GUARDED_SAVE_IF_ABSENT_SCRIPT, + keys, + String.valueOf(ttlMillis), + serialize(trigger), + String.valueOf(trigger.getTriggerAt()), + triggerId, + String.valueOf(requiredLockFencingToken)); + } else if (requiredClaimGeneration > 0L) { + throw new IllegalArgumentException( + "required lock fencing token is required with claim generation"); + } else { + saved = redisTemplate.execute( + SAVE_IF_ABSENT_SCRIPT, + keys, + String.valueOf(ttlMillis), + serialize(trigger), + String.valueOf(trigger.getTriggerAt()), + triggerId); + } + if (Long.valueOf(1L).equals(saved)) { + return true; + } + if (Long.valueOf(0L).equals(saved)) { + return false; + } + if (Long.valueOf(-1L).equals(saved)) { + throw new TriggerClaimLostException(triggerId); + } + throw new IllegalStateException( + "Trigger create returned no result: " + + triggerId); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean remove(String triggerId) { + List guardKeys = removeLocalClaims(triggerId); + if (!guardKeys.isEmpty()) { + redisTemplate.delete(guardKeys); + } + Long removed = redisTemplate.execute( + REMOVE_SCRIPT, + java.util.Arrays.asList( + dataKey(triggerId), claimKey(triggerId), CacheKey.TRIGGER_PENDING_KEY), + triggerId); + return Long.valueOf(1L).equals(removed); + } + + /** + * {@inheritDoc} + */ + @Override + public Trigger find(String triggerId) { + return deserialize(redisTemplate.opsForValue().get(dataKey(triggerId))); + } + + /** + * {@inheritDoc} + */ + @Override + public List findDue(long uptoTimestamp) { + return findByScore(0L, uptoTimestamp, DUE_BATCH_SIZE); + } + + /** + * {@inheritDoc} + */ + @Override + public List findAllPending() { + return findByScore(0L, Long.MAX_VALUE, RECOVERY_BATCH_SIZE); + } + + /** + * {@inheritDoc} + */ + @Override + public Trigger claim(String triggerId, long leaseMillis) { + return claim(find(triggerId), leaseMillis); + } + + /** + * {@inheritDoc} + */ + @Override + public Trigger claim(Trigger candidate, long leaseMillis) { + if (candidate == null) { + return null; + } + String triggerId = requireText(candidate.getId(), "trigger ID"); + String instanceId = requireText(candidate.getStateInstanceId(), "state instance ID"); + String claimToken = UUID.randomUUID().toString(); + long lease = Math.max(1L, leaseMillis); + String guardKey = executionGuardKey(instanceId, triggerId); + String payload = redisTemplate.execute( + CLAIM_SCRIPT, + Arrays.asList( + dataKey(triggerId), + claimKey(triggerId), + CacheKey.TRIGGER_PENDING_KEY, + guardKey, + claimGenerationKey(instanceId)), + claimToken, + String.valueOf(lease), + triggerId, + String.valueOf(System.currentTimeMillis() + lease), + String.valueOf(CLAIM_GENERATION_TTL.toMillis())); + int separator = payload == null ? -1 : payload.indexOf('\n'); + if (payload != null && separator <= 0) { + redisTemplate.delete(guardKey); + throw new IllegalStateException( + "Claim result is missing generation: " + triggerId); + } + long claimGeneration = payload == null + ? 0L + : Long.parseLong(payload.substring(0, separator)); + Trigger trigger = deserialize( + payload == null ? null : payload.substring(separator + 1)); + if (trigger != null) { + if (!triggerId.equals(trigger.getId()) + || !instanceId.equals(trigger.getStateInstanceId())) { + redisTemplate.delete(guardKey); + throw new IllegalStateException("Claimed trigger identity changed: " + triggerId); + } + trigger.setFencingToken(claimGeneration); + claimContexts.put( + trigger, + new ClaimContext(claimToken, guardKey, claimGeneration)); + } + return trigger; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean renewClaim(Trigger trigger, long leaseMillis) { + ClaimContext claim = claimContexts.get(trigger); + if (claim == null) { + return false; + } + long lease = Math.max(1L, leaseMillis); + Long renewed = redisTemplate.execute( + RENEW_SCRIPT, + Arrays.asList( + claimKey(trigger.getId()), + CacheKey.TRIGGER_PENDING_KEY, + claim.guardKey), + claim.ownerToken, + String.valueOf(lease), + trigger.getId(), + String.valueOf(System.currentTimeMillis() + lease), + String.valueOf(claim.claimGeneration)); + return Long.valueOf(1L).equals(renewed); + } + + /** + * {@inheritDoc} + */ + @Override + public void acknowledge(Trigger trigger) { + ClaimContext claim = claimContexts.get(trigger); + if (claim == null) { + throw new TriggerClaimLostException( + trigger.getId()); + } + Long acknowledged = redisTemplate.execute( + ACK_SCRIPT, + Arrays.asList( + claimKey(trigger.getId()), + dataKey(trigger.getId()), + CacheKey.TRIGGER_PENDING_KEY, + claim.guardKey), + claim.ownerToken, + trigger.getId()); + finishClaimMutation( + trigger, claim, acknowledged); + } + + /** + * {@inheritDoc} + */ + @Override + public void release(Trigger trigger) { + ClaimContext claim = claimContexts.get(trigger); + if (claim == null) { + throw new TriggerClaimLostException( + trigger.getId()); + } + Long released = redisTemplate.execute( + RELEASE_SCRIPT, + Arrays.asList( + claimKey(trigger.getId()), + CacheKey.TRIGGER_PENDING_KEY, + dataKey(trigger.getId()), + claim.guardKey), + claim.ownerToken, + trigger.getId(), + String.valueOf(trigger.getTriggerAt()), + String.valueOf(TRIGGER_TTL.toMillis()), + serialize(trigger)); + finishClaimMutation( + trigger, claim, released); + } + + /** + * {@inheritDoc} + */ + @Override + public void markDeadLetterPending( + Trigger trigger) { + ClaimContext claim = + claimContexts.get(trigger); + if (claim == null) { + throw new TriggerClaimLostException( + trigger.getId()); + } + Long marked = redisTemplate.execute( + MARK_DEAD_LETTER_PENDING_SCRIPT, + Arrays.asList( + claimKey(trigger.getId()), + dataKey(trigger.getId())), + claim.ownerToken, + String.valueOf( + TRIGGER_TTL.toMillis()), + serialize(trigger)); + if (Long.valueOf(1L).equals(marked)) { + return; + } + if (Long.valueOf(0L).equals(marked)) { + claimContexts.remove(trigger, claim); + throw new TriggerClaimLostException( + trigger.getId()); + } + throw new IllegalStateException( + "Dead-letter marker returned no result: " + + trigger.getId()); + } + + /** + * {@inheritDoc} + */ + @Override + public void deadLetter(Trigger trigger, String reason) { + ClaimContext claim = claimContexts.get(trigger); + if (claim == null) { + throw new TriggerClaimLostException( + trigger.getId()); + } + Long moved = redisTemplate.execute( + DEAD_LETTER_SCRIPT, + Arrays.asList( + claimKey(trigger.getId()), + dataKey(trigger.getId()), + CacheKey.TRIGGER_PENDING_KEY, + CacheKey.TRIGGER_DEAD_LETTER_KEY + trigger.getId(), + claim.guardKey), + claim.ownerToken, + trigger.getId(), + String.valueOf(TRIGGER_TTL.toMillis()), + serialize(trigger)); + if (Long.valueOf(1L).equals(moved)) { + claimContexts.remove(trigger, claim); + log.error( + "Workflow trigger moved to dead letter, triggerId={}, reason={}", + trigger.getId(), + reason); + return; + } + if (Long.valueOf(0L).equals(moved)) { + claimContexts.remove(trigger, claim); + throw new TriggerClaimLostException( + trigger.getId()); + } + throw new IllegalStateException( + "Dead-letter operation returned no result: " + + trigger.getId()); + } + + /** + * 校验 claim 变更结果，并在 Redis 已完成或确认失去 owner 后清理本地凭证。 + * + * @param trigger 已认领触发器 + * @param claim 本地认领上下文 + * @param result Redis 原子脚本结果 + */ + private void finishClaimMutation( + Trigger trigger, + ClaimContext claim, + Long result) { + if (Long.valueOf(1L).equals(result)) { + claimContexts.remove(trigger, claim); + return; + } + if (Long.valueOf(0L).equals(result)) { + claimContexts.remove(trigger, claim); + throw new TriggerClaimLostException( + trigger.getId()); + } + throw new IllegalStateException( + "Trigger claim mutation returned no result: " + + trigger.getId()); + } + + /** + * 清理指定触发器 ID 的所有进程内认领凭证。 + * + * @param triggerId 触发器 ID + */ + private List removeLocalClaims(String triggerId) { + List guardKeys = new ArrayList<>(); + synchronized (claimContexts) { + claimContexts.entrySet().removeIf(entry -> { + if (!triggerId.equals(entry.getKey().getId())) { + return false; + } + guardKeys.add(entry.getValue().guardKey); + return true; + }); + } + return guardKeys; + } + + /** + * 按触发时间范围批量加载触发器。 + * + * @param minScore 最小触发时间 + * @param maxScore 最大触发时间 + * @param limit 最大返回数 + * @return 保持触发时间顺序的触发器列表 + */ + private List findByScore(long minScore, long maxScore, int limit) { + java.util.Set triggerIds = redisTemplate.opsForZSet().rangeByScore( + CacheKey.TRIGGER_PENDING_KEY, minScore, maxScore, 0, limit); + if (triggerIds == null || triggerIds.isEmpty()) { + return Collections.emptyList(); + } + List ids = new ArrayList<>(triggerIds); + List keys = new ArrayList<>(ids.size()); + for (String triggerId : ids) { + keys.add(dataKey(triggerId)); + } + List payloads = redisTemplate.opsForValue().multiGet(keys); + List triggers = new ArrayList<>(ids.size()); + if (payloads == null) { + return triggers; + } + for (int index = 0; index < payloads.size(); index++) { + String payload = payloads.get(index); + Trigger trigger; + try { + trigger = deserialize(payload); + } catch (IllegalStateException error) { + quarantine(ids.get(index), payload, error); + continue; + } + if (trigger != null) { + triggers.add(trigger); + } else { + redisTemplate.opsForZSet().remove(CacheKey.TRIGGER_PENDING_KEY, ids.get(index)); + } + } + return triggers; + } + + /** + * 隔离无法反序列化的触发器，避免毒数据持续阻断批量扫描。 + * + * @param triggerId 触发器 ID + * @param payload 原始负载 + * @param error 解析异常 + */ + private void quarantine(String triggerId, String payload, RuntimeException error) { + if (payload != null) { + redisTemplate.opsForValue().set( + CacheKey.TRIGGER_DEAD_LETTER_KEY + triggerId, + payload, + TRIGGER_TTL); + } + remove(triggerId); + log.error("Quarantined invalid workflow trigger payload, triggerId={}", triggerId, error); + } + + /** + * 序列化触发器。 + * + * @param trigger 触发器 + * @return JSON 文本 + */ + private String serialize(Trigger trigger) { + try { + return objectMapper.writeValueAsString(trigger); + } catch (JsonProcessingException error) { + throw new IllegalStateException("Failed to serialize workflow trigger: " + trigger.getId(), error); + } + } + + /** + * 反序列化触发器。 + * + * @param payload JSON 文本 + * @return 触发器；输入为空时返回 null + */ + private Trigger deserialize(String payload) { + if (payload == null) { + return null; + } + try { + return objectMapper.readValue(payload, Trigger.class); + } catch (JsonProcessingException error) { + throw new IllegalStateException("Failed to deserialize workflow trigger", error); + } + } + + /** + * 构建触发器数据键。 + * + * @param triggerId 触发器 ID + * @return Redis 键 + */ + private String dataKey(String triggerId) { + return CacheKey.TRIGGER_DATA_KEY + triggerId; + } + + /** + * 构建触发器认领键。 + * + * @param triggerId 触发器 ID + * @return Redis 键 + */ + private String claimKey(String triggerId) { + return CacheKey.TRIGGER_CLAIM_KEY + triggerId; + } + + /** + * 构建触发器认领代际分配计数器键。 + * + * @param instanceId 工作流实例 ID + * @return 认领代际计数器键 + */ + private String claimGenerationKey(String instanceId) { + return CacheKey.CHAIN_STATE_CAS_KEY + "{" + instanceId + "}:claim-seq"; + } + + /** + * 构建实例锁 fencing token 键。 + * + * @param instanceId 工作流实例 ID + * @return fencing token 键 + */ + private String lockFenceKey(String instanceId) { + return CacheKey.CHAIN_STATE_CAS_KEY + + "{" + + requireText(instanceId, "state instance ID") + + "}:fence"; + } + + /** + * 构建与一次触发器认领绑定的执行守卫键。 + * + * @param instanceId 工作流实例 ID + * @param claimId 触发器 ID + * @return 执行守卫键 + */ + private String executionGuardKey(String instanceId, String claimId) { + return CacheKey.CHAIN_STATE_CAS_KEY + + "{" + + requireText(instanceId, "state instance ID") + + "}:claim:" + + requireText(claimId, "claim ID"); + } + + /** + * 校验必填文本。 + * + * @param value 原始值 + * @param name 字段名称 + * @return 去除首尾空白后的值 + */ + private String requireText(String value, String name) { + if (value == null || value.trim().isEmpty()) { + throw new IllegalArgumentException(name + " must not be blank"); + } + return value.trim(); + } + + /** + * 当前进程持有的一次触发器认领上下文。 + */ + private static final class ClaimContext { + + private final String ownerToken; + private final String guardKey; + private final long claimGeneration; + + /** + * 创建认领上下文。 + * + * @param ownerToken Redis claim owner token + * @param guardKey 执行守卫键 + * @param claimGeneration 本次认领的单调代际 + */ + private ClaimContext( + String ownerToken, String guardKey, long claimGeneration) { + this.ownerToken = ownerToken; + this.guardKey = guardKey; + this.claimGeneration = claimGeneration; + } + } + + /** + * 创建 Long 返回值的 Redis 脚本。 + * + * @param scriptText Lua 脚本文本 + * @return Redis 脚本 + */ + private static DefaultRedisScript longScript(String scriptText) { + DefaultRedisScript script = new DefaultRedisScript<>(); + script.setScriptText(scriptText); + script.setResultType(Long.class); + return script; + } + + /** + * 创建字符串返回值的 Redis 脚本。 + * + * @param scriptText Lua 脚本文本 + * @return Redis 脚本 + */ + private static DefaultRedisScript stringScript(String scriptText) { + DefaultRedisScript script = new DefaultRedisScript<>(); + script.setScriptText(scriptText); + script.setResultType(String.class); + return script; + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/RedisWorkflowDefinitionVersionStore.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/RedisWorkflowDefinitionVersionStore.java new file mode 100644 index 00000000..206fe99f --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/RedisWorkflowDefinitionVersionStore.java @@ -0,0 +1,74 @@ +package tech.easyflow.ai.easyagentsflow.repository; + +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.stereotype.Component; +import tech.easyflow.ai.easyagentsflow.support.PublishedWorkflowDefinitionIds; +import tech.easyflow.common.constant.CacheKey; + +import java.time.Duration; +import java.util.UUID; + +/** + * 基于 Redis 的工作流定义缓存版本令牌仓储。 + */ +@Component +public class RedisWorkflowDefinitionVersionStore implements WorkflowDefinitionVersionStore { + + private static final Duration TOKEN_TTL = Duration.ofDays(7); + + private final StringRedisTemplate redisTemplate; + + /** + * 创建版本令牌仓储。 + * + * @param redisTemplate Redis 字符串模板 + */ + public RedisWorkflowDefinitionVersionStore(StringRedisTemplate redisTemplate) { + this.redisTemplate = redisTemplate; + } + + /** + * {@inheritDoc} + */ + @Override + public String currentToken(String definitionId) { + String key = versionKey(definitionId); + String current = redisTemplate.opsForValue().get(key); + if (current != null) { + return current; + } + String candidate = UUID.randomUUID().toString(); + Boolean created = redisTemplate.opsForValue().setIfAbsent(key, candidate, TOKEN_TTL); + if (Boolean.TRUE.equals(created)) { + return candidate; + } + current = redisTemplate.opsForValue().get(key); + if (current == null) { + throw new IllegalStateException("Workflow definition version token is unavailable: " + definitionId); + } + return current; + } + + /** + * {@inheritDoc} + */ + @Override + public void invalidateWorkflow(String workflowId) { + String token = UUID.randomUUID().toString(); + redisTemplate.opsForValue().set(versionKey(workflowId), token, TOKEN_TTL); + redisTemplate.opsForValue().set( + versionKey(PublishedWorkflowDefinitionIds.published(workflowId)), + token, + TOKEN_TTL); + } + + /** + * 构建版本令牌 Redis 键。 + * + * @param definitionId 定义 ID + * @return Redis 键 + */ + private String versionKey(String definitionId) { + return CacheKey.WORKFLOW_DEFINITION_VERSION_KEY + definitionId; + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/WorkflowDefinitionCache.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/WorkflowDefinitionCache.java new file mode 100644 index 00000000..34968176 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/WorkflowDefinitionCache.java @@ -0,0 +1,224 @@ +package tech.easyflow.ai.easyagentsflow.repository; + +import com.easyagents.flow.core.chain.ChainDefinition; +import org.springframework.stereotype.Component; +import org.springframework.transaction.event.TransactionPhase; +import org.springframework.transaction.event.TransactionalEventListener; +import tech.easyflow.ai.easyagentsflow.config.WorkflowRuntimeProperties; +import tech.easyflow.ai.easyagentsflow.event.WorkflowDefinitionChangedEvent; +import tech.easyflow.ai.easyagentsflow.support.PublishedWorkflowDefinitionIds; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectOutputStream; +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Supplier; + +/** + * 带跨实例版本校验和本地有界 LRU 的工作流编译定义缓存。 + */ +@Component +public class WorkflowDefinitionCache { + + private static final int MAX_COMPILE_RETRIES = 3; + + private final WorkflowDefinitionVersionStore versionStore; + private final int maxEntries; + private final long maxBytes; + private final long expireAfterAccessNanos; + private final Map entries = new LinkedHashMap<>(16, 0.75F, true); + private final ConcurrentMap compileLocks = new ConcurrentHashMap<>(); + private long currentBytes; + + /** + * 创建工作流定义缓存。 + * + * @param versionStore 跨实例版本令牌仓储 + * @param properties 工作流运行时配置 + */ + public WorkflowDefinitionCache( + WorkflowDefinitionVersionStore versionStore, WorkflowRuntimeProperties properties) { + this.versionStore = versionStore; + this.maxEntries = Math.max(1, properties.getDefinitionCacheMaxEntries()); + this.maxBytes = Math.max( + 1L, properties.getDefinitionCacheMaxBytes()); + Duration expireAfterAccess = properties.getDefinitionCacheExpireAfterAccess(); + this.expireAfterAccessNanos = expireAfterAccess == null + ? Duration.ofMinutes(30).toNanos() + : Math.max(1L, expireAfterAccess.toNanos()); + } + + /** + * 获取已编译定义；缓存未命中时只允许一个线程执行加载与编译。 + * + * @param definitionId 定义 ID + * @param loader 定义加载与编译函数 + * @return 已编译工作流定义 + */ + public ChainDefinition get(String definitionId, Supplier loader) { + String token = versionStore.currentToken(definitionId); + ChainDefinition cached = getCached(definitionId, token); + if (cached != null) { + return cached; + } + + Object compileLock = compileLocks.computeIfAbsent(definitionId, ignored -> new Object()); + try { + synchronized (compileLock) { + for (int attempt = 0; attempt < MAX_COMPILE_RETRIES; attempt++) { + token = versionStore.currentToken(definitionId); + cached = getCached(definitionId, token); + if (cached != null) { + return cached; + } + ChainDefinition compiled = loader.get(); + String tokenAfterCompile = versionStore.currentToken(definitionId); + if (token.equals(tokenAfterCompile)) { + put(definitionId, token, compiled); + return compiled; + } + } + } + } finally { + compileLocks.remove(definitionId, compileLock); + } + throw new IllegalStateException( + "Workflow definition changed repeatedly while compiling: " + definitionId); + } + + /** + * 处理定义变更并更新跨实例版本令牌。 + * + * @param event 工作流定义变更事件 + */ + @TransactionalEventListener( + phase = TransactionPhase.AFTER_COMMIT, + fallbackExecution = true) + public void onDefinitionChanged(WorkflowDefinitionChangedEvent event) { + if (event == null || event.workflowId() == null) { + return; + } + versionStore.invalidateWorkflow(event.workflowId()); + synchronized (entries) { + removeEntry(event.workflowId()); + removeEntry(PublishedWorkflowDefinitionIds.published( + event.workflowId())); + } + } + + /** + * 获取仍有效的本地缓存项。 + * + * @param definitionId 定义 ID + * @param token 当前版本令牌 + * @return 命中的定义；未命中时返回 null + */ + private ChainDefinition getCached(String definitionId, String token) { + long now = System.nanoTime(); + synchronized (entries) { + CacheEntry entry = entries.get(definitionId); + if (entry == null) { + return null; + } + if (!entry.token.equals(token) || now - entry.lastAccessNanos > expireAfterAccessNanos) { + removeEntry(definitionId); + return null; + } + entry.lastAccessNanos = now; + return entry.definition; + } + } + + /** + * 保存本地缓存并按 LRU 淘汰。 + * + * @param definitionId 定义 ID + * @param token 版本令牌 + * @param definition 已编译定义 + */ + private void put(String definitionId, String token, ChainDefinition definition) { + synchronized (entries) { + removeEntry(definitionId); + long weight = serializedSize(definition); + entries.put(definitionId, new CacheEntry( + token, + definition, + System.nanoTime(), + weight)); + currentBytes += weight; + while (entries.size() > maxEntries + || (currentBytes > maxBytes + && entries.size() > 1)) { + String eldestKey = entries.keySet().iterator().next(); + removeEntry(eldestKey); + } + } + } + + /** + * 删除缓存项并同步维护重量。 + * + * @param definitionId 定义 ID + */ + private void removeEntry(String definitionId) { + CacheEntry removed = entries.remove(definitionId); + if (removed != null) { + currentBytes = Math.max( + 0L, currentBytes - removed.weightBytes); + } + } + + /** + * 使用实际 Java 序列化大小作为缓存重量。 + * + * @param definition 编译定义 + * @return 序列化字节数 + */ + private long serializedSize(ChainDefinition definition) { + try (ByteArrayOutputStream output = + new ByteArrayOutputStream(); + ObjectOutputStream objects = + new ObjectOutputStream(output)) { + objects.writeObject(definition); + objects.flush(); + return Math.max(1L, output.size()); + } catch (IOException error) { + throw new IllegalStateException( + "Failed to estimate workflow definition size", + error); + } + } + + /** + * 本地定义缓存项。 + */ + private static final class CacheEntry { + + private final String token; + private final ChainDefinition definition; + private final long weightBytes; + private long lastAccessNanos; + + /** + * 创建本地缓存项。 + * + * @param token 版本令牌 + * @param definition 已编译定义 + * @param lastAccessNanos 最近访问时间 + */ + private CacheEntry( + String token, + ChainDefinition definition, + long lastAccessNanos, + long weightBytes) { + this.token = token; + this.definition = definition; + this.lastAccessNanos = lastAccessNanos; + this.weightBytes = weightBytes; + } + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/WorkflowDefinitionVersionStore.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/WorkflowDefinitionVersionStore.java new file mode 100644 index 00000000..0b4a407e --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/WorkflowDefinitionVersionStore.java @@ -0,0 +1,22 @@ +package tech.easyflow.ai.easyagentsflow.repository; + +/** + * 工作流定义缓存版本令牌仓储。 + */ +public interface WorkflowDefinitionVersionStore { + + /** + * 获取定义当前版本令牌，不存在时原子创建。 + * + * @param definitionId 定义 ID，包含可选发布态前缀 + * @return 当前版本令牌 + */ + String currentToken(String definitionId); + + /** + * 同时使指定工作流的草稿和发布态版本令牌失效。 + * + * @param workflowId 工作流 ID + */ + void invalidateWorkflow(String workflowId); +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/WorkflowStateFields.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/WorkflowStateFields.java new file mode 100644 index 00000000..c8574da2 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/repository/WorkflowStateFields.java @@ -0,0 +1,288 @@ +package tech.easyflow.ai.easyagentsflow.repository; + +import com.easyagents.flow.core.chain.ChainState; +import com.easyagents.flow.core.chain.NodeState; +import com.easyagents.flow.core.chain.repository.ChainStateField; +import com.easyagents.flow.core.chain.repository.NodeStateField; +import tech.easyflow.ai.easyagentsflow.support.WorkflowExecutionStepKey; +import tech.easyflow.common.cache.VersionedFields; + +import java.io.Serializable; +import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * 工作流状态对象与 Redis 字段之间的无反射映射。 + */ +final class WorkflowStateFields { + + static final String FORMAT_FIELD = "_format"; + static final String FORMAT_VERSION = "2"; + private static final String NODE_ID_FIELD = "_nodeId"; + private static final String CHAIN_INSTANCE_ID_FIELD = "_chainInstanceId"; + + private WorkflowStateFields() { + } + + /** + * 判断快照是否采用字段化格式。 + * + * @param snapshot Redis 快照 + * @return 字段化格式时为 {@code true} + */ + static boolean isFieldFormat(VersionedFields snapshot) { + return snapshot != null + && FORMAT_VERSION.equals(snapshot.getFields().get(FORMAT_FIELD)); + } + + /** + * 将完整工作流状态编码为字段。 + * + * @param state 工作流状态 + * @return 完整字段映射 + */ + static Map allChainFields(ChainState state) { + EnumSet fields = EnumSet.allOf(ChainStateField.class); + fields.remove(ChainStateField.VERSION); + return chainFields(state, fields); + } + + /** + * 将变化的工作流状态字段编码为可独立提交的值。 + * + * @param state 工作流状态 + * @param fields 变化字段 + * @return 字段映射 + */ + static Map chainFields( + ChainState state, EnumSet fields) { + Map values = new LinkedHashMap<>(); + values.put(FORMAT_FIELD, FORMAT_VERSION); + for (ChainStateField field : fields) { + Serializable value = switch (field) { + case INSTANCE_ID -> state.getInstanceId(); + case STATUS -> state.getStatus(); + case MESSAGE -> state.getMessage(); + case ERROR -> state.getError(); + case MEMORY -> state.getMemory(); + case COMPUTE_COST -> state.getComputeCost(); + case SUSPEND_NODE_IDS -> serializable(state.getSuspendNodeIds()); + case SUSPEND_FOR_PARAMETERS -> serializable(state.getSuspendForParameters()); + case EXECUTE_RESULT -> serializable(state.getExecuteResult()); + case CHAIN_DEFINITION_ID -> state.getChainDefinitionId(); + case ENVIRONMENT -> serializable(state.getEnvironment()); + case PARENT_INSTANCE_ID -> state.getParentInstanceId(); + case AUDIT_INSTANCE_ID -> state.getAuditInstanceId(); + case TRIGGER_NODE_IDS -> serializable(state.getTriggerNodeIds()); + case TRIGGER_EDGE_IDS -> serializable(state.getTriggerEdgeIds()); + case UNCHECKED_EDGE_IDS -> serializable(state.getUncheckedEdgeIds()); + case UNCHECKED_NODE_IDS -> serializable(state.getUncheckedNodeIds()); + case STARTED_AT -> state.getStartedAt(); + case CHILD_EXECUTION_COUNT -> state.getChildExecutionCount(); + case VERSION, PAYLOAD, NODE_STATES, CHILD_STATE_IDS -> null; + }; + if (field != ChainStateField.VERSION + && field != ChainStateField.PAYLOAD + && field != ChainStateField.NODE_STATES + && field != ChainStateField.CHILD_STATE_IDS) { + values.put(field.name(), value); + } + } + return values; + } + + /** + * 从字段快照还原工作流状态。 + * + * @param snapshot Redis 字段快照 + * @return 工作流状态 + */ + @SuppressWarnings("unchecked") + static ChainState decodeChain(VersionedFields snapshot) { + Map fields = snapshot.getFields(); + ChainState state = new ChainState(); + state.setInstanceId((String) fields.get(ChainStateField.INSTANCE_ID.name())); + state.setStatus((com.easyagents.flow.core.chain.ChainStatus) + fields.get(ChainStateField.STATUS.name())); + state.setMessage((String) fields.get(ChainStateField.MESSAGE.name())); + state.setError((com.easyagents.flow.core.chain.ExceptionSummary) + fields.get(ChainStateField.ERROR.name())); + Object memory = fields.get(ChainStateField.MEMORY.name()); + state.setMemory(memory == null + ? new ConcurrentHashMap<>() + : new ConcurrentHashMap<>((Map) memory)); + state.setComputeCost(number(fields.get(ChainStateField.COMPUTE_COST.name()))); + state.setSuspendNodeIds((java.util.Set) + fields.get(ChainStateField.SUSPEND_NODE_IDS.name())); + state.setSuspendForParameters((java.util.List) + fields.get(ChainStateField.SUSPEND_FOR_PARAMETERS.name())); + state.setExecuteResult((Map) + fields.get(ChainStateField.EXECUTE_RESULT.name())); + state.setChainDefinitionId((String) + fields.get(ChainStateField.CHAIN_DEFINITION_ID.name())); + state.setEnvironment((Map) + fields.get(ChainStateField.ENVIRONMENT.name())); + state.setParentInstanceId((String) + fields.get(ChainStateField.PARENT_INSTANCE_ID.name())); + state.setAuditInstanceId((String) + fields.get(ChainStateField.AUDIT_INSTANCE_ID.name())); + state.setTriggerNodeIds((java.util.List) + fields.get(ChainStateField.TRIGGER_NODE_IDS.name())); + state.setTriggerEdgeIds((java.util.List) + fields.get(ChainStateField.TRIGGER_EDGE_IDS.name())); + state.setUncheckedEdgeIds((java.util.List) + fields.get(ChainStateField.UNCHECKED_EDGE_IDS.name())); + state.setUncheckedNodeIds((java.util.List) + fields.get(ChainStateField.UNCHECKED_NODE_IDS.name())); + state.setStartedAt(number(fields.get(ChainStateField.STARTED_AT.name()))); + state.setChildExecutionCount(number( + fields.get(ChainStateField.CHILD_EXECUTION_COUNT.name()))); + state.setVersion(snapshot.getVersion()); + return state; + } + + /** + * 将完整节点状态编码为字段。 + * + * @param state 节点状态 + * @return 完整字段映射 + */ + static Map allNodeFields(NodeState state) { + EnumSet fields = EnumSet.allOf(NodeStateField.class); + fields.remove(NodeStateField.VERSION); + return nodeFields(state, fields); + } + + /** + * 将变化的节点状态字段编码为可独立提交的值。 + * + * @param state 节点状态 + * @param fields 变化字段 + * @return 字段映射 + */ + static Map nodeFields( + NodeState state, EnumSet fields) { + Map values = new LinkedHashMap<>(); + values.put(FORMAT_FIELD, FORMAT_VERSION); + values.put(NODE_ID_FIELD, state.getNodeId()); + values.put(CHAIN_INSTANCE_ID_FIELD, state.getChainInstanceId()); + for (NodeStateField field : fields) { + Serializable value = switch (field) { + case STATUS -> state.getStatus(); + case ERROR -> state.getError(); + case MEMORY -> state.getMemory(); + case RETRY_COUNT -> state.getRetryCount(); + case EXECUTE_COUNT -> state.getExecuteCount(); + case EXECUTE_EDGE_IDS -> serializable(state.getExecuteEdgeIds()); + case EXECUTION_ATTEMPT_KEY -> state.getExecutionAttemptKey(); + case LOOP_COUNT -> state.getLoopCount(); + case TRIGGER_COUNT -> state.getTriggerCount(); + case TRIGGER_EDGE_IDS -> serializable(state.getTriggerEdgeIds()); + case INSTANCE_ID, MESSAGE, PAYLOAD, NODE_STATES, COMPUTE_COST, + SUSPEND_NODE_IDS, SUSPEND_FOR_PARAMETERS, EXECUTE_RESULT, + ENVIRONMENT, VERSION -> null; + }; + if (field == NodeStateField.STATUS + || field == NodeStateField.ERROR + || field == NodeStateField.MEMORY + || field == NodeStateField.RETRY_COUNT + || field == NodeStateField.EXECUTE_COUNT + || field == NodeStateField.EXECUTE_EDGE_IDS + || field == NodeStateField.EXECUTION_ATTEMPT_KEY + || field == NodeStateField.LOOP_COUNT + || field == NodeStateField.TRIGGER_COUNT + || field == NodeStateField.TRIGGER_EDGE_IDS) { + values.put(field.name(), value); + } + } + return values; + } + + /** + * 从字段快照还原节点状态。 + * + * @param snapshot Redis 字段快照 + * @return 节点状态 + */ + @SuppressWarnings("unchecked") + static NodeState decodeNode(VersionedFields snapshot) { + Map fields = snapshot.getFields(); + NodeState state = new NodeState(); + state.setNodeId((String) fields.get(NODE_ID_FIELD)); + state.setChainInstanceId((String) fields.get(CHAIN_INSTANCE_ID_FIELD)); + state.setStatus((com.easyagents.flow.core.chain.NodeStatus) + fields.get(NodeStateField.STATUS.name())); + state.setError((com.easyagents.flow.core.chain.ExceptionSummary) + fields.get(NodeStateField.ERROR.name())); + Object memory = fields.get(NodeStateField.MEMORY.name()); + state.setMemory(memory == null + ? new ConcurrentHashMap<>() + : new ConcurrentHashMap<>((Map) memory)); + state.setRetryCount(integer(fields.get(NodeStateField.RETRY_COUNT.name()))); + state.setExecuteCount(atomic(fields.get(NodeStateField.EXECUTE_COUNT.name()))); + state.setExecuteEdgeIds(defaultList( + (java.util.List) fields.get(NodeStateField.EXECUTE_EDGE_IDS.name()))); + state.setExecutionAttemptKey( + (String) fields.get( + NodeStateField + .EXECUTION_ATTEMPT_KEY + .name())); + state.setLoopCount(integer(fields.get(NodeStateField.LOOP_COUNT.name()))); + state.setTriggerCount(atomic(fields.get(NodeStateField.TRIGGER_COUNT.name()))); + state.setTriggerEdgeIds(defaultList( + (java.util.List) fields.get(NodeStateField.TRIGGER_EDGE_IDS.name()))); + state.setVersion(snapshot.getVersion()); + return normalizeNode(state); + } + + /** + * 补齐升级前节点快照缺失的执行尝试键。 + * + * @param state 字段化或旧对象节点状态 + * @return 原节点状态 + */ + static NodeState normalizeNode(NodeState state) { + if (state == null + || (state.getExecutionAttemptKey() != null + && !state.getExecutionAttemptKey().isBlank()) + || state.getMemory() == null) { + return state; + } + Object legacyExecKey = + state.getMemory().get( + "executeId"); + if (legacyExecKey instanceof String) { + state.setExecutionAttemptKey( + WorkflowExecutionStepKey + .encodeLegacy( + (String) legacyExecKey)); + } + return state; + } + + private static Serializable serializable(Object value) { + return value == null ? null : (Serializable) value; + } + + private static long number(Object value) { + return value instanceof Number ? ((Number) value).longValue() : 0L; + } + + private static int integer(Object value) { + return value instanceof Number ? ((Number) value).intValue() : 0; + } + + private static AtomicInteger atomic(Object value) { + if (value instanceof AtomicInteger) { + return new AtomicInteger(((AtomicInteger) value).get()); + } + return new AtomicInteger(integer(value)); + } + + private static java.util.List defaultList(java.util.List value) { + return value == null ? new java.util.ArrayList<>() : new java.util.ArrayList<>(value); + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/service/TinyFlowService.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/service/TinyFlowService.java index 683a47fd..b6784778 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/service/TinyFlowService.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/service/TinyFlowService.java @@ -3,6 +3,7 @@ package tech.easyflow.ai.easyagentsflow.service; import com.easyagents.flow.core.chain.ChainState; import com.easyagents.flow.core.chain.ExceptionSummary; import com.easyagents.flow.core.chain.NodeState; +import com.easyagents.flow.core.chain.NodeStatus; import com.easyagents.flow.core.chain.repository.ChainStateRepository; import com.easyagents.flow.core.chain.repository.NodeStateRepository; import com.easyagents.flow.core.chain.runtime.ChainExecutor; @@ -14,14 +15,24 @@ import javax.annotation.Resource; import java.util.List; import java.util.Map; +/** + * 为工作流设计器提供执行状态查询与结果解析能力。 + */ @Component public class TinyFlowService { + /** + * 工作流执行器及其状态仓储入口。 + */ @Resource private ChainExecutor chainExecutor; /** - * 获取执行状态 + * 获取工作流及其节点的执行状态。 + * + * @param executeId 工作流执行实例 ID + * @param nodes 设计器中的节点列表 + * @return 工作流执行状态 */ public ChainInfo getChainStatus(String executeId, List nodes) { @@ -33,7 +44,7 @@ public class TinyFlowService { if (nodes != null) { for (NodeInfo node : nodes) { - processNodeState(executeId, node, chainStateRepository, nodeStateRepository); + processNodeState(executeId, node, chainState, nodeStateRepository); res.getNodes().put(node.getNodeId(), node); } } @@ -41,21 +52,31 @@ public class TinyFlowService { } /** - * 处理节点状态 + * 使用同一工作流状态快照补充节点状态，避免轮询期间重复读取工作流状态。 + * + * @param currentExecuteId 工作流执行实例 ID + * @param node 待补充状态的节点 + * @param currentChainState 当前轮询取得的工作流状态快照 + * @param nodeStateRepository 节点状态仓储 */ private void processNodeState(String currentExecuteId, NodeInfo node, - ChainStateRepository chainStateRepository, + ChainState currentChainState, NodeStateRepository nodeStateRepository) { - // 加载当前层的状态 - ChainState currentChainState = chainStateRepository.load(currentExecuteId); NodeState currentNodeState = nodeStateRepository.load(currentExecuteId, node.getNodeId()); setNodeStatus(node, currentNodeState, currentChainState); } - private static ChainInfo getChainInfo(String executeId, ChainState chainState) { + /** + * 将工作流状态转换为设计器响应。 + * + * @param executeId 工作流执行实例 ID + * @param chainState 工作流状态快照 + * @return 设计器工作流状态 + */ + private ChainInfo getChainInfo(String executeId, ChainState chainState) { ChainInfo res = new ChainInfo(); res.setExecuteId(executeId); res.setStatus(chainState.getStatus().getValue()); @@ -65,24 +86,41 @@ public class TinyFlowService { } Map executeResult = chainState.getExecuteResult(); if (executeResult != null && !executeResult.isEmpty()) { - res.setResult(executeResult); + @SuppressWarnings("unchecked") + Map resolved = (Map) + chainExecutor.resolveResultReferences(executeResult); + res.setResult(resolved); } return res; } + /** + * 将节点状态和节点执行结果写入设计器节点。 + * + * @param node 设计器节点 + * @param nodeState 节点状态；节点尚未开始执行时可为空 + * @param chainState 工作流状态快照 + */ private void setNodeStatus(NodeInfo node, NodeState nodeState, ChainState chainState) { String nodeId = node.getNodeId(); - // 如果状态为空或不存在，可能不需要覆盖，这里视具体业务逻辑而定，目前保持原逻辑 - node.setStatus(nodeState.getStatus().getValue()); + // 旧仓储会为未启动节点返回 READY 状态；纯读取仓储返回空时保持相同行为但不产生写入。 + node.setStatus(nodeState == null + ? NodeStatus.READY.getValue() + : nodeState.getStatus().getValue()); - ExceptionSummary error = nodeState.getError(); - if (error != null) { - node.setMessage(error.getRootCauseClass() + " --> " + error.getRootCauseMessage()); + if (nodeState != null) { + ExceptionSummary error = nodeState.getError(); + if (error != null) { + node.setMessage(error.getRootCauseClass() + " --> " + error.getRootCauseMessage()); + } } Map nodeExecuteResult = chainState.getNodeExecuteResult(nodeId); if (nodeExecuteResult != null && !nodeExecuteResult.isEmpty()) { - node.setResult(nodeExecuteResult); + @SuppressWarnings("unchecked") + Map resolved = (Map) + chainExecutor.resolveResultReferences(nodeExecuteResult); + node.setResult(resolved); } // 只有当参数不为空时才覆盖 diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/service/WorkflowCheckService.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/service/WorkflowCheckService.java index 3a678ada..13464c3c 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/service/WorkflowCheckService.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/service/WorkflowCheckService.java @@ -22,6 +22,7 @@ import tech.easyflow.common.web.exceptions.BusinessException; import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse; import javax.annotation.Resource; +import java.math.BigDecimal; import java.math.BigInteger; import java.util.ArrayDeque; import java.util.ArrayList; @@ -50,6 +51,8 @@ public class WorkflowCheckService { private static final String TYPE_PLUGIN = "plugin-node"; private static final String TYPE_MAKE_FILE = "make-file"; private static final String SYSTEM_START_PARAM_NAME = "user_input"; + private static final int MIN_LOOP_COUNT = 1; + private static final int MAX_LOOP_COUNT = 300; @Resource private WorkflowService workflowService; @@ -171,6 +174,7 @@ public class WorkflowCheckService { "父节点不存在: " + node.parentId, node.id, null, node.name); } } + checkLoopConfigurations(nodes, nodeMap, issues, issueKeys); List edges = new ArrayList<>(); Set edgeIds = new HashSet<>(); @@ -221,6 +225,159 @@ public class WorkflowCheckService { return parsedWorkflow; } + /** + * 校验普通循环、显式循环和循环父子层级。 + * + * @param nodes 节点列表 + * @param nodeMap 节点索引 + * @param issues 问题列表 + * @param issueKeys 问题去重键 + */ + private void checkLoopConfigurations( + List nodes, + Map nodeMap, + List issues, + Set issueKeys) { + for (NodeView node : nodes) { + checkConfiguredLoopCount(node, issues, issueKeys); + checkFixedExplicitLoopCount(node, issues, issueKeys); + if (StringUtils.hasText(node.parentId)) { + NodeView parent = nodeMap.get(node.parentId); + if (parent != null && !TYPE_LOOP.equals(parent.type)) { + addIssue( + issues, + issueKeys, + "NODE_PARENT_NOT_LOOP", + "嵌套节点的父节点必须是循环节点", + node.id, + null, + node.name); + } + } + checkLoopParentCycle(node, nodeMap, issues, issueKeys); + } + } + + /** + * 校验普通节点启用循环后的总执行次数。 + * + * @param node 节点 + * @param issues 问题列表 + * @param issueKeys 问题去重键 + */ + private void checkConfiguredLoopCount( + NodeView node, + List issues, + Set issueKeys) { + if (node.data == null + || !Boolean.TRUE.equals(node.data.getBoolean("loopEnable"))) { + return; + } + Object value = node.data.get("maxLoopCount"); + if (value != null) { + addLoopCountIssueIfInvalid( + value, "LOOP_COUNT_INVALID", node, issues, issueKeys); + } + } + + /** + * 校验显式循环节点使用固定数值时的次数范围。 + * + * @param node 节点 + * @param issues 问题列表 + * @param issueKeys 问题去重键 + */ + private void checkFixedExplicitLoopCount( + NodeView node, + List issues, + Set issueKeys) { + if (!TYPE_LOOP.equals(node.type) || node.data == null) { + return; + } + JSONArray loopVars = node.data.getJSONArray("loopVars"); + if (loopVars == null || loopVars.isEmpty()) { + return; + } + JSONObject loopVar = loopVars.getJSONObject(0); + if (loopVar == null || !"fixed".equals(loopVar.getString("refType"))) { + return; + } + Object value = loopVar.get("value"); + if (value != null && StringUtils.hasText(String.valueOf(value))) { + addLoopCountIssueIfInvalid( + value, + "EXPLICIT_LOOP_COUNT_INVALID", + node, + issues, + issueKeys); + } + } + + /** + * 在次数值无效时添加校验问题。 + * + * @param value 原始次数 + * @param code 问题编码 + * @param node 节点 + * @param issues 问题列表 + * @param issueKeys 问题去重键 + */ + private void addLoopCountIssueIfInvalid( + Object value, + String code, + NodeView node, + List issues, + Set issueKeys) { + try { + int count = new BigDecimal(String.valueOf(value).trim()) + .intValueExact(); + if (count >= MIN_LOOP_COUNT && count <= MAX_LOOP_COUNT) { + return; + } + } catch (ArithmeticException | NumberFormatException ignored) { + // 统一在下方返回用户可执行的范围提示。 + } + addIssue( + issues, + issueKeys, + code, + "循环次数必须是 1～300 的整数", + node.id, + null, + node.name); + } + + /** + * 校验 parentId 层级不存在循环引用。 + * + * @param node 起始节点 + * @param nodeMap 节点索引 + * @param issues 问题列表 + * @param issueKeys 问题去重键 + */ + private void checkLoopParentCycle( + NodeView node, + Map nodeMap, + List issues, + Set issueKeys) { + Set visited = new HashSet<>(); + NodeView current = node; + while (current != null && StringUtils.hasText(current.parentId)) { + if (!visited.add(current.id)) { + addIssue( + issues, + issueKeys, + "LOOP_PARENT_CYCLE", + "循环嵌套层级存在循环引用", + node.id, + null, + node.name); + return; + } + current = nodeMap.get(current.parentId); + } + } + private void checkDatacenterNodes(ParsedWorkflow parsed, List issues, Set issueKeys) { for (NodeView node : parsed.nodes) { if (node == null) { diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/support/WorkflowExecutionStepKey.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/support/WorkflowExecutionStepKey.java new file mode 100644 index 00000000..4a23214c --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/support/WorkflowExecutionStepKey.java @@ -0,0 +1,47 @@ +package tech.easyflow.ai.easyagentsflow.support; + +import cn.hutool.core.util.StrUtil; +import cn.hutool.crypto.digest.DigestUtil; + +/** + * 工作流节点执行步骤键转换工具。 + */ +public final class WorkflowExecutionStepKey { + + private static final String LEGACY_PREFIX = + "easyflow-legacy-exec-key:"; + + private WorkflowExecutionStepKey() { + } + + /** + * 将旧快照中的最终执行键编码为可随节点生命周期传递的兼容键。 + * + * @param execKey 旧版最终执行键 + * @return 兼容键；输入为空时为 {@code null} + */ + public static String encodeLegacy(String execKey) { + return StrUtil.isBlank(execKey) + ? null + : LEGACY_PREFIX + execKey; + } + + /** + * 将节点业务尝试键转换为最终执行步骤键。 + * + * @param executionAttemptKey 业务尝试键或旧版兼容键 + * @return 最终执行步骤键；输入为空时为 {@code null} + */ + public static String resolve(String executionAttemptKey) { + if (StrUtil.isBlank(executionAttemptKey)) { + return null; + } + if (executionAttemptKey.startsWith( + LEGACY_PREFIX)) { + return executionAttemptKey.substring( + LEGACY_PREFIX.length()); + } + return DigestUtil.sha256Hex( + executionAttemptKey); + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/PluginItem.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/PluginItem.java index 8873e6a5..2fc2d79f 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/PluginItem.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/PluginItem.java @@ -4,6 +4,7 @@ import com.easyagents.core.model.chat.tool.Tool; import com.mybatisflex.annotation.Column; import com.mybatisflex.annotation.Table; import tech.easyflow.ai.easyagents.tool.PluginTool; +import tech.easyflow.ai.entity.Plugin; import tech.easyflow.ai.entity.base.PluginItemBase; @@ -31,4 +32,14 @@ public class PluginItem extends PluginItemBase { return new PluginTool(this); } + /** + * 使用调用方已经加载的插件快照创建工具，避免执行热路径重复查询。 + * + * @param plugin 插件快照 + * @return 插件工具 + */ + public Tool toFunction(Plugin plugin) { + return new PluginTool(this, plugin); + } + } diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/WorkflowExecResult.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/WorkflowExecResult.java index 89c9e4f1..92858585 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/WorkflowExecResult.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/WorkflowExecResult.java @@ -13,8 +13,14 @@ import tech.easyflow.ai.entity.base.WorkflowExecResultBase; @Table(value = "tb_workflow_exec_result", comment = "工作流执行记录") public class WorkflowExecResult extends WorkflowExecResultBase { + /** + * 获取工作流执行耗时。 + * + * @return 起止时间完整时返回毫秒耗时，否则返回 null + */ public Long getExecTime() { - if (getEndTime() == null) { + if (getStartTime() == null + || getEndTime() == null) { return null; } return getEndTime().getTime() - getStartTime().getTime(); diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/WorkflowExecStep.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/WorkflowExecStep.java index f4f93a21..ddf9a46e 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/WorkflowExecStep.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/WorkflowExecStep.java @@ -20,8 +20,14 @@ public class WorkflowExecStep extends WorkflowExecStepBase { @Column(ignore = true) private String nodeType; + /** + * 获取节点执行耗时。 + * + * @return 起止时间完整时返回毫秒耗时，否则返回 null + */ public Long getExecTime() { - if (getEndTime() == null) { + if (getStartTime() == null + || getEndTime() == null) { return null; } return getEndTime().getTime() - getStartTime().getTime(); diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/ConditionNode.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/ConditionNode.java index b4d6d241..c4c5394c 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/ConditionNode.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/ConditionNode.java @@ -7,6 +7,7 @@ import com.easyagents.flow.core.node.BaseNode; import com.easyagents.flow.core.util.JsConditionUtil; import com.easyagents.flow.core.util.StringUtil; +import java.io.Serializable; import java.math.BigDecimal; import java.util.*; import java.util.regex.Matcher; @@ -16,6 +17,7 @@ import java.util.regex.Pattern; * 条件判断节点：首个命中（if / else-if）语义。 */ public class ConditionNode extends BaseNode { + private static final long serialVersionUID = 1L; private static final Pattern TEMPLATE_PARAM_PATTERN = Pattern.compile("\\{\\{\\s*([^{}]+?)\\s*}}"); private String branchMode = "first_match"; @@ -116,7 +118,10 @@ public class ConditionNode extends BaseNode { while (matcher.find()) { String path = matcher.group(1) == null ? "" : matcher.group(1).trim(); - Object value = StringUtil.noText(path) ? null : chain.getState().resolveValue(path); + Object value = StringUtil.noText(path) + ? null + : chain.getExecutionState() + .resolveValue(path); matcher.appendReplacement(output, Matcher.quoteReplacement(toJsLiteral(value))); } matcher.appendTail(output); @@ -218,7 +223,8 @@ public class ConditionNode extends BaseNode { return null; } - return chain.getState().resolveValue(path); + return chain.getExecutionState() + .resolveValue(path); } private boolean isEmpty(Object value) { @@ -380,7 +386,9 @@ public class ConditionNode extends BaseNode { this.branches = branches; } - public static class ConditionBranch { + public static class ConditionBranch implements Serializable { + private static final long serialVersionUID = 1L; + private String id; private String label; private String mode; @@ -428,7 +436,9 @@ public class ConditionNode extends BaseNode { } } - public static class ConditionRule { + public static class ConditionRule implements Serializable { + private static final long serialVersionUID = 1L; + private String id; private String joiner; private String leftRef; diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/DocNode.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/DocNode.java index 9f47b13e..d04f34f4 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/DocNode.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/DocNode.java @@ -21,6 +21,8 @@ import java.util.Map; * @since 2026-04-14 */ public class DocNode extends BaseNode { + private static final long serialVersionUID = 1L; + /** * 执行文件内容提取。 @@ -30,7 +32,8 @@ public class DocNode extends BaseNode { */ @Override public Map execute(Chain chain) { - Map map = chain.getState().resolveParameters(this); + Map map = + chain.getExecutionState().resolveParameters(this); DocNodeFileContentExtractor extractor = SpringContextUtil.getBean(DocNodeFileContentExtractor.class); List documents = extractor.extractDocuments(map.get("file")); diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/DownloadNode.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/DownloadNode.java index 395d17ce..0d7bfe66 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/DownloadNode.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/DownloadNode.java @@ -3,27 +3,38 @@ package tech.easyflow.ai.node; import cn.hutool.core.io.FileTypeUtil; import cn.hutool.core.util.IdUtil; import com.easyagents.core.util.StringUtil; +import com.mybatisflex.core.query.QueryWrapper; import com.mybatisflex.core.tenant.TenantManager; import com.easyagents.flow.core.chain.Chain; import com.easyagents.flow.core.chain.Parameter; +import com.easyagents.flow.core.chain.runtime.RetryableTriggerException; import com.easyagents.flow.core.node.BaseNode; +import com.easyagents.flow.core.util.IoBulkhead; import tech.easyflow.ai.entity.Resource; +import tech.easyflow.ai.easyagentsflow.config.WorkflowRuntimeProperties; import tech.easyflow.ai.service.ResourceService; import tech.easyflow.ai.utils.DocUtil; import tech.easyflow.ai.utils.WorkFlowUtil; import tech.easyflow.common.constant.enums.EnumResourceOriginType; +import tech.easyflow.common.cache.RedisIdempotencyExecutor; +import tech.easyflow.common.cache.RedisIdempotencyExecutor.IdempotentOperationInProgressException; import tech.easyflow.common.entity.LoginAccount; import tech.easyflow.common.filestorage.FileStorageManager; +import tech.easyflow.common.filestorage.FileStorageWriteHandle; +import tech.easyflow.common.filestorage.FileStorageWriteResult; import tech.easyflow.common.util.SpringContextUtil; -import java.io.ByteArrayInputStream; import java.math.BigInteger; +import java.nio.charset.StandardCharsets; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicReference; public class DownloadNode extends BaseNode { + private static final long serialVersionUID = 1L; private Integer resourceType; @@ -36,57 +47,199 @@ public class DownloadNode extends BaseNode { @Override public Map execute(Chain chain) { - Map map = chain.getState().resolveParameters(this); - Map res = new HashMap<>(); - + Map map = + chain.getExecutionState().resolveParameters(this); String originUrl = map.get("originUrl").toString(); + LoginAccount account = WorkFlowUtil.getOperator(chain); + ResourceService resourceService = + SpringContextUtil.getBean(ResourceService.class); + String idempotencyKey = + chain.currentExecutionIdempotencyKey(this.id); + String resourceName = idempotencyKey == null + ? IdUtil.simpleUUID() + : UUID.nameUUIDFromBytes( + idempotencyKey.getBytes(StandardCharsets.UTF_8)) + .toString() + .replace("-", ""); - byte[] bytes = DocUtil.downloadFile(originUrl); - - String suffix = FileTypeUtil.getType(new ByteArrayInputStream(bytes)); - - if (suffix == null) { - suffix = "unknown"; + Resource existing = findResource( + resourceService, resourceName, account); + if (existing != null) { + return output(existing.getResourceUrl()); } - String fileName = IdUtil.simpleUUID() + "." + suffix; + WorkflowRuntimeProperties runtimeProperties = + SpringContextUtil.getBean(WorkflowRuntimeProperties.class); + AtomicReference resourceUrl = + new AtomicReference<>(); + if (idempotencyKey == null) { + resourceUrl.set(downloadAndPersist( + originUrl, + resourceName, + account, + resourceService, + runtimeProperties)); + } else { + RedisIdempotencyExecutor idempotencyExecutor = + SpringContextUtil.getBean( + RedisIdempotencyExecutor.class); + boolean executed; + try { + executed = idempotencyExecutor.executeOnce( + idempotencyKey + ":download-resource", + () -> resourceUrl.set(downloadAndPersist( + originUrl, + resourceName, + account, + resourceService, + runtimeProperties))); + } catch (IdempotentOperationInProgressException conflict) { + throw new RetryableTriggerException( + "下载幂等操作仍在处理中", conflict); + } + if (!executed) { + Resource completed = findResource( + resourceService, resourceName, account); + if (completed == null) { + throw new IllegalStateException( + "Download idempotency receipt exists " + + "without resource record"); + } + resourceUrl.set(completed.getResourceUrl()); + } + } + return output(resourceUrl.get()); + } - FileStorageManager manager = SpringContextUtil.getBean(FileStorageManager.class); + /** + * 流式下载、稳定上传并持久化素材记录。 + * + * @param originUrl 原始 URL + * @param resourceName 稳定资源名称 + * @param account 操作账号 + * @param resourceService 素材服务 + * @param runtimeProperties 工作流运行配置 + * @return 素材 URL + */ + private String downloadAndPersist( + String originUrl, + String resourceName, + LoginAccount account, + ResourceService resourceService, + WorkflowRuntimeProperties runtimeProperties) { + try (DocUtil.DownloadedFile downloadedFile = + DocUtil.downloadFileToTemp(originUrl, runtimeProperties.getDownloadMaxBytes())) { + String suffix = FileTypeUtil.getType(downloadedFile.path().toFile()); + if (suffix == null) { + suffix = "unknown"; + } - String resourceUrl = manager.save(new CustomFile(fileName, bytes)); + String fileName = resourceName + "." + suffix; + FileStorageManager manager = SpringContextUtil.getBean(FileStorageManager.class); + FileStorageWriteHandle handle = + manager.prepareRecoverableWrite( + "workflow/download", fileName); + boolean existedBefore = manager.existsRecoverable(handle); + FileStorageWriteResult writeResult; + try { + try (IoBulkhead.Permit ignored = + IoBulkhead.storage().acquire( + "storage:upload")) { + writeResult = manager.saveRecoverable( + new TemporaryFileMultipartFile( + fileName, + downloadedFile.path(), + downloadedFile.contentType()), + handle); + } - Resource resource = new Resource(); + Resource alreadySaved = findResource( + resourceService, resourceName, account); + if (alreadySaved != null) { + return alreadySaved.getResourceUrl(); + } + Resource resource = new Resource(); + resource.setDeptId(account.getDeptId()); + resource.setTenantId(account.getTenantId()); + resource.setResourceType(this.resourceType); + resource.setResourceName(resourceName); + resource.setSuffix(suffix); + resource.setResourceUrl(writeResult.getUrl()); + resource.setOrigin( + EnumResourceOriginType.GENERATE.getCode()); + resource.setCreated(new Date()); + resource.setCreatedBy(account.getId()); + resource.setModified(new Date()); + resource.setModifiedBy(account.getId()); + resource.setFileSize( + BigInteger.valueOf(downloadedFile.size())); + try { + TenantManager.ignoreTenantCondition(); + if (!resourceService.save(resource)) { + throw new IllegalStateException( + "素材记录保存失败"); + } + } finally { + TenantManager.restoreTenantCondition(); + } + return writeResult.getUrl(); + } catch (RuntimeException | Error error) { + if (!existedBefore) { + try { + manager.deleteRecoverable(handle); + } catch (RuntimeException cleanupError) { + error.addSuppressed(cleanupError); + } + } + throw error; + } + } + } - LoginAccount account = WorkFlowUtil.getOperator(chain); - - resource.setDeptId(account.getDeptId()); - resource.setTenantId(account.getTenantId()); - resource.setResourceType(this.resourceType); - resource.setResourceName(DocUtil.getFileNameByUrl(resourceUrl).split("\\.")[0]); - resource.setSuffix(suffix); - resource.setResourceUrl(resourceUrl); - resource.setOrigin(EnumResourceOriginType.GENERATE.getCode()); - resource.setCreated(new Date()); - resource.setCreatedBy(account.getId()); - resource.setModified(new Date()); - resource.setModifiedBy(account.getId()); - resource.setFileSize(BigInteger.valueOf(bytes.length)); + /** + * 查询同一稳定执行生成的素材记录。 + * + * @param service 素材服务 + * @param resourceName 稳定资源名 + * @param account 操作账号 + * @return 已存在记录 + */ + private Resource findResource( + ResourceService service, + String resourceName, + LoginAccount account) { try { TenantManager.ignoreTenantCondition(); - ResourceService service = SpringContextUtil.getBean(ResourceService.class); - service.save(resource); + return service.getOne(QueryWrapper.create() + .where(Resource::getResourceName) + .eq(resourceName) + .and(Resource::getTenantId) + .eq(account.getTenantId()) + .and(Resource::getResourceType) + .eq(this.resourceType)); } finally { TenantManager.restoreTenantCondition(); } + } + /** + * 按节点定义的输出名称返回资源 URL。 + * + * @param resourceUrl 资源 URL + * @return 节点输出 + */ + private Map output(String resourceUrl) { + Map result = new HashMap<>(); String key = "resourceUrl"; List outputDefs = getOutputDefs(); if (outputDefs != null && !outputDefs.isEmpty()) { String defName = outputDefs.get(0).getName(); - if (StringUtil.hasText(defName)) key = defName; + if (StringUtil.hasText(defName)) { + key = defName; + } } - res.put(key, resourceUrl); - return res; + result.put(key, resourceUrl); + return result; } public Integer getResourceType() { diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/GiteeParseService.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/GiteeParseService.java index cc19e7c2..0a4bc349 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/GiteeParseService.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/GiteeParseService.java @@ -1,6 +1,5 @@ package tech.easyflow.ai.node; -import cn.hutool.core.thread.ThreadUtil; import cn.hutool.core.util.StrUtil; import com.alibaba.fastjson2.JSON; import com.alibaba.fastjson2.JSONArray; @@ -22,13 +21,35 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; @Component("giteeReader") public class GiteeParseService implements ReadDocService { @Value("${node.gitee.appKey}") private String appKey; + @Value("${node.gitee.parse-timeout-ms:900000}") + private long parseTimeoutMillis; private static final Logger log = LoggerFactory.getLogger(GiteeParseService.class); + private static final int PARSER_THREADS = 5; + private static final int PARSER_QUEUE_CAPACITY = 64; + private static final AtomicInteger THREAD_SEQUENCE = new AtomicInteger(); + private static final ExecutorService PARSER_EXECUTOR = + new ThreadPoolExecutor( + PARSER_THREADS, + PARSER_THREADS, + 0L, + TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(PARSER_QUEUE_CAPACITY), + runnable -> { + Thread thread = new Thread( + runnable, + "gitee-document-parser-" + + THREAD_SEQUENCE.incrementAndGet()); + thread.setDaemon(true); + return thread; + }, + new ThreadPoolExecutor.AbortPolicy()); @Resource(name = "defaultCache") private Cache defaultCache; @@ -45,7 +66,9 @@ public class GiteeParseService implements ReadDocService { return cache.toString(); } String content; - ExecutorService executor = Executors.newFixedThreadPool(5); + long timeoutMillis = Math.max(1_000L, parseTimeoutMillis); + long deadlineNanos = System.nanoTime() + + TimeUnit.MILLISECONDS.toNanos(timeoutMillis); try { byte[] b = DocUtil.readBytes(is); Map split = splitDocFile(DocUtil.getSuffix(fileName), b, 30); @@ -54,12 +77,23 @@ public class GiteeParseService implements ReadDocService { for (Map.Entry entry : split.entrySet()) { int index = entry.getKey(); byte[] splitBytes = entry.getValue(); - tasks.add(() -> splitContent(index + "-" + fileName, splitBytes)); + tasks.add(() -> splitContent( + index + "-" + fileName, + splitBytes, + deadlineNanos)); } - // 提交所有任务并等待完成 - List> futures = executor.invokeAll(tasks); + long remainingNanos = Math.max( + 1L, deadlineNanos - System.nanoTime()); + List> futures = PARSER_EXECUTOR.invokeAll( + tasks, remainingNanos, TimeUnit.NANOSECONDS); StringBuilder res = new StringBuilder(); for (Future future : futures) { + if (future.isCancelled()) { + throw new TimeoutException( + "文档解析超过 " + + timeoutMillis + + "ms"); + } String call = future.get(); if (StrUtil.isEmpty(call)) { throw new RuntimeException("读取文件任务失败：" + call); @@ -69,20 +103,12 @@ public class GiteeParseService implements ReadDocService { content = res.toString(); defaultCache.put(CacheKey.DOC_NODE_CONTENT_KEY + fileName, content); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("读取文档内容被中断", e); } catch (Exception e) { log.error("读取文档内容失败：", e); throw new RuntimeException("读取文档内容失败：", e); - } finally { - // 关闭线程池 - executor.shutdown(); - try { - if (!executor.awaitTermination(60, TimeUnit.SECONDS)) { - executor.shutdownNow(); - } - } catch (InterruptedException e) { - executor.shutdownNow(); - Thread.currentThread().interrupt(); - } } return content; } @@ -104,7 +130,8 @@ public class GiteeParseService implements ReadDocService { .addHeader("Authorization", "Bearer " + appKey) .post(requestBody).build(); - OkHttpClient okHttpClient = OkHttpClientUtil.buildDefaultClient(); + // 创建任务是非幂等 POST，禁止 OkHttp 在连接失败后隐式重发。 + OkHttpClient okHttpClient = OkHttpClientUtil.buildNoRetryClient(); Call call = okHttpClient.newCall(request); try (Response response = call.execute()) { if (response.body() == null) { @@ -112,7 +139,11 @@ public class GiteeParseService implements ReadDocService { } String jsonStr = response.body().string(); JSONObject object = JSON.parseObject(jsonStr); - log.info("读取文件接口返回：{}", jsonStr); + log.info( + "文档解析任务已创建，fileName={}, status={}, taskId={}", + fileName, + object.getString("status"), + object.getString("task_id")); String error = object.getString("error"); if (StrUtil.isNotEmpty(error)) { throw new RuntimeException(object.getString("message")); @@ -154,7 +185,10 @@ public class GiteeParseService implements ReadDocService { } return md.toString(); } else { - System.out.println(taskId + " >>>>>>>>> " + object); + log.debug( + "文档解析任务等待中，taskId={}, status={}", + taskId, + object.getString("status")); } } catch (Exception e) { log.error("请求失败：", e); @@ -175,10 +209,16 @@ public class GiteeParseService implements ReadDocService { } } - private String splitContent(String fileName, byte[] b) { + private String splitContent( + String fileName, byte[] b, long deadlineNanos) + throws InterruptedException, TimeoutException { String taskId = giteeParse(fileName, b); while (true) { - ThreadUtil.sleep(1000); + if (System.nanoTime() >= deadlineNanos) { + throw new TimeoutException( + "文档解析任务超时：" + taskId); + } + Thread.sleep(1_000L); String result = giteeParseResult(taskId); if (!"waiting".equals(result)) { // 去掉 HTML 标签，![images/xx](xxx)的内容，提取纯文本 diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/MakeFileNode.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/MakeFileNode.java index aaecb320..8a28f31a 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/MakeFileNode.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/MakeFileNode.java @@ -21,6 +21,8 @@ import java.util.Map; * @since 2026-04-18 */ public class MakeFileNode extends BaseNode { + private static final long serialVersionUID = 1L; + private String targetFormat; private String sourceFormat; @@ -45,7 +47,8 @@ public class MakeFileNode extends BaseNode { */ @Override public Map execute(Chain chain) { - Map map = chain.getState().resolveParameters(this); + Map map = + chain.getExecutionState().resolveParameters(this); Object rawContent = map.get("content"); if (rawContent == null) { throw new BusinessException("文件生成节点缺少 content 参数"); diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/PluginToolNode.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/PluginToolNode.java index abb00a7a..3157c755 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/PluginToolNode.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/PluginToolNode.java @@ -25,6 +25,8 @@ import java.util.Collections; import java.util.Map; public class PluginToolNode extends BaseNode { + private static final long serialVersionUID = 1L; + private BigInteger pluginId; @@ -38,7 +40,8 @@ public class PluginToolNode extends BaseNode { @SuppressWarnings("unchecked") @Override public Map execute(Chain chain) { - Map map = chain.getState().resolveParameters(this); + Map map = + chain.getExecutionState().resolveParameters(this); PluginItemService bean = SpringContextUtil.getBean(PluginItemService.class); PluginItem tool = bean.getById(pluginId); if (tool == null) { @@ -49,7 +52,7 @@ public class PluginToolNode extends BaseNode { if (plugin != null && PluginType.isWorkflow(plugin.getType())) { return executeWorkflowPlugin(chain, map, plugin); } - Tool function = tool.toFunction(); + Tool function = tool.toFunction(plugin); if (function == null) { return Collections.emptyMap(); } diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/SaveDatasetNode.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/SaveDatasetNode.java index 39ec5188..c9c38cc9 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/SaveDatasetNode.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/SaveDatasetNode.java @@ -3,21 +3,28 @@ package tech.easyflow.ai.node; import com.alibaba.fastjson2.JSONArray; import com.alibaba.fastjson2.JSONObject; import com.easyagents.flow.core.chain.Chain; +import com.easyagents.flow.core.chain.runtime.RetryableTriggerException; import com.easyagents.flow.core.node.BaseNode; +import com.easyagents.flow.core.util.IoBulkhead; import com.mybatisflex.core.tenant.TenantManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import tech.easyflow.ai.utils.WorkFlowUtil; +import tech.easyflow.ai.easyagentsflow.config.WorkflowRuntimeProperties; import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.common.cache.RedisIdempotencyExecutor.IdempotentOperationInProgressException; import tech.easyflow.common.util.SpringContextUtil; import tech.easyflow.datacenter.execution.model.DatasetRef; import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService; import tech.easyflow.datacenter.execution.service.DatacenterDatasetWriteService; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; public class SaveDatasetNode extends BaseNode { + private static final long serialVersionUID = 1L; private static final Logger log = LoggerFactory.getLogger(SaveDatasetNode.class); @@ -32,7 +39,8 @@ public class SaveDatasetNode extends BaseNode { @Override public Map execute(Chain chain) { - Map state = chain.getState().resolveParameters(this); + Map state = + chain.getExecutionState().resolveParameters(this); JSONObject payload = new JSONObject(state); JSONArray saveList = payload.getJSONArray("saveList"); if (saveList == null || saveList.isEmpty()) { @@ -41,22 +49,31 @@ public class SaveDatasetNode extends BaseNode { LoginAccount account = WorkFlowUtil.getOperator(chain); DatacenterDatasetWriteService writeService = SpringContextUtil.getBean(DatacenterDatasetWriteService.class); DatacenterDatasetQueryService queryService = SpringContextUtil.getBean(DatacenterDatasetQueryService.class); - int successRows = 0; + WorkflowRuntimeProperties runtimeProperties = SpringContextUtil.getBean(WorkflowRuntimeProperties.class); + List rows = new ArrayList<>(saveList.size()); + for (Object item : saveList) { + rows.add(item instanceof JSONObject json ? json : JSONObject.from(item)); + } try { TenantManager.ignoreTenantCondition(); - for (Object item : saveList) { - JSONObject row = item instanceof JSONObject json ? json : JSONObject.from(item); - writeService.saveRow(datasetRef, row, account); - successRows++; + try (IoBulkhead.Permit ignored = IoBulkhead.dataset().acquire(resolveIoTarget())) { + writeService.saveRowsIdempotently( + datasetRef, + rows, + account, + runtimeProperties.getDataWriteBatchSize(), + chain.currentExecutionIdempotencyKey(getId())); + var schema = queryService.getLocation(datasetRef); + Map result = new HashMap<>(); + result.put("successRows", rows.size()); + result.put("source", schema.getSource()); + result.put("catalog", schema.getCatalog()); + result.put("table", schema.getTable()); + result.put("version", datasetRef.getVersionId()); + return result; } - var schema = queryService.getSchema(datasetRef); - Map result = new HashMap<>(); - result.put("successRows", successRows); - result.put("source", schema.getSource()); - result.put("catalog", schema.getCatalog()); - result.put("table", schema.getTable()); - result.put("version", datasetRef.getVersionId()); - return result; + } catch (IdempotentOperationInProgressException conflict) { + throw new RetryableTriggerException("数据集写入幂等操作仍在处理中", conflict); } catch (Exception ex) { log.error("工作流保存数据到统一数据集失败，datasetRef={}", datasetRef, ex); throw ex; @@ -65,6 +82,18 @@ public class SaveDatasetNode extends BaseNode { } } + /** + * 获取数据源级 I/O 隔离目标。 + * + * @return 数据源目标键 + */ + private String resolveIoTarget() { + return "dataset:" + + (datasetRef == null || datasetRef.getSourceId() == null + ? "unknown" + : datasetRef.getSourceId()); + } + public DatasetRef getDatasetRef() { return datasetRef; } diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/SearchDatasetNode.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/SearchDatasetNode.java index 96747de3..f5c00320 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/SearchDatasetNode.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/SearchDatasetNode.java @@ -3,7 +3,9 @@ package tech.easyflow.ai.node; import com.easyagents.core.util.StringUtil; import com.easyagents.flow.core.chain.Chain; import com.easyagents.flow.core.chain.Parameter; +import com.easyagents.flow.core.chain.repository.LoopInputReference; import com.easyagents.flow.core.node.BaseNode; +import com.easyagents.flow.core.util.IoBulkhead; import com.mybatisflex.core.row.Row; import com.mybatisflex.core.tenant.TenantManager; import tech.easyflow.common.util.SpringContextUtil; @@ -15,12 +17,20 @@ import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; public class SearchDatasetNode extends BaseNode { + private static final long serialVersionUID = 1L; + private static final Pattern PARAM_PATTERN = Pattern.compile("\\{\\{(.+?)\\}\\}"); + private static final int QUERY_PAGE_SIZE = Math.max( + 1, + Integer.getInteger( + "easyflow.workflow.dataset.page-size", + 1_000)); private DatasetRef datasetRef; private String querySql; @@ -39,20 +49,50 @@ public class SearchDatasetNode extends BaseNode { @Override public Map execute(Chain chain) { - Map params = chain.getState().resolveParameters(this); + Map params = + chain.getExecutionState().resolveParameters(this); DatacenterDatasetQueryService queryService = SpringContextUtil.getBean(DatacenterDatasetQueryService.class); DatacenterSqlQueryRequest request = buildRuntimeRequest(params); Map result = new HashMap<>(); try { TenantManager.ignoreTenantCondition(); - List rows = queryService.queryBySql(request); - result.put(resolveOutputKey("data"), rows); - return result; + try (IoBulkhead.Permit ignored = IoBulkhead.dataset().acquire(resolveIoTarget())) { + String resultId = chain.getStateInstanceId() + + ":dataset:" + + UUID.randomUUID(); + int rowCount = + chain.storeProducedLoopInputOutsideLock( + resultId, + sink -> queryService.consumeBySql( + request, + QUERY_PAGE_SIZE, + sink::accept), + 0L, + chain.currentFencingClaimId(), + chain.currentClaimGeneration()); + result.put( + resolveOutputKey("data"), + new LoopInputReference( + resultId, rowCount)); + return result; + } } finally { TenantManager.restoreTenantCondition(); } } + /** + * 获取数据源级 I/O 隔离目标。 + * + * @return 数据源目标键 + */ + private String resolveIoTarget() { + return "dataset:" + + (datasetRef == null || datasetRef.getSourceId() == null + ? "unknown" + : datasetRef.getSourceId()); + } + private DatacenterSqlQueryRequest buildRuntimeRequest(Map params) { DatacenterSqlQueryRequest request = new DatacenterSqlQueryRequest(); request.setDatasetRef(copyDatasetRef()); diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/TemporaryFileMultipartFile.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/TemporaryFileMultipartFile.java new file mode 100644 index 00000000..61e96dfe --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/TemporaryFileMultipartFile.java @@ -0,0 +1,137 @@ +package tech.easyflow.ai.node; + +import org.apache.tika.Tika; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.multipart.MultipartFile; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Objects; + +/** + * 基于临时文件的 MultipartFile，供工作流大文件上传路径复用文件流。 + */ +public final class TemporaryFileMultipartFile implements MultipartFile { + + private static final Logger log = LoggerFactory.getLogger(TemporaryFileMultipartFile.class); + private static final Tika tika = new Tika(); + + private final String fileName; + private final Path path; + private final String contentType; + + /** + * 创建临时文件上传对象。 + * + * @param fileName 上传文件名 + * @param path 临时文件路径 + * @param contentType 已知媒体类型，可为空 + */ + public TemporaryFileMultipartFile(String fileName, Path path, String contentType) { + this.fileName = Objects.requireNonNull(fileName, "fileName 不能为空"); + this.path = Objects.requireNonNull(path, "path 不能为空"); + this.contentType = contentType; + } + + /** + * 获取表单字段名。 + * + * @return 表单字段名 + */ + @Override + public String getName() { + return fileName; + } + + /** + * 获取原始文件名。 + * + * @return 原始文件名 + */ + @Override + public String getOriginalFilename() { + return fileName; + } + + /** + * 获取媒体类型；响应未提供时从临时文件检测。 + * + * @return 媒体类型，检测失败时返回空字符串 + */ + @Override + public String getContentType() { + if (contentType != null && !contentType.isBlank()) { + return contentType; + } + try { + return tika.detect(path); + } catch (IOException exception) { + log.warn("检测工作流临时文件媒体类型失败，path={}", path, exception); + return ""; + } + } + + /** + * 判断文件是否为空。 + * + * @return 文件为空时返回 true + */ + @Override + public boolean isEmpty() { + return getSize() == 0L; + } + + /** + * 获取文件大小。 + * + * @return 文件字节数 + * @throws IllegalStateException 无法读取文件元数据时抛出 + */ + @Override + public long getSize() { + try { + return Files.size(path); + } catch (IOException exception) { + throw new IllegalStateException("读取工作流临时文件大小失败", exception); + } + } + + /** + * 读取完整字节数组，兼容仅支持字节数组的存储后端。 + * + * @return 文件字节 + * @throws IOException 读取失败时抛出 + */ + @Override + public byte[] getBytes() throws IOException { + return Files.readAllBytes(path); + } + + /** + * 打开文件输入流。 + * + * @return 文件输入流 + * @throws IOException 打开失败时抛出 + */ + @Override + public InputStream getInputStream() throws IOException { + return Files.newInputStream(path); + } + + /** + * 将临时文件复制到目标位置。 + * + * @param destination 目标文件 + * @throws IOException 复制失败时抛出 + * @throws IllegalStateException 目标不可写时抛出 + */ + @Override + public void transferTo(File destination) throws IOException, IllegalStateException { + Files.copy(path, destination.toPath(), StandardCopyOption.REPLACE_EXISTING); + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/WorkflowNode.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/WorkflowNode.java index a6d79a6a..b9f7276f 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/WorkflowNode.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/WorkflowNode.java @@ -9,7 +9,12 @@ import tech.easyflow.common.util.SpringContextUtil; import java.util.Map; +/** + * 在独立子工作流执行通道中同步执行子工作流。 + */ public class WorkflowNode extends BaseNode { + private static final long serialVersionUID = 1L; + private String workflowId; @@ -20,17 +25,27 @@ public class WorkflowNode extends BaseNode { this.workflowId = workflowId; } + /** + * 执行子流程并返回其业务结果。 + * + * @param chain 父工作流 + * @return 子流程完成结果，或保持当前节点运行的控制结果 + */ @Override public Map execute(Chain chain) { - - Map params = chain.getState().resolveParameters(this); - WorkflowService service = SpringContextUtil.getBean(WorkflowService.class); + Map params = + chain.getExecutionState() + .resolveParameters(this); + WorkflowService service = + SpringContextUtil.getBean(WorkflowService.class); Workflow workflow = service.getById(workflowId); if (workflow == null) { throw new RuntimeException("工作流不存在：" + workflowId); } - ChainExecutor executor = SpringContextUtil.getBean(ChainExecutor.class); - return executor.execute(workflowId, params); + ChainExecutor executor = + SpringContextUtil.getBean(ChainExecutor.class); + return executor.executeChild( + workflowId, params, chain, this.id); } public String getWorkflowId() { diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/WorkflowExecResultService.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/WorkflowExecResultService.java index 8d35a006..ef326c32 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/WorkflowExecResultService.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/WorkflowExecResultService.java @@ -11,5 +11,19 @@ import tech.easyflow.ai.entity.WorkflowExecResult; */ public interface WorkflowExecResultService extends IService { + /** + * 根据稳定执行键查询记录。 + * + * @param execKey 执行键 + * @return 执行记录；不存在时为 {@code null} + */ WorkflowExecResult getByExecKey(String execKey); + + /** + * 根据稳定执行键更新非空审计字段。 + * + * @param record 包含执行键和待更新字段的记录 + * @return 受影响行数 + */ + int updateByExecKey(WorkflowExecResult record); } diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/WorkflowExecStepService.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/WorkflowExecStepService.java index 0a700e60..5db08bb0 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/WorkflowExecStepService.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/WorkflowExecStepService.java @@ -11,6 +11,19 @@ import tech.easyflow.ai.entity.WorkflowExecStep; */ public interface WorkflowExecStepService extends IService { - // 根据 execKey 获取记录 + /** + * 根据稳定执行键查询步骤。 + * + * @param execKey 执行键 + * @return 执行步骤；不存在时为 {@code null} + */ WorkflowExecStep getByExecKey(String execKey); + + /** + * 根据稳定执行键更新非空审计字段。 + * + * @param step 包含执行键和待更新字段的步骤 + * @return 受影响行数 + */ + int updateByExecKey(WorkflowExecStep step); } diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/WorkflowExecResultServiceImpl.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/WorkflowExecResultServiceImpl.java index 810888fd..bfbfd5cc 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/WorkflowExecResultServiceImpl.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/WorkflowExecResultServiceImpl.java @@ -22,4 +22,17 @@ public class WorkflowExecResultServiceImpl extends ServiceImpl implements WorkflowService { + @javax.annotation.Resource + private ApplicationEventPublisher eventPublisher; + /** * 根据别名或 id 查询详情 */ @@ -127,7 +134,9 @@ public class WorkflowServiceImpl extends ServiceImpl i } - return super.updateById(workFlow,false); + boolean updated = super.updateById(workFlow,false); + publishDefinitionChanged(updated, workFlow.getId()); + return updated; } /** @@ -141,13 +150,61 @@ public class WorkflowServiceImpl extends ServiceImpl i Date modified, BigInteger modifiedBy ) { - return getMapper().updateContentByRevision( + boolean updated = getMapper().updateContentByRevision( id, content, expectedRevision, modified, modifiedBy ) == 1; + publishDefinitionChanged(updated, id); + return updated; + } + + /** + * 删除工作流后使编译定义缓存失效。 + * + * @param id 工作流 ID + * @return 删除成功时为 true + */ + @Override + public boolean removeById(Serializable id) { + boolean removed = super.removeById(id); + if (removed && id != null && eventPublisher != null) { + eventPublisher.publishEvent(new WorkflowDefinitionChangedEvent(String.valueOf(id))); + } + return removed; + } + + /** + * 批量删除工作流后使对应编译定义缓存失效。 + * + * @param ids 工作流 ID 集合 + * @return 删除成功时为 true + */ + @Override + public boolean removeByIds(Collection ids) { + boolean removed = super.removeByIds(ids); + if (removed && ids != null && eventPublisher != null) { + for (Serializable id : ids) { + if (id != null) { + eventPublisher.publishEvent(new WorkflowDefinitionChangedEvent(String.valueOf(id))); + } + } + } + return removed; + } + + /** + * 在工作流变更成功后发布定义失效事件。 + * + * @param changed 是否已发生变更 + * @param workflowId 工作流 ID + */ + private void publishDefinitionChanged(boolean changed, BigInteger workflowId) { + if (changed && workflowId != null && eventPublisher != null) { + eventPublisher.publishEvent(new WorkflowDefinitionChangedEvent(String.valueOf(workflowId))); + } } } diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/utils/DocUtil.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/utils/DocUtil.java index 3122de39..f5d33b4b 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/utils/DocUtil.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/utils/DocUtil.java @@ -5,6 +5,7 @@ import okhttp3.Call; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response; +import okhttp3.ResponseBody; import org.apache.poi.extractor.ExtractorFactory; import org.apache.poi.extractor.POITextExtractor; import org.apache.pdfbox.multipdf.Splitter; @@ -24,6 +25,10 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; @@ -52,6 +57,103 @@ public class DocUtil { } } + /** + * 将远程文件流式下载到临时文件，避免在工作流热路径持有整文件字节数组。 + * + * @param url 远程文件地址 + * @param maxBytes 最大允许字节数，小于等于 0 时不限制 + * @return 可自动清理的临时下载结果 + * @throws RuntimeException 下载失败或文件超过限制时抛出 + */ + public static DownloadedFile downloadFileToTemp(String url, long maxBytes) { + Request request = new Request.Builder().url(url).build(); + OkHttpClient client = OkHttpClientUtil.buildDefaultClient(); + Path tempFile = null; + // 共享客户端拦截器覆盖完整响应生命周期，避免同一下载重复领取 I/O 许可。 + try (Response response = client.newCall(request).execute()) { + ResponseBody body = response.body(); + if (body == null) { + throw new IOException("下载内容为空"); + } + long contentLength = body.contentLength(); + if (maxBytes > 0 && contentLength > maxBytes) { + throw new IOException("下载文件超过限制: " + maxBytes + " bytes"); + } + + tempFile = Files.createTempFile("easyflow-workflow-download-", ".tmp"); + long size = copyWithLimit(body.byteStream(), tempFile, maxBytes); + String contentType = body.contentType() == null ? null : body.contentType().toString(); + return new DownloadedFile(tempFile, size, contentType); + } catch (Exception exception) { + deleteTempFile(tempFile); + log.error("下载文件失败：", exception); + throw new RuntimeException(exception); + } + } + + /** + * 将输入流复制到临时文件，并在复制过程中执行大小保护。 + * + * @param inputStream 输入流 + * @param target 目标临时文件 + * @param maxBytes 最大允许字节数，小于等于 0 时不限制 + * @return 实际复制字节数 + * @throws IOException 读写失败或超出限制时抛出 + */ + private static long copyWithLimit(InputStream inputStream, Path target, long maxBytes) throws IOException { + long total = 0L; + byte[] buffer = new byte[64 * 1024]; + try (InputStream input = inputStream; + OutputStream output = Files.newOutputStream( + target, + StandardOpenOption.WRITE, + StandardOpenOption.TRUNCATE_EXISTING)) { + int read; + while ((read = input.read(buffer)) != -1) { + if (maxBytes > 0 && total > maxBytes - read) { + throw new IOException("下载文件超过限制: " + maxBytes + " bytes"); + } + output.write(buffer, 0, read); + total += read; + } + } + return total; + } + + /** + * 尽力删除下载临时文件。 + * + * @param path 临时文件路径 + */ + private static void deleteTempFile(Path path) { + if (path == null) { + return; + } + try { + Files.deleteIfExists(path); + } catch (IOException exception) { + log.warn("清理工作流下载临时文件失败，path={}", path, exception); + } + } + + /** + * 工作流流式下载结果。 + * + * @param path 临时文件路径 + * @param size 文件字节数 + * @param contentType 响应媒体类型 + */ + public record DownloadedFile(Path path, long size, String contentType) implements AutoCloseable { + + /** + * 删除临时文件。 + */ + @Override + public void close() { + deleteTempFile(path); + } + } + public static String readWordFile(String suffix, InputStream is) { String content = ""; try { diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/utils/WorkFlowUtil.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/utils/WorkFlowUtil.java index 72c1a529..e7b5b396 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/utils/WorkFlowUtil.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/utils/WorkFlowUtil.java @@ -33,7 +33,9 @@ public class WorkFlowUtil { } public static LoginAccount getOperator(Chain chain) { - Object cache = chain.getState().getMemory().get(Constants.LOGIN_USER_KEY); + Object cache = chain.getExecutionState() + .getMemory() + .get(Constants.LOGIN_USER_KEY); return cache == null ? defaultAccount() : (LoginAccount) cache; } @@ -44,7 +46,9 @@ public class WorkFlowUtil { * @return 执行人标识 */ public static String getCreatedKey(Chain chain) { - Object value = chain.getState().getMemory().get(CREATED_KEY_MEMORY_KEY); + Object value = chain.getExecutionState() + .getMemory() + .get(CREATED_KEY_MEMORY_KEY); return value == null ? USER_KEY : String.valueOf(value); } diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/event/WorkflowExecutionAuditConsumerTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/event/WorkflowExecutionAuditConsumerTest.java new file mode 100644 index 00000000..26db15a2 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/event/WorkflowExecutionAuditConsumerTest.java @@ -0,0 +1,747 @@ +package tech.easyflow.ai.easyagentsflow.event; + +import com.alibaba.fastjson2.JSON; +import com.easyagents.flow.core.chain.repository.InMemoryLoopResultRepository; +import com.easyagents.flow.core.chain.repository.LoopInputReference; +import com.easyagents.flow.core.chain.repository.LoopResultReference; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import tech.easyflow.ai.entity.WorkflowExecResult; +import tech.easyflow.ai.entity.WorkflowExecStep; +import tech.easyflow.ai.service.WorkflowExecResultService; +import tech.easyflow.ai.service.WorkflowExecStepService; +import tech.easyflow.common.mq.config.MQProperties; +import tech.easyflow.common.mq.core.MQDeadLetterService; +import tech.easyflow.common.mq.core.MQMessage; +import tech.easyflow.common.mq.core.MQProducer; + +import java.lang.reflect.Field; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * 工作流执行审计异步持久化测试。 + */ +public class WorkflowExecutionAuditConsumerTest { + + /** + * 验证存在测试构造器时 Spring 仍能选择生产构造器创建 Bean。 + */ + @Test + public void shouldCreateProducerThroughSpringContext() { + MQProducer mqProducer = + Mockito.mock(MQProducer.class); + MQDeadLetterService deadLetterService = + Mockito.mock( + MQDeadLetterService.class); + try (AnnotationConfigApplicationContext context = + new AnnotationConfigApplicationContext()) { + context.registerBean( + MQProducer.class, + () -> mqProducer); + context.registerBean( + MQDeadLetterService.class, + () -> deadLetterService); + context.registerBean( + WorkflowExecutionAuditProducer.class); + context.refresh(); + + Assert.assertNotNull( + context.getBean( + WorkflowExecutionAuditProducer.class)); + } + } + + /** + * 验证生产者固定投递到单一有序分片。 + */ + @Test + public void shouldPublishAuditEventToOrderedShard() { + MQProducer mqProducer = Mockito.mock(MQProducer.class); + MQDeadLetterService deadLetterService = + Mockito.mock(MQDeadLetterService.class); + WorkflowExecutionAuditProducer producer = + new WorkflowExecutionAuditProducer( + mqProducer, deadLetterService); + try { + WorkflowExecutionAuditEvent event = event( + WorkflowExecutionAuditEvent.Type.CHAIN_STARTED, + "instance-1:chain-started", + "instance-1", + new WorkflowExecResult(), + null); + + producer.send(event); + + Mockito.verify(mqProducer).send(Mockito.argThat(message -> + WorkflowExecutionAuditMqConstants.TOPIC.equals( + message.getTopic()) + && "instance-1:chain-started".equals( + message.getMessageId()) + && "instance-1".equals(message.getKey()) + && message.getBody().contains( + "CHAIN_STARTED"))); + } finally { + producer.close(); + } + } + + /** + * 验证结束事件仅携带结束时间时仍可安全序列化投递。 + */ + @Test + public void shouldPublishEndEventsWithoutStartTime() { + MQProducer mqProducer = + Mockito.mock(MQProducer.class); + MQDeadLetterService deadLetterService = + Mockito.mock( + MQDeadLetterService.class); + WorkflowExecutionAuditProducer producer = + new WorkflowExecutionAuditProducer( + mqProducer, + deadLetterService); + Mockito.when(mqProducer.send( + Mockito.any())) + .thenReturn("message-id"); + try { + WorkflowExecStep step = + new WorkflowExecStep(); + step.setExecKey("step-ended"); + step.setEndTime(new Date()); + WorkflowExecResult result = + new WorkflowExecResult(); + result.setExecKey("instance-ended"); + result.setEndTime(new Date()); + + producer.send(event( + WorkflowExecutionAuditEvent.Type.NODE_ENDED, + "step-ended:event", + "instance-ended", + null, + step)); + producer.send(event( + WorkflowExecutionAuditEvent.Type.CHAIN_ENDED, + "instance-ended:event", + "instance-ended", + result, + null)); + + Mockito.verify( + mqProducer, + Mockito.times(2)) + .send(Mockito.argThat(message -> + message.getBody() != null + && (message.getBody() + .contains("NODE_ENDED") + || message.getBody() + .contains("CHAIN_ENDED")))); + } finally { + producer.close(); + } + } + + /** + * 验证一个实例的毒消息退避不会阻塞其他发送 lane。 + * + * @throws Exception 等待健康实例发送失败时抛出 + */ + @Test + public void shouldIsolateRetryHeadBlockingAcrossLanes() + throws Exception { + MQProducer mqProducer = + Mockito.mock(MQProducer.class); + MQDeadLetterService deadLetterService = + Mockito.mock( + MQDeadLetterService.class); + CountDownLatch healthySent = + new CountDownLatch(1); + Mockito.when(mqProducer.send( + Mockito.any())) + .thenAnswer(invocation -> { + MQMessage message = + invocation.getArgument(0); + if ("instance-0".equals( + message.getKey())) { + throw new IllegalStateException( + "poison"); + } + healthySent.countDown(); + return "sent"; + }); + WorkflowExecutionAuditProducer producer = + new WorkflowExecutionAuditProducer( + mqProducer, + deadLetterService, + 8, + 100, + 1024L * 1024L, + 8L * 1024L * 1024L, + 0L); + try { + producer.send(event( + WorkflowExecutionAuditEvent.Type.NODE_STARTED, + "poison", + "instance-0", + null, + new WorkflowExecStep())); + producer.send(event( + WorkflowExecutionAuditEvent.Type.NODE_STARTED, + "healthy", + "instance-1", + null, + new WorkflowExecStep())); + + Assert.assertTrue( + healthySent.await( + 1L, + TimeUnit.SECONDS)); + } finally { + producer.close(); + } + Mockito.verify(deadLetterService) + .deadLetter( + Mockito.argThat(message -> + "poison".equals( + message.getMessageId())), + Mockito.contains("shutdown")); + } + + /** + * 验证同一实例失败恢复后仍按原事件顺序发送。 + * + * @throws Exception 等待重试发送失败时抛出 + */ + @Test + public void shouldPreserveOrderWithinAuditLane() + throws Exception { + MQProducer mqProducer = + Mockito.mock(MQProducer.class); + MQDeadLetterService deadLetterService = + Mockito.mock( + MQDeadLetterService.class); + AtomicInteger firstAttempts = + new AtomicInteger(); + CountDownLatch sent = + new CountDownLatch(2); + List order = + Collections.synchronizedList( + new ArrayList<>()); + Mockito.when(mqProducer.send( + Mockito.any())) + .thenAnswer(invocation -> { + MQMessage message = + invocation.getArgument(0); + if ("first".equals( + message.getMessageId()) + && firstAttempts + .getAndIncrement() == 0) { + throw new IllegalStateException( + "temporary"); + } + order.add( + message.getMessageId()); + sent.countDown(); + return "sent"; + }); + WorkflowExecutionAuditProducer producer = + new WorkflowExecutionAuditProducer( + mqProducer, + deadLetterService, + 2, + 100, + 1024L * 1024L, + 8L * 1024L * 1024L, + 1000L); + try { + producer.send(event( + WorkflowExecutionAuditEvent.Type.NODE_STARTED, + "first", + "same-instance", + null, + new WorkflowExecStep())); + producer.send(event( + WorkflowExecutionAuditEvent.Type.NODE_ENDED, + "second", + "same-instance", + null, + new WorkflowExecStep())); + + Assert.assertTrue( + sent.await( + 2L, + TimeUnit.SECONDS)); + Assert.assertEquals( + List.of("first", "second"), + order); + } finally { + producer.close(); + } + } + + /** + * 验证超出单条字节预算的审计消息直接进入死信并显式失败。 + */ + @Test + public void shouldRejectOversizedAuditMessage() { + MQProducer mqProducer = + Mockito.mock(MQProducer.class); + MQDeadLetterService deadLetterService = + Mockito.mock( + MQDeadLetterService.class); + WorkflowExecutionAuditProducer producer = + new WorkflowExecutionAuditProducer( + mqProducer, + deadLetterService, + 1, + 10, + 512L, + 2048L, + 0L); + WorkflowExecResult result = + new WorkflowExecResult(); + result.setOutput( + "x".repeat(1024)); + try { + producer.send(event( + WorkflowExecutionAuditEvent.Type.CHAIN_ENDED, + "oversized", + "instance", + result, + null)); + Assert.fail( + "oversized audit message should fail"); + } catch (IllegalArgumentException expected) { + Assert.assertTrue( + expected.getMessage() + .contains("byte limit")); + } finally { + producer.close(); + } + Mockito.verify( + deadLetterService) + .deadLetter( + Mockito.argThat(message -> + "oversized".equals( + message.getMessageId())), + Mockito.contains("byte limit")); + Mockito.verifyNoInteractions( + mqProducer); + } + + /** + * 验证关闭期间仍在直发的失败消息会转入死信且不会重新形成孤儿积压。 + * + * @throws Exception 并发关闭、等待或反射读取失败时抛出 + */ + @Test + public void shouldNotEnqueueAfterConcurrentClose() + throws Exception { + MQProducer mqProducer = + Mockito.mock(MQProducer.class); + MQDeadLetterService deadLetterService = + Mockito.mock( + MQDeadLetterService.class); + CountDownLatch sending = + new CountDownLatch(1); + CountDownLatch releaseSend = + new CountDownLatch(1); + Mockito.when(mqProducer.send( + Mockito.any())) + .thenAnswer(invocation -> { + sending.countDown(); + releaseSend.await( + 2L, + TimeUnit.SECONDS); + throw new IllegalStateException( + "send failed during close"); + }); + WorkflowExecutionAuditProducer producer = + new WorkflowExecutionAuditProducer( + mqProducer, + deadLetterService, + 1, + 100, + 1024L * 1024L, + 8L * 1024L * 1024L, + 0L); + ExecutorService callers = + Executors.newFixedThreadPool(2); + try { + Future sender = + callers.submit(() -> { + try { + producer.send(event( + WorkflowExecutionAuditEvent.Type.NODE_STARTED, + "closing", + "instance", + null, + new WorkflowExecStep())); + Assert.fail( + "send should report concurrent close"); + } catch (IllegalStateException expected) { + Assert.assertTrue( + expected.getMessage() + .contains("closed")); + } + }); + Assert.assertTrue( + sending.await( + 1L, + TimeUnit.SECONDS)); + Future closer = + callers.submit( + producer::close); + closer.get( + 1L, + TimeUnit.SECONDS); + releaseSend.countDown(); + sender.get( + 2L, + TimeUnit.SECONDS); + + Field backlogCountField = + WorkflowExecutionAuditProducer.class + .getDeclaredField( + "backlogCount"); + backlogCountField.setAccessible(true); + Assert.assertEquals( + 0, + backlogCountField.getInt( + producer)); + Mockito.verify(deadLetterService) + .deadLetter( + Mockito.argThat(message -> + "closing".equals( + message.getMessageId())), + Mockito.contains( + "closed during send")); + } finally { + releaseSend.countDown(); + producer.close(); + callers.shutdownNow(); + } + } + + /** + * 验证启动、节点开始、节点结束和流程结束事件按顺序幂等落库。 + */ + @Test + public void shouldApplyOrderedExecutionAuditEvents() { + WorkflowExecResultService resultService = + Mockito.mock(WorkflowExecResultService.class); + WorkflowExecStepService stepService = + Mockito.mock(WorkflowExecStepService.class); + InMemoryLoopResultRepository loopRepository = + new InMemoryLoopResultRepository(); + WorkflowExecutionAuditConsumer consumer = + new WorkflowExecutionAuditConsumer( + resultService, + stepService, + new MQProperties(), + loopRepository); + + WorkflowExecResult persistedResult = new WorkflowExecResult(); + persistedResult.setId(BigInteger.ONE); + persistedResult.setExecKey("instance-1"); + Mockito.when(resultService.getByExecKey("instance-1")) + .thenReturn(persistedResult); + Mockito.when(resultService.updateByExecKey(Mockito.any())) + .thenReturn(1); + Mockito.when(stepService.updateByExecKey(Mockito.any())) + .thenReturn(1); + + WorkflowExecResult startRecord = new WorkflowExecResult(); + startRecord.setExecKey("instance-1"); + startRecord.setStatus(1); + WorkflowExecStep startStep = new WorkflowExecStep(); + startStep.setExecKey("step-1"); + startStep.setNodeId("node-1"); + startStep.setNodeName("node"); + startStep.setStatus(1); + WorkflowExecStep endStep = new WorkflowExecStep(); + endStep.setExecKey("step-1"); + endStep.setStatus(2); + endStep.setOutput("{\"value\":1}"); + WorkflowExecResult endRecord = new WorkflowExecResult(); + endRecord.setExecKey("instance-1"); + endRecord.setStatus(2); + endRecord.setOutput("{\"value\":1}"); + + consumer.handle(List.of( + message(event(WorkflowExecutionAuditEvent.Type.CHAIN_STARTED, + "start", "instance-1", startRecord, null)), + message(event(WorkflowExecutionAuditEvent.Type.NODE_STARTED, + "node-start", "instance-1", null, startStep)), + message(event(WorkflowExecutionAuditEvent.Type.NODE_ENDED, + "node-end", "instance-1", null, endStep)), + message(event(WorkflowExecutionAuditEvent.Type.CHAIN_ENDED, + "end", "instance-1", endRecord, null)) + )); + + Mockito.verify(resultService).save(Mockito.argThat(record -> + "instance-1".equals(record.getExecKey()) + && Integer.valueOf(1).equals(record.getStatus()))); + Mockito.verify(stepService).save(Mockito.argThat(step -> + "step-1".equals(step.getExecKey()) + && BigInteger.ONE.equals(step.getRecordId()))); + Mockito.verify(stepService).updateByExecKey( + Mockito.argThat(step -> + "step-1".equals(step.getExecKey()) + && "{\"value\":1}".equals( + step.getOutput()))); + Mockito.verify(resultService).updateByExecKey( + Mockito.argThat(record -> + "instance-1".equals(record.getExecKey()) + && "{\"value\":1}".equals( + record.getOutput()))); + Mockito.verify(stepService, Mockito.never()) + .getByExecKey(Mockito.anyString()); + } + + /** + * 验证大型结果引用在审计消费线程还原，持久记录仍保持完整 JSON。 + */ + @Test + public void shouldResolveLargeReferenceInAuditConsumer() { + WorkflowExecResultService resultService = + Mockito.mock( + WorkflowExecResultService.class); + WorkflowExecStepService stepService = + Mockito.mock( + WorkflowExecStepService.class); + InMemoryLoopResultRepository loopRepository = + new InMemoryLoopResultRepository(); + String resultId = "instance:dataset:rows"; + loopRepository.storeInput( + resultId, List.of(1, 2, 3)); + WorkflowExecutionAuditConsumer consumer = + new WorkflowExecutionAuditConsumer( + resultService, + stepService, + new MQProperties(), + loopRepository); + Mockito.when(stepService.updateByExecKey( + Mockito.any())) + .thenReturn(1); + WorkflowExecStep incoming = + new WorkflowExecStep(); + incoming.setExecKey("step-reference"); + incoming.setOutput(JSON.toJSONString( + Map.of( + "data", + new LoopInputReference( + resultId, 3)))); + + consumer.handle(List.of(message(event( + WorkflowExecutionAuditEvent.Type.NODE_ENDED, + "node-reference", + "instance", + null, + incoming)))); + + Mockito.verify(stepService).updateByExecKey( + Mockito.argThat(step -> + "{\"data\":[1,2,3]}" + .equals(step.getOutput()))); + } + + /** + * 验证节点启动输入引用在审计消费者中还原后再保存。 + */ + @Test + public void shouldResolveLargeInputReferenceWhenCreatingStep() { + WorkflowExecResultService resultService = + Mockito.mock( + WorkflowExecResultService.class); + WorkflowExecStepService stepService = + Mockito.mock( + WorkflowExecStepService.class); + InMemoryLoopResultRepository loopRepository = + new InMemoryLoopResultRepository(); + String resultId = "instance:dataset:input"; + loopRepository.storeInput( + resultId, List.of(1, 2, 3)); + WorkflowExecutionAuditConsumer consumer = + new WorkflowExecutionAuditConsumer( + resultService, + stepService, + new MQProperties(), + loopRepository); + WorkflowExecResult record = + new WorkflowExecResult(); + record.setId(BigInteger.ONE); + Mockito.when(resultService.getByExecKey( + "instance")) + .thenReturn(record); + WorkflowExecStep incoming = + new WorkflowExecStep(); + incoming.setExecKey("step-input"); + incoming.setInput(JSON.toJSONString( + Map.of( + "items", + new LoopInputReference( + resultId, 3)))); + + consumer.handle(List.of(message(event( + WorkflowExecutionAuditEvent.Type.NODE_STARTED, + "node-input", + "instance", + null, + incoming)))); + + Mockito.verify(stepService).save( + Mockito.argThat(step -> + BigInteger.ONE.equals( + step.getRecordId()) + && "{\"items\":[1,2,3]}" + .equals(step.getInput()))); + } + + /** + * 验证节点结束与流程结束审计均还原循环累计输出。 + */ + @Test + @SuppressWarnings("unchecked") + public void shouldResolveLoopResultReferenceForEndedAudits() { + WorkflowExecResultService resultService = + Mockito.mock( + WorkflowExecResultService.class); + WorkflowExecStepService stepService = + Mockito.mock( + WorkflowExecStepService.class); + InMemoryLoopResultRepository loopRepository = + new InMemoryLoopResultRepository(); + String resultId = "instance:loop:result"; + loopRepository.append( + resultId, + 0, + Map.of("answer", "first")); + loopRepository.append( + resultId, + 1, + Map.of("answer", "second")); + WorkflowExecutionAuditConsumer consumer = + new WorkflowExecutionAuditConsumer( + resultService, + stepService, + new MQProperties(), + loopRepository); + Mockito.when(stepService.updateByExecKey( + Mockito.any())) + .thenReturn(1); + Mockito.when(resultService.updateByExecKey( + Mockito.any())) + .thenReturn(1); + Map referenceOutput = + Map.of( + "answers", + new LoopResultReference( + resultId, + 2, + "answer")); + WorkflowExecStep incomingStep = + new WorkflowExecStep(); + incomingStep.setExecKey("step-loop"); + incomingStep.setOutput( + JSON.toJSONString( + referenceOutput)); + WorkflowExecResult incomingResult = + new WorkflowExecResult(); + incomingResult.setExecKey( + "instance-loop"); + incomingResult.setOutput( + JSON.toJSONString( + referenceOutput)); + + consumer.handle(List.of( + message(event( + WorkflowExecutionAuditEvent.Type.NODE_ENDED, + "node-loop-ended", + "instance-loop", + null, + incomingStep)), + message(event( + WorkflowExecutionAuditEvent.Type.CHAIN_ENDED, + "chain-loop-ended", + "instance-loop", + incomingResult, + null)))); + + org.mockito.ArgumentCaptor + stepCaptor = + org.mockito.ArgumentCaptor.forClass( + WorkflowExecStep.class); + org.mockito.ArgumentCaptor + resultCaptor = + org.mockito.ArgumentCaptor.forClass( + WorkflowExecResult.class); + Mockito.verify(stepService) + .updateByExecKey(stepCaptor.capture()); + Mockito.verify(resultService) + .updateByExecKey(resultCaptor.capture()); + Map stepOutput = + JSON.parseObject( + stepCaptor.getValue().getOutput(), + Map.class); + Map resultOutput = + JSON.parseObject( + resultCaptor.getValue().getOutput(), + Map.class); + Assert.assertEquals( + List.of("first", "second"), + stepOutput.get("answers")); + Assert.assertEquals( + List.of("first", "second"), + resultOutput.get("answers")); + } + + /** + * 构造审计事件。 + * + * @param type 事件类型 + * @param eventId 事件 ID + * @param instanceId 实例 ID + * @param result 工作流记录 + * @param step 节点步骤 + * @return 审计事件 + */ + private WorkflowExecutionAuditEvent event(WorkflowExecutionAuditEvent.Type type, + String eventId, + String instanceId, + WorkflowExecResult result, + WorkflowExecStep step) { + WorkflowExecutionAuditEvent event = new WorkflowExecutionAuditEvent(); + event.setType(type); + event.setEventId(eventId); + event.setInstanceId(instanceId); + event.setOccurredAt(new Date()); + event.setResult(result); + event.setStep(step); + return event; + } + + /** + * 将审计事件包装为通用 MQ 消息。 + * + * @param event 审计事件 + * @return MQ 消息 + */ + private MQMessage message(WorkflowExecutionAuditEvent event) { + MQMessage message = new MQMessage(); + message.setMessageId(event.getEventId()); + message.setBody(JSON.toJSONString(event)); + return message; + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/listener/ChainEventListenerForSaveTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/listener/ChainEventListenerForSaveTest.java new file mode 100644 index 00000000..d9c93e8f --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/listener/ChainEventListenerForSaveTest.java @@ -0,0 +1,112 @@ +package tech.easyflow.ai.easyagentsflow.listener; + +import com.easyagents.flow.core.chain.Chain; +import com.easyagents.flow.core.chain.ChainDefinition; +import com.easyagents.flow.core.chain.ChainState; +import com.easyagents.flow.core.chain.NodeStatus; +import com.easyagents.flow.core.chain.event.NodeEndEvent; +import com.easyagents.flow.core.chain.event.NodeStartEvent; +import com.easyagents.flow.core.chain.repository.InMemoryChainStateRepository; +import com.easyagents.flow.core.node.StartNode; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import tech.easyflow.ai.easyagentsflow.event.WorkflowExecutionAuditEvent; +import tech.easyflow.ai.easyagentsflow.event.WorkflowExecutionAuditProducer; + +import java.lang.reflect.Field; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +/** + * {@link ChainEventListenerForSave} 节点审计归属回归测试。 + */ +public class ChainEventListenerForSaveTest { + + /** + * 验证 parent-linked 节点开始与结束事件使用同一顶级实例顺序键。 + * + * @throws Exception 测试依赖注入失败时抛出 + */ + @Test + public void shouldUseSameRootInstanceForNodeStartAndEnd() + throws Exception { + String suffix = + UUID.randomUUID().toString(); + String rootId = + "root-" + suffix; + String childId = + "child-" + suffix; + InMemoryChainStateRepository repository = + new InMemoryChainStateRepository(); + ChainState root = + repository.create(rootId); + ChainState child = + repository.create(childId); + child.setParentInstanceId(rootId); + child.setAuditInstanceId(null); + + ChainDefinition definition = + new ChainDefinition(); + definition.setId("1"); + StartNode node = + new StartNode(); + node.setId("node"); + definition.addNode(node); + Chain chain = + new Chain(definition, childId); + chain.setChainStateRepository(repository); + WorkflowExecutionAuditProducer producer = + Mockito.mock( + WorkflowExecutionAuditProducer.class); + ChainEventListenerForSave listener = + new ChainEventListenerForSave(); + Field producerField = + ChainEventListenerForSave.class + .getDeclaredField("auditProducer"); + producerField.setAccessible(true); + producerField.set(listener, producer); + + listener.onEvent( + new NodeStartEvent( + chain, + node, + "attempt", + NodeStatus.RUNNING, + chain.getAuditInstanceId()), + chain); + listener.onEvent( + new NodeEndEvent( + chain, + node, + Map.of("value", "ok"), + null, + NodeStatus.SUCCEEDED, + "attempt"), + chain); + + ArgumentCaptor captor = + ArgumentCaptor.forClass( + WorkflowExecutionAuditEvent.class); + Mockito.verify( + producer, + Mockito.times(2)) + .send(captor.capture()); + List events = + captor.getAllValues(); + Assert.assertEquals( + WorkflowExecutionAuditEvent.Type.NODE_STARTED, + events.get(0).getType()); + Assert.assertEquals( + WorkflowExecutionAuditEvent.Type.NODE_ENDED, + events.get(1).getType()); + Assert.assertEquals( + rootId, + events.get(0).getInstanceId()); + Assert.assertEquals( + rootId, + events.get(1).getInstanceId()); + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/repository/BaseRepositoryTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/repository/BaseRepositoryTest.java new file mode 100644 index 00000000..84b05162 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/repository/BaseRepositoryTest.java @@ -0,0 +1,100 @@ +package tech.easyflow.ai.easyagentsflow.repository; + +import com.alicp.jetcache.Cache; +import com.alicp.jetcache.CacheException; +import com.alicp.jetcache.CacheResult; +import com.alicp.jetcache.CacheResultCode; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.lang.reflect.Proxy; + +/** + * {@link BaseRepository} 缓存操作语义回归测试。 + */ +public class BaseRepositoryTest { + + /** + * 验证删除不存在的缓存键按幂等成功处理。 + * + * @throws Exception 测试依赖注入失败时抛出 + */ + @Test + public void removeCacheShouldAcceptMissingKey() throws Exception { + TestRepository repository = repository( + new CacheResult(CacheResultCode.NOT_EXISTS, null)); + + repository.remove("missing-key"); + } + + /** + * 验证真实删除错误仍会向上抛出。 + * + * @throws Exception 测试依赖注入失败时抛出 + */ + @Test + public void removeCacheShouldRejectOperationFailure() throws Exception { + TestRepository repository = repository( + new CacheResult(CacheResultCode.FAIL, "redis unavailable")); + + try { + repository.remove("failed-key"); + Assert.fail("cache failure should be propagated"); + } catch (CacheException expected) { + Assert.assertTrue(expected.getMessage().contains("failed-key")); + Assert.assertTrue(expected.getMessage().contains("redis unavailable")); + } + } + + /** + * 创建注入指定删除结果的测试仓储。 + * + * @param removeResult 删除操作结果 + * @return 测试仓储 + * @throws Exception 反射注入失败时抛出 + */ + private TestRepository repository(CacheResult removeResult) throws Exception { + Cache cache = cache(removeResult); + TestRepository repository = new TestRepository(); + Field field = BaseRepository.class.getDeclaredField("cache"); + field.setAccessible(true); + field.set(repository, cache); + return repository; + } + + /** + * 创建只支持删除操作的 JetCache 代理。 + * + * @param removeResult 删除操作结果 + * @return JetCache 测试代理 + */ + @SuppressWarnings("unchecked") + private Cache cache(CacheResult removeResult) { + return (Cache) Proxy.newProxyInstance( + Cache.class.getClassLoader(), + new Class[]{Cache.class}, + (proxy, method, args) -> { + if ("REMOVE".equals(method.getName())) { + return removeResult; + } + throw new UnsupportedOperationException( + "unsupported cache method: " + method.getName()); + }); + } + + /** + * 暴露受保护缓存删除能力的测试仓储。 + */ + private static final class TestRepository extends BaseRepository { + + /** + * 删除指定缓存键。 + * + * @param key 缓存键 + */ + private void remove(String key) { + removeCache(key); + } + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/repository/ChainDefinitionSnapshotSerializationTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/repository/ChainDefinitionSnapshotSerializationTest.java new file mode 100644 index 00000000..c2b90012 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/repository/ChainDefinitionSnapshotSerializationTest.java @@ -0,0 +1,168 @@ +package tech.easyflow.ai.easyagentsflow.repository; + +import com.easyagents.flow.core.chain.ChainDefinition; +import com.easyagents.flow.core.chain.Node; +import com.easyagents.flow.core.chain.Parameter; +import com.easyagents.flow.core.node.BaseNode; +import com.easyagents.flow.core.node.EndNode; +import com.easyagents.flow.core.node.HttpNode; +import com.easyagents.flow.core.node.LlmNode; +import com.easyagents.flow.core.node.LoopNode; +import com.easyagents.flow.core.node.StartNode; +import org.junit.Assert; +import org.junit.Test; +import tech.easyflow.ai.node.ConditionNode; +import tech.easyflow.ai.node.DocNode; +import tech.easyflow.ai.node.DownloadNode; +import tech.easyflow.ai.node.MakeFileNode; +import tech.easyflow.ai.node.PluginToolNode; +import tech.easyflow.ai.node.SaveDatasetNode; +import tech.easyflow.ai.node.SearchDatasetNode; +import tech.easyflow.ai.node.WorkflowNode; +import tech.easyflow.datacenter.execution.model.DatasetRef; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.ObjectStreamClass; +import java.util.List; + +/** + * 工作流定义快照 Java 序列化兼容约束测试。 + */ +public class ChainDefinitionSnapshotSerializationTest { + + /** + * 验证全部运行时业务节点对象图可完成定义快照往返。 + * + * @throws Exception 序列化失败时抛出 + */ + @Test + public void shouldRoundTripAllRuntimeNodeTypes() + throws Exception { + ChainDefinition definition = new ChainDefinition(); + definition.setId("snapshot-all-node-types"); + List nodes = List.of( + node(new StartNode(), "start"), + node(new EndNode(), "end"), + node(new HttpNode(), "http"), + node(new LlmNode(), "llm"), + node(new LoopNode(), "loop"), + node(new PluginToolNode(), "plugin"), + node(new DownloadNode(), "download"), + node(new WorkflowNode(), "workflow"), + node(new DocNode(), "doc"), + datasetNode(new SearchDatasetNode(), "search"), + datasetNode(new SaveDatasetNode(), "save"), + conditionNode(), + node(new MakeFileNode(), "make-file")); + nodes.forEach(definition::addNode); + + byte[] bytes; + try (ByteArrayOutputStream output = + new ByteArrayOutputStream(); + ObjectOutputStream objectOutput = + new ObjectOutputStream(output)) { + objectOutput.writeObject(definition); + objectOutput.flush(); + bytes = output.toByteArray(); + } + ChainDefinition restored; + try (ObjectInputStream input = + new ObjectInputStream( + new ByteArrayInputStream(bytes))) { + restored = (ChainDefinition) input.readObject(); + } + + Assert.assertEquals( + definition.getId(), restored.getId()); + Assert.assertEquals( + nodes.size(), restored.getNodes().size()); + } + + /** + * 验证定义对象图关键类使用显式稳定 UID，防止新增方法导致默认 UID 漂移。 + */ + @Test + public void shouldKeepStableSerialVersionUids() { + List> stableTypes = List.of( + Node.class, + BaseNode.class, + Parameter.class, + StartNode.class, + EndNode.class, + HttpNode.class, + LlmNode.class, + LoopNode.class, + PluginToolNode.class, + DownloadNode.class, + WorkflowNode.class, + DocNode.class, + SearchDatasetNode.class, + SaveDatasetNode.class, + ConditionNode.class, + ConditionNode.ConditionBranch.class, + ConditionNode.ConditionRule.class, + MakeFileNode.class, + DatasetRef.class); + for (Class type : stableTypes) { + Assert.assertEquals( + "unstable serialVersionUID: " + + type.getName(), + 1L, + ObjectStreamClass.lookup(type) + .getSerialVersionUID()); + } + } + + /** + * 设置测试节点 ID。 + * + * @param node 节点 + * @param id 节点 ID + * @return 原节点 + */ + private T node(T node, String id) { + node.setId(id); + return node; + } + + /** + * 设置带数据集引用的节点。 + * + * @param node 数据集节点 + * @param id 节点 ID + * @return 原节点 + */ + private T datasetNode( + T node, String id) { + DatasetRef ref = new DatasetRef(); + ref.setTableName("dataset_table"); + if (node instanceof SearchDatasetNode) { + ((SearchDatasetNode) node).setDatasetRef(ref); + } else { + ((SaveDatasetNode) node).setDatasetRef(ref); + } + return node(node, id); + } + + /** + * 创建带完整嵌套规则对象图的条件节点。 + * + * @return 条件节点 + */ + private ConditionNode conditionNode() { + ConditionNode.ConditionRule rule = + new ConditionNode.ConditionRule(); + rule.setId("rule"); + ConditionNode.ConditionBranch branch = + new ConditionNode.ConditionBranch(); + branch.setId("branch"); + branch.setRules(List.of(rule)); + ConditionNode node = node( + new ConditionNode(), "condition"); + node.setBranches(List.of(branch)); + return node; + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/repository/ChainStateRepositoryImplTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/repository/ChainStateRepositoryImplTest.java index 56950ae1..70433c4f 100644 --- a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/repository/ChainStateRepositoryImplTest.java +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/repository/ChainStateRepositoryImplTest.java @@ -5,36 +5,52 @@ import com.alicp.jetcache.CacheException; import com.alicp.jetcache.CacheGetResult; import com.alicp.jetcache.CacheResult; import com.alicp.jetcache.CacheResultCode; +import com.alicp.jetcache.CacheValueHolder; import com.alicp.jetcache.support.CacheEncodeException; import com.easyagents.flow.core.chain.ChainState; +import com.easyagents.flow.core.chain.ChainStatus; +import com.easyagents.flow.core.chain.repository.ChainLock; +import com.easyagents.flow.core.chain.repository.ChainStateField; import org.junit.Assert; import org.junit.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; +import tech.easyflow.common.cache.RedisLockExecutor; +import tech.easyflow.common.cache.VersionedObjectStore; +import tech.easyflow.common.cache.VersionedFields; +import java.io.Serializable; import java.lang.reflect.Field; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; import java.lang.reflect.Proxy; +import java.time.Duration; +import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.concurrent.ConcurrentHashMap; /** - * {@link ChainStateRepositoryImpl} 缓存异常处理回归测试。 + * {@link ChainStateRepositoryImpl} 缓存迁移和版本提交回归测试。 */ public class ChainStateRepositoryImplTest { /** - * 验证缓存解码失败时抛出异常且不创建空工作流状态。 + * 验证旧缓存解码失败时抛出异常且不创建空工作流状态。 * - * @throws Exception 缓存依赖注入失败时抛出 + * @throws Exception 测试依赖注入失败时抛出 */ @Test public void loadShouldFailWithoutOverwritingStateWhenCacheDecodeFails() throws Exception { String instanceId = "decode-failed-instance"; CacheGetResult failure = new CacheGetResult<>(new CacheEncodeException( - "decode error", - new ClassNotFoundException("com.alicp.jetcache.CacheValueHolder") + "decode error", + new ClassNotFoundException("com.alicp.jetcache.CacheValueHolder") )); - RecordingCache cache = new RecordingCache(failure, CacheResult.SUCCESS_WITHOUT_MSG); - ChainStateRepositoryImpl repository = repository(cache.asCache()); + RecordingCache cache = new RecordingCache(failure); + RecordingVersionedObjectStore stateStore = new RecordingVersionedObjectStore(); + ChainStateRepositoryImpl repository = repository(cache.asCache(), stateStore); try { repository.load(instanceId); @@ -44,142 +60,448 @@ public class ChainStateRepositoryImplTest { Assert.assertTrue(expected.getMessage().contains(instanceId)); } - Assert.assertEquals(0, cache.getPutCount()); + Assert.assertEquals(0, stateStore.getCreateCount()); } /** - * 验证缓存未命中时创建并持久化新的工作流状态。 + * 验证新实例通过版本对象存储显式创建。 * - * @throws Exception 缓存依赖注入失败时抛出 + * @throws Exception 测试依赖注入失败时抛出 */ @Test - public void loadShouldCreateStateWhenCacheDoesNotExist() throws Exception { + public void createShouldPersistStateWhenStateDoesNotExist() throws Exception { String instanceId = "new-instance"; RecordingCache cache = new RecordingCache( - new CacheGetResult<>(CacheResultCode.NOT_EXISTS, null, null), - CacheResult.SUCCESS_WITHOUT_MSG - ); - ChainStateRepositoryImpl repository = repository(cache.asCache()); + new CacheGetResult<>(CacheResultCode.NOT_EXISTS, null, null)); + RecordingVersionedObjectStore stateStore = new RecordingVersionedObjectStore(); + ChainStateRepositoryImpl repository = repository(cache.asCache(), stateStore); - ChainState state = repository.load(instanceId); + ChainState state = repository.create(instanceId); Assert.assertEquals(instanceId, state.getInstanceId()); - Assert.assertEquals(1, cache.getPutCount()); - Assert.assertSame(state, cache.getLastPutValue()); + Assert.assertEquals(1, stateStore.getCreateCount()); + Assert.assertEquals( + instanceId, + stateStore.getLastCreatedFields().get(ChainStateField.INSTANCE_ID.name())); } /** - * 验证缓存写入失败时不会返回未持久化的工作流状态。 + * 验证旧 JetCache 状态首次读取后迁移到版本对象存储。 * - * @throws Exception 缓存依赖注入失败时抛出 + * @throws Exception 测试依赖注入失败时抛出 */ @Test - public void loadShouldFailWhenNewStateCannotBePersisted() throws Exception { + public void loadShouldMigrateLegacyStateOnce() throws Exception { + ChainState legacy = new ChainState(); + legacy.setInstanceId("legacy-instance"); + legacy.setVersion(7L); + legacy.setStatus(ChainStatus.SUCCEEDED); RecordingCache cache = new RecordingCache( - new CacheGetResult<>(CacheResultCode.NOT_EXISTS, null, null), - new CacheResult(new IllegalStateException("redis unavailable")) - ); - ChainStateRepositoryImpl repository = repository(cache.asCache()); + new CacheGetResult<>( + CacheResultCode.SUCCESS, + null, + new CacheValueHolder<>(legacy, Long.MAX_VALUE))); + RecordingVersionedObjectStore stateStore = new RecordingVersionedObjectStore(); + ChainStateRepositoryImpl repository = repository(cache.asCache(), stateStore); - try { - repository.load("write-failed-instance"); - Assert.fail("cache write failure should be propagated"); - } catch (CacheException expected) { - Assert.assertTrue(expected.getMessage().contains("工作流状态缓存写入失败")); - } + ChainState loaded = repository.load(legacy.getInstanceId()); - Assert.assertEquals(1, cache.getPutCount()); + Assert.assertNotSame(legacy, loaded); + Assert.assertEquals(legacy.getInstanceId(), loaded.getInstanceId()); + Assert.assertEquals(legacy.getStatus(), loaded.getStatus()); + Assert.assertEquals(legacy.getVersion(), loaded.getVersion()); + Assert.assertEquals(1, stateStore.getCreateCount()); + Assert.assertEquals(7L, stateStore.getVersionForLastKey()); } /** - * 创建工作流状态仓储并注入缓存。 + * 验证过期版本不能覆盖已经成功提交的新状态。 * - * @param cache 测试缓存 - * @return 已完成依赖注入的工作流状态仓储 + * @throws Exception 测试依赖注入失败时抛出 + */ + @Test + public void tryUpdateShouldRejectStaleVersion() throws Exception { + RecordingCache cache = new RecordingCache( + new CacheGetResult<>(CacheResultCode.NOT_EXISTS, null, null)); + RecordingVersionedObjectStore stateStore = new RecordingVersionedObjectStore(); + ChainStateRepositoryImpl repository = repository(cache.asCache(), stateStore); + ChainState created = repository.create("cas-instance"); + + ChainState firstUpdate = new ChainState(); + firstUpdate.setInstanceId(created.getInstanceId()); + firstUpdate.setVersion(1L); + Assert.assertTrue(repository.tryUpdate( + firstUpdate, EnumSet.of(ChainStateField.VERSION))); + + ChainState staleUpdate = new ChainState(); + staleUpdate.setInstanceId(created.getInstanceId()); + staleUpdate.setVersion(1L); + Assert.assertFalse(repository.tryUpdate( + staleUpdate, EnumSet.of(ChainStateField.VERSION))); + } + + /** + * 验证实例锁成功获取后分配独立的实例 fencing token。 + * + * @throws Exception 测试依赖注入失败时抛出 + */ + @Test + public void getLockShouldAllocateInstanceFencingToken() throws Exception { + RecordingCache cache = new RecordingCache( + new CacheGetResult<>(CacheResultCode.NOT_EXISTS, null, null)); + ChainStateRepositoryImpl repository = repository( + cache.asCache(), new RecordingVersionedObjectStore()); + RedisLockExecutor lockExecutor = Mockito.mock(RedisLockExecutor.class); + RedisLockExecutor.LockHandle handle = + Mockito.mock(RedisLockExecutor.LockHandle.class); + Mockito.when(lockExecutor.tryAcquireFenced( + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), + ArgumentMatchers.any(Duration.class), + ArgumentMatchers.any(Duration.class), + ArgumentMatchers.any(Duration.class) + )).thenReturn(handle); + Mockito.when(handle.getFencingToken()).thenReturn(17L); + setField(repository, "redisLockExecutor", lockExecutor); + + ChainLock lock = repository.getLock("fenced-instance", 10L, TimeUnit.SECONDS); + try { + Assert.assertTrue(lock.isAcquired()); + Assert.assertEquals(17L, lock.getFencingToken()); + } finally { + lock.close(); + } + + Mockito.verify(lockExecutor).tryAcquireFenced( + ArgumentMatchers.eq("chainLock:{fenced-instance}"), + ArgumentMatchers.eq("workflowState:{fenced-instance}:fence"), + ArgumentMatchers.any(Duration.class), + ArgumentMatchers.any(Duration.class), + ArgumentMatchers.eq(Duration.ofDays(4))); + Mockito.verify(handle).release(); + } + + /** + * 验证状态 CAS 同时校验实例锁和 trigger claim 守卫。 + * + * @throws Exception 测试依赖注入失败时抛出 + */ + @Test + public void tryUpdateShouldGuardLockAndSpecificTriggerClaim() throws Exception { + RecordingCache cache = new RecordingCache( + new CacheGetResult<>(CacheResultCode.NOT_EXISTS, null, null)); + VersionedObjectStore stateStore = Mockito.mock(VersionedObjectStore.class); + Mockito.when(stateStore.compareAndSetFieldsAndRefresh( + ArgumentMatchers.anyString(), + ArgumentMatchers.anyLong(), + ArgumentMatchers.anyMap(), + ArgumentMatchers.anyLong(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyLong(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyLong(), + ArgumentMatchers.any(Duration.class), + ArgumentMatchers.anyString(), + ArgumentMatchers.any(Duration.class) + )).thenReturn(true); + ChainStateRepositoryImpl repository = repository(cache.asCache(), stateStore); + ChainState update = new ChainState(); + update.setInstanceId("claim-guard-instance"); + update.setVersion(1L); + + Assert.assertTrue(repository.tryUpdate( + update, + EnumSet.of(ChainStateField.VERSION), + 17L, + "trigger-1", + 42L)); + + Mockito.verify(stateStore).compareAndSetFieldsAndRefresh( + ArgumentMatchers.eq("workflowState:{claim-guard-instance}:chain"), + ArgumentMatchers.eq(0L), + ArgumentMatchers.anyMap(), + ArgumentMatchers.eq(1L), + ArgumentMatchers.eq("workflowState:{claim-guard-instance}:fence"), + ArgumentMatchers.eq(17L), + ArgumentMatchers.eq( + "workflowState:{claim-guard-instance}:claim:trigger-1"), + ArgumentMatchers.eq(42L), + ArgumentMatchers.eq(Duration.ofDays(3)), + ArgumentMatchers.eq("workflowState:{claim-guard-instance}:format"), + ArgumentMatchers.eq(Duration.ofDays(4))); + } + + /** + * 创建工作流状态仓储并注入测试依赖。 + * + * @param cache 旧 JetCache 测试代理 + * @param stateStore 版本对象存储 + * @return 已完成依赖注入的仓储 * @throws Exception 反射注入失败时抛出 */ - private ChainStateRepositoryImpl repository(Cache cache) throws Exception { + private ChainStateRepositoryImpl repository(Cache cache, + VersionedObjectStore stateStore) throws Exception { ChainStateRepositoryImpl repository = new ChainStateRepositoryImpl(); - Field field = BaseRepository.class.getDeclaredField("cache"); - field.setAccessible(true); - field.set(repository, cache); + Field cacheField = BaseRepository.class.getDeclaredField("cache"); + cacheField.setAccessible(true); + cacheField.set(repository, cache); + Field storeField = ChainStateRepositoryImpl.class.getDeclaredField("versionedObjectStore"); + storeField.setAccessible(true); + storeField.set(repository, stateStore); return repository; } /** - * 仅实现当前仓储测试所需操作的 JetCache 调用记录器。 + * 反射注入测试依赖。 + * + * @param target 目标对象 + * @param name 字段名 + * @param value 字段值 + * @throws Exception 字段访问失败时抛出 + */ + private void setField(Object target, String name, Object value) throws Exception { + Field field = target.getClass().getDeclaredField(name); + field.setAccessible(true); + field.set(target, value); + } + + /** + * 仅实现当前仓储测试所需读取操作的 JetCache 代理。 */ private static final class RecordingCache implements InvocationHandler { private final CacheGetResult getResult; - private final CacheResult putResult; - private int putCount; - private Object lastPutValue; /** - * 创建缓存调用记录器。 + * 创建 JetCache 调用记录器。 * * @param getResult 读取操作结果 - * @param putResult 写入操作结果 */ - private RecordingCache(CacheGetResult getResult, CacheResult putResult) { + private RecordingCache(CacheGetResult getResult) { this.getResult = getResult; - this.putResult = putResult; } /** - * 创建实现 JetCache 接口的 JDK 动态代理。 + * 创建实现 JetCache 接口的动态代理。 * * @return JetCache 测试代理 */ @SuppressWarnings("unchecked") private Cache asCache() { return (Cache) Proxy.newProxyInstance( - Cache.class.getClassLoader(), - new Class[]{Cache.class}, - this + Cache.class.getClassLoader(), + new Class[]{Cache.class}, + this ); } /** - * 处理仓储发起的缓存读写操作。 + * 处理仓储发起的缓存调用。 * * @param proxy 代理对象 * @param method 被调用方法 * @param args 调用参数 - * @return 预设的缓存操作结果 + * @return 预设结果 */ @Override public Object invoke(Object proxy, Method method, Object[] args) { if ("GET".equals(method.getName())) { return getResult; } - if ("PUT".equals(method.getName()) && args != null && args.length == 4) { - putCount++; - lastPutValue = args[1]; - Assert.assertEquals(3L, args[2]); - Assert.assertEquals(TimeUnit.DAYS, args[3]); - return putResult; + if ("REMOVE".equals(method.getName())) { + return CacheResult.SUCCESS_WITHOUT_MSG; } - throw new UnsupportedOperationException("unsupported cache method: " + method.getName()); + throw new UnsupportedOperationException( + "unsupported cache method: " + method.getName()); + } + } + + /** + * 以进程内 Map 模拟原子版本对象存储。 + */ + private static final class RecordingVersionedObjectStore implements VersionedObjectStore { + + private final Map values = new ConcurrentHashMap<>(); + private final Map> fieldValues = new ConcurrentHashMap<>(); + private final Map versions = new ConcurrentHashMap<>(); + private int createCount; + private Map lastCreatedFields; + private String lastKey; + + /** + * {@inheritDoc} + */ + @Override + public T load(String key, Class type) { + Serializable value = values.get(key); + return value == null ? null : type.cast(value); } /** - * 获取写入调用次数。 - * - * @return 写入调用次数 + * {@inheritDoc} */ - private int getPutCount() { - return putCount; + @Override + public VersionedFields loadFields(String key) { + Map fields = fieldValues.get(key); + Long version = versions.get(key); + return fields == null || version == null + ? null + : new VersionedFields(version, fields); } /** - * 获取最后一次写入的缓存值。 - * - * @return 最后一次写入的缓存值 + * {@inheritDoc} */ - private Object getLastPutValue() { - return lastPutValue; + @Override + public synchronized boolean createFieldsIfAbsent( + String key, + Map fields, + long version, + Duration ttl) { + if (fieldValues.containsKey(key) || values.containsKey(key)) { + return false; + } + fieldValues.put(key, new LinkedHashMap<>(fields)); + versions.put(key, version); + if (!key.endsWith(":format")) { + createCount++; + lastCreatedFields = new LinkedHashMap<>(fields); + lastKey = key; + } + return true; + } + + /** + * {@inheritDoc} + */ + @Override + public synchronized boolean compareAndSetFields( + String key, + long expectedVersion, + Map fields, + long newVersion, + Duration ttl) { + Long currentVersion = versions.get(key); + if (currentVersion == null || currentVersion != expectedVersion) { + return false; + } + fieldValues.computeIfAbsent(key, ignored -> new LinkedHashMap<>()).putAll(fields); + versions.put(key, newVersion); + return true; + } + + /** + * {@inheritDoc} + */ + @Override + public synchronized boolean rewriteAsFields( + String key, + long expectedVersion, + Map fields, + Duration ttl) { + Long currentVersion = versions.get(key); + if (currentVersion == null || currentVersion != expectedVersion) { + return false; + } + values.remove(key); + fieldValues.put(key, new LinkedHashMap<>(fields)); + return true; + } + + /** + * {@inheritDoc} + */ + @Override + public synchronized boolean createIfAbsent(String key, + Serializable value, + long version, + Duration ttl) { + if (values.containsKey(key)) { + return false; + } + values.put(key, value); + versions.put(key, version); + createCount++; + lastKey = key; + return true; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean createIfAbsent(String key, + Serializable value, + long version, + String guardKey, + long guardVersion, + Duration ttl) { + Long currentGuard = versions.get(guardKey); + return currentGuard != null + && currentGuard == guardVersion + && createIfAbsent(key, value, version, ttl); + } + + /** + * {@inheritDoc} + */ + @Override + public synchronized boolean compareAndSet(String key, + long expectedVersion, + Serializable value, + long newVersion, + Duration ttl) { + Long currentVersion = versions.get(key); + if (currentVersion == null || currentVersion != expectedVersion) { + return false; + } + values.put(key, value); + versions.put(key, newVersion); + return true; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean compareAndSet(String key, + long expectedVersion, + Serializable value, + long newVersion, + String guardKey, + long guardVersion, + Duration ttl) { + Long currentGuard = versions.get(guardKey); + return currentGuard != null + && currentGuard == guardVersion + && compareAndSet(key, expectedVersion, value, newVersion, ttl); + } + + /** + * 获取创建次数。 + * + * @return 创建次数 + */ + private int getCreateCount() { + return createCount; + } + + /** + * 获取最后创建的对象。 + * + * @return 最后创建的对象 + */ + private Map getLastCreatedFields() { + return lastCreatedFields; + } + + /** + * 获取最后写入键的版本。 + * + * @return 最后写入版本 + */ + private long getVersionForLastKey() { + return versions.get(lastKey); } } } diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/repository/LoopResultRepositoryImplTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/repository/LoopResultRepositoryImplTest.java new file mode 100644 index 00000000..26f6cdab --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/repository/LoopResultRepositoryImplTest.java @@ -0,0 +1,621 @@ +package tech.easyflow.ai.easyagentsflow.repository; + +import com.easyagents.flow.core.chain.repository.LoopResultReference; +import com.easyagents.flow.core.chain.repository.LoopInputReference; +import com.easyagents.flow.core.chain.runtime.TriggerClaimLostException; +import org.junit.Assert; +import org.junit.Test; +import tech.easyflow.common.cache.VersionedObjectStore; + +import java.lang.reflect.Field; +import java.io.Serializable; +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * 循环结果分块仓储测试。 + */ +public class LoopResultRepositoryImplTest { + + /** + * 验证跨多个分块的结果顺序、完整性及幂等重放。 + */ + @Test + public void shouldPreserveOrderingAcrossChunkBoundaries() { + InMemoryLoopResultRepository repository = new InMemoryLoopResultRepository(); + String resultId = "loop-result"; + int iterations = LoopResultRepositoryImpl.CHUNK_SIZE * 2 + 1; + + for (int index = 0; index < iterations; index++) { + repository.append(resultId, index, Map.of( + "index", index, + "value", "value-" + index)); + } + repository.append(resultId, iterations - 1, Map.of( + "index", iterations - 1, + "value", "value-" + (iterations - 1))); + + Map result = + repository.load(resultId, iterations, List.of("index", "value")); + Assert.assertEquals(iterations, ((List) result.get("index")).size()); + Assert.assertEquals(0, ((List) result.get("index")).get(0)); + Assert.assertEquals(iterations - 1, ((List) result.get("index")).get(iterations - 1)); + Assert.assertEquals("value-128", ((List) result.get("value")).get(128)); + } + + /** + * 验证同一轮次写入不同结果时拒绝覆盖。 + */ + @Test(expected = IllegalStateException.class) + public void shouldRejectConflictingReplay() { + InMemoryLoopResultRepository repository = new InMemoryLoopResultRepository(); + repository.append("loop-result", 0, Map.of("value", "first")); + repository.append("loop-result", 0, Map.of("value", "changed")); + } + + /** + * 验证热状态只保存轻量引用，业务读取边界仍还原为原有列表结构。 + */ + @Test + public void shouldResolveLightweightReferenceAtReadBoundary() { + InMemoryLoopResultRepository repository = new InMemoryLoopResultRepository(); + String resultId = "instance:loop-result"; + repository.append(resultId, 0, Map.of("value", "first")); + repository.append(resultId, 1, Map.of("value", "second")); + + Map references = repository.references( + resultId, 2, List.of("value")); + Assert.assertTrue(references.get("value") instanceof LoopResultReference); + + @SuppressWarnings("unchecked") + Map resolved = + (Map) repository.resolveReferences(references); + Assert.assertEquals(List.of("first", "second"), resolved.get("value")); + } + + /** + * 验证没有声明输出的长循环跨分块时仍会续期输入生命周期。 + * + * @throws Exception 测试依赖注入失败时抛出 + */ + @Test + public void shouldRefreshInputChunksWhenLoopHasNoOutputs() throws Exception { + LoopResultRepositoryImpl repository = new LoopResultRepositoryImpl(); + VersionedObjectStore store = mock(VersionedObjectStore.class); + Field field = LoopResultRepositoryImpl.class.getDeclaredField( + "versionedObjectStore"); + field.setAccessible(true); + field.set(repository, store); + when(store.load(any(String.class), eq(Integer.class))).thenReturn(256); + when(store.createIfAbsent( + anyString(), + any(Serializable.class), + anyLong(), + anyString(), + anyLong(), + anyString(), + anyLong(), + any(Duration.class))).thenReturn(true); + + repository.append( + "instance", + 1L, + "claim", + 1L, + "instance:loop", + LoopResultRepositoryImpl.CHUNK_SIZE, + Map.of()); + + verify(store).refreshExpirations(anyList(), any(Duration.class)); + } + + /** + * 验证同一输入分块内的多轮读取只访问一次底层对象存储。 + * + * @throws Exception 测试依赖注入失败时抛出 + */ + @Test + public void shouldLoadEachInputChunkOnlyOnce() throws Exception { + LoopResultRepositoryImpl repository = new LoopResultRepositoryImpl(); + VersionedObjectStore store = mock(VersionedObjectStore.class); + Field field = LoopResultRepositoryImpl.class.getDeclaredField( + "versionedObjectStore"); + field.setAccessible(true); + field.set(repository, store); + List values = java.util.stream.IntStream.range(0, 128) + .boxed() + .map(value -> (Object) value) + .toList(); + when(store.load(anyString(), eq(List.class))).thenReturn(values); + + Assert.assertEquals(0, repository.loadInputItem("instance:loop", 0)); + Assert.assertEquals(127, repository.loadInputItem("instance:loop", 127)); + + verify(store, times(1)).load(anyString(), eq(List.class)); + } + + /** + * 验证调用方修改已读取的可变输入时不会污染活动分块缓存。 + * + * @throws Exception 测试依赖注入失败时抛出 + */ + @Test + public void shouldIsolateMutableInputValuesFromActiveCache() + throws Exception { + LoopResultRepositoryImpl repository = + new LoopResultRepositoryImpl(); + VersionedObjectStore store = + mock(VersionedObjectStore.class); + Field field = LoopResultRepositoryImpl.class + .getDeclaredField("versionedObjectStore"); + field.setAccessible(true); + field.set(repository, store); + Map persisted = + new LinkedHashMap<>(); + persisted.put("name", "original"); + when(store.load(anyString(), eq(List.class))) + .thenReturn(List.of(persisted)); + + @SuppressWarnings("unchecked") + Map first = + (Map) + repository.loadInputItem( + "instance:mutable-input", + 0); + first.put("name", "changed"); + @SuppressWarnings("unchecked") + Map second = + (Map) + repository.loadInputItem( + "instance:mutable-input", + 0); + + Assert.assertEquals( + "original", second.get("name")); + verify(store, times(1)).load( + anyString(), eq(List.class)); + } + + /** + * 验证完整输入还原按分块批量读取，并保持不同调用方的可变值隔离。 + * + * @throws Exception 测试依赖注入失败时抛出 + */ + @Test + @SuppressWarnings("unchecked") + public void shouldBulkLoadMutableInputChunks() + throws Exception { + LoopResultRepositoryImpl repository = + new LoopResultRepositoryImpl(); + VersionedObjectStore store = + mock(VersionedObjectStore.class); + Field field = LoopResultRepositoryImpl.class + .getDeclaredField("versionedObjectStore"); + field.setAccessible(true); + field.set(repository, store); + int itemCount = + LoopResultRepositoryImpl.CHUNK_SIZE + * 2 + 1; + when(store.loadAll( + anyList(), eq(List.class))) + .thenAnswer(invocation -> { + List> chunks = + new java.util.ArrayList<>(); + for (int chunkIndex = 0; + chunkIndex < 3; + chunkIndex++) { + int chunkSize = chunkIndex < 2 + ? LoopResultRepositoryImpl.CHUNK_SIZE + : 1; + List chunk = + new java.util.ArrayList<>(); + for (int offset = 0; + offset < chunkSize; + offset++) { + Map value = + new LinkedHashMap<>(); + value.put( + "index", + chunkIndex + * LoopResultRepositoryImpl.CHUNK_SIZE + + offset); + chunk.add(value); + } + chunks.add(chunk); + } + return chunks; + }); + LoopInputReference reference = + new LoopInputReference( + "instance:bulk-input", + itemCount); + + List first = + repository.loadInput(reference); + ((Map) first.get(0)) + .put("index", -1); + List second = + repository.loadInput(reference); + + Assert.assertEquals( + itemCount, second.size()); + Assert.assertEquals( + 0, + ((Map) second.get(0)) + .get("index")); + verify(store, times(2)).loadAll( + anyList(), eq(List.class)); + verify(store, times(0)).load( + anyString(), eq(List.class)); + } + + /** + * 验证连续循环输出命中活动分块缓存时不重复读取 Redis 对象。 + * + * @throws Exception 测试依赖注入失败时抛出 + */ + @Test + public void shouldReuseActiveOutputChunkAfterSuccessfulCommit() + throws Exception { + LoopResultRepositoryImpl repository = + new LoopResultRepositoryImpl(); + VersionedObjectStore store = + mock(VersionedObjectStore.class); + Field field = LoopResultRepositoryImpl.class + .getDeclaredField("versionedObjectStore"); + field.setAccessible(true); + field.set(repository, store); + when(store.load( + anyString(), + eq(LoopResultRepositoryImpl + .LoopResultChunk.class))) + .thenReturn(null); + when(store.createIfAbsent( + anyString(), + any(Serializable.class), + anyLong(), + anyString(), + anyLong(), + anyString(), + anyLong(), + any(Duration.class))) + .thenReturn(true); + when(store.compareAndSet( + anyString(), + anyLong(), + any(Serializable.class), + anyLong(), + anyString(), + anyLong(), + anyString(), + anyLong(), + any(Duration.class))) + .thenReturn(true); + + repository.append( + "instance", + 1L, + "claim", + 1L, + "instance:cached-output", + 0, + Map.of("value", "first")); + repository.append( + "instance", + 1L, + "claim", + 1L, + "instance:cached-output", + 1, + Map.of("value", "second")); + + verify(store, times(1)).load( + anyString(), + eq(LoopResultRepositoryImpl + .LoopResultChunk.class)); + verify(store, times(1)).compareAndSet( + anyString(), + eq(0L), + any(Serializable.class), + eq(1L), + anyString(), + eq(1L), + anyString(), + eq(1L), + any(Duration.class)); + } + + /** + * 验证调用方修改已提交的可变输出时不会污染下一轮活动分块写入。 + * + * @throws Exception 测试依赖注入失败时抛出 + */ + @Test + public void shouldIsolateMutableOutputValuesFromActiveCache() + throws Exception { + LoopResultRepositoryImpl repository = + new LoopResultRepositoryImpl(); + VersionedObjectStore store = + mock(VersionedObjectStore.class); + Field field = LoopResultRepositoryImpl.class + .getDeclaredField("versionedObjectStore"); + field.setAccessible(true); + field.set(repository, store); + when(store.load( + anyString(), + eq(LoopResultRepositoryImpl + .LoopResultChunk.class))) + .thenReturn(null); + when(store.createIfAbsent( + anyString(), + any(Serializable.class), + anyLong(), + anyString(), + anyLong(), + anyString(), + anyLong(), + any(Duration.class))) + .thenReturn(true); + when(store.compareAndSet( + anyString(), + anyLong(), + any(Serializable.class), + anyLong(), + anyString(), + anyLong(), + anyString(), + anyLong(), + any(Duration.class))) + .thenReturn(true); + Map firstValue = + new LinkedHashMap<>(); + firstValue.put("name", "original"); + + repository.append( + "instance", + 1L, + "claim", + 1L, + "instance:mutable-output", + 0, + Map.of("value", firstValue)); + firstValue.put("name", "changed"); + repository.append( + "instance", + 1L, + "claim", + 1L, + "instance:mutable-output", + 1, + Map.of("value", Map.of( + "name", "second"))); + + org.mockito.ArgumentCaptor + chunkCaptor = + org.mockito.ArgumentCaptor.forClass( + Serializable.class); + verify(store).compareAndSet( + anyString(), + eq(0L), + chunkCaptor.capture(), + eq(1L), + anyString(), + eq(1L), + anyString(), + eq(1L), + any(Duration.class)); + LoopResultRepositoryImpl.LoopResultChunk + committed = + (LoopResultRepositoryImpl.LoopResultChunk) + chunkCaptor.getValue(); + @SuppressWarnings("unchecked") + Map committedFirst = + (Map) + committed.getValues() + .get("value") + .get(0); + Assert.assertEquals( + "original", + committedFirst.get("name")); + } + + /** + * 验证跨分块后活动输出缓存只保留当前分块。 + * + * @throws Exception 测试依赖注入或反射读取失败时抛出 + */ + @Test + public void shouldKeepOnlyCurrentOutputChunkInActiveCache() + throws Exception { + LoopResultRepositoryImpl repository = + new LoopResultRepositoryImpl(); + VersionedObjectStore store = + mock(VersionedObjectStore.class); + Field storeField = LoopResultRepositoryImpl.class + .getDeclaredField("versionedObjectStore"); + storeField.setAccessible(true); + storeField.set(repository, store); + when(store.createIfAbsent( + anyString(), + any(Serializable.class), + anyLong(), + anyString(), + anyLong(), + anyString(), + anyLong(), + any(Duration.class))) + .thenReturn(true); + + repository.append( + "instance", + 1L, + "claim", + 1L, + "instance:chunk-release", + 0, + Map.of("value", "first")); + repository.append( + "instance", + 1L, + "claim", + 1L, + "instance:chunk-release", + LoopResultRepositoryImpl.CHUNK_SIZE, + Map.of("value", "next")); + + Field cacheField = LoopResultRepositoryImpl.class + .getDeclaredField("outputChunkCache"); + cacheField.setAccessible(true); + Object cache = cacheField.get(repository); + Field valuesField = cache.getClass() + .getDeclaredField("values"); + valuesField.setAccessible(true); + @SuppressWarnings("unchecked") + Map cachedValues = + (Map) + valuesField.get(cache); + Assert.assertEquals( + 1, cachedValues.size()); + } + + /** + * 验证物化中失去 fencing 守卫后立即停止并清理本 owner 已写分块。 + * + * @throws Exception 测试依赖注入失败时抛出 + */ + @Test + public void shouldCleanupPartialInputWhenClaimIsLost() + throws Exception { + LoopResultRepositoryImpl repository = + new LoopResultRepositoryImpl(); + VersionedObjectStore store = + mock(VersionedObjectStore.class); + Field field = LoopResultRepositoryImpl.class + .getDeclaredField("versionedObjectStore"); + field.setAccessible(true); + field.set(repository, store); + when(store.createIfAbsent( + anyString(), + any(Serializable.class), + anyLong(), + anyString(), + anyLong(), + anyString(), + anyLong(), + any(Duration.class))) + .thenReturn(true, false); + List input = java.util.stream.IntStream + .range(0, LoopResultRepositoryImpl.CHUNK_SIZE + 1) + .boxed() + .toList(); + + try { + repository.storeInput( + "instance", + 1L, + "claim", + 1L, + "instance:guarded-input", + input, + 10_000L); + Assert.fail("lost claim must stop materialization"); + } catch (TriggerClaimLostException expected) { + // 第二个分块守卫失败后立即退出。 + } + + verify(store).deleteAll( + org.mockito.ArgumentMatchers.argThat( + keys -> keys.size() == 1)); + } + + /** + * 验证锁外物化只依赖稳定 claim，合法实例锁代际推进不会中断后续分块。 + * + * @throws Exception 测试依赖注入失败时抛出 + */ + @Test + public void shouldMaterializeAllChunksWithStableClaimGuard() + throws Exception { + LoopResultRepositoryImpl repository = + new LoopResultRepositoryImpl(); + VersionedObjectStore store = + mock(VersionedObjectStore.class); + Field field = LoopResultRepositoryImpl.class + .getDeclaredField("versionedObjectStore"); + field.setAccessible(true); + field.set(repository, store); + when(store.createIfAbsent( + anyString(), + any(Serializable.class), + anyLong(), + anyString(), + anyLong(), + any(Duration.class))).thenReturn(true); + List input = java.util.stream.IntStream + .range(0, LoopResultRepositoryImpl.CHUNK_SIZE + 1) + .boxed() + .toList(); + + int stored = repository.storeProducedInput( + "instance", + 0L, + "claim", + 7L, + "instance:stable-input", + sink -> input.forEach(sink), + 10_000L); + + Assert.assertEquals(input.size(), stored); + verify(store, times(3)).createIfAbsent( + anyString(), + any(Serializable.class), + anyLong(), + anyString(), + eq(7L), + any(Duration.class)); + } + + /** + * 使用内存 Map 隔离 JetCache 的测试仓储。 + */ + private static final class InMemoryLoopResultRepository extends LoopResultRepositoryImpl { + + private final Map values = new LinkedHashMap<>(); + + /** + * 将分块写入测试内存。 + * + * @param key 缓存键 + * @param value 缓存值 + */ + @Override + protected void putCache(String key, Object value) { + values.put(key, value); + } + + /** + * 从测试内存读取分块。 + * + * @param key 缓存键 + * @param clazz 期望类型 + * @param 缓存值类型 + * @return 命中的分块 + */ + @Override + protected T getCache(String key, Class clazz) { + return clazz.cast(values.get(key)); + } + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/repository/RedisTriggerStoreTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/repository/RedisTriggerStoreTest.java new file mode 100644 index 00000000..7deaf7d7 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/repository/RedisTriggerStoreTest.java @@ -0,0 +1,193 @@ +package tech.easyflow.ai.easyagentsflow.repository; + +import com.easyagents.flow.core.chain.runtime.Trigger; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.data.redis.core.ValueOperations; +import org.springframework.data.redis.core.ZSetOperations; +import org.springframework.data.redis.core.script.RedisScript; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * {@link RedisTriggerStore} 分布式认领语义回归测试。 + */ +public class RedisTriggerStoreTest { + + /** + * 验证同一到期窗口超过 200 条任务时仍可一次填充本地调度容量。 + * + * @throws Exception 测试触发器序列化失败时抛出 + */ + @Test + @SuppressWarnings("unchecked") + public void findDueShouldLoadMoreThanLegacyBatchLimit() + throws Exception { + StringRedisTemplate redisTemplate = + Mockito.mock(StringRedisTemplate.class); + ZSetOperations zSetOperations = + Mockito.mock(ZSetOperations.class); + ValueOperations valueOperations = + Mockito.mock(ValueOperations.class); + Mockito.when(redisTemplate.opsForZSet()) + .thenReturn(zSetOperations); + Mockito.when(redisTemplate.opsForValue()) + .thenReturn(valueOperations); + + ObjectMapper objectMapper = new ObjectMapper(); + Set ids = new LinkedHashSet<>(); + List payloads = new ArrayList<>(); + for (int index = 0; index < 512; index++) { + String id = "due-" + index; + Trigger trigger = new Trigger(); + trigger.setId(id); + trigger.setStateInstanceId( + "instance-" + index); + trigger.setTriggerAt(1000L); + ids.add(id); + payloads.add( + objectMapper.writeValueAsString( + trigger)); + } + Mockito.when(zSetOperations.rangeByScore( + ArgumentMatchers.anyString(), + ArgumentMatchers.anyDouble(), + ArgumentMatchers.anyDouble(), + ArgumentMatchers.eq(0L), + ArgumentMatchers.eq(1024L))) + .thenReturn(ids); + Mockito.when(valueOperations.multiGet( + ArgumentMatchers.anyList())) + .thenReturn(payloads); + RedisTriggerStore store = + new RedisTriggerStore( + redisTemplate, + objectMapper); + + List due = + store.findDue(1000L); + + Assert.assertEquals(512, due.size()); + Mockito.verify(zSetOperations) + .rangeByScore( + ArgumentMatchers.anyString(), + ArgumentMatchers.eq(0.0), + ArgumentMatchers.eq(1000.0), + ArgumentMatchers.eq(0L), + ArgumentMatchers.eq(1024L)); + } + + /** + * 验证稳定触发器通过单条 Redis 脚本完成存在性判断和创建。 + */ + @Test + @SuppressWarnings({"unchecked", "rawtypes"}) + public void saveIfAbsentShouldUseAtomicRedisScript() { + StringRedisTemplate redisTemplate = + Mockito.mock(StringRedisTemplate.class); + Mockito.doReturn(1L).when(redisTemplate).execute( + ArgumentMatchers.>any(), + ArgumentMatchers.>any(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString()); + RedisTriggerStore store = + new RedisTriggerStore( + redisTemplate, + new ObjectMapper()); + Trigger trigger = new Trigger(); + trigger.setId("stable-trigger"); + trigger.setTriggerAt( + System.currentTimeMillis()); + + Assert.assertTrue( + store.saveIfAbsent(trigger)); + + ArgumentCaptor> + scriptCaptor = + ArgumentCaptor.forClass( + (Class) RedisScript.class); + Mockito.verify(redisTemplate).execute( + scriptCaptor.capture(), + ArgumentMatchers.>any(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString()); + String script = + scriptCaptor.getValue() + .getScriptAsString(); + Assert.assertTrue(script.contains( + "exists', KEYS[1]")); + Assert.assertTrue(script.contains( + "psetex', KEYS[1]")); + } + + /** + * 验证认领触发器分配一次独立代际，并创建与该 trigger claim 绑定的执行守卫。 + * + *

认领代际与实例锁 fencing token 使用不同计数器，claim 不推进实例锁 fence。

+ * + * @throws Exception JSON 构造失败时抛出 + */ + @Test + @SuppressWarnings({"unchecked", "rawtypes"}) + public void claimShouldCreateTriggerScopedExecutionGuard() throws Exception { + StringRedisTemplate redisTemplate = Mockito.mock(StringRedisTemplate.class); + ObjectMapper objectMapper = new ObjectMapper(); + Trigger stored = new Trigger(); + stored.setId("trigger-1"); + stored.setStateInstanceId("instance-1"); + stored.setTriggerAt(System.currentTimeMillis()); + stored.setFencingToken(7L); + String payload = objectMapper.writeValueAsString(stored); + Mockito.doReturn("8\n" + payload).when(redisTemplate).execute( + ArgumentMatchers.>any(), + ArgumentMatchers.>any(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString()); + RedisTriggerStore store = + new RedisTriggerStore(redisTemplate, objectMapper); + + Trigger claimed = store.claim(stored, 60_000L); + + Assert.assertNotNull(claimed); + Assert.assertEquals(8L, claimed.getFencingToken()); + ArgumentCaptor> scriptCaptor = + ArgumentCaptor.forClass((Class) RedisScript.class); + ArgumentCaptor> keysCaptor = + ArgumentCaptor.forClass((Class) List.class); + Mockito.verify(redisTemplate).execute( + scriptCaptor.capture(), + keysCaptor.capture(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString()); + Assert.assertTrue( + scriptCaptor.getValue().getScriptAsString().contains( + "hset', KEYS[4], 'version'")); + Assert.assertEquals( + "workflowState:{instance-1}:claim:trigger-1", + keysCaptor.getValue().get(3)); + Assert.assertEquals( + "workflowState:{instance-1}:claim-seq", + keysCaptor.getValue().get(4)); + Assert.assertTrue( + scriptCaptor.getValue().getScriptAsString().contains( + "hincrby', KEYS[5], 'version'")); + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/repository/WorkflowDefinitionCacheTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/repository/WorkflowDefinitionCacheTest.java new file mode 100644 index 00000000..3cabf35a --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/repository/WorkflowDefinitionCacheTest.java @@ -0,0 +1,107 @@ +package tech.easyflow.ai.easyagentsflow.repository; + +import com.easyagents.flow.core.chain.ChainDefinition; +import org.junit.Assert; +import org.junit.Test; +import tech.easyflow.ai.easyagentsflow.config.WorkflowRuntimeProperties; +import tech.easyflow.ai.easyagentsflow.event.WorkflowDefinitionChangedEvent; +import tech.easyflow.ai.easyagentsflow.support.PublishedWorkflowDefinitionIds; + +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * {@link WorkflowDefinitionCache} 命中、失效和编译去重回归测试。 + */ +public class WorkflowDefinitionCacheTest { + + /** + * 验证同一版本重复执行只编译一次。 + */ + @Test + public void shouldCompileOnlyOnceForRepeatedReads() { + InMemoryVersionStore versionStore = new InMemoryVersionStore(); + WorkflowDefinitionCache cache = cache(versionStore); + AtomicInteger loads = new AtomicInteger(); + + ChainDefinition first = cache.get("1", () -> definition("1", loads)); + ChainDefinition second = cache.get("1", () -> definition("1", loads)); + + Assert.assertSame(first, second); + Assert.assertEquals(1, loads.get()); + } + + /** + * 验证工作流变更后草稿态和发布态缓存同时失效。 + */ + @Test + public void shouldInvalidateDraftAndPublishedDefinitionsTogether() { + InMemoryVersionStore versionStore = new InMemoryVersionStore(); + WorkflowDefinitionCache cache = cache(versionStore); + AtomicInteger loads = new AtomicInteger(); + String publishedId = PublishedWorkflowDefinitionIds.published("2"); + + ChainDefinition draftBefore = cache.get("2", () -> definition("2", loads)); + ChainDefinition publishedBefore = cache.get(publishedId, () -> definition(publishedId, loads)); + cache.onDefinitionChanged(new WorkflowDefinitionChangedEvent("2")); + ChainDefinition draftAfter = cache.get("2", () -> definition("2", loads)); + ChainDefinition publishedAfter = cache.get(publishedId, () -> definition(publishedId, loads)); + + Assert.assertNotSame(draftBefore, draftAfter); + Assert.assertNotSame(publishedBefore, publishedAfter); + Assert.assertEquals(4, loads.get()); + } + + /** + * 创建测试缓存。 + * + * @param versionStore 版本令牌仓储 + * @return 定义缓存 + */ + private WorkflowDefinitionCache cache(WorkflowDefinitionVersionStore versionStore) { + WorkflowRuntimeProperties properties = new WorkflowRuntimeProperties(); + properties.setDefinitionCacheMaxEntries(4); + return new WorkflowDefinitionCache(versionStore, properties); + } + + /** + * 创建测试定义并记录编译次数。 + * + * @param id 定义 ID + * @param loads 编译计数 + * @return 工作流定义 + */ + private ChainDefinition definition(String id, AtomicInteger loads) { + loads.incrementAndGet(); + ChainDefinition definition = new ChainDefinition(); + definition.setId(id); + return definition; + } + + /** + * 进程内版本令牌测试仓储。 + */ + private static final class InMemoryVersionStore implements WorkflowDefinitionVersionStore { + + private final Map tokens = new ConcurrentHashMap<>(); + + /** + * {@inheritDoc} + */ + @Override + public String currentToken(String definitionId) { + return tokens.computeIfAbsent(definitionId, ignored -> UUID.randomUUID().toString()); + } + + /** + * {@inheritDoc} + */ + @Override + public void invalidateWorkflow(String workflowId) { + tokens.put(workflowId, UUID.randomUUID().toString()); + tokens.put(PublishedWorkflowDefinitionIds.published(workflowId), UUID.randomUUID().toString()); + } + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/repository/WorkflowStateFieldsNodeTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/repository/WorkflowStateFieldsNodeTest.java new file mode 100644 index 00000000..fb6a16a8 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/repository/WorkflowStateFieldsNodeTest.java @@ -0,0 +1,103 @@ +package tech.easyflow.ai.easyagentsflow.repository; + +import com.easyagents.flow.core.chain.NodeState; +import com.easyagents.flow.core.chain.repository.NodeStateField; +import org.junit.Assert; +import org.junit.Test; +import tech.easyflow.ai.easyagentsflow.support.WorkflowExecutionStepKey; +import tech.easyflow.common.cache.VersionedFields; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * 节点字段化状态编码回归测试。 + */ +public class WorkflowStateFieldsNodeTest { + + /** + * 验证节点生命周期业务尝试键可跨 Redis 字段快照恢复。 + */ + @Test + public void shouldPreserveExecutionAttemptKey() { + NodeState state = new NodeState(); + state.setNodeId("loop"); + state.setChainInstanceId("instance"); + state.setExecutionAttemptKey( + "instance:loop:trigger"); + state.setVersion(7L); + + Map encoded = + new LinkedHashMap<>( + WorkflowStateFields + .allNodeFields(state)); + NodeState decoded = + WorkflowStateFields.decodeNode( + new VersionedFields( + 7L, + encoded)); + + Assert.assertEquals( + "instance:loop:trigger", + decoded + .getExecutionAttemptKey()); + Assert.assertEquals( + 7L, decoded.getVersion()); + } + + /** + * 验证升级前在途节点沿用 memory.executeId，避免结束审计关联到新键。 + */ + @Test + public void shouldRestoreLegacyExecutionKey() { + NodeState legacyState = new NodeState(); + legacyState.setNodeId("loop"); + legacyState.setChainInstanceId( + "instance"); + legacyState.getMemory().put( + "executeId", + "legacy-step-key"); + + Map encoded = + new LinkedHashMap<>( + WorkflowStateFields + .allNodeFields( + legacyState)); + encoded.remove( + NodeStateField + .EXECUTION_ATTEMPT_KEY + .name()); + + NodeState decoded = + WorkflowStateFields.decodeNode( + new VersionedFields( + 3L, + encoded)); + + Assert.assertEquals( + "legacy-step-key", + WorkflowExecutionStepKey.resolve( + decoded + .getExecutionAttemptKey())); + } + + /** + * 验证旧对象快照同样补齐最终执行键。 + */ + @Test + public void shouldNormalizeLegacyObjectState() { + NodeState legacyState = new NodeState(); + legacyState.getMemory().put( + "executeId", + "legacy-object-step"); + + WorkflowStateFields.normalizeNode( + legacyState); + + Assert.assertEquals( + "legacy-object-step", + WorkflowExecutionStepKey.resolve( + legacyState + .getExecutionAttemptKey())); + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/service/TinyFlowServiceTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/service/TinyFlowServiceTest.java new file mode 100644 index 00000000..61128426 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/service/TinyFlowServiceTest.java @@ -0,0 +1,138 @@ +package tech.easyflow.ai.easyagentsflow.service; + +import com.easyagents.flow.core.chain.ChainState; +import com.easyagents.flow.core.chain.ChainStatus; +import com.easyagents.flow.core.chain.NodeState; +import com.easyagents.flow.core.chain.NodeStatus; +import com.easyagents.flow.core.chain.repository.ChainStateRepository; +import com.easyagents.flow.core.chain.repository.NodeStateRepository; +import com.easyagents.flow.core.chain.runtime.ChainExecutor; +import org.junit.Assert; +import org.junit.Test; +import tech.easyflow.ai.easyagentsflow.entity.ChainInfo; +import tech.easyflow.ai.easyagentsflow.entity.NodeInfo; + +import java.lang.reflect.Field; +import java.util.List; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * 工作流设计器状态轮询服务测试。 + */ +public class TinyFlowServiceTest { + + private static final String EXECUTE_ID = "execution-1"; + private static final String NODE_ID = "node-1"; + + /** + * 验证尚未启动的节点返回 READY，且一次轮询只读取一次工作流状态。 + * + * @throws Exception 测试依赖注入失败时抛出 + */ + @Test + public void shouldReturnReadyForMissingNodeStateWithoutRepeatedChainReads() + throws Exception { + ChainExecutor chainExecutor = mock(ChainExecutor.class); + ChainStateRepository chainStateRepository = + mock(ChainStateRepository.class); + NodeStateRepository nodeStateRepository = + mock(NodeStateRepository.class); + ChainState chainState = new ChainState(); + chainState.setStatus(ChainStatus.RUNNING); + when(chainExecutor.getChainStateRepository()) + .thenReturn(chainStateRepository); + when(chainExecutor.getNodeStateRepository()) + .thenReturn(nodeStateRepository); + when(chainStateRepository.load(EXECUTE_ID)) + .thenReturn(chainState); + when(nodeStateRepository.load(EXECUTE_ID, NODE_ID)) + .thenReturn(null); + TinyFlowService service = service(chainExecutor); + NodeInfo node = node(NodeStatus.SUCCEEDED); + + ChainInfo result = service.getChainStatus( + EXECUTE_ID, List.of(node)); + + Assert.assertEquals( + Integer.valueOf(ChainStatus.RUNNING.getValue()), + result.getStatus()); + Assert.assertEquals( + Integer.valueOf(NodeStatus.READY.getValue()), + result.getNodes().get(NODE_ID).getStatus()); + verify(chainStateRepository, times(1)).load(EXECUTE_ID); + verify(nodeStateRepository, times(1)) + .load(EXECUTE_ID, NODE_ID); + } + + /** + * 验证已存在节点仍返回仓储中的真实执行状态。 + * + * @throws Exception 测试依赖注入失败时抛出 + */ + @Test + public void shouldReturnPersistedNodeStatus() + throws Exception { + ChainExecutor chainExecutor = mock(ChainExecutor.class); + ChainStateRepository chainStateRepository = + mock(ChainStateRepository.class); + NodeStateRepository nodeStateRepository = + mock(NodeStateRepository.class); + ChainState chainState = new ChainState(); + chainState.setStatus(ChainStatus.RUNNING); + NodeState nodeState = new NodeState(); + nodeState.setStatus(NodeStatus.RUNNING); + when(chainExecutor.getChainStateRepository()) + .thenReturn(chainStateRepository); + when(chainExecutor.getNodeStateRepository()) + .thenReturn(nodeStateRepository); + when(chainStateRepository.load(EXECUTE_ID)) + .thenReturn(chainState); + when(nodeStateRepository.load(EXECUTE_ID, NODE_ID)) + .thenReturn(nodeState); + TinyFlowService service = service(chainExecutor); + + ChainInfo result = service.getChainStatus( + EXECUTE_ID, List.of(node(NodeStatus.READY))); + + Assert.assertEquals( + Integer.valueOf(NodeStatus.RUNNING.getValue()), + result.getNodes().get(NODE_ID).getStatus()); + verify(chainStateRepository, times(1)).load(EXECUTE_ID); + verify(nodeStateRepository, times(1)) + .load(EXECUTE_ID, NODE_ID); + } + + /** + * 创建带指定初始状态的设计器节点。 + * + * @param status 初始节点状态 + * @return 设计器节点 + */ + private NodeInfo node(NodeStatus status) { + NodeInfo node = new NodeInfo(); + node.setNodeId(NODE_ID); + node.setStatus(status.getValue()); + return node; + } + + /** + * 创建并注入执行器的轮询服务。 + * + * @param chainExecutor 工作流执行器 + * @return 已完成依赖注入的服务 + * @throws Exception 反射访问失败时抛出 + */ + private TinyFlowService service(ChainExecutor chainExecutor) + throws Exception { + TinyFlowService service = new TinyFlowService(); + Field field = TinyFlowService.class.getDeclaredField( + "chainExecutor"); + field.setAccessible(true); + field.set(service, chainExecutor); + return service; + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/service/WorkflowCheckServiceTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/service/WorkflowCheckServiceTest.java index 898a8e4e..3533663e 100644 --- a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/service/WorkflowCheckServiceTest.java +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/service/WorkflowCheckServiceTest.java @@ -21,6 +21,89 @@ import java.util.Map; public class WorkflowCheckServiceTest { + /** + * 验证普通节点循环次数必须处于 1～300。 + */ + @Test + public void testSaveShouldBlockConfiguredLoopCountAboveLimit() throws Exception { + WorkflowCheckService service = newService(new HashMap<>()); + JSONObject codeData = data("循环处理"); + codeData.put("loopEnable", true); + codeData.put("maxLoopCount", 301); + String content = workflowJson( + array(node("code-1", "codeNode", null, codeData)), + new JSONArray()); + + WorkflowCheckResult result = service.checkContent( + content, WorkflowCheckStage.SAVE, null); + + Assert.assertFalse(result.isPassed()); + assertHasCode(result, "LOOP_COUNT_INVALID"); + } + + /** + * 验证显式循环节点的固定次数不能为零。 + */ + @Test + public void testSaveShouldBlockFixedExplicitLoopCountZero() throws Exception { + WorkflowCheckService service = newService(new HashMap<>()); + JSONObject loopData = data("循环"); + JSONObject loopVar = new JSONObject(); + loopVar.put("name", "loopVar"); + loopVar.put("refType", "fixed"); + loopVar.put("value", "0"); + loopData.put("loopVars", array(loopVar)); + String content = workflowJson( + array(node("loop-1", "loopNode", null, loopData)), + new JSONArray()); + + WorkflowCheckResult result = service.checkContent( + content, WorkflowCheckStage.SAVE, null); + + Assert.assertFalse(result.isPassed()); + assertHasCode(result, "EXPLICIT_LOOP_COUNT_INVALID"); + } + + /** + * 验证嵌套节点只能挂在显式循环节点下。 + */ + @Test + public void testSaveShouldBlockNonLoopParent() throws Exception { + WorkflowCheckService service = newService(new HashMap<>()); + String content = workflowJson( + array( + node("code-parent", "codeNode", null, data("父节点")), + node("code-child", "codeNode", "code-parent", data("子节点")) + ), + new JSONArray()); + + WorkflowCheckResult result = service.checkContent( + content, WorkflowCheckStage.SAVE, null); + + Assert.assertFalse(result.isPassed()); + assertHasCode(result, "NODE_PARENT_NOT_LOOP"); + } + + /** + * 验证显式循环嵌套层级不能形成 parentId 环。 + */ + @Test + public void testSaveShouldBlockLoopParentCycle() throws Exception { + WorkflowCheckService service = newService(new HashMap<>()); + String content = workflowJson( + array( + node("loop-a", "loopNode", "loop-b", data("循环 A")), + node("loop-b", "loopNode", "loop-a", data("循环 B")) + ), + new JSONArray()); + + WorkflowCheckResult result = service.checkContent( + content, WorkflowCheckStage.SAVE, null); + + Assert.assertFalse(result.isPassed()); + assertHasCode(result, "LOOP_PARENT_CYCLE"); + } + @Test public void testSaveShouldPassForValidDraft() throws Exception { WorkflowCheckService service = newService(new HashMap<>()); diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/node/TemporaryFileMultipartFileTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/node/TemporaryFileMultipartFileTest.java new file mode 100644 index 00000000..ed85275c --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/node/TemporaryFileMultipartFileTest.java @@ -0,0 +1,40 @@ +package tech.easyflow.ai.node; + +import org.junit.Assert; +import org.junit.Test; + +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * 临时文件 MultipartFile 适配测试。 + */ +public class TemporaryFileMultipartFileTest { + + /** + * 验证文件流、大小及 transferTo 均复用磁盘内容。 + * + * @throws Exception 临时文件读写失败时抛出 + */ + @Test + public void shouldExposeTemporaryFileWithoutChangingContent() throws Exception { + byte[] content = "workflow-streaming-file".getBytes(java.nio.charset.StandardCharsets.UTF_8); + Path source = Files.createTempFile("temporary-file-multipart-source-", ".txt"); + Path target = Files.createTempFile("temporary-file-multipart-target-", ".txt"); + try { + Files.write(source, content); + TemporaryFileMultipartFile file = + new TemporaryFileMultipartFile("result.txt", source, "text/plain"); + + Assert.assertEquals(content.length, file.getSize()); + Assert.assertEquals("text/plain", file.getContentType()); + Assert.assertArrayEquals(content, file.getInputStream().readAllBytes()); + + file.transferTo(target.toFile()); + Assert.assertArrayEquals(content, Files.readAllBytes(target)); + } finally { + Files.deleteIfExists(source); + Files.deleteIfExists(target); + } + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/utils/DocUtilStreamingDownloadTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/utils/DocUtilStreamingDownloadTest.java new file mode 100644 index 00000000..6a80c2fb --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/utils/DocUtilStreamingDownloadTest.java @@ -0,0 +1,87 @@ +package tech.easyflow.ai.utils; + +import com.sun.net.httpserver.HttpServer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.file.Files; +import java.util.Arrays; + +/** + * DocUtil 流式下载测试。 + */ +public class DocUtilStreamingDownloadTest { + + /** + * 验证大响应按流落盘、内容完整且关闭后清理临时文件。 + * + * @throws Exception 测试服务器或文件读取失败时抛出 + */ + @Test + public void shouldStreamResponseToTemporaryFileAndCleanup() throws Exception { + byte[] content = new byte[2 * 1024 * 1024 + 17]; + Arrays.fill(content, (byte) 7); + HttpServer server = startServer(content); + try { + String url = "http://127.0.0.1:" + server.getAddress().getPort() + "/download"; + java.nio.file.Path path; + try (DocUtil.DownloadedFile downloadedFile = + DocUtil.downloadFileToTemp(url, content.length + 1L)) { + path = downloadedFile.path(); + Assert.assertEquals(content.length, downloadedFile.size()); + Assert.assertEquals("application/octet-stream", downloadedFile.contentType()); + Assert.assertArrayEquals(content, Files.readAllBytes(path)); + } + Assert.assertFalse(Files.exists(path)); + } finally { + server.stop(0); + } + } + + /** + * 验证超过配置上限时显式失败。 + * + * @throws Exception 测试服务器初始化失败时抛出 + */ + @Test + public void shouldRejectResponseAboveConfiguredLimit() throws Exception { + byte[] content = new byte[1024]; + HttpServer server = startServer(content); + try { + String url = "http://127.0.0.1:" + server.getAddress().getPort() + "/download"; + try { + DocUtil.downloadFileToTemp(url, content.length - 1L); + Assert.fail("expected download limit failure"); + } catch (RuntimeException exception) { + Assert.assertTrue(exception.getCause().getMessage().contains("超过限制")); + } + } finally { + server.stop(0); + } + } + + /** + * 启动仅用于本测试的本地 HTTP 文件服务。 + * + * @param content 响应内容 + * @return 已启动的 HTTP 服务 + * @throws Exception 服务创建失败时抛出 + */ + private HttpServer startServer(byte[] content) throws Exception { + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/download", exchange -> { + exchange.getResponseHeaders().set("Content-Type", "application/octet-stream"); + exchange.sendResponseHeaders(200, content.length); + try (OutputStream output = exchange.getResponseBody()) { + for (int offset = 0; offset < content.length; offset += 8192) { + int length = Math.min(8192, content.length - offset); + output.write(content, offset, length); + } + } + }); + server.start(); + return server; + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/datacenter/connector/impl/ProjectMysqlConnectorBatchTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/datacenter/connector/impl/ProjectMysqlConnectorBatchTest.java new file mode 100644 index 00000000..01f90b8b --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/datacenter/connector/impl/ProjectMysqlConnectorBatchTest.java @@ -0,0 +1,341 @@ +package tech.easyflow.datacenter.connector.impl; + +import com.alibaba.fastjson2.JSONObject; +import org.junit.Test; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.entity.DatacenterTableField; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.common.web.exceptions.BusinessException; + +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * 项目 MySQL 连接器批量写入测试。 + */ +public class ProjectMysqlConnectorBatchTest { + + /** + * 验证原始 SQL 在单连接、单 ResultSet 中原样流式消费。 + * + * @throws Exception JDBC 模拟初始化失败时抛出 + */ + @Test + public void shouldStreamOriginalSqlWithoutPaginationRewrite() + throws Exception { + DataSource dataSource = mock(DataSource.class); + Connection connection = mock(Connection.class); + PreparedStatement statement = + mock(PreparedStatement.class); + ResultSet resultSet = mock(ResultSet.class); + ResultSetMetaData metadata = + mock(ResultSetMetaData.class); + String sql = + "SELECT id FROM sample LIMIT 10 FOR UPDATE"; + when(dataSource.getConnection()) + .thenReturn(connection); + when(connection.prepareStatement( + eq(sql), + eq(ResultSet.TYPE_FORWARD_ONLY), + eq(ResultSet.CONCUR_READ_ONLY))) + .thenReturn(statement); + when(statement.executeQuery()) + .thenReturn(resultSet); + when(resultSet.getMetaData()) + .thenReturn(metadata); + when(metadata.getColumnCount()).thenReturn(1); + when(metadata.getColumnLabel(1)).thenReturn("id"); + when(resultSet.next()) + .thenReturn(true, false); + when(resultSet.getObject(1)) + .thenReturn(1L); + List ids = new ArrayList<>(); + + new ProjectMysqlConnector(dataSource) + .consumeBySql( + source(), + sql, + 1_000, + row -> ids.add( + row.getString("id"))); + + org.junit.Assert.assertEquals( + List.of("1"), ids); + verify(dataSource).getConnection(); + verify(connection).prepareStatement( + sql, + ResultSet.TYPE_FORWARD_ONLY, + ResultSet.CONCUR_READ_ONLY); + verify(statement).setFetchSize( + Integer.MIN_VALUE); + verify(statement).executeQuery(); + } + + /** + * 验证多行写入仅获取一次连接，并按批次执行 JDBC batch。 + * + * @throws Exception JDBC 模拟初始化失败时抛出 + */ + @Test + public void shouldReuseSingleConnectionAndExecuteConfiguredBatches() throws Exception { + DataSource dataSource = mock(DataSource.class); + Connection connection = mock(Connection.class); + List statements = new ArrayList<>(); + when(dataSource.getConnection()).thenReturn(connection); + when(connection.prepareStatement(anyString())).thenAnswer(invocation -> { + PreparedStatement statement = mock(PreparedStatement.class); + statements.add(statement); + return statement; + }); + + ProjectMysqlConnector connector = new ProjectMysqlConnector(dataSource); + DatacenterSource source = new DatacenterSource(); + source.setDatabaseName("easyflow"); + DatacenterTable table = new DatacenterTable(); + table.setTableName("sample"); + DatacenterTableField nameField = new DatacenterTableField(); + nameField.setFieldName("name"); + nameField.setWritable(1); + table.setFields(List.of(nameField)); + + List rows = new ArrayList<>(); + for (int index = 0; index < 5; index++) { + JSONObject row = new JSONObject(); + row.put("name", "row-" + index); + rows.add(row); + } + + connector.saveRows(source, table, rows, null, 2); + + verify(dataSource, times(1)).getConnection(); + if (statements.size() != 3) { + throw new AssertionError("expected 3 JDBC batches but got " + statements.size()); + } + int addBatchCalls = 0; + for (PreparedStatement statement : statements) { + verify(statement, times(1)).executeBatch(); + addBatchCalls += org.mockito.Mockito.mockingDetails(statement) + .getInvocations() + .stream() + .filter(invocation -> "addBatch".equals(invocation.getMethod().getName())) + .count(); + } + if (addBatchCalls != rows.size()) { + throw new AssertionError("expected " + rows.size() + " addBatch calls but got " + addBatchCalls); + } + } + + /** + * 验证回执和业务批量写入在同一 JDBC 事务中提交。 + * + * @throws Exception JDBC 模拟初始化失败时抛出 + */ + @Test + public void shouldCommitReceiptAndRowsInSingleTransaction() throws Exception { + DataSource dataSource = mock(DataSource.class); + Connection connection = mock(Connection.class); + PreparedStatement receiptStatement = mock(PreparedStatement.class); + PreparedStatement queryStatement = mock(PreparedStatement.class); + PreparedStatement rowStatement = mock(PreparedStatement.class); + ResultSet resultSet = mock(ResultSet.class); + when(dataSource.getConnection()).thenReturn(connection); + when(connection.getAutoCommit()).thenReturn(true); + when(connection.prepareStatement(anyString())).thenAnswer(invocation -> { + String sql = invocation.getArgument(0); + if (sql.startsWith("SELECT")) { + return queryStatement; + } + return sql.contains("tb_datacenter_write_receipt") + ? receiptStatement + : rowStatement; + }); + when(queryStatement.executeQuery()).thenReturn(resultSet); + when(resultSet.next()).thenReturn(false); + + ProjectMysqlConnector connector = new ProjectMysqlConnector(dataSource); + DatacenterSource source = source(); + DatacenterTable table = table(); + JSONObject row = new JSONObject(); + row.put("name", "row-1"); + + boolean written = connector.saveRowsIdempotently( + source, + table, + List.of(row), + null, + 100, + "receipt-key", + "payload-hash"); + + assertTrue(written); + verify(connection).setAutoCommit(false); + verify(receiptStatement).executeBatch(); + verify(receiptStatement).executeUpdate(); + verify(rowStatement).executeBatch(); + verify(connection, times(2)).commit(); + verify(connection, never()).rollback(); + verify(connection).setAutoCommit(true); + } + + /** + * 验证业务批量失败时回执与业务数据一并回滚。 + * + * @throws Exception JDBC 模拟初始化失败时抛出 + */ + @Test + public void shouldRollbackReceiptWhenBatchWriteFails() throws Exception { + DataSource dataSource = mock(DataSource.class); + Connection connection = mock(Connection.class); + PreparedStatement receiptStatement = mock(PreparedStatement.class); + PreparedStatement queryStatement = mock(PreparedStatement.class); + PreparedStatement rowStatement = mock(PreparedStatement.class); + ResultSet resultSet = mock(ResultSet.class); + when(dataSource.getConnection()).thenReturn(connection); + when(connection.getAutoCommit()).thenReturn(true); + when(connection.prepareStatement(anyString())).thenAnswer(invocation -> { + String sql = invocation.getArgument(0); + if (sql.startsWith("SELECT")) { + return queryStatement; + } + return sql.contains("tb_datacenter_write_receipt") + ? receiptStatement + : rowStatement; + }); + when(queryStatement.executeQuery()).thenReturn(resultSet); + when(resultSet.next()).thenReturn(false); + when(rowStatement.executeBatch()).thenThrow(new SQLException("write failed")); + + ProjectMysqlConnector connector = new ProjectMysqlConnector(dataSource); + JSONObject row = new JSONObject(); + row.put("name", "row-1"); + + try { + connector.saveRowsIdempotently( + source(), + table(), + List.of(row), + null, + 100, + "receipt-key", + "payload-hash"); + throw new AssertionError("failed business batch must rollback"); + } catch (BusinessException expected) { + assertTrue(expected.getMessage().contains("write failed")); + } + + verify(connection, atLeastOnce()).rollback(); + verify(connection, never()).commit(); + verify(connection).setAutoCommit(true); + } + + /** + * 验证中间行失败时前序行已经提交，后续行不会执行。 + * + * @throws Exception JDBC 模拟初始化失败时抛出 + */ + @Test + public void shouldKeepEarlierRowsCommittedWhenMiddleRowFails() throws Exception { + DataSource dataSource = mock(DataSource.class); + Connection connection = mock(Connection.class); + PreparedStatement receiptStatement = mock(PreparedStatement.class); + PreparedStatement queryStatement = mock(PreparedStatement.class); + PreparedStatement rowStatement = mock(PreparedStatement.class); + ResultSet resultSet = mock(ResultSet.class); + when(dataSource.getConnection()).thenReturn(connection); + when(connection.getAutoCommit()).thenReturn(true); + when(connection.prepareStatement(anyString())).thenAnswer(invocation -> { + String sql = invocation.getArgument(0); + if (sql.startsWith("SELECT")) { + return queryStatement; + } + return sql.contains("tb_datacenter_write_receipt") + ? receiptStatement + : rowStatement; + }); + when(queryStatement.executeQuery()).thenReturn(resultSet); + when(resultSet.next()).thenReturn(false); + when(rowStatement.executeBatch()) + .thenThrow(new SQLException("batch failed")) + .thenReturn(new int[]{1}) + .thenThrow(new SQLException("middle row failed")); + + ProjectMysqlConnector connector = new ProjectMysqlConnector(dataSource); + List rows = List.of( + row("row-0"), row("row-1"), row("row-2")); + + try { + connector.saveRowsIdempotently( + source(), + table(), + rows, + null, + 100, + "receipt-key", + "payload-hash"); + throw new AssertionError("middle row failure must be propagated"); + } catch (BusinessException expected) { + assertTrue(expected.getMessage().contains("middle row failed")); + } + + verify(dataSource, times(1)).getConnection(); + verify(connection, times(1)).commit(); + verify(connection, atLeastOnce()).rollback(); + verify(rowStatement, times(3)).executeBatch(); + verify(receiptStatement).executeBatch(); + verify(receiptStatement, times(2)).executeUpdate(); + } + + /** + * 创建测试数据源元数据。 + * + * @return 项目 MySQL 数据源 + */ + private DatacenterSource source() { + DatacenterSource source = new DatacenterSource(); + source.setDatabaseName("easyflow"); + return source; + } + + /** + * 创建包含一个可写字段的测试表。 + * + * @return 测试数据表 + */ + private DatacenterTable table() { + DatacenterTable table = new DatacenterTable(); + table.setTableName("sample"); + DatacenterTableField nameField = new DatacenterTableField(); + nameField.setFieldName("name"); + nameField.setWritable(1); + table.setFields(List.of(nameField)); + return table; + } + + /** + * 创建测试数据行。 + * + * @param name 行名称 + * @return JSON 行 + */ + private JSONObject row(String name) { + JSONObject row = new JSONObject(); + row.put("name", name); + return row; + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnectorBatchTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnectorBatchTest.java new file mode 100644 index 00000000..4f847fb5 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnectorBatchTest.java @@ -0,0 +1,113 @@ +package tech.easyflow.datacenter.connector.support; + +import com.alibaba.fastjson2.JSONObject; +import com.mybatisflex.core.row.Db; +import com.mybatisflex.core.row.Row; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.MockedStatic; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.entity.DatacenterTableField; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterCapability; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; + +import javax.sql.DataSource; +import java.math.BigInteger; +import java.util.Collections; +import java.util.List; +import java.util.function.Supplier; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; + +/** + * 内部动态表幂等批量写入回归测试。 + */ +public class AbstractInternalTableConnectorBatchTest { + + /** + * 验证正常路径按批次写入回执和数据，不退化为逐行 SQL。 + */ + @Test + public void shouldBatchReceiptsAndRowsOnNormalPath() { + DatacenterTable table = mock(DatacenterTable.class); + DatacenterTableField field = + mock(DatacenterTableField.class); + org.mockito.Mockito.when(table.getFields()) + .thenReturn(List.of(field)); + org.mockito.Mockito.when(table.getMaterializedTable()) + .thenReturn("tb_internal_test"); + org.mockito.Mockito.when(field.getFieldName()) + .thenReturn("name"); + LoginAccount account = mock(LoginAccount.class); + org.mockito.Mockito.when(account.getId()) + .thenReturn(BigInteger.ONE); + org.mockito.Mockito.when(account.getDeptId()) + .thenReturn(BigInteger.ONE); + org.mockito.Mockito.when(account.getTenantId()) + .thenReturn(BigInteger.ONE); + JSONObject first = JSONObject.of("name", "first"); + JSONObject second = JSONObject.of("name", "second"); + + try (MockedStatic db = mockStatic(Db.class)) { + db.when(() -> Db.selectOneByMap( + eq("tb_datacenter_write_receipt"), + anyMap())) + .thenReturn(null); + db.when(() -> Db.txWithResult( + org.mockito.ArgumentMatchers + .>any())) + .thenAnswer(invocation -> invocation + .>getArgument(0) + .get()); + + boolean written = new TestInternalConnector() + .saveRowsIdempotently( + new DatacenterSource(), + table, + List.of(first, second), + account, + 2, + "receipt", + "hash"); + + Assert.assertTrue(written); + db.verify(() -> Db.insertBatch( + eq("tb_datacenter_write_receipt"), + anyCollection(), + eq(2))); + db.verify(() -> Db.insertBatch( + eq("tb_internal_test"), + anyCollection(), + eq(2))); + db.verify(() -> Db.updateBatchById( + eq("tb_internal_test"), + anyList()), never()); + } + } + + /** + * 仅用于测试内部动态表批量协议的最小连接器。 + */ + private static final class TestInternalConnector + extends AbstractInternalTableConnector { + + /** + * 创建测试连接器。 + */ + private TestInternalConnector() { + super( + DatacenterSourceType.EXCEL, + Collections.emptySet(), + mock(DataSource.class)); + } + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/datacenter/connector/support/PostgresqlStreamingConnectorTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/datacenter/connector/support/PostgresqlStreamingConnectorTest.java new file mode 100644 index 00000000..9f65a715 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/datacenter/connector/support/PostgresqlStreamingConnectorTest.java @@ -0,0 +1,125 @@ +package tech.easyflow.datacenter.connector.support; + +import org.junit.Test; +import org.mockito.InOrder; +import org.mockito.Mockito; +import tech.easyflow.datacenter.connector.dialect.PostgresqlSqlDialect; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterCapability; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.util.EnumSet; + +/** + * PostgreSQL 服务端游标连接状态回归测试。 + */ +public class PostgresqlStreamingConnectorTest { + + /** + * 验证自动提交连接进入游标事务并在消费完成后恢复。 + * + * @throws Exception JDBC 模拟调用失败时抛出 + */ + @Test + public void shouldRestoreAutoCommitAfterStreaming() + throws Exception { + Connection connection = + Mockito.mock(Connection.class); + PreparedStatement statement = + Mockito.mock( + PreparedStatement.class); + ResultSet resultSet = + Mockito.mock(ResultSet.class); + ResultSetMetaData metaData = + Mockito.mock( + ResultSetMetaData.class); + Mockito.when(connection.getAutoCommit()) + .thenReturn(true); + Mockito.when(connection.prepareStatement( + "SELECT id FROM sample", + ResultSet.TYPE_FORWARD_ONLY, + ResultSet.CONCUR_READ_ONLY)) + .thenReturn(statement); + Mockito.when(statement.executeQuery()) + .thenReturn(resultSet); + Mockito.when(resultSet.getMetaData()) + .thenReturn(metaData); + Mockito.when(resultSet.next()) + .thenReturn(false); + TestConnector connector = + new TestConnector(connection); + + connector.consumeBySql( + new DatacenterSource(), + "SELECT id FROM sample", + 512, + row -> { + }); + + InOrder order = Mockito.inOrder( + connection, + statement, + resultSet); + order.verify(connection) + .getAutoCommit(); + order.verify(connection) + .setAutoCommit(false); + order.verify(connection) + .prepareStatement( + "SELECT id FROM sample", + ResultSet.TYPE_FORWARD_ONLY, + ResultSet.CONCUR_READ_ONLY); + order.verify(statement) + .setFetchSize(512); + order.verify(statement) + .executeQuery(); + order.verify(resultSet) + .close(); + order.verify(statement) + .close(); + order.verify(connection) + .rollback(); + order.verify(connection) + .setAutoCommit(true); + } + + /** + * 使用测试连接执行 PostgreSQL 查询。 + */ + private static final class TestConnector + extends AbstractJdbcConnector { + + private final Connection connection; + + /** + * 创建测试连接器。 + * + * @param connection 测试 JDBC 连接 + */ + private TestConnector( + Connection connection) { + super( + DatacenterSourceType.POSTGRESQL, + new PostgresqlSqlDialect(), + EnumSet.of( + DatacenterCapability.READ_QUERY)); + this.connection = connection; + } + + /** + * {@inheritDoc} + */ + @Override + protected T withConnection( + DatacenterSource source, + boolean cacheable, + JdbcCallback callback) + throws Exception { + return callback.apply(connection); + } + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetWriteServiceImplTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetWriteServiceImplTest.java new file mode 100644 index 00000000..16ce556a --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetWriteServiceImplTest.java @@ -0,0 +1,122 @@ +package tech.easyflow.datacenter.execution.service.impl; + +import com.alibaba.fastjson2.JSONObject; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import tech.easyflow.common.cache.RedisIdempotencyExecutor; +import tech.easyflow.datacenter.connector.DatacenterConnector; +import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.execution.model.DatasetRef; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; +import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService; + +import java.lang.reflect.Field; +import java.math.BigInteger; +import java.util.List; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * 数据集写入服务的幂等批写测试。 + */ +public class DatacenterDatasetWriteServiceImplTest { + + /** + * 验证服务只调用一次连接器，并保留配置的批大小供连接器复用连接处理。 + * + * @throws Exception 测试依赖注入失败时抛出 + */ + @Test + public void shouldDelegateIdempotentRowsInSingleConnectorCall() throws Exception { + DatacenterDatasetRegistryService registryService = + mock(DatacenterDatasetRegistryService.class); + DatacenterConnectorRegistry connectorRegistry = + mock(DatacenterConnectorRegistry.class); + DatacenterConnector connector = mock(DatacenterConnector.class); + RedisIdempotencyExecutor idempotencyExecutor = + mock(RedisIdempotencyExecutor.class); + + BigInteger tableId = BigInteger.ONE; + BigInteger sourceId = BigInteger.TWO; + DatasetRef datasetRef = new DatasetRef(); + datasetRef.setTableId(tableId); + DatacenterTable table = new DatacenterTable(); + table.setSourceId(sourceId); + DatacenterSource source = new DatacenterSource(); + source.setSourceType(DatacenterSourceType.PROJECT_MYSQL.name()); + when(registryService.getTableWithFields(tableId)).thenReturn(table); + when(registryService.getSourceRequired(sourceId)).thenReturn(source); + when(connectorRegistry.getConnector( + DatacenterSourceType.PROJECT_MYSQL.name())).thenReturn(connector); + when(idempotencyExecutor.executeOnce( + anyString(), anyString(), any(Runnable.class))) + .thenAnswer(invocation -> { + invocation.getArgument(2).run(); + return true; + }); + when(connector.saveRowsIdempotently( + any(), any(), anyList(), any(), anyInt(), anyString(), anyString())) + .thenReturn(true); + + DatacenterDatasetWriteServiceImpl service = + new DatacenterDatasetWriteServiceImpl(); + inject(service, "registryService", registryService); + inject(service, "connectorRegistry", connectorRegistry); + inject(service, "idempotencyExecutor", idempotencyExecutor); + List rows = List.of( + row("row-0"), row("row-1"), row("row-2")); + + Assert.assertTrue(service.saveRowsIdempotently( + datasetRef, rows, null, 64, "stable-execution-key")); + + @SuppressWarnings("unchecked") + ArgumentCaptor> rowsCaptor = + ArgumentCaptor.forClass(List.class); + verify(connector, times(1)).saveRowsIdempotently( + eq(source), + eq(table), + rowsCaptor.capture(), + any(), + eq(64), + anyString(), + anyString()); + Assert.assertEquals(rows, rowsCaptor.getValue()); + } + + /** + * 创建测试行。 + * + * @param name 行名称 + * @return JSON 行 + */ + private JSONObject row(String name) { + JSONObject row = new JSONObject(); + row.put("name", name); + return row; + } + + /** + * 注入服务测试依赖。 + * + * @param target 目标服务 + * @param fieldName 字段名 + * @param value 字段值 + * @throws Exception 反射访问失败时抛出 + */ + private void inject(Object target, String fieldName, Object value) throws Exception { + Field field = target.getClass().getDeclaredField(fieldName); + field.setAccessible(true); + field.set(target, value); + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/datacenter/schedule/DatacenterWriteReceiptCleanupJobTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/datacenter/schedule/DatacenterWriteReceiptCleanupJobTest.java new file mode 100644 index 00000000..52877102 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/datacenter/schedule/DatacenterWriteReceiptCleanupJobTest.java @@ -0,0 +1,36 @@ +package tech.easyflow.datacenter.schedule; + +import org.junit.Test; +import org.springframework.jdbc.core.JdbcTemplate; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * 数据集写入回执清理任务测试。 + */ +public class DatacenterWriteReceiptCleanupJobTest { + + /** + * 验证清理任务按固定大小分批，并在最后一个非满批次后停止。 + */ + @Test + public void shouldDeleteExpiredReceiptsInBoundedBatches() { + JdbcTemplate jdbcTemplate = mock(JdbcTemplate.class); + when(jdbcTemplate.update(anyString(), any(), anyInt())) + .thenReturn(1000, 7); + DatacenterWriteReceiptCleanupJob job = + new DatacenterWriteReceiptCleanupJob( + jdbcTemplate, 14L, 1000, 20); + + job.cleanup(); + + verify(jdbcTemplate, times(2)).update( + anyString(), any(), anyInt()); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/pom.xml b/easyflow-modules/easyflow-module-datacenter/pom.xml index 1bcbc07c..5effd619 100644 --- a/easyflow-modules/easyflow-module-datacenter/pom.xml +++ b/easyflow-modules/easyflow-module-datacenter/pom.xml @@ -45,6 +45,18 @@ tech.easyflow easyflow-common-web + + junit + junit + ${junit.version} + test + + + org.mockito + mockito-core + 5.12.0 + test + diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/QueryExecutor.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/QueryExecutor.java index 09927cc3..24e02d53 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/QueryExecutor.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/QueryExecutor.java @@ -7,9 +7,45 @@ import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest; import tech.easyflow.datacenter.meta.entity.DatacenterSource; import java.util.List; +import java.util.function.Consumer; public interface QueryExecutor { + + /** + * 分页查询结构化数据集。 + * + * @param source 数据源 + * @param table 数据表 + * @param request 查询请求 + * @return 分页结果 + */ Page queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request); + /** + * 执行原生 SQL 并返回完整结果。 + * + * @param source 数据源 + * @param sql 已校验 SQL + * @return 完整结果 + */ List queryBySql(DatacenterSource source, String sql); + + /** + * 在单次查询中按结果顺序消费原生 SQL 返回行。 + * + *

缺省实现保持第三方连接器兼容；JDBC 连接器应覆盖此方法并使用单连接、 + * 单 ResultSet 流式读取。

+ * + * @param source 数据源 + * @param sql 已校验 SQL + * @param fetchSize JDBC 建议拉取行数 + * @param consumer 单行消费者 + */ + default void consumeBySql( + DatacenterSource source, + String sql, + int fetchSize, + Consumer consumer) { + queryBySql(source, sql).forEach(consumer); + } } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/WriteExecutor.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/WriteExecutor.java index 8b1ac0d4..f0661eb0 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/WriteExecutor.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/WriteExecutor.java @@ -6,9 +6,59 @@ import tech.easyflow.datacenter.entity.DatacenterTable; import tech.easyflow.datacenter.meta.entity.DatacenterSource; import java.math.BigInteger; +import java.util.List; public interface WriteExecutor { + void saveRow(DatacenterSource source, DatacenterTable table, JSONObject data, LoginAccount account); + /** + * 批量保存数据行。 + *

+ * 缺省实现保持逐行语义；支持批处理的连接器应覆盖此方法。 + * + * @param source 数据源 + * @param table 数据表 + * @param rows 待保存数据行 + * @param account 当前操作账号 + * @param batchSize 单批最大行数 + */ + default void saveRows(DatacenterSource source, + DatacenterTable table, + List rows, + LoginAccount account, + int batchSize) { + for (JSONObject row : rows) { + saveRow(source, table, row, account); + } + } + + /** + * 在目标数据库中以唯一回执和业务写入同事务保存数据。 + * + *

缺省实现用于不支持目标库事务回执的连接器，仍保持普通批量写入语义。支持写能力 + * 的连接器应覆盖该方法。

+ * + * @param source 数据源 + * @param table 数据表 + * @param rows 待保存行 + * @param account 操作账号 + * @param batchSize 单批最大行数 + * @param receiptKey 有界幂等回执键 + * @param payloadHash 输入负载摘要 + * @return 本次实际写入时为 {@code true}，同负载回执已存在时为 {@code false} + */ + default boolean saveRowsIdempotently( + DatacenterSource source, + DatacenterTable table, + List rows, + LoginAccount account, + int batchSize, + String receiptKey, + String payloadHash) { + saveRows(source, table, rows, account, batchSize); + return true; + } + void deleteRow(DatacenterSource source, DatacenterTable table, BigInteger id, LoginAccount account); } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/ExcelConnector.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/ExcelConnector.java index cd53b647..d22db3f0 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/ExcelConnector.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/ExcelConnector.java @@ -5,11 +5,12 @@ import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector import tech.easyflow.datacenter.meta.enums.DatacenterCapability; import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; +import javax.sql.DataSource; import java.util.EnumSet; @Component public class ExcelConnector extends AbstractInternalTableConnector { - public ExcelConnector() { + public ExcelConnector(DataSource dataSource) { super(DatacenterSourceType.EXCEL, EnumSet.of( DatacenterCapability.TEST_CONNECTION, DatacenterCapability.BROWSE_METADATA, @@ -17,6 +18,6 @@ public class ExcelConnector extends AbstractInternalTableConnector { DatacenterCapability.WRITE_MUTATION, DatacenterCapability.MATERIALIZE, DatacenterCapability.EXPORT - )); + ), dataSource); } } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/ExcelMaterializedConnector.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/ExcelMaterializedConnector.java index 17ad5b41..afcfd7c0 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/ExcelMaterializedConnector.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/ExcelMaterializedConnector.java @@ -5,11 +5,13 @@ import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector import tech.easyflow.datacenter.meta.enums.DatacenterCapability; import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; +import javax.sql.DataSource; import java.util.EnumSet; @Component public class ExcelMaterializedConnector extends AbstractInternalTableConnector { - public ExcelMaterializedConnector() { + public ExcelMaterializedConnector( + DataSource dataSource) { super(DatacenterSourceType.EXCEL_MATERIALIZED, EnumSet.of( DatacenterCapability.TEST_CONNECTION, DatacenterCapability.BROWSE_METADATA, @@ -17,6 +19,6 @@ public class ExcelMaterializedConnector extends AbstractInternalTableConnector { DatacenterCapability.WRITE_MUTATION, DatacenterCapability.MATERIALIZE, DatacenterCapability.EXPORT - )); + ), dataSource); } } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/ProjectMysqlConnector.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/ProjectMysqlConnector.java index 9a8ffc2e..18ad320b 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/ProjectMysqlConnector.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/ProjectMysqlConnector.java @@ -9,6 +9,7 @@ import tech.easyflow.common.entity.LoginAccount; import tech.easyflow.common.web.exceptions.BusinessException; import tech.easyflow.datacenter.connector.dialect.MysqlSqlDialect; import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector; +import tech.easyflow.datacenter.connector.support.WriteReceiptSupport; import tech.easyflow.datacenter.entity.DatacenterTable; import tech.easyflow.datacenter.entity.DatacenterTableField; import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest; @@ -20,9 +21,15 @@ import javax.sql.DataSource; import java.math.BigInteger; import java.sql.Connection; import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.EnumSet; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; @Component @@ -116,6 +123,444 @@ public class ProjectMysqlConnector extends AbstractJdbcConnector { } } + /** + * 使用单个连接按相同 SQL 连续分组批量保存数据。 + * + * @param source 数据源 + * @param table 数据表 + * @param rows 待保存数据行 + * @param account 当前操作账号 + * @param batchSize 单批最大行数 + */ + @Override + public void saveRows(DatacenterSource source, + DatacenterTable table, + List rows, + LoginAccount account, + int batchSize) { + if (rows == null || rows.isEmpty()) { + return; + } + try (Connection connection = dataSource.getConnection()) { + saveRows(connection, source, table, rows, batchSize); + } catch (Exception ex) { + throw new BusinessException("项目 MySQL 批量写入失败: " + ex.getMessage()); + } + } + + /** + * {@inheritDoc} + */ + @Override + public boolean saveRowsIdempotently( + DatacenterSource source, + DatacenterTable table, + List rows, + LoginAccount account, + int batchSize, + String receiptKey, + String payloadHash) { + if (StrUtil.isBlank(receiptKey)) { + saveRows(source, table, rows, account, batchSize); + return true; + } + try (Connection connection = dataSource.getConnection()) { + boolean originalAutoCommit = connection.getAutoCommit(); + connection.setAutoCommit(false); + try { + if (hasMatchingReceipt(connection, receiptKey, payloadHash)) { + connection.rollback(); + return false; + } + int effectiveBatchSize = Math.max(1, batchSize); + for (int offset = 0; offset < rows.size(); offset += effectiveBatchSize) { + int end = Math.min(rows.size(), offset + effectiveBatchSize); + List batch = pendingRows( + connection, + rows, + offset, + end, + receiptKey, + payloadHash); + if (batch.isEmpty()) { + connection.rollback(); + continue; + } + try { + insertReceiptsBatch(connection, batch, payloadHash); + saveRows( + connection, + source, + table, + batch.stream() + .map(PendingRow::row) + .collect(Collectors.toList()), + effectiveBatchSize); + connection.commit(); + } catch (Exception batchError) { + connection.rollback(); + /* + * 批失败才逐行回放，精确保留旧实现“失败前行已提交、失败后不再执行” + * 的可观察语义，同时让正常路径按 batchSize 真正批量提交。 + */ + replayRowsIndividually( + connection, + source, + table, + batch, + payloadHash); + } + } + if (!insertReceipt(connection, receiptKey, payloadHash)) { + connection.rollback(); + return false; + } + connection.commit(); + return true; + } catch (Exception error) { + connection.rollback(); + throw error; + } finally { + connection.setAutoCommit(originalAutoCommit); + } + } catch (Exception error) { + throw new BusinessException("项目 MySQL 幂等批量写入失败: " + error.getMessage()); + } + } + + /** + * 批量读取子回执并筛出尚未写入的行。 + * + * @param connection JDBC 连接 + * @param rows 全部数据行 + * @param startInclusive 批起始下标 + * @param endExclusive 批结束下标 + * @param receiptKey 根回执键 + * @param payloadHash 负载摘要 + * @return 尚未提交的行 + * @throws SQLException 查询失败或回执负载冲突 + */ + private List pendingRows( + Connection connection, + List rows, + int startInclusive, + int endExclusive, + String receiptKey, + String payloadHash) throws SQLException { + List candidates = + new ArrayList<>(endExclusive - startInclusive); + for (int rowIndex = startInclusive; + rowIndex < endExclusive; + rowIndex++) { + candidates.add(new PendingRow( + WriteReceiptSupport.childKey(receiptKey, rowIndex), + rows.get(rowIndex))); + } + String placeholders = candidates.stream() + .map(candidate -> "?") + .collect(Collectors.joining(",")); + String sql = "SELECT idempotency_key, payload_hash " + + "FROM tb_datacenter_write_receipt " + + "WHERE idempotency_key IN (" + placeholders + ")"; + Map existing = new HashMap<>(); + try (PreparedStatement statement = connection.prepareStatement(sql)) { + for (int index = 0; index < candidates.size(); index++) { + statement.setString(index + 1, candidates.get(index).receiptKey()); + } + try (ResultSet resultSet = statement.executeQuery()) { + while (resultSet.next()) { + existing.put(resultSet.getString(1), resultSet.getString(2)); + } + } + } + List pending = new ArrayList<>(candidates.size()); + for (PendingRow candidate : candidates) { + String existingHash = existing.get(candidate.receiptKey()); + if (existingHash == null) { + pending.add(candidate); + } else if (!java.util.Objects.equals(payloadHash, existingHash)) { + throw new SQLException( + "相同幂等键对应的数据内容不一致", "23000"); + } + } + return pending; + } + + /** + * 在当前事务中批量创建子回执。 + * + * @param connection JDBC 连接 + * @param rows 待写行 + * @param payloadHash 负载摘要 + * @throws SQLException 回执批写失败 + */ + private void insertReceiptsBatch( + Connection connection, + List rows, + String payloadHash) throws SQLException { + String sql = "INSERT INTO tb_datacenter_write_receipt " + + "(idempotency_key, payload_hash, created) " + + "VALUES (?, ?, CURRENT_TIMESTAMP)"; + try (PreparedStatement statement = connection.prepareStatement(sql)) { + for (PendingRow row : rows) { + statement.setString(1, row.receiptKey()); + statement.setString(2, payloadHash); + statement.addBatch(); + } + statement.executeBatch(); + } + } + + /** + * 批失败后逐行回放，定位首个业务失败并保留旧部分成功边界。 + * + * @param connection JDBC 连接 + * @param source 数据源 + * @param table 数据表 + * @param rows 本批待写行 + * @param payloadHash 负载摘要 + * @throws Exception 首个真实行写入失败 + */ + private void replayRowsIndividually( + Connection connection, + DatacenterSource source, + DatacenterTable table, + List rows, + String payloadHash) throws Exception { + for (PendingRow pendingRow : rows) { + if (!insertReceipt( + connection, + pendingRow.receiptKey(), + payloadHash)) { + connection.rollback(); + continue; + } + try { + saveRows( + connection, + source, + table, + List.of(pendingRow.row()), + 1); + connection.commit(); + } catch (Exception rowError) { + connection.rollback(); + throw rowError; + } + } + } + + /** + * 待写行及其稳定子回执键。 + * + * @param receiptKey 子回执键 + * @param row 数据行 + */ + private record PendingRow(String receiptKey, JSONObject row) { + } + + /** + * 检查整次写入回执是否已经存在并校验负载。 + * + * @param connection JDBC 连接 + * @param receiptKey 回执键 + * @param payloadHash 负载摘要 + * @return 相同负载的回执存在时为 {@code true} + * @throws SQLException 数据库访问失败或负载冲突 + */ + private boolean hasMatchingReceipt( + Connection connection, String receiptKey, String payloadHash) + throws SQLException { + String querySql = "SELECT payload_hash FROM tb_datacenter_write_receipt " + + "WHERE idempotency_key = ?"; + try (PreparedStatement statement = connection.prepareStatement(querySql)) { + statement.setString(1, receiptKey); + try (ResultSet resultSet = statement.executeQuery()) { + if (!resultSet.next()) { + return false; + } + if (!java.util.Objects.equals(payloadHash, resultSet.getString(1))) { + throw new SQLException( + "相同幂等键对应的数据内容不一致", "23000"); + } + return true; + } + } + } + + /** + * 在当前连接中写入唯一回执。 + * + * @param connection JDBC 连接 + * @param receiptKey 回执键 + * @param payloadHash 负载摘要 + * @return 新建回执时为 {@code true}，相同负载回执已存在时为 {@code false} + * @throws SQLException 数据库访问失败或负载摘要冲突 + */ + private boolean insertReceipt( + Connection connection, String receiptKey, String payloadHash) throws SQLException { + String insertSql = "INSERT INTO tb_datacenter_write_receipt " + + "(idempotency_key, payload_hash, created) VALUES (?, ?, CURRENT_TIMESTAMP)"; + try (PreparedStatement statement = connection.prepareStatement(insertSql)) { + statement.setString(1, receiptKey); + statement.setString(2, payloadHash); + statement.executeUpdate(); + return true; + } catch (SQLException duplicate) { + if (!"23000".equals(duplicate.getSQLState()) && duplicate.getErrorCode() != 1062) { + throw duplicate; + } + String querySql = "SELECT payload_hash FROM tb_datacenter_write_receipt " + + "WHERE idempotency_key = ?"; + try (PreparedStatement statement = connection.prepareStatement(querySql)) { + statement.setString(1, receiptKey); + try (ResultSet resultSet = statement.executeQuery()) { + if (!resultSet.next()) { + throw duplicate; + } + if (!java.util.Objects.equals(payloadHash, resultSet.getString(1))) { + throw new SQLException("相同幂等键对应的数据内容不一致", "23000"); + } + return false; + } + } + } + } + + /** + * 在给定连接上执行完整批量，供普通和事务幂等写入复用。 + * + * @param connection JDBC 连接 + * @param source 数据源 + * @param table 数据表 + * @param rows 待保存行 + * @param batchSize 单批最大行数 + * @throws Exception JDBC 批处理失败 + */ + private void saveRows( + Connection connection, + DatacenterSource source, + DatacenterTable table, + List rows, + int batchSize) throws Exception { + List writableFields = table.getFields().stream() + .filter(field -> field.getWritable() == null || field.getWritable() == 1) + .collect(Collectors.toList()); + int effectiveBatchSize = Math.max(1, batchSize); + List batch = new ArrayList<>(Math.min(rows.size(), effectiveBatchSize)); + String batchSql = null; + for (JSONObject row : rows) { + SqlMutation mutation = buildMutation(source, table, writableFields, row); + if (mutation == null) { + continue; + } + if (batchSql != null + && (!batchSql.equals(mutation.sql) || batch.size() >= effectiveBatchSize)) { + executeBatch(connection, batchSql, batch); + batch.clear(); + } + batchSql = mutation.sql; + batch.add(mutation); + } + if (!batch.isEmpty()) { + executeBatch(connection, batchSql, batch); + } + } + + /** + * 构建单行参数化写入。 + * + * @param source 数据源 + * @param table 数据表 + * @param writableFields 可写字段 + * @param data 数据行 + * @return SQL 与参数；无可更新字段时返回 null + */ + private SqlMutation buildMutation(DatacenterSource source, + DatacenterTable table, + List writableFields, + JSONObject data) { + Object id = data.get("id"); + if (id == null) { + List columns = new ArrayList<>(); + List values = new ArrayList<>(); + for (DatacenterTableField field : writableFields) { + Object value = data.get(field.getFieldName()); + if (value != null) { + columns.add(field.getFieldName()); + values.add(value); + } + } + if (columns.isEmpty()) { + throw new BusinessException("没有可写字段"); + } + String sql = "INSERT INTO " + + dialect.qualifyTable(source.getDatabaseName(), resolvePhysicalTableName(table)) + + " (" + columns.stream().map(dialect::quoteIdentifier).collect(Collectors.joining(",")) + + ") VALUES (" + columns.stream().map(item -> "?").collect(Collectors.joining(",")) + ")"; + return new SqlMutation(sql, values); + } + + List setClauses = new ArrayList<>(); + List values = new ArrayList<>(); + for (DatacenterTableField field : writableFields) { + if (!data.containsKey(field.getFieldName())) { + continue; + } + setClauses.add(dialect.quoteIdentifier(field.getFieldName()) + " = ?"); + values.add(data.get(field.getFieldName())); + } + if (setClauses.isEmpty()) { + return null; + } + String sql = "UPDATE " + + dialect.qualifyTable(source.getDatabaseName(), resolvePhysicalTableName(table)) + + " SET " + String.join(",", setClauses) + + " WHERE " + dialect.quoteIdentifier("id") + " = ?"; + values.add(id); + return new SqlMutation(sql, values); + } + + /** + * 执行同构 SQL 批次。 + * + * @param connection 数据库连接 + * @param sql 参数化 SQL + * @param mutations 待执行参数 + * @throws Exception JDBC 批处理失败时抛出 + */ + private void executeBatch(Connection connection, String sql, List mutations) throws Exception { + try (PreparedStatement statement = connection.prepareStatement(sql)) { + for (SqlMutation mutation : mutations) { + for (int index = 0; index < mutation.parameters.size(); index++) { + statement.setObject(index + 1, mutation.parameters.get(index)); + } + statement.addBatch(); + } + statement.executeBatch(); + } + } + + /** + * 参数化写入描述。 + */ + private static final class SqlMutation { + + private final String sql; + private final List parameters; + + /** + * 创建参数化写入。 + * + * @param sql SQL 文本 + * @param parameters SQL 参数 + */ + private SqlMutation(String sql, List parameters) { + this.sql = sql; + this.parameters = parameters; + } + } + @Override public void deleteRow(DatacenterSource source, DatacenterTable table, BigInteger id, LoginAccount account) { String sql = "DELETE FROM " + dialect.qualifyTable(source.getDatabaseName(), resolvePhysicalTableName(table)) diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnector.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnector.java index e6251e07..f65d130c 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnector.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnector.java @@ -8,6 +8,7 @@ import com.mybatisflex.core.row.Db; import com.mybatisflex.core.row.Row; import com.mybatisflex.core.row.RowKey; import org.springframework.util.CollectionUtils; +import org.springframework.dao.DuplicateKeyException; import tech.easyflow.common.entity.LoginAccount; import tech.easyflow.common.web.exceptions.BusinessException; import tech.easyflow.datacenter.connector.DatacenterConnector; @@ -21,18 +22,30 @@ import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta; import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta; +import javax.sql.DataSource; import java.math.BigDecimal; import java.math.BigInteger; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; import java.util.*; +import java.util.function.Consumer; public abstract class AbstractInternalTableConnector implements DatacenterConnector { + private static final String WRITE_RECEIPT_TABLE = "tb_datacenter_write_receipt"; private final DatacenterSourceType sourceType; private final Set capabilities; + private final DataSource dataSource; - protected AbstractInternalTableConnector(DatacenterSourceType sourceType, Set capabilities) { + protected AbstractInternalTableConnector( + DatacenterSourceType sourceType, + Set capabilities, + DataSource dataSource) { this.sourceType = sourceType; this.capabilities = capabilities; + this.dataSource = dataSource; } @Override @@ -93,6 +106,58 @@ public abstract class AbstractInternalTableConnector implements DatacenterConnec return rows; } + /** + * {@inheritDoc} + */ + @Override + public void consumeBySql( + DatacenterSource source, + String sql, + int fetchSize, + Consumer consumer) { + if (fetchSize <= 0 || consumer == null) { + throw new IllegalArgumentException( + "fetchSize and consumer must be valid"); + } + try (Connection connection = + dataSource.getConnection(); + PreparedStatement statement = + connection.prepareStatement( + sql, + ResultSet.TYPE_FORWARD_ONLY, + ResultSet.CONCUR_READ_ONLY)) { + // 内部数据源使用项目 MySQL，启用驱动前向流式结果。 + statement.setFetchSize(Integer.MIN_VALUE); + int timeoutSeconds = Integer.getInteger( + "easyflow.datacenter.query.timeout-seconds", + 300); + if (timeoutSeconds > 0) { + statement.setQueryTimeout(timeoutSeconds); + } + try (ResultSet resultSet = + statement.executeQuery()) { + ResultSetMetaData metaData = + resultSet.getMetaData(); + while (resultSet.next()) { + Row row = new Row(); + for (int index = 1; + index <= metaData.getColumnCount(); + index++) { + row.put( + metaData.getColumnLabel(index), + normalizeValue( + resultSet.getObject(index))); + } + consumer.accept(row); + } + } + } catch (Exception error) { + throw DatacenterConnectorExceptionSupport + .wrapAccessException( + "SQL 流式查询失败", error); + } + } + @Override public void saveRow(DatacenterSource source, DatacenterTable table, JSONObject data, LoginAccount account) { List fields = table.getFields(); @@ -100,21 +165,314 @@ public abstract class AbstractInternalTableConnector implements DatacenterConnec throw new BusinessException("数据集字段为空，无法写入"); } String actualTable = resolveTableName(table); + RowMutation mutation = buildRowMutation(fields, data, account); + if (mutation.insert) { + Db.insert(actualTable, mutation.row); + } else { + Db.updateById(actualTable, mutation.row); + } + } + + /** + * 使用 MyBatis-Flex 动态表批处理保存数据行。 + * + * @param source 数据源 + * @param table 数据表 + * @param rows 待保存数据行 + * @param account 当前操作账号 + * @param batchSize 单批最大行数 + */ + @Override + public void saveRows(DatacenterSource source, + DatacenterTable table, + List rows, + LoginAccount account, + int batchSize) { + List fields = table.getFields(); + if (CollectionUtils.isEmpty(fields)) { + throw new BusinessException("数据集字段为空，无法写入"); + } + if (rows == null || rows.isEmpty()) { + return; + } + String actualTable = resolveTableName(table); + int effectiveBatchSize = Math.max(1, batchSize); + List batch = new ArrayList<>(Math.min(rows.size(), effectiveBatchSize)); + Boolean insertBatch = null; + for (JSONObject data : rows) { + RowMutation mutation = buildRowMutation(fields, data, account); + if (insertBatch != null + && (insertBatch != mutation.insert || batch.size() >= effectiveBatchSize)) { + executeRowBatch(actualTable, batch, insertBatch); + batch.clear(); + } + insertBatch = mutation.insert; + batch.add(mutation.row); + } + if (!batch.isEmpty()) { + executeRowBatch(actualTable, batch, Boolean.TRUE.equals(insertBatch)); + } + } + + /** + * {@inheritDoc} + */ + @Override + public boolean saveRowsIdempotently( + DatacenterSource source, + DatacenterTable table, + List rows, + LoginAccount account, + int batchSize, + String receiptKey, + String payloadHash) { + if (StrUtil.isBlank(receiptKey)) { + saveRows(source, table, rows, account, batchSize); + return true; + } + Row completedReceipt = Db.selectOneByMap( + WRITE_RECEIPT_TABLE, + Collections.singletonMap("idempotency_key", receiptKey)); + if (completedReceipt != null) { + Object existingHash = completedReceipt.get("payload_hash"); + if (!Objects.equals(payloadHash, existingHash)) { + throw new BusinessException("相同幂等键对应的数据内容不一致"); + } + return false; + } + List fields = table.getFields(); + if (CollectionUtils.isEmpty(fields)) { + throw new BusinessException("数据集字段为空，无法写入"); + } + String actualTable = resolveTableName(table); + int effectiveBatchSize = Math.max(1, batchSize); + for (int offset = 0; offset < rows.size(); offset += effectiveBatchSize) { + int end = Math.min(rows.size(), offset + effectiveBatchSize); + List batch = new ArrayList<>(end - offset); + for (int index = offset; index < end; index++) { + batch.add(new PendingInternalRow( + WriteReceiptSupport.childKey(receiptKey, index), + rows.get(index), + buildRowMutation( + fields, + rows.get(index), + account))); + } + try { + Db.txWithResult(() -> { + insertReceiptsBatch(batch, payloadHash); + executePendingMutations(actualTable, batch); + return true; + }); + } catch (RuntimeException batchError) { + /* + * 正常路径每批一个事务；批失败后才逐行回放，继续保持旧实现的 + * 部分成功顺序边界，并利用子回执跳过已提交行。 + */ + replayInternalRows( + actualTable, + fields, + batch, + account, + payloadHash); + } + } + return Db.txWithResult(() -> { + Row receipt = new Row(); + receipt.put("idempotency_key", receiptKey); + receipt.put("payload_hash", payloadHash); + receipt.put("created", new Date()); + try { + Db.insert(WRITE_RECEIPT_TABLE, receipt); + } catch (DuplicateKeyException duplicate) { + Row existing = Db.selectOneByMap( + WRITE_RECEIPT_TABLE, + Collections.singletonMap("idempotency_key", receiptKey)); + Object existingHash = existing == null ? null : existing.get("payload_hash"); + if (!Objects.equals(payloadHash, existingHash)) { + throw new BusinessException("相同幂等键对应的数据内容不一致"); + } + return false; + } + return true; + }); + } + + /** + * 在当前 MyBatis-Flex 事务中写入单行动态表数据。 + * + * @param actualTable 实际表名 + * @param fields 可写字段 + * @param data 原始行 + * @param account 当前账号 + */ + private void saveRowInCurrentTransaction( + String actualTable, + List fields, + JSONObject data, + LoginAccount account) { + RowMutation mutation = buildRowMutation(fields, data, account); + if (mutation.insert) { + Db.insert(actualTable, mutation.row); + } else { + Db.updateById(actualTable, mutation.row); + } + } + + /** + * 创建写入回执。 + * + * @param receiptKey 回执键 + * @param payloadHash 负载摘要 + */ + private void insertReceipt(String receiptKey, String payloadHash) { + Row receipt = new Row(); + receipt.put("idempotency_key", receiptKey); + receipt.put("payload_hash", payloadHash); + receipt.put("created", new Date()); + Db.insert(WRITE_RECEIPT_TABLE, receipt); + } + + /** + * 在当前事务中批量创建子回执。 + * + * @param rows 本批待写行 + * @param payloadHash 负载摘要 + */ + private void insertReceiptsBatch( + List rows, + String payloadHash) { + Date created = new Date(); + List receipts = new ArrayList<>(rows.size()); + for (PendingInternalRow pendingRow : rows) { + Row receipt = new Row(); + receipt.put( + "idempotency_key", + pendingRow.receiptKey()); + receipt.put("payload_hash", payloadHash); + receipt.put("created", created); + receipts.add(receipt); + } + Db.insertBatch( + WRITE_RECEIPT_TABLE, + receipts, + receipts.size()); + } + + /** + * 按原始顺序合并相邻同类型写入，减少正常路径 SQL 往返。 + * + * @param actualTable 实际表名 + * @param rows 本批待写行 + */ + private void executePendingMutations( + String actualTable, + List rows) { + List batch = new ArrayList<>(rows.size()); + Boolean insertBatch = null; + for (PendingInternalRow pendingRow : rows) { + RowMutation mutation = pendingRow.mutation(); + if (insertBatch != null + && insertBatch != mutation.insert) { + executeRowBatch( + actualTable, + batch, + insertBatch); + batch.clear(); + } + insertBatch = mutation.insert; + batch.add(mutation.row); + } + if (!batch.isEmpty()) { + executeRowBatch( + actualTable, + batch, + Boolean.TRUE.equals(insertBatch)); + } + } + + /** + * 批失败后逐行回放，并校验重复回执的负载摘要。 + * + * @param actualTable 实际表名 + * @param fields 可写字段 + * @param rows 本批数据 + * @param account 当前账号 + * @param payloadHash 负载摘要 + */ + private void replayInternalRows( + String actualTable, + List fields, + List rows, + LoginAccount account, + String payloadHash) { + for (PendingInternalRow pendingRow : rows) { + Db.txWithResult(() -> { + try { + insertReceipt(pendingRow.receiptKey(), payloadHash); + } catch (DuplicateKeyException duplicate) { + Row existing = Db.selectOneByMap( + WRITE_RECEIPT_TABLE, + Collections.singletonMap( + "idempotency_key", + pendingRow.receiptKey())); + Object existingHash = existing == null + ? null + : existing.get("payload_hash"); + if (!Objects.equals(payloadHash, existingHash)) { + throw new BusinessException( + "相同幂等键对应的数据内容不一致"); + } + return false; + } + saveRowInCurrentTransaction( + actualTable, + fields, + pendingRow.data(), + account); + return true; + }); + } + } + + /** + * 内部动态表待写行。 + * + * @param receiptKey 子回执键 + * @param data 原始行 + * @param mutation 已构建的写入对象 + */ + private record PendingInternalRow( + String receiptKey, + JSONObject data, + RowMutation mutation) { + } + + /** + * 构建动态表单行写入对象。 + * + * @param fields 数据表字段 + * @param data 输入数据 + * @param account 当前操作账号 + * @return 行数据与写入类型 + */ + private RowMutation buildRowMutation( + List fields, JSONObject data, LoginAccount account) { Object id = data.get("id"); if (id == null) { + Date now = new Date(); Row row = Row.ofKey(RowKey.SNOW_FLAKE_ID); row.put("dept_id", account.getDeptId()); row.put("tenant_id", account.getTenantId()); - row.put("created", new Date()); + row.put("created", now); row.put("created_by", account.getId()); - row.put("modified", new Date()); + row.put("modified", now); row.put("modified_by", account.getId()); row.put("remark", data.get("remark")); for (DatacenterTableField field : fields) { row.put(field.getFieldName(), data.get(field.getFieldName())); } - Db.insert(actualTable, row); - return; + return new RowMutation(true, row); } Row row = Row.ofKey("id", id); row.put("modified", new Date()); @@ -122,7 +480,42 @@ public abstract class AbstractInternalTableConnector implements DatacenterConnec for (DatacenterTableField field : fields) { row.put(field.getFieldName(), data.get(field.getFieldName())); } - Db.updateById(actualTable, row); + return new RowMutation(false, row); + } + + /** + * 执行动态表同类型批次。 + * + * @param actualTable 实际表名 + * @param rows 行数据 + * @param insert 是否为新增批次 + */ + private void executeRowBatch(String actualTable, List rows, boolean insert) { + if (insert) { + Db.insertBatch(actualTable, rows, rows.size()); + } else { + Db.updateBatchById(actualTable, rows); + } + } + + /** + * 动态表单行写入描述。 + */ + private static final class RowMutation { + + private final boolean insert; + private final Row row; + + /** + * 创建动态表行写入描述。 + * + * @param insert 是否新增 + * @param row 行数据 + */ + private RowMutation(boolean insert, Row row) { + this.insert = insert; + this.row = row; + } } @Override @@ -138,15 +531,27 @@ public abstract class AbstractInternalTableConnector implements DatacenterConnec for (Row record : records) { Map converted = new LinkedHashMap<>(); for (Map.Entry entry : record.entrySet()) { - Object value = entry.getValue(); - if (value instanceof BigInteger || value instanceof BigDecimal || value instanceof Long) { - converted.put(entry.getKey(), value.toString()); - } else { - converted.put(entry.getKey(), value); - } + converted.put( + entry.getKey(), + normalizeValue(entry.getValue())); } record.clear(); record.putAll(converted); } } + + /** + * 统一内部查询的数值 JSON 表现。 + * + * @param value JDBC 原始值 + * @return 兼容既有查询接口的值 + */ + private Object normalizeValue(Object value) { + if (value instanceof BigInteger + || value instanceof BigDecimal + || value instanceof Long) { + return value.toString(); + } + return value; + } } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractJdbcConnector.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractJdbcConnector.java index 7ddeb569..54435d3e 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractJdbcConnector.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractJdbcConnector.java @@ -28,6 +28,7 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.sql.*; import java.util.*; +import java.util.function.Consumer; import java.util.stream.Collectors; public abstract class AbstractJdbcConnector implements DatacenterConnector { @@ -265,6 +266,43 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector { } } + /** + * {@inheritDoc} + */ + @Override + public void consumeBySql( + DatacenterSource source, + String sql, + int fetchSize, + Consumer consumer) { + if (!capabilities.contains( + DatacenterCapability.READ_QUERY)) { + throw new BusinessException( + "当前数据源暂不支持查询"); + } + if (fetchSize <= 0 || consumer == null) { + throw new IllegalArgumentException( + "fetchSize and consumer must be valid"); + } + try { + withConnection( + source, + true, + connection -> { + consumeBySql( + connection, + sql, + fetchSize, + consumer); + return null; + }); + } catch (Exception ex) { + throw DatacenterConnectorExceptionSupport + .wrapAccessException( + "SQL 流式查询失败", ex); + } + } + @Override public void saveRow(DatacenterSource source, DatacenterTable table, JSONObject data, LoginAccount account) { throw new BusinessException("当前数据源不支持写入"); @@ -330,9 +368,163 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector { } protected List doQueryBySql(Connection connection, String sql) throws SQLException { - try (PreparedStatement statement = connection.prepareStatement(sql); - ResultSet resultSet = statement.executeQuery()) { - return readRows(resultSet); + try (PreparedStatement statement = connection.prepareStatement(sql)) { + configureStreamingQuery(statement); + try (ResultSet resultSet = statement.executeQuery()) { + return readRows(resultSet); + } + } + } + + /** + * 使用单连接、单 ResultSet 顺序消费查询结果。 + * + * @param connection JDBC 连接 + * @param sql 已校验 SQL + * @param fetchSize JDBC 建议拉取行数 + * @param consumer 单行消费者 + * @throws SQLException 查询失败 + */ + protected void consumeBySql( + Connection connection, + String sql, + int fetchSize, + Consumer consumer) throws SQLException { + boolean localCursorTransaction = + usesPostgresqlCursor() + && connection.getAutoCommit(); + if (localCursorTransaction) { + // PostgreSQL 协议仅在事务内按 fetchSize 使用服务端游标。 + connection.setAutoCommit(false); + } + Throwable queryFailure = null; + try { + try (PreparedStatement statement = + connection.prepareStatement( + sql, + ResultSet.TYPE_FORWARD_ONLY, + ResultSet.CONCUR_READ_ONLY)) { + configureCursorQuery(statement, fetchSize); + try (ResultSet resultSet = + statement.executeQuery()) { + ResultSetMetaData metaData = + resultSet.getMetaData(); + while (resultSet.next()) { + Row row = new Row(); + for (int index = 1; + index <= metaData.getColumnCount(); + index++) { + row.put( + metaData.getColumnLabel(index), + normalizeValue( + resultSet.getObject(index))); + } + consumer.accept(row); + } + } + } + } catch (SQLException | RuntimeException | Error ex) { + queryFailure = ex; + throw ex; + } finally { + if (localCursorTransaction) { + restoreCursorConnection( + connection, queryFailure); + } + } + } + + /** + * 回滚只读游标事务并恢复连接池连接状态。 + * + * @param connection JDBC 连接 + * @param queryFailure 查询阶段异常；为空表示查询成功 + * @throws SQLException 清理失败且查询本身成功 + */ + private void restoreCursorConnection( + Connection connection, + Throwable queryFailure) throws SQLException { + SQLException cleanupFailure = null; + try { + connection.rollback(); + } catch (SQLException ex) { + cleanupFailure = ex; + } + try { + connection.setAutoCommit(true); + } catch (SQLException ex) { + if (cleanupFailure == null) { + cleanupFailure = ex; + } else { + cleanupFailure.addSuppressed(ex); + } + } + if (cleanupFailure == null) { + return; + } + if (queryFailure != null) { + queryFailure.addSuppressed(cleanupFailure); + return; + } + throw cleanupFailure; + } + + /** + * 判断当前连接器是否使用 PostgreSQL 游标协议。 + * + * @return PostgreSQL 或 GaussDB 原生连接器返回 true + */ + private boolean usesPostgresqlCursor() { + return sourceType == DatacenterSourceType.POSTGRESQL + || sourceType + == DatacenterSourceType.GAUSSDB_NATIVE; + } + + /** + * 配置长结果游标的拉取策略和超时。 + * + * @param statement JDBC 语句 + * @param fetchSize 建议拉取行数 + * @throws SQLException 配置失败 + */ + private void configureCursorQuery( + PreparedStatement statement, + int fetchSize) throws SQLException { + if (sourceType == DatacenterSourceType.MYSQL + || sourceType + == DatacenterSourceType.PROJECT_MYSQL + || sourceType == DatacenterSourceType.GBASE_8A + || sourceType == DatacenterSourceType.GBASE_8S) { + // MySQL 协议以该值启用前向只读流，避免驱动先缓存完整结果。 + statement.setFetchSize(Integer.MIN_VALUE); + } else { + statement.setFetchSize(fetchSize); + } + int timeoutSeconds = Integer.getInteger( + "easyflow.datacenter.query.timeout-seconds", + 300); + if (timeoutSeconds > 0) { + statement.setQueryTimeout(timeoutSeconds); + } + } + + /** + * 为原生查询配置宽松但有限的流式拉取和超时。 + * + * @param statement JDBC 语句 + * @throws SQLException 配置失败 + */ + private void configureStreamingQuery( + PreparedStatement statement) throws SQLException { + int fetchSize = Integer.getInteger( + "easyflow.datacenter.query.fetch-size", 1_000); + int timeoutSeconds = Integer.getInteger( + "easyflow.datacenter.query.timeout-seconds", 300); + if (fetchSize > 0) { + statement.setFetchSize(fetchSize); + } + if (timeoutSeconds > 0) { + statement.setQueryTimeout(timeoutSeconds); } } @@ -349,17 +541,61 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector { protected List readRows(ResultSet resultSet) throws SQLException { List records = new ArrayList<>(); ResultSetMetaData metaData = resultSet.getMetaData(); + int maxRows = Integer.getInteger( + "easyflow.datacenter.query.max-rows", 1_000_000); + long maxBytes = Long.getLong( + "easyflow.datacenter.query.max-bytes", + 512L * 1024L * 1024L); + long estimatedBytes = 0L; while (resultSet.next()) { + if (maxRows > 0 && records.size() >= maxRows) { + throw new SQLException( + "数据集查询结果超过行数上限: " + maxRows); + } Row row = new Row(); for (int i = 1; i <= metaData.getColumnCount(); i++) { String columnLabel = metaData.getColumnLabel(i); - row.put(columnLabel, normalizeValue(resultSet.getObject(i))); + Object value = normalizeValue(resultSet.getObject(i)); + row.put(columnLabel, value); + estimatedBytes += estimateQueryValueBytes( + columnLabel, value); + if (maxBytes > 0L && estimatedBytes > maxBytes) { + throw new SQLException( + "数据集查询结果超过字节上限: " + + maxBytes); + } } records.add(row); } return records; } + /** + * 估算查询结果在 JVM 中的最低占用，作为失控保护。 + * + * @param columnLabel 列名 + * @param value 列值 + * @return 估算字节数 + */ + private long estimateQueryValueBytes( + String columnLabel, Object value) { + long bytes = columnLabel == null + ? 0L + : (long) columnLabel.length() * Character.BYTES; + if (value == null) { + return bytes + 8L; + } + if (value instanceof byte[]) { + return bytes + ((byte[]) value).length; + } + if (value instanceof CharSequence) { + return bytes + + (long) value.toString().length() + * Character.BYTES; + } + return bytes + 64L; + } + protected String resolveCatalogArgument(DatacenterSource source, String catalogName) { return usesCatalogNamespace() ? resolveCatalogName(source, catalogName) : source.getDatabaseName(); } @@ -488,7 +724,7 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector { } } - private Object normalizeValue(Object value) { + protected Object normalizeValue(Object value) { if (value instanceof BigDecimal || value instanceof BigInteger || value instanceof Long) { return value.toString(); } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/WriteReceiptSupport.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/WriteReceiptSupport.java new file mode 100644 index 00000000..d6a9b04b --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/WriteReceiptSupport.java @@ -0,0 +1,33 @@ +package tech.easyflow.datacenter.connector.support; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; + +/** + * 数据集写入回执键工具。 + */ +public final class WriteReceiptSupport { + + private WriteReceiptSupport() { + } + + /** + * 为一行数据派生固定长度的稳定回执键。 + * + * @param operationReceiptKey 整次写入的回执键 + * @param rowIndex 行序号 + * @return SHA-256 行回执键 + */ + public static String childKey(String operationReceiptKey, int rowIndex) { + String value = operationReceiptKey + ':' + rowIndex; + try { + return HexFormat.of().formatHex( + MessageDigest.getInstance("SHA-256").digest( + value.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException error) { + throw new IllegalStateException("SHA-256 is unavailable", error); + } + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatasetRef.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatasetRef.java index 8a362102..e334a7d1 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatasetRef.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatasetRef.java @@ -2,7 +2,9 @@ package tech.easyflow.datacenter.execution.model; import java.math.BigInteger; -public class DatasetRef { +public class DatasetRef implements java.io.Serializable { + private static final long serialVersionUID = 1L; + private BigInteger sourceId; private BigInteger catalogId; private String catalogName; diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/DatacenterDatasetQueryService.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/DatacenterDatasetQueryService.java index 62a73740..74766ccd 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/DatacenterDatasetQueryService.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/DatacenterDatasetQueryService.java @@ -8,11 +8,51 @@ import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest; import tech.easyflow.datacenter.execution.model.DatasetRef; import java.util.List; +import java.util.function.Consumer; public interface DatacenterDatasetQueryService { + + /** + * 分页查询结构化数据集。 + * + * @param request 查询请求 + * @return 分页结果 + */ Page queryPage(DatacenterQueryRequest request); + /** + * 执行原生 SQL 并返回完整结果。 + * + * @param request SQL 查询请求 + * @return 完整结果 + */ List queryBySql(DatacenterSqlQueryRequest request); + /** + * 使用单次数据库查询流式消费原生 SQL 结果。 + * + * @param request SQL 查询请求 + * @param fetchSize JDBC 建议拉取行数 + * @param consumer 单行消费者 + */ + void consumeBySql( + DatacenterSqlQueryRequest request, + int fetchSize, + Consumer consumer); + + /** + * 获取数据集结构。 + * + * @param datasetRef 数据集引用 + * @return 数据集结构 + */ DatacenterSchemaResponse getSchema(DatasetRef datasetRef); + + /** + * 仅解析数据集定位信息，不加载版本和血缘。 + * + * @param datasetRef 数据集引用 + * @return 包含 source、catalog、table 的轻量响应 + */ + DatacenterSchemaResponse getLocation(DatasetRef datasetRef); } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/DatacenterDatasetWriteService.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/DatacenterDatasetWriteService.java index 7ad357f5..31edcbc8 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/DatacenterDatasetWriteService.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/DatacenterDatasetWriteService.java @@ -5,9 +5,54 @@ import tech.easyflow.common.entity.LoginAccount; import tech.easyflow.datacenter.execution.model.DatasetRef; import java.math.BigInteger; +import java.util.List; public interface DatacenterDatasetWriteService { + + /** + * 保存单行数据。 + * + * @param datasetRef 数据集引用 + * @param data 待保存数据 + * @param account 当前操作账号 + */ void saveRow(DatasetRef datasetRef, JSONObject data, LoginAccount account); + /** + * 批量保存数据集行。 + * + * @param datasetRef 数据集引用 + * @param rows 待保存数据行 + * @param account 当前操作账号 + * @param batchSize 单批最大行数 + */ + void saveRows(DatasetRef datasetRef, List rows, LoginAccount account, int batchSize); + + /** + * 使用稳定幂等键批量保存数据集行。 + * + * @param datasetRef 数据集引用 + * @param rows 待保存数据行 + * @param account 当前操作账号 + * @param batchSize 单批最大行数 + * @param idempotencyKey 稳定业务幂等键；为空时保持普通写入语义 + * @return 本次实际执行写入时为 {@code true}，已有成功记录时为 {@code false} + */ + default boolean saveRowsIdempotently(DatasetRef datasetRef, + List rows, + LoginAccount account, + int batchSize, + String idempotencyKey) { + saveRows(datasetRef, rows, account, batchSize); + return true; + } + + /** + * 删除单行数据。 + * + * @param datasetRef 数据集引用 + * @param id 数据主键 + * @param account 当前操作账号 + */ void deleteRow(DatasetRef datasetRef, BigInteger id, LoginAccount account); } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImpl.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImpl.java index f25bf3cd..eea5b8a2 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImpl.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImpl.java @@ -30,8 +30,13 @@ import tech.easyflow.datacenter.utils.SqlSupportUtils; import javax.annotation.Resource; import java.math.BigInteger; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Consumer; +import java.util.stream.Collectors; @Service public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQueryService { @@ -73,6 +78,98 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery @Override public List queryBySql(DatacenterSqlQueryRequest request) { + ResolvedSqlQuery query = resolveSqlQuery(request); + return query.connector.queryBySql( + query.source, query.sql); + } + + /** + * {@inheritDoc} + */ + @Override + public void consumeBySql( + DatacenterSqlQueryRequest request, + int fetchSize, + Consumer consumer) { + if (fetchSize <= 0 || consumer == null) { + throw new IllegalArgumentException( + "fetchSize and consumer must be valid"); + } + ResolvedSqlQuery query = resolveSqlQuery(request); + int maxRows = Integer.getInteger( + "easyflow.datacenter.query.max-rows", + 1_000_000); + long maxBytes = Long.getLong( + "easyflow.datacenter.query.max-bytes", + 512L * 1024L * 1024L); + long[] accumulatedRows = {0L}; + long[] accumulatedBytes = {0L}; + query.connector.consumeBySql( + query.source, + query.sql, + fetchSize, + row -> { + accumulatedRows[0]++; + if (maxRows > 0 + && accumulatedRows[0] > maxRows) { + throw new BusinessException( + "数据集查询结果超过行数上限: " + + maxRows); + } + for (Map.Entry entry + : row.entrySet()) { + accumulatedBytes[0] += + estimateQueryValueBytes( + entry.getKey(), + entry.getValue()); + if (maxBytes > 0L + && accumulatedBytes[0] > maxBytes) { + throw new BusinessException( + "数据集查询结果超过字节上限: " + + maxBytes); + } + } + consumer.accept(row); + } + ); + } + + /** + * 估算查询值在 JVM 中的最低占用，用于跨页累计保护。 + * + * @param columnLabel 列名 + * @param value 列值 + * @return 估算字节数 + */ + private long estimateQueryValueBytes( + String columnLabel, + Object value) { + long bytes = columnLabel == null + ? 0L + : (long) columnLabel.length() + * Character.BYTES; + if (value == null) { + return bytes + 8L; + } + if (value instanceof byte[] binary) { + return bytes + binary.length; + } + if (value instanceof CharSequence text) { + return bytes + + (long) text.length() + * Character.BYTES; + } + return bytes + 64L; + } + + /** + * 校验请求并解析实际连接器与可执行 SQL。 + * + * @param request SQL 查询请求 + * @return 已解析查询 + */ + private ResolvedSqlQuery resolveSqlQuery( + DatacenterSqlQueryRequest request) { if (request == null || request.getDatasetRef() == null) { throw new BusinessException("datasetRef 不能为空"); } @@ -90,12 +187,33 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery if (CollectionUtils.isEmpty(managedTables)) { throw new BusinessException("当前连接下没有已接入表"); } + Map catalogsById = + loadCatalogsById(managedTables); SqlSupportUtils.ResolvedSql resolvedSql = SqlSupportUtils.resolve( sql, - managedTables.stream().map(this::toManagedSqlTable).toList() + managedTables.stream() + .map(table -> toManagedSqlTable( + table, catalogsById)) + .toList() ); DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType()); - return connector.queryBySql(source, resolvedSql.getExecutableSql()); + return new ResolvedSqlQuery( + source, + connector, + resolvedSql.getExecutableSql()); + } + + /** + * 一次已校验的 SQL 查询上下文。 + * + * @param source 数据源 + * @param connector 数据连接器 + * @param sql 可执行 SQL + */ + private record ResolvedSqlQuery( + DatacenterSource source, + DatacenterConnector connector, + String sql) { } @Override @@ -115,6 +233,23 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery return response; } + /** + * {@inheritDoc} + */ + @Override + public DatacenterSchemaResponse getLocation(DatasetRef datasetRef) { + DatacenterTable table = resolveTable(datasetRef); + DatacenterSchemaResponse response = + new DatacenterSchemaResponse(); + response.setDatasetRef(datasetRef); + response.setSource( + registryService.getSourceRequired(table.getSourceId())); + response.setCatalog( + registryService.getCatalogById(table.getCatalogId())); + response.setTable(table); + return response; + } + private DatacenterTable resolveTable(DatasetRef datasetRef) { if (datasetRef.getTableId() != null) { return registryService.getTableWithFields(datasetRef.getTableId()); @@ -174,8 +309,13 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery return null; } - private SqlSupportUtils.ManagedTable toManagedSqlTable(DatacenterTable table) { - DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId()); + private SqlSupportUtils.ManagedTable toManagedSqlTable( + DatacenterTable table, + Map catalogsById) { + BigInteger catalogId = table.getCatalogId(); + DatacenterCatalog catalog = catalogId == null + ? null + : catalogsById.get(catalogId); return new SqlSupportUtils.ManagedTable( catalog == null ? null : catalog.getCatalogName(), table.getTableName(), @@ -183,6 +323,32 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery ); } + /** + * 一次批量加载 SQL 白名单表关联的目录，避免逐表查询。 + * + * @param managedTables 已接入表 + * @return 目录 ID 到目录实体 + */ + private Map loadCatalogsById( + List managedTables) { + Set catalogIds = managedTables.stream() + .map(DatacenterTable::getCatalogId) + .filter(java.util.Objects::nonNull) + .collect(Collectors.toCollection( + LinkedHashSet::new)); + if (catalogIds.isEmpty()) { + return Map.of(); + } + QueryWrapper wrapper = QueryWrapper.create(); + wrapper.in(DatacenterCatalog::getId, catalogIds); + return catalogMapper.selectListByQuery(wrapper).stream() + .collect(Collectors.toMap( + DatacenterCatalog::getId, + Function.identity(), + (first, ignored) -> first, + LinkedHashMap::new)); + } + private String resolvePhysicalTableName(DatacenterTable table) { if (table == null) { return null; diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetWriteServiceImpl.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetWriteServiceImpl.java index 46d403f8..3da8ba7c 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetWriteServiceImpl.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetWriteServiceImpl.java @@ -1,7 +1,10 @@ package tech.easyflow.datacenter.execution.service.impl; import com.alibaba.fastjson2.JSONObject; +import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONWriter; import org.springframework.stereotype.Service; +import tech.easyflow.common.cache.RedisIdempotencyExecutor; import tech.easyflow.common.entity.LoginAccount; import tech.easyflow.common.web.exceptions.BusinessException; import tech.easyflow.datacenter.connector.DatacenterConnector; @@ -14,6 +17,11 @@ import tech.easyflow.datacenter.execution.service.DatacenterDatasetWriteService; import javax.annotation.Resource; import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; +import java.util.List; @Service public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWriteService { @@ -22,7 +30,12 @@ public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWrite private DatacenterDatasetRegistryService registryService; @Resource private DatacenterConnectorRegistry connectorRegistry; + @Resource + private RedisIdempotencyExecutor idempotencyExecutor; + /** + * {@inheritDoc} + */ @Override public void saveRow(DatasetRef datasetRef, JSONObject data, LoginAccount account) { DatacenterTable table = resolveTable(datasetRef); @@ -31,6 +44,55 @@ public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWrite connector.saveRow(source, table, data, account); } + /** + * {@inheritDoc} + */ + @Override + public void saveRows(DatasetRef datasetRef, List rows, LoginAccount account, int batchSize) { + if (rows == null || rows.isEmpty()) { + return; + } + DatacenterTable table = resolveTable(datasetRef); + DatacenterSource source = registryService.getSourceRequired(table.getSourceId()); + DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType()); + connector.saveRows(source, table, rows, account, Math.max(1, batchSize)); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean saveRowsIdempotently(DatasetRef datasetRef, + List rows, + LoginAccount account, + int batchSize, + String idempotencyKey) { + if (rows == null || rows.isEmpty()) { + return true; + } + DatacenterTable table = resolveTable(datasetRef); + DatacenterSource source = registryService.getSourceRequired(table.getSourceId()); + DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType()); + String payloadHash = sha256Rows(rows); + if (idempotencyKey == null || idempotencyKey.isBlank()) { + connector.saveRows(source, table, rows, account, Math.max(1, batchSize)); + return true; + } + String receiptKey = sha256(idempotencyKey); + return idempotencyExecutor.executeOnce(idempotencyKey, payloadHash, () -> + connector.saveRowsIdempotently( + source, + table, + rows, + account, + Math.max(1, batchSize), + receiptKey, + payloadHash)); + } + + /** + * {@inheritDoc} + */ @Override public void deleteRow(DatasetRef datasetRef, BigInteger id, LoginAccount account) { DatacenterTable table = resolveTable(datasetRef); @@ -45,4 +107,46 @@ public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWrite } return registryService.getTableWithFields(datasetRef.getTableId()); } + + /** + * 计算稳定的 SHA-256 摘要。 + * + * @param value 原始文本 + * @return 十六进制摘要 + */ + private String sha256(String value) { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(value.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException error) { + throw new IllegalStateException("SHA-256 is unavailable", error); + } + } + + /** + * 逐行计算与排序字段 JSON 数组等价的 SHA-256，避免构造整批字符串副本。 + * + * @param rows 待写入行 + * @return 十六进制摘要 + */ + private String sha256Rows(List rows) { + try { + MessageDigest digest = + MessageDigest.getInstance("SHA-256"); + digest.update((byte) '['); + for (int index = 0; index < rows.size(); index++) { + if (index > 0) { + digest.update((byte) ','); + } + digest.update(JSON.toJSONBytes( + rows.get(index), + JSONWriter.Feature.MapSortField)); + } + digest.update((byte) ']'); + return HexFormat.of().formatHex(digest.digest()); + } catch (NoSuchAlgorithmException error) { + throw new IllegalStateException( + "SHA-256 is unavailable", error); + } + } } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/schedule/DatacenterWriteReceiptCleanupJob.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/schedule/DatacenterWriteReceiptCleanupJob.java new file mode 100644 index 00000000..e66e1304 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/schedule/DatacenterWriteReceiptCleanupJob.java @@ -0,0 +1,88 @@ +package tech.easyflow.datacenter.schedule; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import tech.easyflow.common.cache.DistributedScheduledLock; + +import java.sql.Timestamp; +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +/** + * 定期分批清理过期的数据集写入幂等回执。 + */ +@Component +public class DatacenterWriteReceiptCleanupJob { + + private static final Logger log = + LoggerFactory.getLogger(DatacenterWriteReceiptCleanupJob.class); + private static final String DELETE_SQL = + "DELETE FROM tb_datacenter_write_receipt " + + "WHERE created < ? ORDER BY created LIMIT ?"; + private static final long MIN_RETENTION_DAYS = 7L; + + private final JdbcTemplate jdbcTemplate; + private final long retentionDays; + private final int batchSize; + private final int maxBatches; + + /** + * 创建回执清理任务。 + * + * @param jdbcTemplate JDBC 操作模板 + * @param retentionDays 回执保留天数，最低七天 + * @param batchSize 单批删除行数 + * @param maxBatches 单次调度最多删除批次 + */ + public DatacenterWriteReceiptCleanupJob( + JdbcTemplate jdbcTemplate, + @Value("${easyflow.workflow.data-write-receipt-retention-days:14}") + long retentionDays, + @Value("${easyflow.workflow.data-write-receipt-cleanup-batch-size:1000}") + int batchSize, + @Value("${easyflow.workflow.data-write-receipt-cleanup-max-batches:20}") + int maxBatches) { + this.jdbcTemplate = jdbcTemplate; + this.retentionDays = Math.max(MIN_RETENTION_DAYS, retentionDays); + this.batchSize = Math.max(1, batchSize); + this.maxBatches = Math.max(1, maxBatches); + } + + /** + * 在单个集群节点上删除一批超过保留期的回执。 + */ + @Scheduled( + fixedDelayString = + "${easyflow.workflow.data-write-receipt-cleanup-interval:1h}", + initialDelayString = + "${easyflow.workflow.data-write-receipt-cleanup-initial-delay:10m}") + @DistributedScheduledLock( + key = "easyflow:schedule:datacenter-write-receipt-cleanup", + leaseSeconds = 300L) + public void cleanup() { + Timestamp cutoff = Timestamp.from( + Instant.now().minus(retentionDays, ChronoUnit.DAYS)); + int totalDeleted = 0; + try { + for (int batch = 0; batch < maxBatches; batch++) { + int deleted = jdbcTemplate.update( + DELETE_SQL, cutoff, batchSize); + totalDeleted += deleted; + if (deleted < batchSize) { + break; + } + } + if (totalDeleted > 0) { + log.info( + "Cleaned {} expired datacenter write receipts", + totalDeleted); + } + } catch (RuntimeException error) { + log.error("Datacenter write receipt cleanup failed", error); + } + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImplTest.java b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImplTest.java new file mode 100644 index 00000000..2dabfbea --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImplTest.java @@ -0,0 +1,133 @@ +package tech.easyflow.datacenter.execution.service.impl; + +import com.mybatisflex.core.row.Row; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; +import tech.easyflow.datacenter.connector.DatacenterConnector; +import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest; +import tech.easyflow.datacenter.execution.model.DatasetRef; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService; + +import java.lang.reflect.Field; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; + +/** + * {@link DatacenterDatasetQueryServiceImpl} 分页 SQL 读取回归测试。 + */ +public class DatacenterDatasetQueryServiceImplTest { + + /** + * 验证惰性迭代器逐页读取并保持原始行顺序。 + * + * @throws Exception 测试依赖注入失败时抛出 + */ + @Test + public void consumeBySqlShouldStreamSingleQueryInOrder() + throws Exception { + BigInteger sourceId = BigInteger.valueOf(1001L); + DatacenterSource source = new DatacenterSource(); + source.setId(sourceId); + source.setSourceType("MYSQL"); + DatacenterTable table = new DatacenterTable(); + table.setId(BigInteger.valueOf(2001L)); + table.setSourceId(sourceId); + table.setTableName("orders"); + table.setActualTable("orders_actual"); + + DatacenterDatasetRegistryService registry = + Mockito.mock( + DatacenterDatasetRegistryService.class); + Mockito.when(registry.getSourceRequired(sourceId)) + .thenReturn(source); + Mockito.when(registry.listManagedTables( + sourceId, null)) + .thenReturn(List.of(table)); + DatacenterConnector connector = + Mockito.mock(DatacenterConnector.class); + Mockito.doAnswer(invocation -> { + @SuppressWarnings("unchecked") + Consumer consumer = + invocation.getArgument(3); + consumer.accept(row(1)); + consumer.accept(row(2)); + consumer.accept(row(3)); + return null; + }) + .when(connector) + .consumeBySql( + ArgumentMatchers.eq(source), + ArgumentMatchers.anyString(), + ArgumentMatchers.eq(2), + ArgumentMatchers.any()); + DatacenterConnectorRegistry connectors = + Mockito.mock( + DatacenterConnectorRegistry.class); + Mockito.when(connectors.getConnector("MYSQL")) + .thenReturn(connector); + + DatacenterDatasetQueryServiceImpl service = + new DatacenterDatasetQueryServiceImpl(); + setField(service, "registryService", registry); + setField(service, "connectorRegistry", connectors); + DatacenterSqlQueryRequest request = + new DatacenterSqlQueryRequest(); + DatasetRef datasetRef = new DatasetRef(); + datasetRef.setSourceId(sourceId); + request.setDatasetRef(datasetRef); + request.setSql("SELECT * FROM orders ORDER BY id"); + + List ids = new ArrayList<>(); + service.consumeBySql( + request, + 2, + current -> ids.add( + current.getInt("id"))); + + Assert.assertEquals( + List.of(1, 2, 3), ids); + Mockito.verify(connector, Mockito.times(1)) + .consumeBySql( + ArgumentMatchers.eq(source), + ArgumentMatchers.anyString(), + ArgumentMatchers.eq(2), + ArgumentMatchers.any()); + } + + /** + * 创建测试数据行。 + * + * @param id 行 ID + * @return 数据行 + */ + private Row row(int id) { + Row row = new Row(); + row.put("id", id); + return row; + } + + /** + * 注入被测服务依赖。 + * + * @param target 被测对象 + * @param name 字段名 + * @param value 字段值 + * @throws Exception 反射失败时抛出 + */ + private void setField( + Object target, + String name, + Object value) throws Exception { + Field field = target.getClass() + .getDeclaredField(name); + field.setAccessible(true); + field.set(target, value); + } +} diff --git a/easyflow-modules/easyflow-module-log/src/main/java/tech/easyflow/log/reporter/ActionLogReporterProperties.java b/easyflow-modules/easyflow-module-log/src/main/java/tech/easyflow/log/reporter/ActionLogReporterProperties.java index 3a49fc68..f7992d08 100644 --- a/easyflow-modules/easyflow-module-log/src/main/java/tech/easyflow/log/reporter/ActionLogReporterProperties.java +++ b/easyflow-modules/easyflow-module-log/src/main/java/tech/easyflow/log/reporter/ActionLogReporterProperties.java @@ -6,6 +6,9 @@ import org.springframework.stereotype.Component; import java.util.Arrays; import java.util.List; +/** + * Action 请求与响应日志采集配置。 + */ @Component @ConfigurationProperties(prefix = "easyflow.log.reporter") public class ActionLogReporterProperties { @@ -35,6 +38,7 @@ public class ActionLogReporterProperties { "/css/**", "/images/**", "/favicon.ico", + "/api/v1/agent/media/**", "/actuator/**", "*.js", "*.css", @@ -45,36 +49,75 @@ public class ActionLogReporterProperties { ); - // getter and setter + /** + * 判断 Action 报告是否启用。 + * + * @return 是否启用 + */ public boolean isEnabled() { return enabled; } + /** + * 设置 Action 报告开关。 + * + * @param enabled 是否启用 + */ public void setEnabled(boolean enabled) { this.enabled = enabled; } + /** + * 获取日志采样率。 + * + * @return 采样率 + */ public double getSampleRate() { return sampleRate; } + /** + * 设置日志采样率。 + * + * @param sampleRate 采样率 + */ public void setSampleRate(double sampleRate) { this.sampleRate = sampleRate; } + /** + * 获取需要采集的路径模式。 + * + * @return 包含路径模式 + */ public List getIncludePatterns() { return includePatterns; } + /** + * 设置需要采集的路径模式。 + * + * @param includePatterns 包含路径模式 + */ public void setIncludePatterns(List includePatterns) { this.includePatterns = includePatterns; } + /** + * 获取禁止缓存正文的路径模式。 + * + * @return 排除路径模式 + */ public List getExcludePatterns() { return excludePatterns; } + /** + * 设置禁止缓存正文的路径模式。 + * + * @param excludePatterns 排除路径模式 + */ public void setExcludePatterns(List excludePatterns) { this.excludePatterns = excludePatterns; } -} \ No newline at end of file +} diff --git a/easyflow-modules/easyflow-module-log/src/main/java/tech/easyflow/log/reporter/ResponseCachingFilter.java b/easyflow-modules/easyflow-module-log/src/main/java/tech/easyflow/log/reporter/ResponseCachingFilter.java index 7368b8ea..cef04de0 100644 --- a/easyflow-modules/easyflow-module-log/src/main/java/tech/easyflow/log/reporter/ResponseCachingFilter.java +++ b/easyflow-modules/easyflow-module-log/src/main/java/tech/easyflow/log/reporter/ResponseCachingFilter.java @@ -3,7 +3,6 @@ package tech.easyflow.log.reporter; import jakarta.servlet.*; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; @@ -15,7 +14,7 @@ import java.io.IOException; import static org.springframework.core.Ordered.HIGHEST_PRECEDENCE; /** - * 响应缓存 Filter，支持基于路径的排除规则 + * 缓存需要记录的请求和响应正文，并绕过流式或敏感路径。 */ @Component @Order(HIGHEST_PRECEDENCE) @@ -27,10 +26,20 @@ import static org.springframework.core.Ordered.HIGHEST_PRECEDENCE; ) public class ResponseCachingFilter implements Filter { - @Autowired - private ActionLogReporterProperties logProperties; + private final ActionLogReporterProperties logProperties; + /** + * 创建响应缓存过滤器。 + * + * @param logProperties 日志采集路径配置 + */ + public ResponseCachingFilter(ActionLogReporterProperties logProperties) { + this.logProperties = logProperties; + } + /** + * {@inheritDoc} + */ @Override public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { @@ -39,25 +48,19 @@ public class ResponseCachingFilter implements Filter { String uri = httpRequest.getRequestURI(); String method = httpRequest.getMethod(); - // 1如果是 OPTIONS 请求，跳过（通常为预检） + // OPTIONS 请求通常是预检，不需要缓存正文。 if ("OPTIONS".equalsIgnoreCase(method)) { chain.doFilter(request, response); return; } -// // 检查是否为 SSE 请求 -// if (isSseRequest(httpRequest)) { -// chain.doFilter(request, response); -// return; -// } - - // 检查是否匹配排除路径 + // 流式下载和敏感媒体路径必须在包装响应前排除，避免截断异步响应或缓存文件正文。 if (isExcluded(uri)) { chain.doFilter(request, response); return; } - // 检查是否匹配包含路径（一般为 /**，可省略） + // 检查是否匹配包含路径（一般为 /**）。 if (!isIncluded(uri)) { chain.doFilter(request, response); return; @@ -65,31 +68,45 @@ public class ResponseCachingFilter implements Filter { ContentCachingRequestWrapper requestWrapper = new ContentCachingRequestWrapper(httpRequest); if (isSseRequest(httpRequest)) { - // SSE 请求不缓存 + // SSE 响应不能经过响应缓存，否则会破坏实时输出。 chain.doFilter(requestWrapper, response); return; } - HttpServletResponse httpResponse = (HttpServletResponse) response; ContentCachingResponseWrapper responseWrapper = new ContentCachingResponseWrapper(httpResponse); try { chain.doFilter(requestWrapper, responseWrapper); } finally { - responseWrapper.copyBodyToResponse(); // 必须调用 + responseWrapper.copyBodyToResponse(); } } + /** + * 判断请求路径是否禁止缓存。 + * + * @param uri 请求路径 + * @return 是否排除 + */ private boolean isExcluded(String uri) { return logProperties.getExcludePatterns().stream().anyMatch(p -> match(uri, p)); } + /** + * 判断请求路径是否需要采集。 + * + * @param uri 请求路径 + * @return 是否包含 + */ private boolean isIncluded(String uri) { return logProperties.getIncludePatterns().stream().anyMatch(p -> match(uri, p)); } /** - * 判断是否为 SSE 请求（基于标准 Accept 头） + * 根据标准 Accept 请求头判断是否为 SSE 请求。 + * + * @param request HTTP 请求 + * @return 是否为 SSE 请求 */ private boolean isSseRequest(HttpServletRequest request) { String accept = request.getHeader("Accept"); @@ -97,8 +114,11 @@ public class ResponseCachingFilter implements Filter { } /** - * 简单的路径匹配（支持 * 和 **） - * 注意：这里简化实现，生产可替换为 AntPathMatcher + * 匹配支持星号和双星号的简单路径模式。 + * + * @param path 请求路径 + * @param pattern 路径模式 + * @return 是否匹配 */ private boolean match(String path, String pattern) { if (pattern.equals("/**")) { @@ -119,4 +139,4 @@ public class ResponseCachingFilter implements Filter { } return path.equals(pattern); } -} \ No newline at end of file +} diff --git a/easyflow-modules/easyflow-module-log/src/test/java/tech/easyflow/log/reporter/ResponseCachingFilterTest.java b/easyflow-modules/easyflow-module-log/src/test/java/tech/easyflow/log/reporter/ResponseCachingFilterTest.java new file mode 100644 index 00000000..36546454 --- /dev/null +++ b/easyflow-modules/easyflow-module-log/src/test/java/tech/easyflow/log/reporter/ResponseCachingFilterTest.java @@ -0,0 +1,35 @@ +package tech.easyflow.log.reporter; + +import jakarta.servlet.FilterChain; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.junit.Test; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * {@link ResponseCachingFilter} 路径排除行为测试。 + */ +public class ResponseCachingFilterTest { + + /** + * 验证 Agent 媒体下载保持原始响应，避免缓存包装器截断异步文件流。 + * + * @throws Exception Filter 执行失败 + */ + @Test + public void agentMediaDownloadShouldBypassResponseCaching() throws Exception { + HttpServletRequest request = mock(HttpServletRequest.class); + HttpServletResponse response = mock(HttpServletResponse.class); + FilterChain chain = mock(FilterChain.class); + when(request.getMethod()).thenReturn("GET"); + when(request.getRequestURI()).thenReturn("/api/v1/agent/media/document/content"); + + ResponseCachingFilter filter = new ResponseCachingFilter(new ActionLogReporterProperties()); + filter.doFilter(request, response, chain); + + verify(chain).doFilter(request, response); + } +} diff --git a/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V35__mysql_datacenter_write_receipt.sql b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V35__mysql_datacenter_write_receipt.sql new file mode 100644 index 00000000..19fa7174 --- /dev/null +++ b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V35__mysql_datacenter_write_receipt.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS `tb_datacenter_write_receipt` +( + `idempotency_key` VARCHAR(64) NOT NULL COMMENT '稳定幂等键摘要', + `payload_hash` VARCHAR(64) NOT NULL COMMENT '写入负载摘要', + `created` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + PRIMARY KEY (`idempotency_key`), + KEY `idx_datacenter_write_receipt_created` (`created`) +) ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4 + COLLATE = utf8mb4_general_ci + COMMENT = '数据中心工作流写入幂等回执'; diff --git a/easyflow-starter/easyflow-starter-all/src/main/resources/logback-spring.xml b/easyflow-starter/easyflow-starter-all/src/main/resources/logback-spring.xml index b3720f14..a5241fc4 100644 --- a/easyflow-starter/easyflow-starter-all/src/main/resources/logback-spring.xml +++ b/easyflow-starter/easyflow-starter-all/src/main/resources/logback-spring.xml @@ -36,16 +36,33 @@ 30 - %d{MM-dd HH:mm:ss.SSS} |-%-5level %logger{36}:%L - %m%n + %d{MM-dd HH:mm:ss.SSS} |-%-5level %logger{36} - %m%n + + + 8192 + 1638 + false + 0 + false + + + + 8192 + 1638 + false + 0 + false + + + - - + + - - - - - \ No newline at end of file + diff --git a/easyflow-ui-admin/packages/tinyflow-ui/src/components/core/NodeWrapper.svelte b/easyflow-ui-admin/packages/tinyflow-ui/src/components/core/NodeWrapper.svelte index 45da85a2..c73185b5 100644 --- a/easyflow-ui-admin/packages/tinyflow-ui/src/components/core/NodeWrapper.svelte +++ b/easyflow-ui-admin/packages/tinyflow-ui/src/components/core/NodeWrapper.svelte @@ -84,6 +84,53 @@ let currentNodeId = getCurrentNodeId(); let wrapperElement: HTMLDivElement | null = null; const nodeSizeObserver = useTinyflowNodeSizeObserver(); + const MIN_LOOP_COUNT = 1; + const MAX_LOOP_COUNT = 300; + let loopCountHint = $state(''); + + const normalizeLoopCount = (value: unknown) => { + const parsed = Number(value); + if (!Number.isFinite(parsed) || !Number.isInteger(parsed) || parsed < MIN_LOOP_COUNT) { + return MIN_LOOP_COUNT; + } + return Math.min(parsed, MAX_LOOP_COUNT); + }; + + const updateLoopEnabled = (event: Event) => { + const loopEnable = (event.target as HTMLInputElement).checked; + updateNodeData(currentNodeId, { + loopEnable, + ...(loopEnable ? {maxLoopCount: normalizeLoopCount(data.maxLoopCount)} : {}) + }); + }; + + const updateMaxLoopCount = (event: Event) => { + const input = event.target as HTMLInputElement; + const parsed = Number(input.value); + const normalized = normalizeLoopCount(input.value); + if (Number.isFinite(parsed) && parsed > MAX_LOOP_COUNT) { + loopCountHint = `最大支持 ${MAX_LOOP_COUNT} 次，已调整为 ${MAX_LOOP_COUNT}`; + } else if (!Number.isInteger(parsed) || parsed < MIN_LOOP_COUNT) { + loopCountHint = `请输入 ${MIN_LOOP_COUNT}～${MAX_LOOP_COUNT} 的整数`; + } else { + loopCountHint = ''; + } + updateNodeData(currentNodeId, {maxLoopCount: normalized}); + }; + + $effect(() => { + if (!data.loopEnable) { + loopCountHint = ''; + return; + } + const normalized = normalizeLoopCount(data.maxLoopCount); + if (Number(data.maxLoopCount) !== normalized) { + loopCountHint = Number(data.maxLoopCount) > MAX_LOOP_COUNT + ? `最大支持 ${MAX_LOOP_COUNT} 次，已调整为 ${MAX_LOOP_COUNT}` + : `请输入 ${MIN_LOOP_COUNT}～${MAX_LOOP_COUNT} 的整数`; + updateNodeData(currentNodeId, {maxLoopCount: normalized}); + } + }); onMount(() => { if (!wrapperElement) { @@ -187,12 +234,7 @@ {#if !!data.loopEnable} @@ -207,13 +249,19 @@
- 最大循环次数（0 表示不限制）： -